Export libtextclassifier
Test: atest TextClassifierService
Change-Id: I99b1b4e492f004f542914a3d022d048765260960
diff --git a/java/src/com/android/textclassifier/ActionsSuggestionsHelper.java b/java/src/com/android/textclassifier/ActionsSuggestionsHelper.java
index 0acca43..ca348bf 100644
--- a/java/src/com/android/textclassifier/ActionsSuggestionsHelper.java
+++ b/java/src/com/android/textclassifier/ActionsSuggestionsHelper.java
@@ -29,9 +29,9 @@
import android.view.textclassifier.ConversationActions.Message;
import com.android.textclassifier.ModelFileManager.ModelFile;
import com.android.textclassifier.common.base.TcLog;
+import com.android.textclassifier.common.intent.LabeledIntent;
+import com.android.textclassifier.common.intent.TemplateIntentFactory;
import com.android.textclassifier.common.statsd.ResultIdUtils;
-import com.android.textclassifier.intent.LabeledIntent;
-import com.android.textclassifier.intent.TemplateIntentFactory;
import com.google.android.textclassifier.ActionsSuggestionsModel;
import com.google.android.textclassifier.RemoteActionTemplate;
import com.google.common.base.Equivalence;
diff --git a/java/src/com/android/textclassifier/TextClassifierImpl.java b/java/src/com/android/textclassifier/TextClassifierImpl.java
index 5e7f044..4d0b7cc 100644
--- a/java/src/com/android/textclassifier/TextClassifierImpl.java
+++ b/java/src/com/android/textclassifier/TextClassifierImpl.java
@@ -42,6 +42,8 @@
import androidx.collection.ArraySet;
import androidx.core.util.Pair;
import com.android.textclassifier.common.base.TcLog;
+import com.android.textclassifier.common.intent.LabeledIntent;
+import com.android.textclassifier.common.intent.TemplateIntentFactory;
import com.android.textclassifier.common.statsd.GenerateLinksLogger;
import com.android.textclassifier.common.statsd.ResultIdUtils;
import com.android.textclassifier.common.statsd.SelectionEventConverter;
@@ -49,10 +51,8 @@
import com.android.textclassifier.common.statsd.TextClassifierEventConverter;
import com.android.textclassifier.common.statsd.TextClassifierEventLogger;
import com.android.textclassifier.intent.ClassificationIntentFactory;
-import com.android.textclassifier.intent.LabeledIntent;
import com.android.textclassifier.intent.LegacyClassificationIntentFactory;
import com.android.textclassifier.intent.TemplateClassificationIntentFactory;
-import com.android.textclassifier.intent.TemplateIntentFactory;
import com.android.textclassifier.utils.IndentingPrintWriter;
import com.google.android.textclassifier.ActionsSuggestionsModel;
import com.google.android.textclassifier.AnnotatorModel;
@@ -471,7 +471,7 @@
RemoteAction remoteAction = null;
Bundle extras = new Bundle();
if (labeledIntentResult != null) {
- remoteAction = labeledIntentResult.remoteAction;
+ remoteAction = labeledIntentResult.remoteAction.toRemoteAction();
ExtrasUtils.putActionIntent(extras, labeledIntentResult.resolvedIntent);
}
ExtrasUtils.putSerializedEntityData(extras, nativeSuggestion.getSerializedEntityData());
@@ -641,7 +641,7 @@
final Bundle foreignLanguageBundle = languagesBundles.second;
boolean isPrimaryAction = true;
- final List<LabeledIntent> labeledIntents =
+ final ImmutableList<LabeledIntent> labeledIntents =
classificationIntentFactory.create(
context,
classifiedText,
@@ -660,7 +660,7 @@
}
final Intent intent = result.resolvedIntent;
- final RemoteAction action = result.remoteAction;
+ final RemoteAction action = result.remoteAction.toRemoteAction();
if (isPrimaryAction) {
// For O backwards compatibility, the first RemoteAction is also written to the
// legacy API fields.
diff --git a/java/src/com/android/textclassifier/intent/LabeledIntent.java b/java/src/com/android/textclassifier/common/intent/LabeledIntent.java
similarity index 74%
rename from java/src/com/android/textclassifier/intent/LabeledIntent.java
rename to java/src/com/android/textclassifier/common/intent/LabeledIntent.java
index e3b001b..d6041b9 100644
--- a/java/src/com/android/textclassifier/intent/LabeledIntent.java
+++ b/java/src/com/android/textclassifier/common/intent/LabeledIntent.java
@@ -14,30 +14,27 @@
* limitations under the License.
*/
-package com.android.textclassifier.intent;
+package com.android.textclassifier.common.intent;
import android.app.PendingIntent;
-import android.app.RemoteAction;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
+import android.content.pm.ActivityInfo;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
-import android.graphics.drawable.Icon;
import android.os.Bundle;
import android.text.TextUtils;
import android.view.textclassifier.TextClassifier;
-import com.android.textclassifier.ExtrasUtils;
-import com.android.textclassifier.R;
+import androidx.annotation.DrawableRes;
+import androidx.core.app.RemoteActionCompat;
+import androidx.core.content.ContextCompat;
+import androidx.core.graphics.drawable.IconCompat;
import com.android.textclassifier.common.base.TcLog;
import com.google.common.base.Preconditions;
import javax.annotation.Nullable;
-/**
- * Helper class to store the information from which RemoteActions are built.
- *
- * @hide
- */
+/** Helper class to store the information from which RemoteActions are built. */
public final class LabeledIntent {
private static final String TAG = "LabeledIntent";
public static final int DEFAULT_REQUEST_CODE = 0;
@@ -104,6 +101,11 @@
TcLog.w(TAG, "resolveInfo or activityInfo is null");
return null;
}
+ if (!hasPermission(context, resolveInfo.activityInfo)) {
+ TcLog.d(TAG, "No permission to access: " + resolveInfo.activityInfo);
+ return null;
+ }
+
final String packageName = resolveInfo.activityInfo.packageName;
final String className = resolveInfo.activityInfo.name;
if (packageName == null || className == null) {
@@ -112,22 +114,24 @@
}
Intent resolvedIntent = new Intent(intent);
resolvedIntent.putExtra(
- TextClassifier.EXTRA_FROM_TEXT_CLASSIFIER, getFromTextClassifierExtra(textLanguagesBundle));
+ TextClassifier.EXTRA_FROM_TEXT_CLASSIFIER,
+ createFromTextClassifierExtra(textLanguagesBundle));
boolean shouldShowIcon = false;
- Icon icon = null;
+ IconCompat icon = null;
if (!"android".equals(packageName)) {
// We only set the component name when the package name is not resolved to "android"
// to workaround a bug that explicit intent with component name == ResolverActivity
// can't be launched on keyguard.
resolvedIntent.setComponent(new ComponentName(packageName, className));
if (resolveInfo.activityInfo.getIconResource() != 0) {
- icon = Icon.createWithResource(packageName, resolveInfo.activityInfo.getIconResource());
+ icon =
+ createIconFromPackage(context, packageName, resolveInfo.activityInfo.getIconResource());
shouldShowIcon = true;
}
}
if (icon == null) {
// RemoteAction requires that there be an icon.
- icon = Icon.createWithResource(context, R.drawable.tcs_app_icon);
+ icon = IconCompat.createWithResource(context, android.R.drawable.ic_menu_more);
}
final PendingIntent pendingIntent = createPendingIntent(context, resolvedIntent, requestCode);
titleChooser = titleChooser == null ? DEFAULT_TITLE_CHOOSER : titleChooser;
@@ -136,8 +140,8 @@
TcLog.w(TAG, "Custom titleChooser return null, fallback to the default titleChooser");
title = DEFAULT_TITLE_CHOOSER.chooseTitle(this, resolveInfo);
}
- final RemoteAction action =
- new RemoteAction(icon, title, resolveDescription(resolveInfo, pm), pendingIntent);
+ final RemoteActionCompat action =
+ new RemoteActionCompat(icon, title, resolveDescription(resolveInfo, pm), pendingIntent);
action.setShouldShowIcon(shouldShowIcon);
return new Result(resolvedIntent, action);
}
@@ -153,6 +157,29 @@
return description;
}
+ // TODO(b/149018167) Remove this once we have moved this to C++.
+ private static void putTextLanguagesExtra(Bundle container, Bundle extra) {
+ container.putBundle("text-languages", extra);
+ }
+
+ @Nullable
+ private static IconCompat createIconFromPackage(
+ Context context, String packageName, @DrawableRes int iconRes) {
+ try {
+ Context packageContext = context.createPackageContext(packageName, 0);
+ return IconCompat.createWithResource(packageContext, iconRes);
+ } catch (PackageManager.NameNotFoundException e) {
+ TcLog.e(TAG, "createIconFromPackage: failed to create package context", e);
+ }
+ return null;
+ }
+
+ private static PendingIntent createPendingIntent(
+ final Context context, final Intent intent, int requestCode) {
+ return PendingIntent.getActivity(
+ context, requestCode, intent, PendingIntent.FLAG_UPDATE_CURRENT);
+ }
+
@Nullable
private static String getApplicationName(ResolveInfo resolveInfo, PackageManager packageManager) {
if (resolveInfo.activityInfo == null) {
@@ -164,31 +191,36 @@
if (resolveInfo.activityInfo.applicationInfo == null) {
return null;
}
- return (String) packageManager.getApplicationLabel(resolveInfo.activityInfo.applicationInfo);
+ return packageManager.getApplicationLabel(resolveInfo.activityInfo.applicationInfo).toString();
}
- private static Bundle getFromTextClassifierExtra(@Nullable Bundle textLanguagesBundle) {
- if (textLanguagesBundle != null) {
- final Bundle bundle = new Bundle();
- ExtrasUtils.putTextLanguagesExtra(bundle, textLanguagesBundle);
- return bundle;
- } else {
+ private static Bundle createFromTextClassifierExtra(@Nullable Bundle textLanguagesBundle) {
+ if (textLanguagesBundle == null) {
return Bundle.EMPTY;
+ } else {
+ Bundle bundle = new Bundle();
+ putTextLanguagesExtra(bundle, textLanguagesBundle);
+ return bundle;
}
}
- private static PendingIntent createPendingIntent(
- final Context context, final Intent intent, int requestCode) {
- return PendingIntent.getActivity(
- context, requestCode, intent, PendingIntent.FLAG_UPDATE_CURRENT);
+ private static boolean hasPermission(Context context, ActivityInfo info) {
+ if (!info.exported) {
+ return false;
+ }
+ if (info.permission == null) {
+ return true;
+ }
+ return ContextCompat.checkSelfPermission(context, info.permission)
+ == PackageManager.PERMISSION_GRANTED;
}
/** Data class that holds the result. */
public static final class Result {
public final Intent resolvedIntent;
- public final RemoteAction remoteAction;
+ public final RemoteActionCompat remoteAction;
- public Result(Intent resolvedIntent, RemoteAction remoteAction) {
+ public Result(Intent resolvedIntent, RemoteActionCompat remoteAction) {
this.resolvedIntent = Preconditions.checkNotNull(resolvedIntent);
this.remoteAction = Preconditions.checkNotNull(remoteAction);
}
diff --git a/java/src/com/android/textclassifier/intent/TemplateIntentFactory.java b/java/src/com/android/textclassifier/common/intent/TemplateIntentFactory.java
similarity index 83%
rename from java/src/com/android/textclassifier/intent/TemplateIntentFactory.java
rename to java/src/com/android/textclassifier/common/intent/TemplateIntentFactory.java
index d74c276..ec95e4e 100644
--- a/java/src/com/android/textclassifier/intent/TemplateIntentFactory.java
+++ b/java/src/com/android/textclassifier/common/intent/TemplateIntentFactory.java
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.android.textclassifier.intent;
+package com.android.textclassifier.common.intent;
import android.content.Intent;
import android.net.Uri;
@@ -23,25 +23,19 @@
import com.android.textclassifier.common.base.TcLog;
import com.google.android.textclassifier.NamedVariant;
import com.google.android.textclassifier.RemoteActionTemplate;
-import java.util.ArrayList;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
import javax.annotation.Nullable;
-/**
- * Creates intents based on {@link RemoteActionTemplate} objects.
- *
- * @hide
- */
+/** Creates intents based on {@link RemoteActionTemplate} objects. */
public final class TemplateIntentFactory {
private static final String TAG = "TemplateIntentFactory";
/** Constructs and returns a list of {@link LabeledIntent} based on the given templates. */
- @Nullable
- public List<LabeledIntent> create(RemoteActionTemplate[] remoteActionTemplates) {
+ public ImmutableList<LabeledIntent> create(RemoteActionTemplate[] remoteActionTemplates) {
if (remoteActionTemplates.length == 0) {
- return new ArrayList<>();
+ return ImmutableList.of();
}
- final List<LabeledIntent> labeledIntents = new ArrayList<>();
+ final ImmutableList.Builder<LabeledIntent> labeledIntents = ImmutableList.builder();
for (RemoteActionTemplate remoteActionTemplate : remoteActionTemplates) {
if (!isValidTemplate(remoteActionTemplate)) {
TcLog.w(TAG, "Invalid RemoteActionTemplate skipped.");
@@ -58,7 +52,7 @@
? LabeledIntent.DEFAULT_REQUEST_CODE
: remoteActionTemplate.requestCode));
}
- return labeledIntents;
+ return labeledIntents.build();
}
private static boolean isValidTemplate(@Nullable RemoteActionTemplate remoteActionTemplate) {
@@ -98,6 +92,9 @@
: Intent.normalizeMimeType(remoteActionTemplate.type);
intent.setDataAndType(uri, type);
intent.setFlags(remoteActionTemplate.flags == null ? 0 : remoteActionTemplate.flags);
+ if (!TextUtils.isEmpty(remoteActionTemplate.packageName)) {
+ intent.setPackage(remoteActionTemplate.packageName);
+ }
if (remoteActionTemplate.category != null) {
for (String category : remoteActionTemplate.category) {
if (category != null) {
@@ -138,6 +135,16 @@
case NamedVariant.TYPE_STRING:
bundle.putString(namedVariant.getName(), namedVariant.getString());
break;
+ case NamedVariant.TYPE_STRING_ARRAY:
+ bundle.putStringArray(namedVariant.getName(), namedVariant.getStringArray());
+ break;
+ case NamedVariant.TYPE_FLOAT_ARRAY:
+ bundle.putFloatArray(namedVariant.getName(), namedVariant.getFloatArray());
+ break;
+ case NamedVariant.TYPE_INT_ARRAY:
+ bundle.putIntArray(namedVariant.getName(), namedVariant.getIntArray());
+ break;
+
default:
TcLog.w(
TAG, "Unsupported type found in nameVariantsToBundle : " + namedVariant.getType());
diff --git a/java/src/com/android/textclassifier/common/statsd/GenerateLinksLogger.java b/java/src/com/android/textclassifier/common/statsd/GenerateLinksLogger.java
index 7db5003..80321f7 100644
--- a/java/src/com/android/textclassifier/common/statsd/GenerateLinksLogger.java
+++ b/java/src/com/android/textclassifier/common/statsd/GenerateLinksLogger.java
@@ -16,10 +16,11 @@
package com.android.textclassifier.common.statsd;
+import android.util.StatsEvent;
+import android.util.StatsLog;
import android.view.textclassifier.TextClassifier;
import android.view.textclassifier.TextLinks;
import androidx.collection.ArrayMap;
-import com.android.textclassifier.TextClassifierStatsLog;
import com.android.textclassifier.common.base.TcLog;
import com.android.textclassifier.common.logging.TextClassifierEvent;
import com.google.common.annotations.VisibleForTesting;
@@ -127,19 +128,24 @@
LinkifyStats stats,
CharSequence text,
long latencyMs) {
- TextClassifierStatsLog.write(
- TextClassifierStatsLog.TEXT_LINKIFY_EVENT,
- callId,
- TextClassifierEvent.TYPE_LINKS_GENERATED,
- /*modelName=*/ null,
- TextClassifierEventLogger.WidgetType.WIDGET_TYPE_UNKNOWN,
- /*eventIndex=*/ 0,
- entityType,
- stats.numLinks,
- stats.numLinksTextLength,
- text.length(),
- latencyMs,
- callingPackageName);
+ StatsEvent statsEvent =
+ StatsEvent.newBuilder()
+ .setAtomId(TextClassifierEventLogger.TEXT_LINKIFY_EVENT_ATOM_ID)
+ .writeString(callId)
+ .writeInt(TextClassifierEvent.TYPE_LINKS_GENERATED)
+ .writeString(/* modelName */ null)
+ .writeInt(TextClassifierEventLogger.WidgetType.WIDGET_TYPE_UNKNOWN)
+ .writeInt(/* eventIndex */ 0)
+ .writeString(entityType)
+ .writeInt(stats.numLinks)
+ .writeInt(stats.numLinksTextLength)
+ .writeInt(text.length())
+ .writeLong(latencyMs)
+ .writeString(callingPackageName)
+ .usePooledBuffer()
+ .build();
+ StatsLog.write(statsEvent);
+
if (TcLog.ENABLE_FULL_LOGGING) {
TcLog.v(
LOG_TAG,
diff --git a/java/src/com/android/textclassifier/common/statsd/TextClassifierEventLogger.java b/java/src/com/android/textclassifier/common/statsd/TextClassifierEventLogger.java
index f42ea82..64426ef 100644
--- a/java/src/com/android/textclassifier/common/statsd/TextClassifierEventLogger.java
+++ b/java/src/com/android/textclassifier/common/statsd/TextClassifierEventLogger.java
@@ -16,8 +16,9 @@
package com.android.textclassifier.common.statsd;
+import android.util.StatsEvent;
+import android.util.StatsLog;
import android.view.textclassifier.TextClassifier;
-import com.android.textclassifier.TextClassifierStatsLog;
import com.android.textclassifier.common.base.TcLog;
import com.android.textclassifier.common.logging.TextClassificationContext;
import com.android.textclassifier.common.logging.TextClassificationSessionId;
@@ -29,8 +30,12 @@
/** Logs {@link android.view.textclassifier.TextClassifierEvent}. */
public final class TextClassifierEventLogger {
-
private static final String TAG = "TCEventLogger";
+ // These constants are defined in atoms.proto.
+ private static final int TEXT_SELECTION_EVENT_ATOM_ID = 219;
+ static final int TEXT_LINKIFY_EVENT_ATOM_ID = 220;
+ private static final int CONVERSATION_ACTIONS_EVENT_ATOM_ID = 221;
+ private static final int LANGUAGE_DETECTION_EVENT_ATOM_ID = 222;
/** Emits a text classifier event to the logs. */
public void writeEvent(
@@ -58,71 +63,89 @@
private static void logTextSelectionEvent(
@Nullable TextClassificationSessionId sessionId,
TextClassifierEvent.TextSelectionEvent event) {
- TextClassifierStatsLog.write(
- TextClassifierStatsLog.TEXT_SELECTION_EVENT,
- sessionId == null ? null : sessionId.flattenToString(),
- event.getEventType(),
- getModelName(event),
- getWidgetType(event),
- event.getEventIndex(),
- getItemAt(event.getEntityTypes(), /* index= */ 0),
- event.getRelativeWordStartIndex(),
- event.getRelativeWordEndIndex(),
- event.getRelativeSuggestedWordStartIndex(),
- event.getRelativeSuggestedWordEndIndex(),
- getPackageName(event));
+ StatsEvent statsEvent =
+ StatsEvent.newBuilder()
+ .setAtomId(TEXT_SELECTION_EVENT_ATOM_ID)
+ .writeString(sessionId == null ? null : sessionId.flattenToString())
+ .writeInt(event.getEventType())
+ .writeString(getModelName(event))
+ .writeInt(getWidgetType(event))
+ .writeInt(event.getEventIndex())
+ .writeString(getItemAt(event.getEntityTypes(), /* index= */ 0))
+ .writeInt(event.getRelativeWordStartIndex())
+ .writeInt(event.getRelativeWordEndIndex())
+ .writeInt(event.getRelativeSuggestedWordStartIndex())
+ .writeInt(event.getRelativeSuggestedWordEndIndex())
+ .writeString(getPackageName(event))
+ .usePooledBuffer()
+ .build();
+ StatsLog.write(statsEvent);
}
private static void logTextLinkifyEvent(
TextClassificationSessionId sessionId, TextClassifierEvent.TextLinkifyEvent event) {
- TextClassifierStatsLog.write(
- TextClassifierStatsLog.TEXT_LINKIFY_EVENT,
- sessionId == null ? null : sessionId.flattenToString(),
- event.getEventType(),
- getModelName(event),
- getWidgetType(event),
- event.getEventIndex(),
- getItemAt(event.getEntityTypes(), /* index= */ 0),
- /*numOfLinks=*/ 0,
- /*linkedTextLength=*/ 0,
- /*textLength=*/ 0,
- /*latencyInMillis=*/ 0L,
- getPackageName(event));
+ StatsEvent statsEvent =
+ StatsEvent.newBuilder()
+ .setAtomId(TEXT_LINKIFY_EVENT_ATOM_ID)
+ .writeString(sessionId == null ? null : sessionId.flattenToString())
+ .writeInt(event.getEventType())
+ .writeString(getModelName(event))
+ .writeInt(getWidgetType(event))
+ .writeInt(event.getEventIndex())
+ .writeString(getItemAt(event.getEntityTypes(), /* index= */ 0))
+ .writeInt(/* numOfLinks */ 0)
+ .writeInt(/* linkedTextLength */ 0)
+ .writeInt(/* textLength */ 0)
+ .writeLong(/* latencyInMillis */ 0L)
+ .writeString(getPackageName(event))
+ .usePooledBuffer()
+ .build();
+ StatsLog.write(statsEvent);
}
private static void logConversationActionsEvent(
@Nullable TextClassificationSessionId sessionId,
TextClassifierEvent.ConversationActionsEvent event) {
ImmutableList<String> modelNames = ResultIdUtils.getModelNames(event.getResultId());
- TextClassifierStatsLog.write(
- TextClassifierStatsLog.CONVERSATION_ACTIONS_EVENT,
- sessionId == null
- ? event.getResultId() // TODO: Update ExtServices to set the session id.
- : sessionId.flattenToString(),
- event.getEventType(),
- getItemAt(modelNames, 0, null),
- getWidgetType(event),
- getItemAt(event.getEntityTypes(), /* index= */ 0),
- getItemAt(event.getEntityTypes(), /* index= */ 1),
- getItemAt(event.getEntityTypes(), /* index= */ 2),
- getFloatAt(event.getScores(), /* index= */ 0),
- getPackageName(event),
- getItemAt(modelNames, 1, null));
+
+ StatsEvent statsEvent =
+ StatsEvent.newBuilder()
+ .setAtomId(CONVERSATION_ACTIONS_EVENT_ATOM_ID)
+ .writeString(
+ sessionId == null
+ ? event.getResultId() // TODO: Update ExtServices to set the session id.
+ : sessionId.flattenToString())
+ .writeInt(event.getEventType())
+ .writeString(getItemAt(modelNames, 0, null))
+ .writeInt(getWidgetType(event))
+ .writeString(getItemAt(event.getEntityTypes(), /* index= */ 0))
+ .writeString(getItemAt(event.getEntityTypes(), /* index= */ 1))
+ .writeString(getItemAt(event.getEntityTypes(), /* index= */ 2))
+ .writeFloat(getFloatAt(event.getScores(), /* index= */ 0))
+ .writeString(getPackageName(event))
+ .writeString(getItemAt(modelNames, 1, null))
+ .usePooledBuffer()
+ .build();
+ StatsLog.write(statsEvent);
}
private static void logLanguageDetectionEvent(
@Nullable TextClassificationSessionId sessionId,
TextClassifierEvent.LanguageDetectionEvent event) {
- TextClassifierStatsLog.write(
- TextClassifierStatsLog.LANGUAGE_DETECTION_EVENT,
- sessionId == null ? null : sessionId.flattenToString(),
- event.getEventType(),
- getModelName(event),
- getWidgetType(event),
- getItemAt(event.getEntityTypes(), /* index= */ 0),
- getFloatAt(event.getScores(), /* index= */ 0),
- getIntAt(event.getActionIndices(), /* index= */ 0),
- getPackageName(event));
+ StatsEvent statsEvent =
+ StatsEvent.newBuilder()
+ .setAtomId(LANGUAGE_DETECTION_EVENT_ATOM_ID)
+ .writeString(sessionId == null ? null : sessionId.flattenToString())
+ .writeInt(event.getEventType())
+ .writeString(getModelName(event))
+ .writeInt(getWidgetType(event))
+ .writeString(getItemAt(event.getEntityTypes(), /* index= */ 0))
+ .writeFloat(getFloatAt(event.getScores(), /* index= */ 0))
+ .writeInt(getIntAt(event.getActionIndices(), /* index= */ 0))
+ .writeString(getPackageName(event))
+ .usePooledBuffer()
+ .build();
+ StatsLog.write(statsEvent);
}
@Nullable
diff --git a/java/src/com/android/textclassifier/intent/ClassificationIntentFactory.java b/java/src/com/android/textclassifier/intent/ClassificationIntentFactory.java
index 450ffc4..cb38c95 100644
--- a/java/src/com/android/textclassifier/intent/ClassificationIntentFactory.java
+++ b/java/src/com/android/textclassifier/intent/ClassificationIntentFactory.java
@@ -19,7 +19,9 @@
import android.content.Context;
import android.content.Intent;
import com.android.textclassifier.R;
+import com.android.textclassifier.common.intent.LabeledIntent;
import com.google.android.textclassifier.AnnotatorModel;
+import com.google.common.collect.ImmutableList;
import java.time.Instant;
import java.util.List;
import javax.annotation.Nullable;
@@ -28,7 +30,7 @@
public interface ClassificationIntentFactory {
/** Return a list of LabeledIntent from the classification result. */
- List<LabeledIntent> create(
+ ImmutableList<LabeledIntent> create(
Context context,
String text,
boolean foreignText,
diff --git a/java/src/com/android/textclassifier/intent/LegacyClassificationIntentFactory.java b/java/src/com/android/textclassifier/intent/LegacyClassificationIntentFactory.java
index 74aa099..f1659dc 100644
--- a/java/src/com/android/textclassifier/intent/LegacyClassificationIntentFactory.java
+++ b/java/src/com/android/textclassifier/intent/LegacyClassificationIntentFactory.java
@@ -31,7 +31,9 @@
import android.view.textclassifier.TextClassifier;
import com.android.textclassifier.R;
import com.android.textclassifier.common.base.TcLog;
+import com.android.textclassifier.common.intent.LabeledIntent;
import com.google.android.textclassifier.AnnotatorModel;
+import com.google.common.collect.ImmutableList;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.time.Instant;
@@ -53,7 +55,7 @@
private static final String TYPE_DICTIONARY = "dictionary";
@Override
- public List<LabeledIntent> create(
+ public ImmutableList<LabeledIntent> create(
Context context,
String text,
boolean foreignText,
@@ -102,7 +104,7 @@
if (foreignText) {
ClassificationIntentFactory.insertTranslateAction(actions, context, text);
}
- return actions;
+ return ImmutableList.copyOf(actions);
}
private static List<LabeledIntent> createForEmail(Context context, String text) {
diff --git a/java/src/com/android/textclassifier/intent/TemplateClassificationIntentFactory.java b/java/src/com/android/textclassifier/intent/TemplateClassificationIntentFactory.java
index fbd742e..f3f6fd3 100644
--- a/java/src/com/android/textclassifier/intent/TemplateClassificationIntentFactory.java
+++ b/java/src/com/android/textclassifier/intent/TemplateClassificationIntentFactory.java
@@ -18,11 +18,14 @@
import android.content.Context;
import com.android.textclassifier.common.base.TcLog;
+import com.android.textclassifier.common.intent.LabeledIntent;
+import com.android.textclassifier.common.intent.TemplateIntentFactory;
import com.google.android.textclassifier.AnnotatorModel;
import com.google.android.textclassifier.RemoteActionTemplate;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import java.time.Instant;
-import java.util.Collections;
+import java.util.ArrayList;
import java.util.List;
import javax.annotation.Nullable;
@@ -46,14 +49,14 @@
* Returns a list of {@link LabeledIntent} that are constructed from the classification result.
*/
@Override
- public List<LabeledIntent> create(
+ public ImmutableList<LabeledIntent> create(
Context context,
String text,
boolean foreignText,
@Nullable Instant referenceTime,
@Nullable AnnotatorModel.ClassificationResult classification) {
if (classification == null) {
- return Collections.emptyList();
+ return ImmutableList.of();
}
RemoteActionTemplate[] remoteActionTemplates = classification.getRemoteActionTemplates();
if (remoteActionTemplates == null) {
@@ -63,10 +66,11 @@
"RemoteActionTemplate is missing, fallback to" + " LegacyClassificationIntentFactory.");
return fallback.create(context, text, foreignText, referenceTime, classification);
}
- final List<LabeledIntent> labeledIntents = templateIntentFactory.create(remoteActionTemplates);
+ final List<LabeledIntent> labeledIntents =
+ new ArrayList<>(templateIntentFactory.create(remoteActionTemplates));
if (foreignText) {
ClassificationIntentFactory.insertTranslateAction(labeledIntents, context, text.trim());
}
- return labeledIntents;
+ return ImmutableList.copyOf(labeledIntents);
}
}
diff --git a/java/tests/instrumentation/AndroidManifest.xml b/java/tests/instrumentation/AndroidManifest.xml
index 5de247c..129b909 100644
--- a/java/tests/instrumentation/AndroidManifest.xml
+++ b/java/tests/instrumentation/AndroidManifest.xml
@@ -1,8 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
- package="com.android.textclassifier.tests">
+ package="com.android.textclassifier.common.tests"
+ android:versionCode="1"
+ android:versionName="1.0">
- <uses-sdk android:minSdkVersion="28"/>
+ <uses-sdk android:minSdkVersion="16"/>
<application>
<uses-library android:name="android.test.runner"/>
@@ -10,5 +12,5 @@
<instrumentation
android:name="androidx.test.runner.AndroidJUnitRunner"
- android:targetPackage="com.android.textclassifier.tests"/>
+ android:targetPackage="com.android.textclassifier.common.tests"/>
</manifest>
diff --git a/java/tests/instrumentation/src/com/android/textclassifier/ActionsSuggestionsHelperTest.java b/java/tests/instrumentation/src/com/android/textclassifier/ActionsSuggestionsHelperTest.java
index d25d97e..59dc41a 100644
--- a/java/tests/instrumentation/src/com/android/textclassifier/ActionsSuggestionsHelperTest.java
+++ b/java/tests/instrumentation/src/com/android/textclassifier/ActionsSuggestionsHelperTest.java
@@ -33,8 +33,8 @@
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest;
-import com.android.textclassifier.intent.LabeledIntent;
-import com.android.textclassifier.intent.TemplateIntentFactory;
+import com.android.textclassifier.common.intent.LabeledIntent;
+import com.android.textclassifier.common.intent.TemplateIntentFactory;
import com.google.android.textclassifier.ActionsSuggestionsModel;
import com.google.android.textclassifier.RemoteActionTemplate;
import com.google.common.collect.ImmutableList;
diff --git a/java/tests/instrumentation/src/com/android/textclassifier/TextClassifierImplTest.java b/java/tests/instrumentation/src/com/android/textclassifier/TextClassifierImplTest.java
index a61ea36..9e31c7e 100644
--- a/java/tests/instrumentation/src/com/android/textclassifier/TextClassifierImplTest.java
+++ b/java/tests/instrumentation/src/com/android/textclassifier/TextClassifierImplTest.java
@@ -41,6 +41,7 @@
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest;
+import com.android.textclassifier.testing.FakeContextBuilder;
import com.google.common.collect.ImmutableList;
import java.util.Arrays;
import java.util.Collections;
diff --git a/java/tests/instrumentation/src/com/android/textclassifier/intent/LabeledIntentTest.java b/java/tests/instrumentation/src/com/android/textclassifier/common/intent/LabeledIntentTest.java
similarity index 97%
rename from java/tests/instrumentation/src/com/android/textclassifier/intent/LabeledIntentTest.java
rename to java/tests/instrumentation/src/com/android/textclassifier/common/intent/LabeledIntentTest.java
index 3840823..fc0393a 100644
--- a/java/tests/instrumentation/src/com/android/textclassifier/intent/LabeledIntentTest.java
+++ b/java/tests/instrumentation/src/com/android/textclassifier/common/intent/LabeledIntentTest.java
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.android.textclassifier.intent;
+package com.android.textclassifier.common.intent;
import static com.google.common.truth.Truth.assertThat;
import static org.testng.Assert.assertThrows;
@@ -27,7 +27,7 @@
import android.view.textclassifier.TextClassifier;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest;
-import com.android.textclassifier.FakeContextBuilder;
+import com.android.textclassifier.testing.FakeContextBuilder;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
diff --git a/java/tests/instrumentation/src/com/android/textclassifier/intent/TemplateIntentFactoryTest.java b/java/tests/instrumentation/src/com/android/textclassifier/common/intent/TemplateIntentFactoryTest.java
similarity index 89%
rename from java/tests/instrumentation/src/com/android/textclassifier/intent/TemplateIntentFactoryTest.java
rename to java/tests/instrumentation/src/com/android/textclassifier/common/intent/TemplateIntentFactoryTest.java
index ee45f18..fc36f1a 100644
--- a/java/tests/instrumentation/src/com/android/textclassifier/intent/TemplateIntentFactoryTest.java
+++ b/java/tests/instrumentation/src/com/android/textclassifier/common/intent/TemplateIntentFactoryTest.java
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.android.textclassifier.intent;
+package com.android.textclassifier.common.intent;
import static com.google.common.truth.Truth.assertThat;
@@ -49,10 +49,20 @@
private static final String VALUE_ONE = "value1";
private static final String KEY_TWO = "key2";
private static final int VALUE_TWO = 42;
+ private static final String KEY_STRING_ARRAY = "string_array_key";
+ private static final String[] VALUE_STRING_ARRAY = new String[] {"a", "b"};
+ private static final String KEY_FLOAT_ARRAY = "float_array_key";
+ private static final float[] VALUE_FLOAT_ARRAY = new float[] {3.14f, 2.718f};
+ private static final String KEY_INT_ARRAY = "int_array_key";
+ private static final int[] VALUE_INT_ARRAY = new int[] {7, 2, 1};
private static final NamedVariant[] NAMED_VARIANTS =
new NamedVariant[] {
- new NamedVariant(KEY_ONE, VALUE_ONE), new NamedVariant(KEY_TWO, VALUE_TWO)
+ new NamedVariant(KEY_ONE, VALUE_ONE),
+ new NamedVariant(KEY_TWO, VALUE_TWO),
+ new NamedVariant(KEY_STRING_ARRAY, VALUE_STRING_ARRAY),
+ new NamedVariant(KEY_FLOAT_ARRAY, VALUE_FLOAT_ARRAY),
+ new NamedVariant(KEY_INT_ARRAY, VALUE_INT_ARRAY)
};
private static final Integer REQUEST_CODE = 10;
@@ -100,6 +110,9 @@
assertThat(intent.getPackage()).isNull();
assertThat(intent.getStringExtra(KEY_ONE)).isEqualTo(VALUE_ONE);
assertThat(intent.getIntExtra(KEY_TWO, 0)).isEqualTo(VALUE_TWO);
+ assertThat(intent.getStringArrayExtra(KEY_STRING_ARRAY)).isEqualTo(VALUE_STRING_ARRAY);
+ assertThat(intent.getFloatArrayExtra(KEY_FLOAT_ARRAY)).isEqualTo(VALUE_FLOAT_ARRAY);
+ assertThat(intent.getIntArrayExtra(KEY_INT_ARRAY)).isEqualTo(VALUE_INT_ARRAY);
}
@Test
diff --git a/java/tests/instrumentation/src/com/android/textclassifier/intent/LegacyIntentClassificationFactoryTest.java b/java/tests/instrumentation/src/com/android/textclassifier/intent/LegacyIntentClassificationFactoryTest.java
index b976849..33c5085 100644
--- a/java/tests/instrumentation/src/com/android/textclassifier/intent/LegacyIntentClassificationFactoryTest.java
+++ b/java/tests/instrumentation/src/com/android/textclassifier/intent/LegacyIntentClassificationFactoryTest.java
@@ -23,6 +23,7 @@
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest;
+import com.android.textclassifier.common.intent.LabeledIntent;
import com.google.android.textclassifier.AnnotatorModel;
import com.google.android.textclassifier.AnnotatorModel.ClassificationResult;
import com.google.android.textclassifier.AnnotatorModel.DatetimeResult;
diff --git a/java/tests/instrumentation/src/com/android/textclassifier/intent/TemplateClassificationIntentFactoryTest.java b/java/tests/instrumentation/src/com/android/textclassifier/intent/TemplateClassificationIntentFactoryTest.java
index 42176bd..098d930 100644
--- a/java/tests/instrumentation/src/com/android/textclassifier/intent/TemplateClassificationIntentFactoryTest.java
+++ b/java/tests/instrumentation/src/com/android/textclassifier/intent/TemplateClassificationIntentFactoryTest.java
@@ -29,6 +29,8 @@
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest;
+import com.android.textclassifier.common.intent.LabeledIntent;
+import com.android.textclassifier.common.intent.TemplateIntentFactory;
import com.google.android.textclassifier.AnnotatorModel;
import com.google.android.textclassifier.RemoteActionTemplate;
import java.util.List;
diff --git a/java/tests/instrumentation/src/com/android/textclassifier/FakeContextBuilder.java b/java/tests/instrumentation/src/com/android/textclassifier/testing/FakeContextBuilder.java
similarity index 98%
rename from java/tests/instrumentation/src/com/android/textclassifier/FakeContextBuilder.java
rename to java/tests/instrumentation/src/com/android/textclassifier/testing/FakeContextBuilder.java
index 17b6e0a..f3ad833 100644
--- a/java/tests/instrumentation/src/com/android/textclassifier/FakeContextBuilder.java
+++ b/java/tests/instrumentation/src/com/android/textclassifier/testing/FakeContextBuilder.java
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.android.textclassifier;
+package com.android.textclassifier.testing;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
diff --git a/jni/com/google/android/textclassifier/ActionsSuggestionsModel.java b/jni/com/google/android/textclassifier/ActionsSuggestionsModel.java
index 84b5c3d..3af04e8 100644
--- a/jni/com/google/android/textclassifier/ActionsSuggestionsModel.java
+++ b/jni/com/google/android/textclassifier/ActionsSuggestionsModel.java
@@ -70,7 +70,7 @@
actionsModelPtr,
conversation,
options,
- (annotator != null ? annotator.getNativeAnnotator() : 0),
+ (annotator != null ? annotator.getNativeAnnotatorPointer() : 0),
/* appContext= */ null,
/* deviceLocales= */ null,
/* generateAndroidIntents= */ false);
@@ -86,7 +86,7 @@
actionsModelPtr,
conversation,
options,
- (annotator != null ? annotator.getNativeAnnotator() : 0),
+ (annotator != null ? annotator.getNativeAnnotatorPointer() : 0),
appContext,
deviceLocales,
/* generateAndroidIntents= */ true);
diff --git a/jni/com/google/android/textclassifier/AnnotatorModel.java b/jni/com/google/android/textclassifier/AnnotatorModel.java
index a94e97e..39ad1ae 100644
--- a/jni/com/google/android/textclassifier/AnnotatorModel.java
+++ b/jni/com/google/android/textclassifier/AnnotatorModel.java
@@ -650,7 +650,7 @@
* Retrieves the pointer to the native object. Note: Need to keep the AnnotatorModel alive as long
* as the pointer is used.
*/
- long getNativeAnnotator() {
+ long getNativeAnnotatorPointer() {
return nativeGetNativeModelPtr(annotatorPtr);
}
diff --git a/jni/com/google/android/textclassifier/LangIdModel.java b/jni/com/google/android/textclassifier/LangIdModel.java
index 9701492..c9c47f1 100644
--- a/jni/com/google/android/textclassifier/LangIdModel.java
+++ b/jni/com/google/android/textclassifier/LangIdModel.java
@@ -113,6 +113,14 @@
return nativeGetMinTextSizeInBytes(modelPtr);
}
+ /**
+ * Returns the pointer to the native object. Note: Need to keep the LangIdModel alive as long as
+ * the pointer is used.
+ */
+ long getNativeLangIdPointer() {
+ return modelPtr;
+ }
+
private static native long nativeNew(int fd);
private static native long nativeNewFromPath(String path);
diff --git a/jni/com/google/android/textclassifier/NamedVariant.java b/jni/com/google/android/textclassifier/NamedVariant.java
index d04bb11..23f2616 100644
--- a/jni/com/google/android/textclassifier/NamedVariant.java
+++ b/jni/com/google/android/textclassifier/NamedVariant.java
@@ -29,6 +29,9 @@
public static final int TYPE_DOUBLE = 4;
public static final int TYPE_BOOL = 5;
public static final int TYPE_STRING = 6;
+ public static final int TYPE_STRING_ARRAY = 7;
+ public static final int TYPE_FLOAT_ARRAY = 8;
+ public static final int TYPE_INT_ARRAY = 9;
public NamedVariant(String name, int value) {
this.name = name;
@@ -66,6 +69,24 @@
this.type = TYPE_STRING;
}
+ public NamedVariant(String name, String[] value) {
+ this.name = name;
+ this.stringArrValue = value;
+ this.type = TYPE_STRING_ARRAY;
+ }
+
+ public NamedVariant(String name, float[] value) {
+ this.name = name;
+ this.floatArrValue = value;
+ this.type = TYPE_FLOAT_ARRAY;
+ }
+
+ public NamedVariant(String name, int[] value) {
+ this.name = name;
+ this.intArrValue = value;
+ this.type = TYPE_INT_ARRAY;
+ }
+
public String getName() {
return name;
}
@@ -104,6 +125,21 @@
return stringValue;
}
+ public String[] getStringArray() {
+ assert (type == TYPE_STRING_ARRAY);
+ return stringArrValue;
+ }
+
+ public float[] getFloatArray() {
+ assert (type == TYPE_FLOAT_ARRAY);
+ return floatArrValue;
+ }
+
+ public int[] getIntArray() {
+ assert (type == TYPE_INT_ARRAY);
+ return intArrValue;
+ }
+
private final String name;
private final int type;
private int intValue;
@@ -112,4 +148,7 @@
private double doubleValue;
private boolean boolValue;
private String stringValue;
+ private String[] stringArrValue;
+ private float[] floatArrValue;
+ private int[] intArrValue;
}
diff --git a/native/actions/actions-suggestions.cc b/native/actions/actions-suggestions.cc
index 711d10c..17ba96a 100644
--- a/native/actions/actions-suggestions.cc
+++ b/native/actions/actions-suggestions.cc
@@ -866,9 +866,45 @@
return options;
}
+// Run annotator on the messages of a conversation.
+Conversation ActionsSuggestions::AnnotateConversation(
+ const Conversation& conversation, const Annotator* annotator) const {
+ if (annotator == nullptr) {
+ return conversation;
+ }
+ const int num_messages_grammar =
+ ((model_->rules() && model_->rules()->grammar_rules() &&
+ model_->rules()->grammar_rules()->annotation_nonterminal())
+ ? 1
+ : 0);
+ const int num_messages_mapping =
+ (model_->annotation_actions_spec()
+ ? std::max(model_->annotation_actions_spec()
+ ->max_history_from_any_person(),
+ model_->annotation_actions_spec()
+ ->max_history_from_last_person())
+ : 0);
+ const int num_messages = std::max(num_messages_grammar, num_messages_mapping);
+ if (num_messages == 0) {
+ // No annotations are used.
+ return conversation;
+ }
+ Conversation annotated_conversation = conversation;
+ for (int i = 0, message_index = annotated_conversation.messages.size() - 1;
+ i < num_messages && message_index >= 0; i++, message_index--) {
+ ConversationMessage* message =
+ &annotated_conversation.messages[message_index];
+ if (message->annotations.empty()) {
+ message->annotations = annotator->Annotate(
+ message->text, AnnotationOptionsForMessage(*message));
+ }
+ }
+ return annotated_conversation;
+}
+
void ActionsSuggestions::SuggestActionsFromAnnotations(
- const Conversation& conversation, const ActionSuggestionOptions& options,
- const Annotator* annotator, std::vector<ActionSuggestion>* actions) const {
+ const Conversation& conversation,
+ std::vector<ActionSuggestion>* actions) const {
if (model_->annotation_actions_spec() == nullptr ||
model_->annotation_actions_spec()->annotation_mapping() == nullptr ||
model_->annotation_actions_spec()->annotation_mapping()->size() == 0) {
@@ -914,10 +950,6 @@
}
}
- if (annotations.empty() && annotator != nullptr) {
- annotations = annotator->Annotate(message.text,
- AnnotationOptionsForMessage(message));
- }
std::vector<ActionSuggestionAnnotation> action_annotations;
action_annotations.reserve(annotations.size());
for (const AnnotatedSpan& annotation : annotations) {
@@ -1057,25 +1089,29 @@
return true;
}
+ // Run annotator against messages.
+ const Conversation annotated_conversation =
+ AnnotateConversation(conversation, annotator);
+
const int num_messages = NumMessagesToConsider(
- conversation, model_->max_conversation_history_length());
+ annotated_conversation, model_->max_conversation_history_length());
if (num_messages <= 0) {
TC3_LOG(INFO) << "No messages provided for actions suggestions.";
return false;
}
- SuggestActionsFromAnnotations(conversation, options, annotator,
- &response->actions);
+ SuggestActionsFromAnnotations(annotated_conversation, &response->actions);
int input_text_length = 0;
int num_matching_locales = 0;
- for (int i = conversation.messages.size() - num_messages;
- i < conversation.messages.size(); i++) {
- input_text_length += conversation.messages[i].text.length();
+ for (int i = annotated_conversation.messages.size() - num_messages;
+ i < annotated_conversation.messages.size(); i++) {
+ input_text_length += annotated_conversation.messages[i].text.length();
std::vector<Locale> message_languages;
- if (!ParseLocales(conversation.messages[i].detected_text_language_tags,
- &message_languages)) {
+ if (!ParseLocales(
+ annotated_conversation.messages[i].detected_text_language_tags,
+ &message_languages)) {
continue;
}
if (Locale::IsAnyLocaleSupported(
@@ -1105,17 +1141,18 @@
std::vector<const UniLib::RegexPattern*> post_check_rules;
if (preconditions_.suppress_on_low_confidence_input) {
if ((ngram_model_ != nullptr &&
- ngram_model_->EvalConversation(conversation, num_messages)) ||
- regex_actions_->IsLowConfidenceInput(conversation, num_messages,
- &post_check_rules)) {
+ ngram_model_->EvalConversation(annotated_conversation,
+ num_messages)) ||
+ regex_actions_->IsLowConfidenceInput(annotated_conversation,
+ num_messages, &post_check_rules)) {
response->output_filtered_low_confidence = true;
return true;
}
}
std::unique_ptr<tflite::Interpreter> interpreter;
- if (!SuggestActionsFromModel(conversation, num_messages, options, response,
- &interpreter)) {
+ if (!SuggestActionsFromModel(annotated_conversation, num_messages, options,
+ response, &interpreter)) {
TC3_LOG(ERROR) << "Could not run model.";
return false;
}
@@ -1127,21 +1164,23 @@
}
if (!SuggestActionsFromLua(
- conversation, model_executor_.get(), interpreter.get(),
+ annotated_conversation, model_executor_.get(), interpreter.get(),
annotator != nullptr ? annotator->entity_data_schema() : nullptr,
&response->actions)) {
TC3_LOG(ERROR) << "Could not suggest actions from script.";
return false;
}
- if (!regex_actions_->SuggestActions(conversation, entity_data_builder_.get(),
+ if (!regex_actions_->SuggestActions(annotated_conversation,
+ entity_data_builder_.get(),
&response->actions)) {
TC3_LOG(ERROR) << "Could not suggest actions from regex rules.";
return false;
}
if (grammar_actions_ != nullptr &&
- !grammar_actions_->SuggestActions(conversation, &response->actions)) {
+ !grammar_actions_->SuggestActions(annotated_conversation,
+ &response->actions)) {
TC3_LOG(ERROR) << "Could not suggest actions from grammar rules.";
return false;
}
diff --git a/native/actions/actions-suggestions.h b/native/actions/actions-suggestions.h
index 86d91f7..cd0714a 100644
--- a/native/actions/actions-suggestions.h
+++ b/native/actions/actions-suggestions.h
@@ -197,13 +197,17 @@
const ConversationMessage& message) const;
void SuggestActionsFromAnnotations(
- const Conversation& conversation, const ActionSuggestionOptions& options,
- const Annotator* annotator, std::vector<ActionSuggestion>* actions) const;
+ const Conversation& conversation,
+ std::vector<ActionSuggestion>* actions) const;
void SuggestActionsFromAnnotation(
const int message_index, const ActionSuggestionAnnotation& annotation,
std::vector<ActionSuggestion>* actions) const;
+ // Run annotator on the messages of a conversation.
+ Conversation AnnotateConversation(const Conversation& conversation,
+ const Annotator* annotator) const;
+
// Deduplicates equivalent annotations - annotations that have the same type
// and same span text.
// Returns the indices of the deduplicated annotations.
diff --git a/native/actions/actions-suggestions_test.cc b/native/actions/actions-suggestions_test.cc
deleted file mode 100644
index 3dfefa3..0000000
--- a/native/actions/actions-suggestions_test.cc
+++ /dev/null
@@ -1,1512 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "actions/actions-suggestions.h"
-
-#include <fstream>
-#include <iterator>
-#include <memory>
-
-#include "actions/actions_model_generated.h"
-#include "actions/test-utils.h"
-#include "actions/zlib-utils.h"
-#include "annotator/collections.h"
-#include "annotator/types.h"
-#include "utils/flatbuffers.h"
-#include "utils/flatbuffers_generated.h"
-#include "utils/grammar/utils/rules.h"
-#include "utils/hash/farmhash.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-#include "flatbuffers/flatbuffers.h"
-#include "flatbuffers/reflection.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::ElementsAre;
-
-constexpr char kModelFileName[] = "actions_suggestions_test.model";
-constexpr char kHashGramModelFileName[] =
- "actions_suggestions_test.hashgram.model";
-
-std::string ReadFile(const std::string& file_name) {
- std::ifstream file_stream(file_name);
- return std::string(std::istreambuf_iterator<char>(file_stream), {});
-}
-
-std::string GetModelPath() {
- return "";
-}
-
-class ActionsSuggestionsTest : public testing::Test {
- protected:
- ActionsSuggestionsTest() : INIT_UNILIB_FOR_TESTING(unilib_) {}
- std::unique_ptr<ActionsSuggestions> LoadTestModel() {
- return ActionsSuggestions::FromPath(GetModelPath() + kModelFileName,
- &unilib_);
- }
- std::unique_ptr<ActionsSuggestions> LoadHashGramTestModel() {
- return ActionsSuggestions::FromPath(GetModelPath() + kHashGramModelFileName,
- &unilib_);
- }
- UniLib unilib_;
-};
-
-TEST_F(ActionsSuggestionsTest, InstantiateActionSuggestions) {
- EXPECT_THAT(LoadTestModel(), testing::NotNull());
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActions) {
- std::unique_ptr<ActionsSuggestions> actions_suggestions = LoadTestModel();
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "Where are you?", /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
- EXPECT_EQ(response.actions.size(), 3 /* share_location + 2 smart replies*/);
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestNoActionsForUnknownLocale) {
- std::unique_ptr<ActionsSuggestions> actions_suggestions = LoadTestModel();
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "Where are you?", /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"zz"}}});
- EXPECT_THAT(response.actions, testing::IsEmpty());
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsFromAnnotations) {
- std::unique_ptr<ActionsSuggestions> actions_suggestions = LoadTestModel();
- AnnotatedSpan annotation;
- annotation.span = {11, 15};
- annotation.classification = {ClassificationResult("address", 1.0)};
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "are you at home?",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{annotation},
- /*locales=*/"en"}}});
- ASSERT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions.front().type, "view_map");
- EXPECT_EQ(response.actions.front().score, 1.0);
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsFromAnnotationsWithEntityData) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- SetTestEntityDataSchema(actions_model.get());
-
- // Set custom actions from annotations config.
- actions_model->annotation_actions_spec->annotation_mapping.clear();
- actions_model->annotation_actions_spec->annotation_mapping.emplace_back(
- new AnnotationActionsSpec_::AnnotationMappingT);
- AnnotationActionsSpec_::AnnotationMappingT* mapping =
- actions_model->annotation_actions_spec->annotation_mapping.back().get();
- mapping->annotation_collection = "address";
- mapping->action.reset(new ActionSuggestionSpecT);
- mapping->action->type = "save_location";
- mapping->action->score = 1.0;
- mapping->action->priority_score = 2.0;
- mapping->entity_field.reset(new FlatbufferFieldPathT);
- mapping->entity_field->field.emplace_back(new FlatbufferFieldT);
- mapping->entity_field->field.back()->field_name = "location";
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
-
- AnnotatedSpan annotation;
- annotation.span = {11, 15};
- annotation.classification = {ClassificationResult("address", 1.0)};
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "are you at home?",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{annotation},
- /*locales=*/"en"}}});
- ASSERT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions.front().type, "save_location");
- EXPECT_EQ(response.actions.front().score, 1.0);
-
- // Check that the `location` entity field holds the text from the address
- // annotation.
- const flatbuffers::Table* entity =
- flatbuffers::GetAnyRoot(reinterpret_cast<const unsigned char*>(
- response.actions.front().serialized_entity_data.data()));
- EXPECT_EQ(entity->GetPointer<const flatbuffers::String*>(/*field=*/6)->str(),
- "home");
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsFromAnnotationsNormalization) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- SetTestEntityDataSchema(actions_model.get());
-
- // Set custom actions from annotations config.
- actions_model->annotation_actions_spec->annotation_mapping.clear();
- actions_model->annotation_actions_spec->annotation_mapping.emplace_back(
- new AnnotationActionsSpec_::AnnotationMappingT);
- AnnotationActionsSpec_::AnnotationMappingT* mapping =
- actions_model->annotation_actions_spec->annotation_mapping.back().get();
- mapping->annotation_collection = "address";
- mapping->action.reset(new ActionSuggestionSpecT);
- mapping->action->type = "save_location";
- mapping->action->score = 1.0;
- mapping->action->priority_score = 2.0;
- mapping->entity_field.reset(new FlatbufferFieldPathT);
- mapping->entity_field->field.emplace_back(new FlatbufferFieldT);
- mapping->entity_field->field.back()->field_name = "location";
- mapping->normalization_options.reset(new NormalizationOptionsT);
- mapping->normalization_options->codepointwise_normalization =
- NormalizationOptions_::CodepointwiseNormalizationOp_UPPERCASE;
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
-
- AnnotatedSpan annotation;
- annotation.span = {11, 15};
- annotation.classification = {ClassificationResult("address", 1.0)};
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "are you at home?",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{annotation},
- /*locales=*/"en"}}});
- ASSERT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions.front().type, "save_location");
- EXPECT_EQ(response.actions.front().score, 1.0);
-
- // Check that the `location` entity field holds the normalized text of the
- // annotation.
- const flatbuffers::Table* entity =
- flatbuffers::GetAnyRoot(reinterpret_cast<const unsigned char*>(
- response.actions.front().serialized_entity_data.data()));
- EXPECT_EQ(entity->GetPointer<const flatbuffers::String*>(/*field=*/6)->str(),
- "HOME");
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsFromDuplicatedAnnotations) {
- std::unique_ptr<ActionsSuggestions> actions_suggestions = LoadTestModel();
- AnnotatedSpan flight_annotation;
- flight_annotation.span = {11, 15};
- flight_annotation.classification = {ClassificationResult("flight", 2.5)};
- AnnotatedSpan flight_annotation2;
- flight_annotation2.span = {35, 39};
- flight_annotation2.classification = {ClassificationResult("flight", 3.0)};
- AnnotatedSpan email_annotation;
- email_annotation.span = {43, 56};
- email_annotation.classification = {ClassificationResult("email", 2.0)};
-
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1,
- "call me at LX38 or send message to LX38 or test@test.com.",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/
- {flight_annotation, flight_annotation2, email_annotation},
- /*locales=*/"en"}}});
-
- ASSERT_GE(response.actions.size(), 2);
- EXPECT_EQ(response.actions[0].type, "track_flight");
- EXPECT_EQ(response.actions[0].score, 3.0);
- EXPECT_EQ(response.actions[1].type, "send_email");
- EXPECT_EQ(response.actions[1].score, 2.0);
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsAnnotationsNoDeduplication) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- // Disable deduplication.
- actions_model->annotation_actions_spec->deduplicate_annotations = false;
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
- AnnotatedSpan flight_annotation;
- flight_annotation.span = {11, 15};
- flight_annotation.classification = {ClassificationResult("flight", 2.5)};
- AnnotatedSpan flight_annotation2;
- flight_annotation2.span = {35, 39};
- flight_annotation2.classification = {ClassificationResult("flight", 3.0)};
- AnnotatedSpan email_annotation;
- email_annotation.span = {43, 56};
- email_annotation.classification = {ClassificationResult("email", 2.0)};
-
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1,
- "call me at LX38 or send message to LX38 or test@test.com.",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/
- {flight_annotation, flight_annotation2, email_annotation},
- /*locales=*/"en"}}});
-
- ASSERT_GE(response.actions.size(), 3);
- EXPECT_EQ(response.actions[0].type, "track_flight");
- EXPECT_EQ(response.actions[0].score, 3.0);
- EXPECT_EQ(response.actions[1].type, "track_flight");
- EXPECT_EQ(response.actions[1].score, 2.5);
- EXPECT_EQ(response.actions[2].type, "send_email");
- EXPECT_EQ(response.actions[2].score, 2.0);
-}
-
-ActionsSuggestionsResponse TestSuggestActionsFromAnnotations(
- const std::function<void(ActionsModelT*)>& set_config_fn,
- const UniLib* unilib = nullptr) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
-
- // Set custom config.
- set_config_fn(actions_model.get());
-
- // Disable smart reply for easier testing.
- actions_model->preconditions->min_smart_reply_triggering_score = 1.0;
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), unilib);
-
- AnnotatedSpan flight_annotation;
- flight_annotation.span = {15, 19};
- flight_annotation.classification = {ClassificationResult("flight", 2.0)};
- AnnotatedSpan email_annotation;
- email_annotation.span = {0, 16};
- email_annotation.classification = {ClassificationResult("email", 1.0)};
-
- return actions_suggestions->SuggestActions(
- {{{/*user_id=*/ActionsSuggestions::kLocalUserId,
- "hehe@android.com",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/
- {email_annotation},
- /*locales=*/"en"},
- {/*user_id=*/2,
- "yoyo@android.com",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/
- {email_annotation},
- /*locales=*/"en"},
- {/*user_id=*/1,
- "test@android.com",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/
- {email_annotation},
- /*locales=*/"en"},
- {/*user_id=*/1,
- "I am on flight LX38.",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/
- {flight_annotation},
- /*locales=*/"en"}}});
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithAnnotationsOnlyLastMessage) {
- const ActionsSuggestionsResponse response = TestSuggestActionsFromAnnotations(
- [](ActionsModelT* actions_model) {
- actions_model->annotation_actions_spec->include_local_user_messages =
- false;
- actions_model->annotation_actions_spec->only_until_last_sent = true;
- actions_model->annotation_actions_spec->max_history_from_any_person = 1;
- actions_model->annotation_actions_spec->max_history_from_last_person =
- 1;
- },
- &unilib_);
- EXPECT_EQ(response.actions.size(), 1);
- EXPECT_EQ(response.actions[0].type, "track_flight");
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithAnnotationsOnlyLastPerson) {
- const ActionsSuggestionsResponse response = TestSuggestActionsFromAnnotations(
- [](ActionsModelT* actions_model) {
- actions_model->annotation_actions_spec->include_local_user_messages =
- false;
- actions_model->annotation_actions_spec->only_until_last_sent = true;
- actions_model->annotation_actions_spec->max_history_from_any_person = 1;
- actions_model->annotation_actions_spec->max_history_from_last_person =
- 3;
- },
- &unilib_);
- EXPECT_EQ(response.actions.size(), 2);
- EXPECT_EQ(response.actions[0].type, "track_flight");
- EXPECT_EQ(response.actions[1].type, "send_email");
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithAnnotationsFromAny) {
- const ActionsSuggestionsResponse response = TestSuggestActionsFromAnnotations(
- [](ActionsModelT* actions_model) {
- actions_model->annotation_actions_spec->include_local_user_messages =
- false;
- actions_model->annotation_actions_spec->only_until_last_sent = true;
- actions_model->annotation_actions_spec->max_history_from_any_person = 2;
- actions_model->annotation_actions_spec->max_history_from_last_person =
- 1;
- },
- &unilib_);
- EXPECT_EQ(response.actions.size(), 2);
- EXPECT_EQ(response.actions[0].type, "track_flight");
- EXPECT_EQ(response.actions[1].type, "send_email");
-}
-
-TEST_F(ActionsSuggestionsTest,
- SuggestActionsWithAnnotationsFromAnyManyMessages) {
- const ActionsSuggestionsResponse response = TestSuggestActionsFromAnnotations(
- [](ActionsModelT* actions_model) {
- actions_model->annotation_actions_spec->include_local_user_messages =
- false;
- actions_model->annotation_actions_spec->only_until_last_sent = true;
- actions_model->annotation_actions_spec->max_history_from_any_person = 3;
- actions_model->annotation_actions_spec->max_history_from_last_person =
- 1;
- },
- &unilib_);
- EXPECT_EQ(response.actions.size(), 3);
- EXPECT_EQ(response.actions[0].type, "track_flight");
- EXPECT_EQ(response.actions[1].type, "send_email");
- EXPECT_EQ(response.actions[2].type, "send_email");
-}
-
-TEST_F(ActionsSuggestionsTest,
- SuggestActionsWithAnnotationsFromAnyManyMessagesButNotLocalUser) {
- const ActionsSuggestionsResponse response = TestSuggestActionsFromAnnotations(
- [](ActionsModelT* actions_model) {
- actions_model->annotation_actions_spec->include_local_user_messages =
- false;
- actions_model->annotation_actions_spec->only_until_last_sent = true;
- actions_model->annotation_actions_spec->max_history_from_any_person = 5;
- actions_model->annotation_actions_spec->max_history_from_last_person =
- 1;
- },
- &unilib_);
- EXPECT_EQ(response.actions.size(), 3);
- EXPECT_EQ(response.actions[0].type, "track_flight");
- EXPECT_EQ(response.actions[1].type, "send_email");
- EXPECT_EQ(response.actions[2].type, "send_email");
-}
-
-TEST_F(ActionsSuggestionsTest,
- SuggestActionsWithAnnotationsFromAnyManyMessagesAlsoFromLocalUser) {
- const ActionsSuggestionsResponse response = TestSuggestActionsFromAnnotations(
- [](ActionsModelT* actions_model) {
- actions_model->annotation_actions_spec->include_local_user_messages =
- true;
- actions_model->annotation_actions_spec->only_until_last_sent = false;
- actions_model->annotation_actions_spec->max_history_from_any_person = 5;
- actions_model->annotation_actions_spec->max_history_from_last_person =
- 1;
- },
- &unilib_);
- EXPECT_EQ(response.actions.size(), 4);
- EXPECT_EQ(response.actions[0].type, "track_flight");
- EXPECT_EQ(response.actions[1].type, "send_email");
- EXPECT_EQ(response.actions[2].type, "send_email");
- EXPECT_EQ(response.actions[3].type, "send_email");
-}
-
-void TestSuggestActionsWithThreshold(
- const std::function<void(ActionsModelT*)>& set_value_fn,
- const UniLib* unilib = nullptr, const int expected_size = 0,
- const std::string& preconditions_overwrite = "") {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- set_value_fn(actions_model.get());
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), unilib, preconditions_overwrite);
- ASSERT_TRUE(actions_suggestions);
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "I have the low-ground. Where are you?",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
- EXPECT_LE(response.actions.size(), expected_size);
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithTriggeringScore) {
- TestSuggestActionsWithThreshold(
- [](ActionsModelT* actions_model) {
- actions_model->preconditions->min_smart_reply_triggering_score = 1.0;
- },
- &unilib_,
- /*expected_size=*/1 /*no smart reply, only actions*/
- );
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithMinReplyScore) {
- TestSuggestActionsWithThreshold(
- [](ActionsModelT* actions_model) {
- actions_model->preconditions->min_reply_score_threshold = 1.0;
- },
- &unilib_,
- /*expected_size=*/1 /*no smart reply, only actions*/
- );
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithSensitiveTopicScore) {
- TestSuggestActionsWithThreshold(
- [](ActionsModelT* actions_model) {
- actions_model->preconditions->max_sensitive_topic_score = 0.0;
- },
- &unilib_,
- /*expected_size=*/4 /* no sensitive prediction in test model*/);
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithMaxInputLength) {
- TestSuggestActionsWithThreshold(
- [](ActionsModelT* actions_model) {
- actions_model->preconditions->max_input_length = 0;
- },
- &unilib_);
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithMinInputLength) {
- TestSuggestActionsWithThreshold(
- [](ActionsModelT* actions_model) {
- actions_model->preconditions->min_input_length = 100;
- },
- &unilib_);
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithPreconditionsOverwrite) {
- TriggeringPreconditionsT preconditions_overwrite;
- preconditions_overwrite.max_input_length = 0;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(
- TriggeringPreconditions::Pack(builder, &preconditions_overwrite));
- TestSuggestActionsWithThreshold(
- // Keep model untouched.
- [](ActionsModelT* actions_model) {}, &unilib_,
- /*expected_size=*/0,
- std::string(reinterpret_cast<const char*>(builder.GetBufferPointer()),
- builder.GetSize()));
-}
-
-#ifdef TC3_UNILIB_ICU
-TEST_F(ActionsSuggestionsTest, SuggestActionsLowConfidence) {
- TestSuggestActionsWithThreshold(
- [](ActionsModelT* actions_model) {
- actions_model->preconditions->suppress_on_low_confidence_input = true;
- actions_model->low_confidence_rules.reset(new RulesModelT);
- actions_model->low_confidence_rules->regex_rule.emplace_back(
- new RulesModel_::RegexRuleT);
- actions_model->low_confidence_rules->regex_rule.back()->pattern =
- "low-ground";
- },
- &unilib_);
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsLowConfidenceInputOutput) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- // Add custom triggering rule.
- actions_model->rules.reset(new RulesModelT());
- actions_model->rules->regex_rule.emplace_back(new RulesModel_::RegexRuleT);
- RulesModel_::RegexRuleT* rule = actions_model->rules->regex_rule.back().get();
- rule->pattern = "^(?i:hello\\s(there))$";
- {
- std::unique_ptr<RulesModel_::RuleActionSpecT> rule_action(
- new RulesModel_::RuleActionSpecT);
- rule_action->action.reset(new ActionSuggestionSpecT);
- rule_action->action->type = "text_reply";
- rule_action->action->response_text = "General Desaster!";
- rule_action->action->score = 1.0f;
- rule_action->action->priority_score = 1.0f;
- rule->actions.push_back(std::move(rule_action));
- }
- {
- std::unique_ptr<RulesModel_::RuleActionSpecT> rule_action(
- new RulesModel_::RuleActionSpecT);
- rule_action->action.reset(new ActionSuggestionSpecT);
- rule_action->action->type = "text_reply";
- rule_action->action->response_text = "General Kenobi!";
- rule_action->action->score = 1.0f;
- rule_action->action->priority_score = 1.0f;
- rule->actions.push_back(std::move(rule_action));
- }
-
- // Add input-output low confidence rule.
- actions_model->preconditions->suppress_on_low_confidence_input = true;
- actions_model->low_confidence_rules.reset(new RulesModelT);
- actions_model->low_confidence_rules->regex_rule.emplace_back(
- new RulesModel_::RegexRuleT);
- actions_model->low_confidence_rules->regex_rule.back()->pattern = "hello";
- actions_model->low_confidence_rules->regex_rule.back()->output_pattern =
- "(?i:desaster)";
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
- ASSERT_TRUE(actions_suggestions);
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "hello there",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
- ASSERT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions[0].response_text, "General Kenobi!");
-}
-
-TEST_F(ActionsSuggestionsTest,
- SuggestActionsLowConfidenceInputOutputOverwrite) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- actions_model->low_confidence_rules.reset();
-
- // Add custom triggering rule.
- actions_model->rules.reset(new RulesModelT());
- actions_model->rules->regex_rule.emplace_back(new RulesModel_::RegexRuleT);
- RulesModel_::RegexRuleT* rule = actions_model->rules->regex_rule.back().get();
- rule->pattern = "^(?i:hello\\s(there))$";
- {
- std::unique_ptr<RulesModel_::RuleActionSpecT> rule_action(
- new RulesModel_::RuleActionSpecT);
- rule_action->action.reset(new ActionSuggestionSpecT);
- rule_action->action->type = "text_reply";
- rule_action->action->response_text = "General Desaster!";
- rule_action->action->score = 1.0f;
- rule_action->action->priority_score = 1.0f;
- rule->actions.push_back(std::move(rule_action));
- }
- {
- std::unique_ptr<RulesModel_::RuleActionSpecT> rule_action(
- new RulesModel_::RuleActionSpecT);
- rule_action->action.reset(new ActionSuggestionSpecT);
- rule_action->action->type = "text_reply";
- rule_action->action->response_text = "General Kenobi!";
- rule_action->action->score = 1.0f;
- rule_action->action->priority_score = 1.0f;
- rule->actions.push_back(std::move(rule_action));
- }
-
- // Add custom triggering rule via overwrite.
- actions_model->preconditions->low_confidence_rules.reset();
- TriggeringPreconditionsT preconditions;
- preconditions.suppress_on_low_confidence_input = true;
- preconditions.low_confidence_rules.reset(new RulesModelT);
- preconditions.low_confidence_rules->regex_rule.emplace_back(
- new RulesModel_::RegexRuleT);
- preconditions.low_confidence_rules->regex_rule.back()->pattern = "hello";
- preconditions.low_confidence_rules->regex_rule.back()->output_pattern =
- "(?i:desaster)";
- flatbuffers::FlatBufferBuilder preconditions_builder;
- preconditions_builder.Finish(
- TriggeringPreconditions::Pack(preconditions_builder, &preconditions));
- std::string serialize_preconditions = std::string(
- reinterpret_cast<const char*>(preconditions_builder.GetBufferPointer()),
- preconditions_builder.GetSize());
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_, serialize_preconditions);
-
- ASSERT_TRUE(actions_suggestions);
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "hello there",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
- ASSERT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions[0].response_text, "General Kenobi!");
-}
-#endif
-
-TEST_F(ActionsSuggestionsTest, SuppressActionsFromAnnotationsOnSensitiveTopic) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
-
- // Don't test if no sensitivity score is produced
- if (actions_model->tflite_model_spec->output_sensitive_topic_score < 0) {
- return;
- }
-
- actions_model->preconditions->max_sensitive_topic_score = 0.0;
- actions_model->preconditions->suppress_on_sensitive_topic = true;
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
- AnnotatedSpan annotation;
- annotation.span = {11, 15};
- annotation.classification = {
- ClassificationResult(Collections::Address(), 1.0)};
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "are you at home?",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{annotation},
- /*locales=*/"en"}}});
- EXPECT_THAT(response.actions, testing::IsEmpty());
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithLongerConversation) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
-
- // Allow a larger conversation context.
- actions_model->max_conversation_history_length = 10;
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
- AnnotatedSpan annotation;
- annotation.span = {11, 15};
- annotation.classification = {
- ClassificationResult(Collections::Address(), 1.0)};
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/ActionsSuggestions::kLocalUserId, "hi, how are you?",
- /*reference_time_ms_utc=*/10000,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"},
- {/*user_id=*/1, "good! are you at home?",
- /*reference_time_ms_utc=*/15000,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{annotation},
- /*locales=*/"en"}}});
- ASSERT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions[0].type, "view_map");
- EXPECT_EQ(response.actions[0].score, 1.0);
-}
-
-TEST_F(ActionsSuggestionsTest, CreateActionsFromClassificationResult) {
- std::unique_ptr<ActionsSuggestions> actions_suggestions = LoadTestModel();
- AnnotatedSpan annotation;
- annotation.span = {8, 12};
- annotation.classification = {
- ClassificationResult(Collections::Flight(), 1.0)};
-
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "I'm on LX38?",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{annotation},
- /*locales=*/"en"}}});
-
- ASSERT_GE(response.actions.size(), 2);
- EXPECT_EQ(response.actions[0].type, "track_flight");
- EXPECT_EQ(response.actions[0].score, 1.0);
- EXPECT_EQ(response.actions[0].annotations.size(), 1);
- EXPECT_EQ(response.actions[0].annotations[0].span.message_index, 0);
- EXPECT_EQ(response.actions[0].annotations[0].span.span, annotation.span);
-}
-
-#ifdef TC3_UNILIB_ICU
-TEST_F(ActionsSuggestionsTest, CreateActionsFromRules) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- ASSERT_TRUE(DecompressActionsModel(actions_model.get()));
-
- actions_model->rules.reset(new RulesModelT());
- actions_model->rules->regex_rule.emplace_back(new RulesModel_::RegexRuleT);
- RulesModel_::RegexRuleT* rule = actions_model->rules->regex_rule.back().get();
- rule->pattern = "^(?i:hello\\s(there))$";
- rule->actions.emplace_back(new RulesModel_::RuleActionSpecT);
- rule->actions.back()->action.reset(new ActionSuggestionSpecT);
- ActionSuggestionSpecT* action = rule->actions.back()->action.get();
- action->type = "text_reply";
- action->response_text = "General Kenobi!";
- action->score = 1.0f;
- action->priority_score = 1.0f;
-
- // Set capturing groups for entity data.
- rule->actions.back()->capturing_group.emplace_back(
- new RulesModel_::RuleActionSpec_::RuleCapturingGroupT);
- RulesModel_::RuleActionSpec_::RuleCapturingGroupT* greeting_group =
- rule->actions.back()->capturing_group.back().get();
- greeting_group->group_id = 0;
- greeting_group->entity_field.reset(new FlatbufferFieldPathT);
- greeting_group->entity_field->field.emplace_back(new FlatbufferFieldT);
- greeting_group->entity_field->field.back()->field_name = "greeting";
- rule->actions.back()->capturing_group.emplace_back(
- new RulesModel_::RuleActionSpec_::RuleCapturingGroupT);
- RulesModel_::RuleActionSpec_::RuleCapturingGroupT* location_group =
- rule->actions.back()->capturing_group.back().get();
- location_group->group_id = 1;
- location_group->entity_field.reset(new FlatbufferFieldPathT);
- location_group->entity_field->field.emplace_back(new FlatbufferFieldT);
- location_group->entity_field->field.back()->field_name = "location";
-
- // Set test entity data schema.
- SetTestEntityDataSchema(actions_model.get());
-
- // Use meta data to generate custom serialized entity data.
- ReflectiveFlatbufferBuilder entity_data_builder(
- flatbuffers::GetRoot<reflection::Schema>(
- actions_model->actions_entity_data_schema.data()));
- std::unique_ptr<ReflectiveFlatbuffer> entity_data =
- entity_data_builder.NewRoot();
- entity_data->Set("person", "Kenobi");
- action->serialized_entity_data = entity_data->Serialize();
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
-
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "hello there", /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
- EXPECT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions[0].response_text, "General Kenobi!");
-
- // Check entity data.
- const flatbuffers::Table* entity =
- flatbuffers::GetAnyRoot(reinterpret_cast<const unsigned char*>(
- response.actions[0].serialized_entity_data.data()));
- EXPECT_EQ(entity->GetPointer<const flatbuffers::String*>(/*field=*/4)->str(),
- "hello there");
- EXPECT_EQ(entity->GetPointer<const flatbuffers::String*>(/*field=*/6)->str(),
- "there");
- EXPECT_EQ(entity->GetPointer<const flatbuffers::String*>(/*field=*/8)->str(),
- "Kenobi");
-}
-
-TEST_F(ActionsSuggestionsTest, CreateActionsFromRulesWithNormalization) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- ASSERT_TRUE(DecompressActionsModel(actions_model.get()));
-
- actions_model->rules.reset(new RulesModelT());
- actions_model->rules->regex_rule.emplace_back(new RulesModel_::RegexRuleT);
- RulesModel_::RegexRuleT* rule = actions_model->rules->regex_rule.back().get();
- rule->pattern = "^(?i:hello\\sthere)$";
- rule->actions.emplace_back(new RulesModel_::RuleActionSpecT);
- rule->actions.back()->action.reset(new ActionSuggestionSpecT);
- ActionSuggestionSpecT* action = rule->actions.back()->action.get();
- action->type = "text_reply";
- action->response_text = "General Kenobi!";
- action->score = 1.0f;
- action->priority_score = 1.0f;
-
- // Set capturing groups for entity data.
- rule->actions.back()->capturing_group.emplace_back(
- new RulesModel_::RuleActionSpec_::RuleCapturingGroupT);
- RulesModel_::RuleActionSpec_::RuleCapturingGroupT* greeting_group =
- rule->actions.back()->capturing_group.back().get();
- greeting_group->group_id = 0;
- greeting_group->entity_field.reset(new FlatbufferFieldPathT);
- greeting_group->entity_field->field.emplace_back(new FlatbufferFieldT);
- greeting_group->entity_field->field.back()->field_name = "greeting";
- greeting_group->normalization_options.reset(new NormalizationOptionsT);
- greeting_group->normalization_options->codepointwise_normalization =
- NormalizationOptions_::CodepointwiseNormalizationOp_DROP_WHITESPACE |
- NormalizationOptions_::CodepointwiseNormalizationOp_UPPERCASE;
-
- // Set test entity data schema.
- SetTestEntityDataSchema(actions_model.get());
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
-
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "hello there", /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
- EXPECT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions[0].response_text, "General Kenobi!");
-
- // Check entity data.
- const flatbuffers::Table* entity =
- flatbuffers::GetAnyRoot(reinterpret_cast<const unsigned char*>(
- response.actions[0].serialized_entity_data.data()));
- EXPECT_EQ(entity->GetPointer<const flatbuffers::String*>(/*field=*/4)->str(),
- "HELLOTHERE");
-}
-
-TEST_F(ActionsSuggestionsTest, CreatesTextRepliesFromRules) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- ASSERT_TRUE(DecompressActionsModel(actions_model.get()));
-
- actions_model->rules.reset(new RulesModelT());
- actions_model->rules->regex_rule.emplace_back(new RulesModel_::RegexRuleT);
- RulesModel_::RegexRuleT* rule = actions_model->rules->regex_rule.back().get();
- rule->pattern = "(?i:reply (stop|quit|end) (?:to|for) )";
- rule->actions.emplace_back(new RulesModel_::RuleActionSpecT);
-
- // Set capturing groups for entity data.
- rule->actions.back()->capturing_group.emplace_back(
- new RulesModel_::RuleActionSpec_::RuleCapturingGroupT);
- RulesModel_::RuleActionSpec_::RuleCapturingGroupT* code_group =
- rule->actions.back()->capturing_group.back().get();
- code_group->group_id = 1;
- code_group->text_reply.reset(new ActionSuggestionSpecT);
- code_group->text_reply->score = 1.0f;
- code_group->text_reply->priority_score = 1.0f;
- code_group->normalization_options.reset(new NormalizationOptionsT);
- code_group->normalization_options->codepointwise_normalization =
- NormalizationOptions_::CodepointwiseNormalizationOp_LOWERCASE;
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
-
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1,
- "visit test.com or reply STOP to cancel your subscription",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
- EXPECT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions[0].response_text, "stop");
-}
-
-TEST_F(ActionsSuggestionsTest, CreatesActionsFromGrammarRules) {
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- ASSERT_TRUE(DecompressActionsModel(actions_model.get()));
-
- actions_model->rules->grammar_rules.reset(new RulesModel_::GrammarRulesT);
-
- // Set tokenizer options.
- RulesModel_::GrammarRulesT* action_grammar_rules =
- actions_model->rules->grammar_rules.get();
- action_grammar_rules->tokenizer_options.reset(new ActionsTokenizerOptionsT);
- action_grammar_rules->tokenizer_options->type = TokenizationType_ICU;
- action_grammar_rules->tokenizer_options->icu_preserve_whitespace_tokens =
- true;
-
- // Setup test rules.
- action_grammar_rules->rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
- rules.Add("<knock>", {"<^>", "ventura", "!?", "<$>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kActionRuleMatch),
- /*callback_param=*/0);
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- action_grammar_rules->rules.get());
- action_grammar_rules->actions.emplace_back(new RulesModel_::RuleActionSpecT);
- RulesModel_::RuleActionSpecT* actions_spec =
- action_grammar_rules->actions.back().get();
- actions_spec->action.reset(new ActionSuggestionSpecT);
- actions_spec->action->response_text = "Yes, Satan?";
- actions_spec->action->priority_score = 1.0;
- actions_spec->action->score = 1.0;
- actions_spec->action->type = "text_reply";
- action_grammar_rules->rule_match.emplace_back(
- new RulesModel_::GrammarRules_::RuleMatchT);
- action_grammar_rules->rule_match.back()->action_id.push_back(0);
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
-
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "Ventura!",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
-
- EXPECT_THAT(response.actions, ElementsAre(IsSmartReply("Yes, Satan?")));
-}
-
-TEST_F(ActionsSuggestionsTest, DeduplicateActions) {
- std::unique_ptr<ActionsSuggestions> actions_suggestions = LoadTestModel();
- ActionsSuggestionsResponse response = actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "Where are you?", /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
-
- // Check that the location sharing model triggered.
- bool has_location_sharing_action = false;
- for (const ActionSuggestion action : response.actions) {
- if (action.type == ActionsSuggestions::kShareLocation) {
- has_location_sharing_action = true;
- break;
- }
- }
- EXPECT_TRUE(has_location_sharing_action);
- const int num_actions = response.actions.size();
-
- // Add custom rule for location sharing.
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- ASSERT_TRUE(DecompressActionsModel(actions_model.get()));
-
- actions_model->rules.reset(new RulesModelT());
- actions_model->rules->regex_rule.emplace_back(new RulesModel_::RegexRuleT);
- actions_model->rules->regex_rule.back()->pattern =
- "^(?i:where are you[.?]?)$";
- actions_model->rules->regex_rule.back()->actions.emplace_back(
- new RulesModel_::RuleActionSpecT);
- actions_model->rules->regex_rule.back()->actions.back()->action.reset(
- new ActionSuggestionSpecT);
- ActionSuggestionSpecT* action =
- actions_model->rules->regex_rule.back()->actions.back()->action.get();
- action->score = 1.0f;
- action->type = ActionsSuggestions::kShareLocation;
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- actions_suggestions = ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
-
- response = actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "Where are you?", /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{}, /*locales=*/"en"}}});
- EXPECT_EQ(response.actions.size(), num_actions);
-}
-
-TEST_F(ActionsSuggestionsTest, DeduplicateConflictingActions) {
- std::unique_ptr<ActionsSuggestions> actions_suggestions = LoadTestModel();
- AnnotatedSpan annotation;
- annotation.span = {7, 11};
- annotation.classification = {
- ClassificationResult(Collections::Flight(), 1.0)};
- ActionsSuggestionsResponse response = actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "I'm on LX38",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{annotation},
- /*locales=*/"en"}}});
-
- // Check that the phone actions are present.
- EXPECT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions[0].type, "track_flight");
-
- // Add custom rule.
- const std::string actions_model_string =
- ReadFile(GetModelPath() + kModelFileName);
- std::unique_ptr<ActionsModelT> actions_model =
- UnPackActionsModel(actions_model_string.c_str());
- ASSERT_TRUE(DecompressActionsModel(actions_model.get()));
-
- actions_model->rules.reset(new RulesModelT());
- actions_model->rules->regex_rule.emplace_back(new RulesModel_::RegexRuleT);
- RulesModel_::RegexRuleT* rule = actions_model->rules->regex_rule.back().get();
- rule->pattern = "^(?i:I'm on ([a-z0-9]+))$";
- rule->actions.emplace_back(new RulesModel_::RuleActionSpecT);
- rule->actions.back()->action.reset(new ActionSuggestionSpecT);
- ActionSuggestionSpecT* action = rule->actions.back()->action.get();
- action->score = 1.0f;
- action->priority_score = 2.0f;
- action->type = "test_code";
- rule->actions.back()->capturing_group.emplace_back(
- new RulesModel_::RuleActionSpec_::RuleCapturingGroupT);
- RulesModel_::RuleActionSpec_::RuleCapturingGroupT* code_group =
- rule->actions.back()->capturing_group.back().get();
- code_group->group_id = 1;
- code_group->annotation_name = "code";
- code_group->annotation_type = "code";
-
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder,
- ActionsModel::Pack(builder, actions_model.get()));
- actions_suggestions = ActionsSuggestions::FromUnownedBuffer(
- reinterpret_cast<const uint8_t*>(builder.GetBufferPointer()),
- builder.GetSize(), &unilib_);
-
- response = actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "I'm on LX38",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{annotation},
- /*locales=*/"en"}}});
- EXPECT_GE(response.actions.size(), 1);
- EXPECT_EQ(response.actions[0].type, "test_code");
-}
-#endif
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsRanking) {
- std::unique_ptr<ActionsSuggestions> actions_suggestions = LoadTestModel();
- std::vector<AnnotatedSpan> annotations(2);
- annotations[0].span = {11, 15};
- annotations[0].classification = {ClassificationResult("address", 1.0)};
- annotations[1].span = {19, 23};
- annotations[1].classification = {ClassificationResult("address", 2.0)};
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "are you at home or work?",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/annotations,
- /*locales=*/"en"}}});
- EXPECT_GE(response.actions.size(), 2);
- EXPECT_EQ(response.actions[0].type, "view_map");
- EXPECT_EQ(response.actions[0].score, 2.0);
- EXPECT_EQ(response.actions[1].type, "view_map");
- EXPECT_EQ(response.actions[1].score, 1.0);
-}
-
-TEST_F(ActionsSuggestionsTest, VisitActionsModel) {
- EXPECT_TRUE(VisitActionsModel<bool>(GetModelPath() + kModelFileName,
- [](const ActionsModel* model) {
- if (model == nullptr) {
- return false;
- }
- return true;
- }));
- EXPECT_FALSE(VisitActionsModel<bool>(GetModelPath() + "non_existing_model.fb",
- [](const ActionsModel* model) {
- if (model == nullptr) {
- return false;
- }
- return true;
- }));
-}
-
-TEST_F(ActionsSuggestionsTest, SuggestActionsWithHashGramModel) {
- std::unique_ptr<ActionsSuggestions> actions_suggestions =
- LoadHashGramTestModel();
- ASSERT_TRUE(actions_suggestions != nullptr);
- {
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "hello",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{},
- /*locales=*/"en"}}});
- EXPECT_THAT(response.actions, testing::IsEmpty());
- }
- {
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "where are you",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{},
- /*locales=*/"en"}}});
- EXPECT_THAT(
- response.actions,
- ElementsAre(testing::Field(&ActionSuggestion::type, "share_location")));
- }
- {
- const ActionsSuggestionsResponse& response =
- actions_suggestions->SuggestActions(
- {{{/*user_id=*/1, "do you know johns number",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{},
- /*locales=*/"en"}}});
- EXPECT_THAT(
- response.actions,
- ElementsAre(testing::Field(&ActionSuggestion::type, "share_contact")));
- }
-}
-
-// Test class to expose token embedding methods for testing.
-class TestingMessageEmbedder : private ActionsSuggestions {
- public:
- explicit TestingMessageEmbedder(const ActionsModel* model);
-
- using ActionsSuggestions::EmbedAndFlattenTokens;
- using ActionsSuggestions::EmbedTokensPerMessage;
-
- protected:
- // EmbeddingExecutor that always returns features based on
- // the id of the sparse features.
- class FakeEmbeddingExecutor : public EmbeddingExecutor {
- public:
- bool AddEmbedding(const TensorView<int>& sparse_features, float* dest,
- const int dest_size) const override {
- TC3_CHECK_GE(dest_size, 1);
- EXPECT_EQ(sparse_features.size(), 1);
- dest[0] = sparse_features.data()[0];
- return true;
- }
- };
-};
-
-TestingMessageEmbedder::TestingMessageEmbedder(const ActionsModel* model) {
- model_ = model;
- const ActionsTokenFeatureProcessorOptions* options =
- model->feature_processor_options();
- feature_processor_.reset(
- new ActionsFeatureProcessor(options, /*unilib=*/nullptr));
- embedding_executor_.reset(new FakeEmbeddingExecutor());
- EXPECT_TRUE(
- EmbedTokenId(options->padding_token_id(), &embedded_padding_token_));
- EXPECT_TRUE(EmbedTokenId(options->start_token_id(), &embedded_start_token_));
- EXPECT_TRUE(EmbedTokenId(options->end_token_id(), &embedded_end_token_));
- token_embedding_size_ = feature_processor_->GetTokenEmbeddingSize();
- EXPECT_EQ(token_embedding_size_, 1);
-}
-
-class EmbeddingTest : public testing::Test {
- protected:
- EmbeddingTest() {
- model_.feature_processor_options.reset(
- new ActionsTokenFeatureProcessorOptionsT);
- options_ = model_.feature_processor_options.get();
- options_->chargram_orders = {1};
- options_->num_buckets = 1000;
- options_->embedding_size = 1;
- options_->start_token_id = 0;
- options_->end_token_id = 1;
- options_->padding_token_id = 2;
- options_->tokenizer_options.reset(new ActionsTokenizerOptionsT);
- }
-
- TestingMessageEmbedder CreateTestingMessageEmbedder() {
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder, ActionsModel::Pack(builder, &model_));
- buffer_ = builder.Release();
- return TestingMessageEmbedder(
- flatbuffers::GetRoot<ActionsModel>(buffer_.data()));
- }
-
- flatbuffers::DetachedBuffer buffer_;
- ActionsModelT model_;
- ActionsTokenFeatureProcessorOptionsT* options_;
-};
-
-TEST_F(EmbeddingTest, EmbedsTokensPerMessageWithNoBounds) {
- const TestingMessageEmbedder embedder = CreateTestingMessageEmbedder();
- std::vector<std::vector<Token>> tokens = {
- {Token("a", 0, 1), Token("b", 2, 3), Token("c", 4, 5)}};
- std::vector<float> embeddings;
- int max_num_tokens_per_message = 0;
-
- EXPECT_TRUE(embedder.EmbedTokensPerMessage(tokens, &embeddings,
- &max_num_tokens_per_message));
-
- EXPECT_EQ(max_num_tokens_per_message, 3);
- EXPECT_EQ(embeddings.size(), 3);
- EXPECT_THAT(embeddings[0],
- testing::FloatEq(tc3farmhash::Fingerprint64("a", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[1],
- testing::FloatEq(tc3farmhash::Fingerprint64("b", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[2],
- testing::FloatEq(tc3farmhash::Fingerprint64("c", 1) %
- options_->num_buckets));
-}
-
-TEST_F(EmbeddingTest, EmbedsTokensPerMessageWithPadding) {
- options_->min_num_tokens_per_message = 5;
- const TestingMessageEmbedder embedder = CreateTestingMessageEmbedder();
- std::vector<std::vector<Token>> tokens = {
- {Token("a", 0, 1), Token("b", 2, 3), Token("c", 4, 5)}};
- std::vector<float> embeddings;
- int max_num_tokens_per_message = 0;
-
- EXPECT_TRUE(embedder.EmbedTokensPerMessage(tokens, &embeddings,
- &max_num_tokens_per_message));
-
- EXPECT_EQ(max_num_tokens_per_message, 5);
- EXPECT_EQ(embeddings.size(), 5);
- EXPECT_THAT(embeddings[0],
- testing::FloatEq(tc3farmhash::Fingerprint64("a", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[1],
- testing::FloatEq(tc3farmhash::Fingerprint64("b", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[2],
- testing::FloatEq(tc3farmhash::Fingerprint64("c", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[3], testing::FloatEq(options_->padding_token_id));
- EXPECT_THAT(embeddings[4], testing::FloatEq(options_->padding_token_id));
-}
-
-TEST_F(EmbeddingTest, EmbedsTokensPerMessageDropsAtBeginning) {
- options_->max_num_tokens_per_message = 2;
- const TestingMessageEmbedder embedder = CreateTestingMessageEmbedder();
- std::vector<std::vector<Token>> tokens = {
- {Token("a", 0, 1), Token("b", 2, 3), Token("c", 4, 5)}};
- std::vector<float> embeddings;
- int max_num_tokens_per_message = 0;
-
- EXPECT_TRUE(embedder.EmbedTokensPerMessage(tokens, &embeddings,
- &max_num_tokens_per_message));
-
- EXPECT_EQ(max_num_tokens_per_message, 2);
- EXPECT_EQ(embeddings.size(), 2);
- EXPECT_THAT(embeddings[0],
- testing::FloatEq(tc3farmhash::Fingerprint64("b", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[1],
- testing::FloatEq(tc3farmhash::Fingerprint64("c", 1) %
- options_->num_buckets));
-}
-
-TEST_F(EmbeddingTest, EmbedsTokensPerMessageWithMultipleMessagesNoBounds) {
- const TestingMessageEmbedder embedder = CreateTestingMessageEmbedder();
- std::vector<std::vector<Token>> tokens = {
- {Token("a", 0, 1), Token("b", 2, 3), Token("c", 4, 5)},
- {Token("d", 0, 1), Token("e", 2, 3)}};
- std::vector<float> embeddings;
- int max_num_tokens_per_message = 0;
-
- EXPECT_TRUE(embedder.EmbedTokensPerMessage(tokens, &embeddings,
- &max_num_tokens_per_message));
-
- EXPECT_EQ(max_num_tokens_per_message, 3);
- EXPECT_THAT(embeddings[0],
- testing::FloatEq(tc3farmhash::Fingerprint64("a", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[1],
- testing::FloatEq(tc3farmhash::Fingerprint64("b", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[2],
- testing::FloatEq(tc3farmhash::Fingerprint64("c", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[3],
- testing::FloatEq(tc3farmhash::Fingerprint64("d", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[4],
- testing::FloatEq(tc3farmhash::Fingerprint64("e", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[5], testing::FloatEq(options_->padding_token_id));
-}
-
-TEST_F(EmbeddingTest, EmbedsFlattenedTokensWithNoBounds) {
- const TestingMessageEmbedder embedder = CreateTestingMessageEmbedder();
- std::vector<std::vector<Token>> tokens = {
- {Token("a", 0, 1), Token("b", 2, 3), Token("c", 4, 5)}};
- std::vector<float> embeddings;
- int total_token_count = 0;
-
- EXPECT_TRUE(
- embedder.EmbedAndFlattenTokens(tokens, &embeddings, &total_token_count));
-
- EXPECT_EQ(total_token_count, 5);
- EXPECT_EQ(embeddings.size(), 5);
- EXPECT_THAT(embeddings[0], testing::FloatEq(options_->start_token_id));
- EXPECT_THAT(embeddings[1],
- testing::FloatEq(tc3farmhash::Fingerprint64("a", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[2],
- testing::FloatEq(tc3farmhash::Fingerprint64("b", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[3],
- testing::FloatEq(tc3farmhash::Fingerprint64("c", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[4], testing::FloatEq(options_->end_token_id));
-}
-
-TEST_F(EmbeddingTest, EmbedsFlattenedTokensWithPadding) {
- options_->min_num_total_tokens = 7;
- const TestingMessageEmbedder embedder = CreateTestingMessageEmbedder();
- std::vector<std::vector<Token>> tokens = {
- {Token("a", 0, 1), Token("b", 2, 3), Token("c", 4, 5)}};
- std::vector<float> embeddings;
- int total_token_count = 0;
-
- EXPECT_TRUE(
- embedder.EmbedAndFlattenTokens(tokens, &embeddings, &total_token_count));
-
- EXPECT_EQ(total_token_count, 7);
- EXPECT_EQ(embeddings.size(), 7);
- EXPECT_THAT(embeddings[0], testing::FloatEq(options_->start_token_id));
- EXPECT_THAT(embeddings[1],
- testing::FloatEq(tc3farmhash::Fingerprint64("a", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[2],
- testing::FloatEq(tc3farmhash::Fingerprint64("b", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[3],
- testing::FloatEq(tc3farmhash::Fingerprint64("c", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[4], testing::FloatEq(options_->end_token_id));
- EXPECT_THAT(embeddings[5], testing::FloatEq(options_->padding_token_id));
- EXPECT_THAT(embeddings[6], testing::FloatEq(options_->padding_token_id));
-}
-
-TEST_F(EmbeddingTest, EmbedsFlattenedTokensDropsAtBeginning) {
- options_->max_num_total_tokens = 3;
- const TestingMessageEmbedder embedder = CreateTestingMessageEmbedder();
- std::vector<std::vector<Token>> tokens = {
- {Token("a", 0, 1), Token("b", 2, 3), Token("c", 4, 5)}};
- std::vector<float> embeddings;
- int total_token_count = 0;
-
- EXPECT_TRUE(
- embedder.EmbedAndFlattenTokens(tokens, &embeddings, &total_token_count));
-
- EXPECT_EQ(total_token_count, 3);
- EXPECT_EQ(embeddings.size(), 3);
- EXPECT_THAT(embeddings[0],
- testing::FloatEq(tc3farmhash::Fingerprint64("b", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[1],
- testing::FloatEq(tc3farmhash::Fingerprint64("c", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[2], testing::FloatEq(options_->end_token_id));
-}
-
-TEST_F(EmbeddingTest, EmbedsFlattenedTokensWithMultipleMessagesNoBounds) {
- const TestingMessageEmbedder embedder = CreateTestingMessageEmbedder();
- std::vector<std::vector<Token>> tokens = {
- {Token("a", 0, 1), Token("b", 2, 3), Token("c", 4, 5)},
- {Token("d", 0, 1), Token("e", 2, 3)}};
- std::vector<float> embeddings;
- int total_token_count = 0;
-
- EXPECT_TRUE(
- embedder.EmbedAndFlattenTokens(tokens, &embeddings, &total_token_count));
-
- EXPECT_EQ(total_token_count, 9);
- EXPECT_EQ(embeddings.size(), 9);
- EXPECT_THAT(embeddings[0], testing::FloatEq(options_->start_token_id));
- EXPECT_THAT(embeddings[1],
- testing::FloatEq(tc3farmhash::Fingerprint64("a", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[2],
- testing::FloatEq(tc3farmhash::Fingerprint64("b", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[3],
- testing::FloatEq(tc3farmhash::Fingerprint64("c", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[4], testing::FloatEq(options_->end_token_id));
- EXPECT_THAT(embeddings[5], testing::FloatEq(options_->start_token_id));
- EXPECT_THAT(embeddings[6],
- testing::FloatEq(tc3farmhash::Fingerprint64("d", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[7],
- testing::FloatEq(tc3farmhash::Fingerprint64("e", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[8], testing::FloatEq(options_->end_token_id));
-}
-
-TEST_F(EmbeddingTest,
- EmbedsFlattenedTokensWithMultipleMessagesDropsAtBeginning) {
- options_->max_num_total_tokens = 7;
- const TestingMessageEmbedder embedder = CreateTestingMessageEmbedder();
- std::vector<std::vector<Token>> tokens = {
- {Token("a", 0, 1), Token("b", 2, 3), Token("c", 4, 5)},
- {Token("d", 0, 1), Token("e", 2, 3), Token("f", 4, 5)}};
- std::vector<float> embeddings;
- int total_token_count = 0;
-
- EXPECT_TRUE(
- embedder.EmbedAndFlattenTokens(tokens, &embeddings, &total_token_count));
-
- EXPECT_EQ(total_token_count, 7);
- EXPECT_EQ(embeddings.size(), 7);
- EXPECT_THAT(embeddings[0],
- testing::FloatEq(tc3farmhash::Fingerprint64("c", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[1], testing::FloatEq(options_->end_token_id));
- EXPECT_THAT(embeddings[2], testing::FloatEq(options_->start_token_id));
- EXPECT_THAT(embeddings[3],
- testing::FloatEq(tc3farmhash::Fingerprint64("d", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[4],
- testing::FloatEq(tc3farmhash::Fingerprint64("e", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[5],
- testing::FloatEq(tc3farmhash::Fingerprint64("f", 1) %
- options_->num_buckets));
- EXPECT_THAT(embeddings[6], testing::FloatEq(options_->end_token_id));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/actions/actions_model.fbs b/native/actions/actions_model.fbs
index 51fe2b4..6f666eb 100755
--- a/native/actions/actions_model.fbs
+++ b/native/actions/actions_model.fbs
@@ -391,6 +391,10 @@
// If set, normalization to apply to the capturing group text.
normalization_options:NormalizationOptions;
+
+ // If set to true, an existing annotator annotation will be used to
+ // create the actions suggestions text annotation.
+ use_annotation_match:bool;
}
// The actions to produce upon triggering.
@@ -426,6 +430,12 @@
action_id:[uint];
}
+namespace libtextclassifier3.RulesModel_.GrammarRules_;
+table AnnotationNonterminalEntry {
+ key:string (key, shared);
+ value:int;
+}
+
// Configuration for actions based on context-free grammars.
namespace libtextclassifier3.RulesModel_;
table GrammarRules {
@@ -439,6 +449,10 @@
// The action specifications used by the rule matches.
actions:[RuleActionSpec];
+
+ // Predefined nonterminals for annotations.
+ // Maps annotation/collection names to non-terminal ids.
+ annotation_nonterminal:[GrammarRules_.AnnotationNonterminalEntry];
}
// Rule based actions.
diff --git a/native/actions/feature-processor_test.cc b/native/actions/feature-processor_test.cc
deleted file mode 100644
index 0a1e3ac..0000000
--- a/native/actions/feature-processor_test.cc
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "actions/feature-processor.h"
-
-#include "actions/actions_model_generated.h"
-#include "annotator/model-executor.h"
-#include "utils/tensor-view.h"
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::FloatEq;
-
-// EmbeddingExecutor that always returns features based on
-// the id of the sparse features.
-class FakeEmbeddingExecutor : public EmbeddingExecutor {
- public:
- bool AddEmbedding(const TensorView<int>& sparse_features, float* dest,
- const int dest_size) const override {
- TC3_CHECK_GE(dest_size, 4);
- EXPECT_EQ(sparse_features.size(), 1);
- dest[0] = sparse_features.data()[0];
- dest[1] = sparse_features.data()[0];
- dest[2] = -sparse_features.data()[0];
- dest[3] = -sparse_features.data()[0];
- return true;
- }
-
- private:
- std::vector<float> storage_;
-};
-
-class FeatureProcessorTest : public ::testing::Test {
- protected:
- FeatureProcessorTest() : INIT_UNILIB_FOR_TESTING(unilib_) {}
-
- flatbuffers::DetachedBuffer PackFeatureProcessorOptions(
- ActionsTokenFeatureProcessorOptionsT* options) const {
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(CreateActionsTokenFeatureProcessorOptions(builder, options));
- return builder.Release();
- }
-
- FakeEmbeddingExecutor embedding_executor_;
- UniLib unilib_;
-};
-
-TEST_F(FeatureProcessorTest, TokenEmbeddings) {
- ActionsTokenFeatureProcessorOptionsT options;
- options.embedding_size = 4;
- options.tokenizer_options.reset(new ActionsTokenizerOptionsT);
-
- flatbuffers::DetachedBuffer options_fb =
- PackFeatureProcessorOptions(&options);
- ActionsFeatureProcessor feature_processor(
- flatbuffers::GetRoot<ActionsTokenFeatureProcessorOptions>(
- options_fb.data()),
- &unilib_);
-
- Token token("aaa", 0, 3);
- std::vector<float> token_features;
- EXPECT_TRUE(feature_processor.AppendTokenFeatures(token, &embedding_executor_,
- &token_features));
- EXPECT_EQ(token_features.size(), 4);
-}
-
-TEST_F(FeatureProcessorTest, TokenEmbeddingsCaseFeature) {
- ActionsTokenFeatureProcessorOptionsT options;
- options.embedding_size = 4;
- options.extract_case_feature = true;
- options.tokenizer_options.reset(new ActionsTokenizerOptionsT);
-
- flatbuffers::DetachedBuffer options_fb =
- PackFeatureProcessorOptions(&options);
- ActionsFeatureProcessor feature_processor(
- flatbuffers::GetRoot<ActionsTokenFeatureProcessorOptions>(
- options_fb.data()),
- &unilib_);
-
- Token token("Aaa", 0, 3);
- std::vector<float> token_features;
- EXPECT_TRUE(feature_processor.AppendTokenFeatures(token, &embedding_executor_,
- &token_features));
- EXPECT_EQ(token_features.size(), 5);
- EXPECT_THAT(token_features[4], FloatEq(1.0));
-}
-
-TEST_F(FeatureProcessorTest, MultipleTokenEmbeddingsCaseFeature) {
- ActionsTokenFeatureProcessorOptionsT options;
- options.embedding_size = 4;
- options.extract_case_feature = true;
- options.tokenizer_options.reset(new ActionsTokenizerOptionsT);
-
- flatbuffers::DetachedBuffer options_fb =
- PackFeatureProcessorOptions(&options);
- ActionsFeatureProcessor feature_processor(
- flatbuffers::GetRoot<ActionsTokenFeatureProcessorOptions>(
- options_fb.data()),
- &unilib_);
-
- const std::vector<Token> tokens = {Token("Aaa", 0, 3), Token("bbb", 4, 7),
- Token("Cccc", 8, 12)};
- std::vector<float> token_features;
- EXPECT_TRUE(feature_processor.AppendTokenFeatures(
- tokens, &embedding_executor_, &token_features));
- EXPECT_EQ(token_features.size(), 15);
- EXPECT_THAT(token_features[4], FloatEq(1.0));
- EXPECT_THAT(token_features[9], FloatEq(-1.0));
- EXPECT_THAT(token_features[14], FloatEq(1.0));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/actions/grammar-actions.cc b/native/actions/grammar-actions.cc
index 84208aa..05cae46 100644
--- a/native/actions/grammar-actions.cc
+++ b/native/actions/grammar-actions.cc
@@ -21,6 +21,7 @@
#include "actions/feature-processor.h"
#include "actions/utils.h"
+#include "annotator/types.h"
#include "utils/grammar/callback-delegate.h"
#include "utils/grammar/match.h"
#include "utils/grammar/matcher.h"
@@ -31,6 +32,12 @@
namespace libtextclassifier3 {
namespace {
+// Represents an annotator annotated span in the grammar.
+struct AnnotationMatch : public grammar::Match {
+ static const int16 kType = 1;
+ ClassificationResult annotation;
+};
+
class GrammarActionsCallbackDelegate : public grammar::CallbackDelegate {
public:
GrammarActionsCallbackDelegate(const UniLib* unilib,
@@ -180,6 +187,16 @@
if (FillAnnotationFromCapturingMatch(
/*span=*/capturing_match->codepoint_span, group,
/*message_index=*/message_index, match_text, &annotation)) {
+ if (group->use_annotation_match()) {
+ const AnnotationMatch* annotation_match =
+ grammar::SelectFirstOfType<AnnotationMatch>(
+ capturing_match, AnnotationMatch::kType);
+ if (!annotation_match) {
+ TC3_LOG(ERROR) << "Could not get annotation for match.";
+ return false;
+ }
+ annotation.entity = annotation_match->annotation;
+ }
annotations.push_back(annotation);
}
}
@@ -247,11 +264,42 @@
{grammar_rules_->rules()->rules()->Get(i), &callback_handler});
}
}
+
+ if (locale_rules.empty()) {
+ // Nothing to do.
+ return true;
+ }
+
+ std::vector<AnnotationMatch> matches;
+ if (auto annotation_nonterminals = grammar_rules_->annotation_nonterminal()) {
+ for (const AnnotatedSpan& annotation :
+ conversation.messages.back().annotations) {
+ if (annotation.classification.empty()) {
+ continue;
+ }
+ const ClassificationResult& classification =
+ annotation.classification.front();
+ if (auto entry = annotation_nonterminals->LookupByKey(
+ classification.collection.c_str())) {
+ AnnotationMatch match;
+ match.Init(entry->value(), annotation.span, annotation.span.first,
+ AnnotationMatch::kType);
+ match.annotation = classification;
+ matches.push_back(match);
+ }
+ }
+ }
+
+ std::vector<grammar::Match*> annotation_matches(matches.size());
+ for (int i = 0; i < matches.size(); i++) {
+ annotation_matches[i] = &matches[i];
+ }
+
grammar::Matcher matcher(*unilib_, grammar_rules_->rules(), locale_rules);
// Run grammar on last message.
lexer_.Process(tokenizer_->Tokenize(conversation.messages.back().text),
- &matcher);
+ /*matches=*/annotation_matches, &matcher);
// Populate results.
return callback_handler.GetActions(conversation, smart_reply_action_type_,
diff --git a/native/actions/grammar-actions_test.cc b/native/actions/grammar-actions_test.cc
deleted file mode 100644
index 3df1d63..0000000
--- a/native/actions/grammar-actions_test.cc
+++ /dev/null
@@ -1,575 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "actions/grammar-actions.h"
-
-#include <iostream>
-#include <memory>
-
-#include "actions/actions_model_generated.h"
-#include "actions/test-utils.h"
-#include "actions/types.h"
-#include "utils/flatbuffers.h"
-#include "utils/grammar/rules_generated.h"
-#include "utils/grammar/utils/rules.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::ElementsAre;
-
-class TestGrammarActions : public GrammarActions {
- public:
- explicit TestGrammarActions(
- const UniLib* unilib, const RulesModel_::GrammarRules* grammar_rules,
- const ReflectiveFlatbufferBuilder* entity_data_builder = nullptr)
- : GrammarActions(unilib, grammar_rules, entity_data_builder,
-
- /*smart_reply_action_type=*/"text_reply") {}
-};
-
-class GrammarActionsTest : public testing::Test {
- protected:
- GrammarActionsTest()
- : INIT_UNILIB_FOR_TESTING(unilib_),
- serialized_entity_data_schema_(TestEntityDataSchema()),
- entity_data_builder_(new ReflectiveFlatbufferBuilder(
- flatbuffers::GetRoot<reflection::Schema>(
- serialized_entity_data_schema_.data()))) {}
-
- void SetTokenizerOptions(
- RulesModel_::GrammarRulesT* action_grammar_rules) const {
- action_grammar_rules->tokenizer_options.reset(new ActionsTokenizerOptionsT);
- action_grammar_rules->tokenizer_options->type = TokenizationType_ICU;
- action_grammar_rules->tokenizer_options->icu_preserve_whitespace_tokens =
- true;
- }
-
- flatbuffers::DetachedBuffer PackRules(
- const RulesModel_::GrammarRulesT& action_grammar_rules) const {
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(
- RulesModel_::GrammarRules::Pack(builder, &action_grammar_rules));
- return builder.Release();
- }
-
- int AddActionSpec(const std::string& type, const std::string& response_text,
- const std::vector<std::pair<int, std::string>>& annotations,
- RulesModel_::GrammarRulesT* action_grammar_rules) const {
- const int action_id = action_grammar_rules->actions.size();
- action_grammar_rules->actions.emplace_back(
- new RulesModel_::RuleActionSpecT);
- RulesModel_::RuleActionSpecT* actions_spec =
- action_grammar_rules->actions.back().get();
- actions_spec->action.reset(new ActionSuggestionSpecT);
- actions_spec->action->response_text = response_text;
- actions_spec->action->priority_score = 1.0;
- actions_spec->action->score = 1.0;
- actions_spec->action->type = type;
- // Create annotations for specified capturing groups.
- for (const auto& it : annotations) {
- actions_spec->capturing_group.emplace_back(
- new RulesModel_::RuleActionSpec_::RuleCapturingGroupT);
- actions_spec->capturing_group.back()->group_id = it.first;
- actions_spec->capturing_group.back()->annotation_name = it.second;
- actions_spec->capturing_group.back()->annotation_type = it.second;
- }
-
- return action_id;
- }
-
- int AddSmartReplySpec(
- const std::string& response_text,
- RulesModel_::GrammarRulesT* action_grammar_rules) const {
- return AddActionSpec("text_reply", response_text, {}, action_grammar_rules);
- }
-
- int AddCapturingMatchSmartReplySpec(
- const int match_id,
- RulesModel_::GrammarRulesT* action_grammar_rules) const {
- const int action_id = action_grammar_rules->actions.size();
- action_grammar_rules->actions.emplace_back(
- new RulesModel_::RuleActionSpecT);
- RulesModel_::RuleActionSpecT* actions_spec =
- action_grammar_rules->actions.back().get();
- actions_spec->capturing_group.emplace_back(
- new RulesModel_::RuleActionSpec_::RuleCapturingGroupT);
- actions_spec->capturing_group.back()->group_id = 0;
- actions_spec->capturing_group.back()->text_reply.reset(
- new ActionSuggestionSpecT);
- actions_spec->capturing_group.back()->text_reply->priority_score = 1.0;
- actions_spec->capturing_group.back()->text_reply->score = 1.0;
- return action_id;
- }
-
- int AddRuleMatch(const std::vector<int>& action_ids,
- RulesModel_::GrammarRulesT* action_grammar_rules) const {
- const int rule_match_id = action_grammar_rules->rule_match.size();
- action_grammar_rules->rule_match.emplace_back(
- new RulesModel_::GrammarRules_::RuleMatchT);
- action_grammar_rules->rule_match.back()->action_id.insert(
- action_grammar_rules->rule_match.back()->action_id.end(),
- action_ids.begin(), action_ids.end());
- return rule_match_id;
- }
-
- UniLib unilib_;
- const std::string serialized_entity_data_schema_;
- std::unique_ptr<ReflectiveFlatbufferBuilder> entity_data_builder_;
-};
-
-TEST_F(GrammarActionsTest, ProducesSmartReplies) {
- // Create test rules.
- // Rule: ^knock knock.?$ -> "Who's there?", "Yes?"
- RulesModel_::GrammarRulesT action_grammar_rules;
- SetTokenizerOptions(&action_grammar_rules);
- action_grammar_rules.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
- rules.Add(
- "<knock>", {"<^>", "knock", "knock", ".?", "<$>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kActionRuleMatch),
- /*callback_param=*/
- AddRuleMatch({AddSmartReplySpec("Who's there?", &action_grammar_rules),
- AddSmartReplySpec("Yes?", &action_grammar_rules)},
- &action_grammar_rules));
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- action_grammar_rules.rules.get());
- flatbuffers::DetachedBuffer serialized_rules =
- PackRules(action_grammar_rules);
- TestGrammarActions grammar_actions(
- &unilib_,
- flatbuffers::GetRoot<RulesModel_::GrammarRules>(serialized_rules.data()));
-
- std::vector<ActionSuggestion> result;
- EXPECT_TRUE(grammar_actions.SuggestActions(
- {/*messages=*/{{/*user_id=*/0, /*text=*/"Knock knock"}}}, &result));
-
- EXPECT_THAT(result,
- ElementsAre(IsSmartReply("Who's there?"), IsSmartReply("Yes?")));
-}
-
-TEST_F(GrammarActionsTest, ProducesSmartRepliesFromCapturingMatches) {
- // Create test rules.
- // Rule: ^Text <reply> to <command>
- RulesModel_::GrammarRulesT action_grammar_rules;
- SetTokenizerOptions(&action_grammar_rules);
- action_grammar_rules.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
-
- // Capturing matches will create their own match objects to keep track of
- // match ids, so we declare the handler as a filter so that the grammar system
- // knows that we handle this ourselves.
- rules.DefineFilter(static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kCapturingMatch));
-
- rules.Add("<scripted_reply>",
- {"<^>", "text", "<captured_reply>", "to", "<command>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kActionRuleMatch),
- /*callback_param=*/
- AddRuleMatch({AddCapturingMatchSmartReplySpec(
- /*match_id=*/0, &action_grammar_rules)},
- &action_grammar_rules));
-
- // <command> ::= unsubscribe | cancel | confirm | receive
- rules.Add("<command>", {"unsubscribe"});
- rules.Add("<command>", {"cancel"});
- rules.Add("<command>", {"confirm"});
- rules.Add("<command>", {"receive"});
-
- // <reply> ::= help | stop | cancel | yes
- rules.Add("<reply>", {"help"});
- rules.Add("<reply>", {"stop"});
- rules.Add("<reply>", {"cancel"});
- rules.Add("<reply>", {"yes"});
- rules.Add("<captured_reply>", {"<reply>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kCapturingMatch),
- /*callback_param=*/0 /*match_id*/);
-
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- action_grammar_rules.rules.get());
- flatbuffers::DetachedBuffer serialized_rules =
- PackRules(action_grammar_rules);
- TestGrammarActions grammar_actions(
- &unilib_,
- flatbuffers::GetRoot<RulesModel_::GrammarRules>(serialized_rules.data()));
-
- {
- std::vector<ActionSuggestion> result;
- EXPECT_TRUE(grammar_actions.SuggestActions(
- {/*messages=*/{{/*user_id=*/0,
- /*text=*/"Text YES to confirm your subscription"}}},
- &result));
- EXPECT_THAT(result, ElementsAre(IsSmartReply("YES")));
- }
-
- {
- std::vector<ActionSuggestion> result;
- EXPECT_TRUE(grammar_actions.SuggestActions(
- {/*messages=*/{{/*user_id=*/0,
- /*text=*/"text Stop to cancel your order"}}},
- &result));
- EXPECT_THAT(result, ElementsAre(IsSmartReply("Stop")));
- }
-}
-
-TEST_F(GrammarActionsTest, ProducesAnnotationsForActions) {
- // Create test rules.
- // Rule: please dial <phone>
- RulesModel_::GrammarRulesT action_grammar_rules;
- SetTokenizerOptions(&action_grammar_rules);
- action_grammar_rules.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
-
- // Capturing matches will create their own match objects to keep track of
- // match ids, so we declare the handler as a filter so that the grammar system
- // knows that we handle this ourselves.
- rules.DefineFilter(static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kCapturingMatch));
-
- rules.Add(
- "<call_phone>", {"please", "dial", "<phone>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kActionRuleMatch),
- /*callback_param=*/
- AddRuleMatch({AddActionSpec("call_phone", /*response_text=*/"",
- /*annotations=*/{{0 /*match_id*/, "phone"}},
- &action_grammar_rules)},
- &action_grammar_rules));
- // phone ::= +00 00 000 00 00
- rules.Add("<phone>",
- {"+", "<2_digits>", "<2_digits>", "<3_digits>", "<2_digits>",
- "<2_digits>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kCapturingMatch),
- /*callback_param=*/0 /*match_id*/);
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- action_grammar_rules.rules.get());
- flatbuffers::DetachedBuffer serialized_rules =
- PackRules(action_grammar_rules);
- TestGrammarActions grammar_actions(
- &unilib_,
- flatbuffers::GetRoot<RulesModel_::GrammarRules>(serialized_rules.data()));
-
- std::vector<ActionSuggestion> result;
- EXPECT_TRUE(grammar_actions.SuggestActions(
- {/*messages=*/{{/*user_id=*/0, /*text=*/"Please dial +41 79 123 45 67"}}},
- &result));
-
- EXPECT_THAT(result, ElementsAre(IsActionOfType("call_phone")));
- EXPECT_THAT(result.front().annotations,
- ElementsAre(IsActionSuggestionAnnotation(
- "phone", "+41 79 123 45 67", CodepointSpan{12, 28})));
-}
-
-TEST_F(GrammarActionsTest, HandlesLocales) {
- // Create test rules.
- // Rule: ^knock knock.?$ -> "Who's there?"
- RulesModel_::GrammarRulesT action_grammar_rules;
- SetTokenizerOptions(&action_grammar_rules);
- action_grammar_rules.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules(/*num_shards=*/2);
- rules.Add(
- "<knock>", {"<^>", "knock", "knock", ".?", "<$>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kActionRuleMatch),
- /*callback_param=*/
- AddRuleMatch({AddSmartReplySpec("Who's there?", &action_grammar_rules)},
- &action_grammar_rules));
- rules.Add(
- "<toc>", {"<knock>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kActionRuleMatch),
- /*callback_param=*/
- AddRuleMatch({AddSmartReplySpec("Qui est là?", &action_grammar_rules)},
- &action_grammar_rules),
- /*max_whitespace_gap=*/-1,
- /*case_sensitive=*/false,
- /*shard=*/1);
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- action_grammar_rules.rules.get());
- // Set locales for rules.
- action_grammar_rules.rules->rules.back()->locale.emplace_back(
- new LanguageTagT);
- action_grammar_rules.rules->rules.back()->locale.back()->language = "fr";
-
- flatbuffers::DetachedBuffer serialized_rules =
- PackRules(action_grammar_rules);
- TestGrammarActions grammar_actions(
- &unilib_,
- flatbuffers::GetRoot<RulesModel_::GrammarRules>(serialized_rules.data()));
-
- // Check default.
- {
- std::vector<ActionSuggestion> result;
- EXPECT_TRUE(grammar_actions.SuggestActions(
- {/*messages=*/{{/*user_id=*/0, /*text=*/"knock knock",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"UTC", /*annotations=*/{},
- /*detected_text_language_tags=*/"en"}}},
- &result));
-
- EXPECT_THAT(result, ElementsAre(IsSmartReply("Who's there?")));
- }
-
- // Check fr.
- {
- std::vector<ActionSuggestion> result;
- EXPECT_TRUE(grammar_actions.SuggestActions(
- {/*messages=*/{{/*user_id=*/0, /*text=*/"knock knock",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"UTC", /*annotations=*/{},
- /*detected_text_language_tags=*/"fr-CH"}}},
- &result));
-
- EXPECT_THAT(result, ElementsAre(IsSmartReply("Who's there?"),
- IsSmartReply("Qui est là?")));
- }
-}
-
-TEST_F(GrammarActionsTest, HandlesAssertions) {
- // Create test rules.
- // Rule: <flight> -> Track flight.
- RulesModel_::GrammarRulesT action_grammar_rules;
- SetTokenizerOptions(&action_grammar_rules);
- action_grammar_rules.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
-
- // Capturing matches will create their own match objects to keep track of
- // match ids, so we declare the handler as a filter so that the grammar system
- // knows that we handle this ourselves.
- rules.DefineFilter(static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kCapturingMatch));
-
- // Assertion matches will create their own match objects.
- // We declare the handler as a filter so that the grammar system knows that we
- // handle this ourselves.
- rules.DefineFilter(static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kAssertionMatch));
-
- rules.Add("<carrier>", {"lx"});
- rules.Add("<carrier>", {"aa"});
- rules.Add("<flight_code>", {"<2_digits>"});
- rules.Add("<flight_code>", {"<3_digits>"});
- rules.Add("<flight_code>", {"<4_digits>"});
-
- // Capture flight code.
- rules.Add("<flight>", {"<carrier>", "<flight_code>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kCapturingMatch),
- /*callback_param=*/0 /*match_id*/);
-
- // Flight: carrier + flight code and check right context.
- rules.Add(
- "<track_flight>", {"<flight>", "<context_assertion>?"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kActionRuleMatch),
- /*callback_param=*/
- AddRuleMatch({AddActionSpec("track_flight", /*response_text=*/"",
- /*annotations=*/{{0 /*match_id*/, "flight"}},
- &action_grammar_rules)},
- &action_grammar_rules));
-
- // Exclude matches like: LX 38.00 etc.
- rules.Add("<context_assertion>", {".?", "<digits>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kAssertionMatch),
- /*callback_param=*/true /*negative*/);
-
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- action_grammar_rules.rules.get());
-
- flatbuffers::DetachedBuffer serialized_rules =
- PackRules(action_grammar_rules);
- TestGrammarActions grammar_actions(
- &unilib_,
- flatbuffers::GetRoot<RulesModel_::GrammarRules>(serialized_rules.data()));
-
- std::vector<ActionSuggestion> result;
- EXPECT_TRUE(grammar_actions.SuggestActions(
- {/*messages=*/{{/*user_id=*/0, /*text=*/"LX38 aa 44 LX 38.38"}}},
- &result));
-
- EXPECT_THAT(result, ElementsAre(IsActionOfType("track_flight"),
- IsActionOfType("track_flight")));
- EXPECT_THAT(result[0].annotations,
- ElementsAre(IsActionSuggestionAnnotation("flight", "LX38",
- CodepointSpan{0, 4})));
- EXPECT_THAT(result[1].annotations,
- ElementsAre(IsActionSuggestionAnnotation("flight", "aa 44",
- CodepointSpan{5, 10})));
-}
-
-TEST_F(GrammarActionsTest, SetsStaticEntityData) {
- // Create test rules.
- // Rule: ^hello there$
- RulesModel_::GrammarRulesT action_grammar_rules;
- SetTokenizerOptions(&action_grammar_rules);
- action_grammar_rules.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
-
- // Create smart reply and static entity data.
- const int spec_id =
- AddSmartReplySpec("General Kenobi!", &action_grammar_rules);
- std::unique_ptr<ReflectiveFlatbuffer> entity_data =
- entity_data_builder_->NewRoot();
- entity_data->Set("person", "Kenobi");
- action_grammar_rules.actions[spec_id]->action->serialized_entity_data =
- entity_data->Serialize();
-
- rules.Add("<greeting>", {"<^>", "hello", "there", "<$>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kActionRuleMatch),
- /*callback_param=*/
- AddRuleMatch({spec_id}, &action_grammar_rules));
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- action_grammar_rules.rules.get());
- flatbuffers::DetachedBuffer serialized_rules =
- PackRules(action_grammar_rules);
- TestGrammarActions grammar_actions(
- &unilib_,
- flatbuffers::GetRoot<RulesModel_::GrammarRules>(serialized_rules.data()),
- entity_data_builder_.get());
-
- std::vector<ActionSuggestion> result;
- EXPECT_TRUE(grammar_actions.SuggestActions(
- {/*messages=*/{{/*user_id=*/0, /*text=*/"Hello there"}}}, &result));
-
- // Check the produces smart replies.
- EXPECT_THAT(result, ElementsAre(IsSmartReply("General Kenobi!")));
-
- // Check entity data.
- const flatbuffers::Table* entity =
- flatbuffers::GetAnyRoot(reinterpret_cast<const unsigned char*>(
- result[0].serialized_entity_data.data()));
- EXPECT_THAT(
- entity->GetPointer<const flatbuffers::String*>(/*field=*/8)->str(),
- "Kenobi");
-}
-
-TEST_F(GrammarActionsTest, SetsEntityDataFromCapturingMatches) {
- // Create test rules.
- // Rule: ^hello there$
- RulesModel_::GrammarRulesT action_grammar_rules;
- SetTokenizerOptions(&action_grammar_rules);
- action_grammar_rules.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
-
- // Capturing matches will create their own match objects to keep track of
- // match ids, so we declare the handler as a filter so that the grammar system
- // knows that we handle this ourselves.
- rules.DefineFilter(static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kCapturingMatch));
-
- // Create smart reply and static entity data.
- const int spec_id =
- AddSmartReplySpec("General Kenobi!", &action_grammar_rules);
- std::unique_ptr<ReflectiveFlatbuffer> entity_data =
- entity_data_builder_->NewRoot();
- entity_data->Set("person", "Kenobi");
- action_grammar_rules.actions[spec_id]->action->serialized_entity_data =
- entity_data->Serialize();
-
- // Specify results for capturing matches.
- const int greeting_match_id = 0;
- const int location_match_id = 1;
- {
- action_grammar_rules.actions[spec_id]->capturing_group.emplace_back(
- new RulesModel_::RuleActionSpec_::RuleCapturingGroupT);
- RulesModel_::RuleActionSpec_::RuleCapturingGroupT* group =
- action_grammar_rules.actions[spec_id]->capturing_group.back().get();
- group->group_id = greeting_match_id;
- group->entity_field.reset(new FlatbufferFieldPathT);
- group->entity_field->field.emplace_back(new FlatbufferFieldT);
- group->entity_field->field.back()->field_name = "greeting";
- }
- {
- action_grammar_rules.actions[spec_id]->capturing_group.emplace_back(
- new RulesModel_::RuleActionSpec_::RuleCapturingGroupT);
- RulesModel_::RuleActionSpec_::RuleCapturingGroupT* group =
- action_grammar_rules.actions[spec_id]->capturing_group.back().get();
- group->group_id = 1;
- group->entity_field.reset(new FlatbufferFieldPathT);
- group->entity_field->field.emplace_back(new FlatbufferFieldT);
- group->entity_field->field.back()->field_name = "location";
- }
-
- rules.Add("<location>", {"there"});
- rules.Add("<location>", {"here"});
- rules.Add("<captured_location>", {"<location>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kCapturingMatch),
- /*callback_param=*/location_match_id);
- rules.Add("<greeting>", {"hello", "<captured_location>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kCapturingMatch),
- /*callback_param=*/greeting_match_id);
- rules.Add("<test>", {"<^>", "<greeting>", "<$>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarActions::Callback::kActionRuleMatch),
- /*callback_param=*/
- AddRuleMatch({spec_id}, &action_grammar_rules));
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- action_grammar_rules.rules.get());
- flatbuffers::DetachedBuffer serialized_rules =
- PackRules(action_grammar_rules);
- TestGrammarActions grammar_actions(
- &unilib_,
- flatbuffers::GetRoot<RulesModel_::GrammarRules>(serialized_rules.data()),
- entity_data_builder_.get());
-
- std::vector<ActionSuggestion> result;
- EXPECT_TRUE(grammar_actions.SuggestActions(
- {/*messages=*/{{/*user_id=*/0, /*text=*/"Hello there"}}}, &result));
-
- // Check the produces smart replies.
- EXPECT_THAT(result, ElementsAre(IsSmartReply("General Kenobi!")));
-
- // Check entity data.
- const flatbuffers::Table* entity =
- flatbuffers::GetAnyRoot(reinterpret_cast<const unsigned char*>(
- result[0].serialized_entity_data.data()));
- EXPECT_THAT(
- entity->GetPointer<const flatbuffers::String*>(/*field=*/4)->str(),
- "Hello there");
- EXPECT_THAT(
- entity->GetPointer<const flatbuffers::String*>(/*field=*/6)->str(),
- "there");
- EXPECT_THAT(
- entity->GetPointer<const flatbuffers::String*>(/*field=*/8)->str(),
- "Kenobi");
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/actions/lua-actions_test.cc b/native/actions/lua-actions_test.cc
deleted file mode 100644
index b371387..0000000
--- a/native/actions/lua-actions_test.cc
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "actions/lua-actions.h"
-
-#include <map>
-#include <string>
-
-#include "actions/test-utils.h"
-#include "actions/types.h"
-#include "utils/tflite-model-executor.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::ElementsAre;
-
-TEST(LuaActions, SimpleAction) {
- Conversation conversation;
- const std::string test_snippet = R"(
- return {{ type = "test_action" }}
- )";
- std::vector<ActionSuggestion> actions;
- EXPECT_TRUE(LuaActionsSuggestions::CreateLuaActionsSuggestions(
- test_snippet, conversation,
- /*model_executor=*/nullptr,
- /*model_spec=*/nullptr,
- /*interpreter=*/nullptr,
- /*actions_entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr)
- ->SuggestActions(&actions));
- EXPECT_THAT(actions, ElementsAre(IsActionOfType("test_action")));
-}
-
-TEST(LuaActions, ConversationActions) {
- Conversation conversation;
- conversation.messages.push_back({/*user_id=*/0, "hello there!"});
- conversation.messages.push_back({/*user_id=*/1, "general kenobi!"});
- const std::string test_snippet = R"(
- local actions = {}
- for i, message in pairs(messages) do
- if i < #messages then
- if message.text == "hello there!" and
- messages[i+1].text == "general kenobi!" then
- table.insert(actions, {
- type = "text_reply",
- response_text = "you are a bold one!"
- })
- end
- if message.text == "i am the senate!" and
- messages[i+1].text == "not yet!" then
- table.insert(actions, {
- type = "text_reply",
- response_text = "it's treason then"
- })
- end
- end
- end
- return actions;
- )";
- std::vector<ActionSuggestion> actions;
- EXPECT_TRUE(LuaActionsSuggestions::CreateLuaActionsSuggestions(
- test_snippet, conversation,
- /*model_executor=*/nullptr,
- /*model_spec=*/nullptr,
- /*interpreter=*/nullptr,
- /*actions_entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr)
- ->SuggestActions(&actions));
- EXPECT_THAT(actions, ElementsAre(IsSmartReply("you are a bold one!")));
-}
-
-TEST(LuaActions, SimpleModelAction) {
- Conversation conversation;
- const std::string test_snippet = R"(
- if #model.actions_scores == 0 then
- return {{ type = "test_action" }}
- end
- return {}
- )";
- std::vector<ActionSuggestion> actions;
- EXPECT_TRUE(LuaActionsSuggestions::CreateLuaActionsSuggestions(
- test_snippet, conversation,
- /*model_executor=*/nullptr,
- /*model_spec=*/nullptr,
- /*interpreter=*/nullptr,
- /*actions_entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr)
- ->SuggestActions(&actions));
- EXPECT_THAT(actions, ElementsAre(IsActionOfType("test_action")));
-}
-
-TEST(LuaActions, AnnotationActions) {
- AnnotatedSpan annotation;
- annotation.span = {11, 15};
- annotation.classification = {ClassificationResult("address", 1.0)};
- Conversation conversation = {{{/*user_id=*/1, "are you at home?",
- /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"Europe/Zurich",
- /*annotations=*/{annotation},
- /*locales=*/"en"}}};
- const std::string test_snippet = R"(
- local actions = {}
- local last_message = messages[#messages]
- for i, annotation in pairs(last_message.annotation) do
- if #annotation.classification > 0 then
- if annotation.classification[1].collection == "address" then
- local text = string.sub(last_message.text,
- annotation.span["begin"] + 1,
- annotation.span["end"])
- table.insert(actions, {
- type = "text_reply",
- response_text = "i am at " .. text,
- annotation = {{
- name = "location",
- span = {
- text = text
- },
- entity = annotation.classification[1]
- }},
- })
- end
- end
- end
- return actions;
- )";
- std::vector<ActionSuggestion> actions;
- EXPECT_TRUE(LuaActionsSuggestions::CreateLuaActionsSuggestions(
- test_snippet, conversation,
- /*model_executor=*/nullptr,
- /*model_spec=*/nullptr,
- /*interpreter=*/nullptr,
- /*actions_entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr)
- ->SuggestActions(&actions));
- EXPECT_THAT(actions, ElementsAre(IsSmartReply("i am at home")));
- EXPECT_EQ("address", actions[0].annotations[0].entity.collection);
-}
-
-TEST(LuaActions, EntityData) {
- std::string test_schema = TestEntityDataSchema();
- Conversation conversation = {{{/*user_id=*/1, "hello there"}}};
- const std::string test_snippet = R"(
- return {{
- type = "test",
- entity = {
- greeting = "hello",
- location = "there",
- person = "Kenobi",
- },
- }};
- )";
- std::vector<ActionSuggestion> actions;
- EXPECT_TRUE(LuaActionsSuggestions::CreateLuaActionsSuggestions(
- test_snippet, conversation,
- /*model_executor=*/nullptr,
- /*model_spec=*/nullptr,
- /*interpreter=*/nullptr,
- /*actions_entity_data_schema=*/
- flatbuffers::GetRoot<reflection::Schema>(test_schema.data()),
- /*annotations_entity_data_schema=*/nullptr)
- ->SuggestActions(&actions));
- EXPECT_THAT(actions, testing::SizeIs(1));
- EXPECT_EQ("test", actions.front().type);
- const flatbuffers::Table* entity =
- flatbuffers::GetAnyRoot(reinterpret_cast<const unsigned char*>(
- actions.front().serialized_entity_data.data()));
- EXPECT_EQ(entity->GetPointer<const flatbuffers::String*>(/*field=*/4)->str(),
- "hello");
- EXPECT_EQ(entity->GetPointer<const flatbuffers::String*>(/*field=*/6)->str(),
- "there");
- EXPECT_EQ(entity->GetPointer<const flatbuffers::String*>(/*field=*/8)->str(),
- "Kenobi");
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/actions/lua-ranker_test.cc b/native/actions/lua-ranker_test.cc
deleted file mode 100644
index a790042..0000000
--- a/native/actions/lua-ranker_test.cc
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "actions/lua-ranker.h"
-
-#include <string>
-
-#include "actions/types.h"
-#include "utils/flatbuffers.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-MATCHER_P2(IsAction, type, response_text, "") {
- return testing::Value(arg.type, type) &&
- testing::Value(arg.response_text, response_text);
-}
-
-MATCHER_P(IsActionType, type, "") { return testing::Value(arg.type, type); }
-
-std::string TestEntitySchema() {
- // Create fake entity data schema meta data.
- // Cannot use object oriented API here as that is not available for the
- // reflection schema.
- flatbuffers::FlatBufferBuilder schema_builder;
- std::vector<flatbuffers::Offset<reflection::Field>> fields = {
- reflection::CreateField(
- schema_builder,
- /*name=*/schema_builder.CreateString("test"),
- /*type=*/
- reflection::CreateType(schema_builder,
- /*base_type=*/reflection::String),
- /*id=*/0,
- /*offset=*/4)};
- std::vector<flatbuffers::Offset<reflection::Enum>> enums;
- std::vector<flatbuffers::Offset<reflection::Object>> objects = {
- reflection::CreateObject(
- schema_builder,
- /*name=*/schema_builder.CreateString("EntityData"),
- /*fields=*/
- schema_builder.CreateVectorOfSortedTables(&fields))};
- schema_builder.Finish(reflection::CreateSchema(
- schema_builder, schema_builder.CreateVectorOfSortedTables(&objects),
- schema_builder.CreateVectorOfSortedTables(&enums),
- /*(unused) file_ident=*/0,
- /*(unused) file_ext=*/0,
- /*root_table*/ objects[0]));
- return std::string(
- reinterpret_cast<const char*>(schema_builder.GetBufferPointer()),
- schema_builder.GetSize());
-}
-
-TEST(LuaRankingTest, PassThrough) {
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"hello there", /*type=*/"text_reply",
- /*score=*/1.0},
- {/*response_text=*/"", /*type=*/"share_location", /*score=*/0.5},
- {/*response_text=*/"", /*type=*/"add_to_collection", /*score=*/0.1}};
- const std::string test_snippet = R"(
- local result = {}
- for i=1,#actions do
- table.insert(result, i)
- end
- return result
- )";
-
- EXPECT_TRUE(ActionsSuggestionsLuaRanker::Create(
- conversation, test_snippet, /*entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr, &response)
- ->RankActions());
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsActionType("text_reply"),
- IsActionType("share_location"),
- IsActionType("add_to_collection")}));
-}
-
-TEST(LuaRankingTest, Filtering) {
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"hello there", /*type=*/"text_reply",
- /*score=*/1.0},
- {/*response_text=*/"", /*type=*/"share_location", /*score=*/0.5},
- {/*response_text=*/"", /*type=*/"add_to_collection", /*score=*/0.1}};
- const std::string test_snippet = R"(
- return {}
- )";
-
- EXPECT_TRUE(ActionsSuggestionsLuaRanker::Create(
- conversation, test_snippet, /*entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr, &response)
- ->RankActions());
- EXPECT_THAT(response.actions, testing::IsEmpty());
-}
-
-TEST(LuaRankingTest, Duplication) {
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"hello there", /*type=*/"text_reply",
- /*score=*/1.0},
- {/*response_text=*/"", /*type=*/"share_location", /*score=*/0.5},
- {/*response_text=*/"", /*type=*/"add_to_collection", /*score=*/0.1}};
- const std::string test_snippet = R"(
- local result = {}
- for i=1,#actions do
- table.insert(result, 1)
- end
- return result
- )";
-
- EXPECT_TRUE(ActionsSuggestionsLuaRanker::Create(
- conversation, test_snippet, /*entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr, &response)
- ->RankActions());
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsActionType("text_reply"),
- IsActionType("text_reply"),
- IsActionType("text_reply")}));
-}
-
-TEST(LuaRankingTest, SortByScore) {
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"hello there", /*type=*/"text_reply",
- /*score=*/1.0},
- {/*response_text=*/"", /*type=*/"share_location", /*score=*/0.5},
- {/*response_text=*/"", /*type=*/"add_to_collection", /*score=*/0.1}};
- const std::string test_snippet = R"(
- function testScoreSorter(a, b)
- return actions[a].score < actions[b].score
- end
- local result = {}
- for i=1,#actions do
- result[i] = i
- end
- table.sort(result, testScoreSorter)
- return result
- )";
-
- EXPECT_TRUE(ActionsSuggestionsLuaRanker::Create(
- conversation, test_snippet, /*entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr, &response)
- ->RankActions());
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsActionType("add_to_collection"),
- IsActionType("share_location"),
- IsActionType("text_reply")}));
-}
-
-TEST(LuaRankingTest, SuppressType) {
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"hello there", /*type=*/"text_reply",
- /*score=*/1.0},
- {/*response_text=*/"", /*type=*/"share_location", /*score=*/0.5},
- {/*response_text=*/"", /*type=*/"add_to_collection", /*score=*/0.1}};
- const std::string test_snippet = R"(
- local result = {}
- for id, action in pairs(actions) do
- if action.type ~= "text_reply" then
- table.insert(result, id)
- end
- end
- return result
- )";
-
- EXPECT_TRUE(ActionsSuggestionsLuaRanker::Create(
- conversation, test_snippet, /*entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr, &response)
- ->RankActions());
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsActionType("share_location"),
- IsActionType("add_to_collection")}));
-}
-
-TEST(LuaRankingTest, HandlesConversation) {
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"hello there", /*type=*/"text_reply",
- /*score=*/1.0},
- {/*response_text=*/"", /*type=*/"share_location", /*score=*/0.5},
- {/*response_text=*/"", /*type=*/"add_to_collection", /*score=*/0.1}};
- const std::string test_snippet = R"(
- local result = {}
- if messages[1].text ~= "hello hello" then
- return result
- end
- for id, action in pairs(actions) do
- if action.type ~= "text_reply" then
- table.insert(result, id)
- end
- end
- return result
- )";
-
- EXPECT_TRUE(ActionsSuggestionsLuaRanker::Create(
- conversation, test_snippet, /*entity_data_schema=*/nullptr,
- /*annotations_entity_data_schema=*/nullptr, &response)
- ->RankActions());
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsActionType("share_location"),
- IsActionType("add_to_collection")}));
-}
-
-TEST(LuaRankingTest, HandlesEntityData) {
- std::string serialized_schema = TestEntitySchema();
- const reflection::Schema* entity_data_schema =
- flatbuffers::GetRoot<reflection::Schema>(serialized_schema.data());
-
- // Create test entity data.
- ReflectiveFlatbufferBuilder builder(entity_data_schema);
- std::unique_ptr<ReflectiveFlatbuffer> buffer = builder.NewRoot();
- buffer->Set("test", "value_a");
- const std::string serialized_entity_data_a = buffer->Serialize();
- buffer->Set("test", "value_b");
- const std::string serialized_entity_data_b = buffer->Serialize();
-
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"", /*type=*/"test",
- /*score=*/1.0, /*priority_score=*/1.0, /*annotations=*/{},
- /*serialized_entity_data=*/serialized_entity_data_a},
- {/*response_text=*/"", /*type=*/"test",
- /*score=*/1.0, /*priority_score=*/1.0, /*annotations=*/{},
- /*serialized_entity_data=*/serialized_entity_data_b},
- {/*response_text=*/"", /*type=*/"share_location", /*score=*/0.5},
- {/*response_text=*/"", /*type=*/"add_to_collection", /*score=*/0.1}};
- const std::string test_snippet = R"(
- local result = {}
- for id, action in pairs(actions) do
- if action.type == "test" and action.test == "value_a" then
- table.insert(result, id)
- end
- end
- return result
- )";
-
- EXPECT_TRUE(ActionsSuggestionsLuaRanker::Create(
- conversation, test_snippet, entity_data_schema,
- /*annotations_entity_data_schema=*/nullptr, &response)
- ->RankActions());
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsActionType("test")}));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/actions/ranker_test.cc b/native/actions/ranker_test.cc
deleted file mode 100644
index b52cf45..0000000
--- a/native/actions/ranker_test.cc
+++ /dev/null
@@ -1,382 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "actions/ranker.h"
-
-#include <string>
-
-#include "actions/types.h"
-#include "utils/zlib/zlib.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-MATCHER_P3(IsAction, type, response_text, score, "") {
- return testing::Value(arg.type, type) &&
- testing::Value(arg.response_text, response_text) &&
- testing::Value(arg.score, score);
-}
-
-MATCHER_P(IsActionType, type, "") { return testing::Value(arg.type, type); }
-
-TEST(RankingTest, DeduplicationSmartReply) {
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"hello there", /*type=*/"text_reply",
- /*score=*/1.0},
- {/*response_text=*/"hello there", /*type=*/"text_reply", /*score=*/0.5}};
-
- RankingOptionsT options;
- options.deduplicate_suggestions = true;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(RankingOptions::Pack(builder, &options));
- auto ranker = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
- flatbuffers::GetRoot<RankingOptions>(builder.GetBufferPointer()),
- /*decompressor=*/nullptr, /*smart_reply_action_type=*/"text_reply");
-
- ranker->RankActions(conversation, &response);
- EXPECT_THAT(
- response.actions,
- testing::ElementsAreArray({IsAction("text_reply", "hello there", 1.0)}));
-}
-
-TEST(RankingTest, DeduplicationExtraData) {
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"hello there", /*type=*/"text_reply",
- /*score=*/1.0, /*priority_score=*/0.0},
- {/*response_text=*/"hello there", /*type=*/"text_reply", /*score=*/0.5,
- /*priority_score=*/0.0},
- {/*response_text=*/"hello there", /*type=*/"text_reply", /*score=*/0.6,
- /*priority_score=*/0.0,
- /*annotations=*/{}, /*serialized_entity_data=*/"test"},
- };
-
- RankingOptionsT options;
- options.deduplicate_suggestions = true;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(RankingOptions::Pack(builder, &options));
- auto ranker = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
- flatbuffers::GetRoot<RankingOptions>(builder.GetBufferPointer()),
- /*decompressor=*/nullptr, /*smart_reply_action_type=*/"text_reply");
-
- ranker->RankActions(conversation, &response);
- EXPECT_THAT(
- response.actions,
- testing::ElementsAreArray({IsAction("text_reply", "hello there", 1.0),
- // Is kept as it has different entity data.
- IsAction("text_reply", "hello there", 0.6)}));
-}
-
-TEST(RankingTest, DeduplicationAnnotations) {
- const Conversation conversation = {
- {{/*user_id=*/1, "742 Evergreen Terrace, the number is 1-800-TESTING"}}};
- ActionsSuggestionsResponse response;
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{0, 21},
- /*text=*/"742 Evergreen Terrace"};
- annotation.entity = ClassificationResult("address", 0.5);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"view_map",
- /*score=*/0.5,
- /*priority_score=*/1.0,
- /*annotations=*/{annotation}});
- }
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{0, 21},
- /*text=*/"742 Evergreen Terrace"};
- annotation.entity = ClassificationResult("address", 1.0);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"view_map",
- /*score=*/1.0,
- /*priority_score=*/2.0,
- /*annotations=*/{annotation}});
- }
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{37, 50},
- /*text=*/"1-800-TESTING"};
- annotation.entity = ClassificationResult("phone", 0.5);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"call_phone",
- /*score=*/0.5,
- /*priority_score=*/1.0,
- /*annotations=*/{annotation}});
- }
-
- RankingOptionsT options;
- options.deduplicate_suggestions = true;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(RankingOptions::Pack(builder, &options));
- auto ranker = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
- flatbuffers::GetRoot<RankingOptions>(builder.GetBufferPointer()),
- /*decompressor=*/nullptr, /*smart_reply_action_type=*/"text_reply");
-
- ranker->RankActions(conversation, &response);
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsAction("view_map", "", 1.0),
- IsAction("call_phone", "", 0.5)}));
-}
-
-TEST(RankingTest, DeduplicationAnnotationsByPriorityScore) {
- const Conversation conversation = {
- {{/*user_id=*/1, "742 Evergreen Terrace, the number is 1-800-TESTING"}}};
- ActionsSuggestionsResponse response;
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{0, 21},
- /*text=*/"742 Evergreen Terrace"};
- annotation.entity = ClassificationResult("address", 0.5);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"view_map",
- /*score=*/0.6,
- /*priority_score=*/2.0,
- /*annotations=*/{annotation}});
- }
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{0, 21},
- /*text=*/"742 Evergreen Terrace"};
- annotation.entity = ClassificationResult("address", 1.0);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"view_map",
- /*score=*/1.0,
- /*priority_score=*/1.0,
- /*annotations=*/{annotation}});
- }
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{37, 50},
- /*text=*/"1-800-TESTING"};
- annotation.entity = ClassificationResult("phone", 0.5);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"call_phone",
- /*score=*/0.5,
- /*priority_score=*/1.0,
- /*annotations=*/{annotation}});
- }
-
- RankingOptionsT options;
- options.deduplicate_suggestions = true;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(RankingOptions::Pack(builder, &options));
- auto ranker = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
- flatbuffers::GetRoot<RankingOptions>(builder.GetBufferPointer()),
- /*decompressor=*/nullptr, /*smart_reply_action_type=*/"text_reply");
-
- ranker->RankActions(conversation, &response);
- EXPECT_THAT(
- response.actions,
- testing::ElementsAreArray(
- {IsAction("view_map", "",
- 0.6), // lower score wins, as priority score is higher
- IsAction("call_phone", "", 0.5)}));
-}
-
-TEST(RankingTest, DeduplicatesConflictingActions) {
- const Conversation conversation = {{{/*user_id=*/1, "code A-911"}}};
- ActionsSuggestionsResponse response;
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{7, 10},
- /*text=*/"911"};
- annotation.entity = ClassificationResult("phone", 1.0);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"call_phone",
- /*score=*/1.0,
- /*priority_score=*/1.0,
- /*annotations=*/{annotation}});
- }
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{5, 10},
- /*text=*/"A-911"};
- annotation.entity = ClassificationResult("code", 1.0);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"copy_code",
- /*score=*/1.0,
- /*priority_score=*/2.0,
- /*annotations=*/{annotation}});
- }
- RankingOptionsT options;
- options.deduplicate_suggestions = true;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(RankingOptions::Pack(builder, &options));
- auto ranker = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
- flatbuffers::GetRoot<RankingOptions>(builder.GetBufferPointer()),
- /*decompressor=*/nullptr, /*smart_reply_action_type=*/"text_reply");
-
- ranker->RankActions(conversation, &response);
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsAction("copy_code", "", 1.0)}));
-}
-
-TEST(RankingTest, HandlesCompressedLuaScript) {
- const Conversation conversation = {{{/*user_id=*/1, "hello hello"}}};
- ActionsSuggestionsResponse response;
- response.actions = {
- {/*response_text=*/"hello there", /*type=*/"text_reply",
- /*score=*/1.0},
- {/*response_text=*/"", /*type=*/"share_location", /*score=*/0.5},
- {/*response_text=*/"", /*type=*/"add_to_collection", /*score=*/0.1}};
- const std::string test_snippet = R"(
- local result = {}
- for id, action in pairs(actions) do
- if action.type ~= "text_reply" then
- table.insert(result, id)
- end
- end
- return result
- )";
- RankingOptionsT options;
- options.compressed_lua_ranking_script.reset(new CompressedBufferT);
- std::unique_ptr<ZlibCompressor> compressor = ZlibCompressor::Instance();
- compressor->Compress(test_snippet,
- options.compressed_lua_ranking_script.get());
- options.deduplicate_suggestions = true;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(RankingOptions::Pack(builder, &options));
-
- std::unique_ptr<ZlibDecompressor> decompressor = ZlibDecompressor::Instance();
- auto ranker = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
- flatbuffers::GetRoot<RankingOptions>(builder.GetBufferPointer()),
- decompressor.get(), /*smart_reply_action_type=*/"text_reply");
-
- ranker->RankActions(conversation, &response);
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsActionType("share_location"),
- IsActionType("add_to_collection")}));
-}
-
-TEST(RankingTest, SuppressSmartRepliesWithAction) {
- const Conversation conversation = {{{/*user_id=*/1, "should i call 911"}}};
- ActionsSuggestionsResponse response;
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{5, 8},
- /*text=*/"911"};
- annotation.entity = ClassificationResult("phone", 1.0);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"call_phone",
- /*score=*/1.0,
- /*priority_score=*/1.0,
- /*annotations=*/{annotation}});
- }
- response.actions.push_back({/*response_text=*/"How are you?",
- /*type=*/"text_reply"});
- RankingOptionsT options;
- options.suppress_smart_replies_with_actions = true;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(RankingOptions::Pack(builder, &options));
- auto ranker = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
- flatbuffers::GetRoot<RankingOptions>(builder.GetBufferPointer()),
- /*decompressor=*/nullptr, /*smart_reply_action_type=*/"text_reply");
-
- ranker->RankActions(conversation, &response);
-
- EXPECT_THAT(response.actions,
- testing::ElementsAreArray({IsAction("call_phone", "", 1.0)}));
-}
-
-TEST(RankingTest, GroupsActionsByAnnotations) {
- const Conversation conversation = {{{/*user_id=*/1, "should i call 911"}}};
- ActionsSuggestionsResponse response;
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{5, 8},
- /*text=*/"911"};
- annotation.entity = ClassificationResult("phone", 1.0);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"call_phone",
- /*score=*/1.0,
- /*priority_score=*/1.0,
- /*annotations=*/{annotation}});
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"add_contact",
- /*score=*/0.0,
- /*priority_score=*/0.0,
- /*annotations=*/{annotation}});
- }
- response.actions.push_back({/*response_text=*/"How are you?",
- /*type=*/"text_reply",
- /*score=*/0.5});
- RankingOptionsT options;
- options.group_by_annotations = true;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(RankingOptions::Pack(builder, &options));
- auto ranker = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
- flatbuffers::GetRoot<RankingOptions>(builder.GetBufferPointer()),
- /*decompressor=*/nullptr, /*smart_reply_action_type=*/"text_reply");
-
- ranker->RankActions(conversation, &response);
-
- // The text reply should be last, even though it has a higher score than the
- // `add_contact` action.
- EXPECT_THAT(
- response.actions,
- testing::ElementsAreArray({IsAction("call_phone", "", 1.0),
- IsAction("add_contact", "", 0.0),
- IsAction("text_reply", "How are you?", 0.5)}));
-}
-
-TEST(RankingTest, SortsActionsByScore) {
- const Conversation conversation = {{{/*user_id=*/1, "should i call 911"}}};
- ActionsSuggestionsResponse response;
- {
- ActionSuggestionAnnotation annotation;
- annotation.span = {/*message_index=*/0, /*span=*/{5, 8},
- /*text=*/"911"};
- annotation.entity = ClassificationResult("phone", 1.0);
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"call_phone",
- /*score=*/1.0,
- /*priority_score=*/1.0,
- /*annotations=*/{annotation}});
- response.actions.push_back({/*response_text=*/"",
- /*type=*/"add_contact",
- /*score=*/0.0,
- /*priority_score=*/0.0,
- /*annotations=*/{annotation}});
- }
- response.actions.push_back({/*response_text=*/"How are you?",
- /*type=*/"text_reply",
- /*score=*/0.5});
- RankingOptionsT options;
- // Don't group by annotation.
- options.group_by_annotations = false;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(RankingOptions::Pack(builder, &options));
- auto ranker = ActionsSuggestionsRanker::CreateActionsSuggestionsRanker(
- flatbuffers::GetRoot<RankingOptions>(builder.GetBufferPointer()),
- /*decompressor=*/nullptr, /*smart_reply_action_type=*/"text_reply");
-
- ranker->RankActions(conversation, &response);
-
- EXPECT_THAT(
- response.actions,
- testing::ElementsAreArray({IsAction("call_phone", "", 1.0),
- IsAction("text_reply", "How are you?", 0.5),
- IsAction("add_contact", "", 0.0)}));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/actions/test-utils.cc b/native/actions/test-utils.cc
deleted file mode 100644
index 9b003dd..0000000
--- a/native/actions/test-utils.cc
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "actions/test-utils.h"
-
-namespace libtextclassifier3 {
-
-std::string TestEntityDataSchema() {
- // Create fake entity data schema meta data.
- // Cannot use object oriented API here as that is not available for the
- // reflection schema.
- flatbuffers::FlatBufferBuilder schema_builder;
- std::vector<flatbuffers::Offset<reflection::Field>> fields = {
- reflection::CreateField(
- schema_builder,
- /*name=*/schema_builder.CreateString("greeting"),
- /*type=*/
- reflection::CreateType(schema_builder,
- /*base_type=*/reflection::String),
- /*id=*/0,
- /*offset=*/4),
- reflection::CreateField(
- schema_builder,
- /*name=*/schema_builder.CreateString("location"),
- /*type=*/
- reflection::CreateType(schema_builder,
- /*base_type=*/reflection::String),
- /*id=*/1,
- /*offset=*/6),
- reflection::CreateField(
- schema_builder,
- /*name=*/schema_builder.CreateString("person"),
- /*type=*/
- reflection::CreateType(schema_builder,
- /*base_type=*/reflection::String),
- /*id=*/2,
- /*offset=*/8)};
- std::vector<flatbuffers::Offset<reflection::Enum>> enums;
- std::vector<flatbuffers::Offset<reflection::Object>> objects = {
- reflection::CreateObject(
- schema_builder,
- /*name=*/schema_builder.CreateString("EntityData"),
- /*fields=*/
- schema_builder.CreateVectorOfSortedTables(&fields))};
- schema_builder.Finish(reflection::CreateSchema(
- schema_builder, schema_builder.CreateVectorOfSortedTables(&objects),
- schema_builder.CreateVectorOfSortedTables(&enums),
- /*(unused) file_ident=*/0,
- /*(unused) file_ext=*/0,
- /*root_table*/ objects[0]));
-
- return std::string(
- reinterpret_cast<const char*>(schema_builder.GetBufferPointer()),
- schema_builder.GetSize());
-}
-
-void SetTestEntityDataSchema(ActionsModelT* test_model) {
- const std::string serialized_schema = TestEntityDataSchema();
-
- test_model->actions_entity_data_schema.assign(
- serialized_schema.data(),
- serialized_schema.data() + serialized_schema.size());
-}
-
-} // namespace libtextclassifier3
diff --git a/native/actions/test-utils.h b/native/actions/test-utils.h
deleted file mode 100644
index c05d6a9..0000000
--- a/native/actions/test-utils.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LIBTEXTCLASSIFIER_ACTIONS_TEST_UTILS_H_
-#define LIBTEXTCLASSIFIER_ACTIONS_TEST_UTILS_H_
-
-#include <string>
-
-#include "actions/actions_model_generated.h"
-#include "utils/flatbuffers.h"
-#include "gmock/gmock.h"
-
-namespace libtextclassifier3 {
-
-using testing::ExplainMatchResult;
-using testing::Value;
-
-// Create test entity data schema.
-std::string TestEntityDataSchema();
-void SetTestEntityDataSchema(ActionsModelT* test_model);
-
-MATCHER_P(IsActionOfType, type, "") { return Value(arg.type, type); }
-MATCHER_P(IsSmartReply, response_text, "") {
- return ExplainMatchResult(IsActionOfType("text_reply"), arg,
- result_listener) &&
- Value(arg.response_text, response_text);
-}
-MATCHER_P(IsSpan, span, "") {
- return Value(arg.first, span.first) && Value(arg.second, span.second);
-}
-MATCHER_P3(IsActionSuggestionAnnotation, name, text, span, "") {
- return Value(arg.name, name) && Value(arg.span.text, text) &&
- ExplainMatchResult(IsSpan(span), arg.span.span, result_listener);
-}
-
-} // namespace libtextclassifier3
-
-#endif // LIBTEXTCLASSIFIER_ACTIONS_TEST_UTILS_H_
diff --git a/native/actions/zlib-utils_test.cc b/native/actions/zlib-utils_test.cc
deleted file mode 100644
index befee31..0000000
--- a/native/actions/zlib-utils_test.cc
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "actions/zlib-utils.h"
-
-#include <memory>
-
-#include "actions/actions_model_generated.h"
-#include "utils/zlib/zlib.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::ElementsAre;
-using testing::Field;
-using testing::Pointee;
-
-TEST(ZlibUtilsTest, CompressModel) {
- ActionsModelT model;
- constexpr char kTestPattern1[] = "this is a test pattern";
- constexpr char kTestPattern2[] = "this is a second test pattern";
- model.rules.reset(new RulesModelT);
- model.rules->regex_rule.emplace_back(new RulesModel_::RegexRuleT);
- model.rules->regex_rule.back()->pattern = kTestPattern1;
- model.rules->regex_rule.emplace_back(new RulesModel_::RegexRuleT);
- model.rules->regex_rule.back()->pattern = kTestPattern2;
-
- // Compress the model.
- EXPECT_TRUE(CompressActionsModel(&model));
-
- // Sanity check that uncompressed field is removed.
- const auto is_empty_pattern =
- Pointee(Field(&libtextclassifier3::RulesModel_::RegexRuleT::pattern,
- testing::IsEmpty()));
- EXPECT_THAT(model.rules->regex_rule,
- ElementsAre(is_empty_pattern, is_empty_pattern));
- // Pack and load the model.
- flatbuffers::FlatBufferBuilder builder;
- FinishActionsModelBuffer(builder, ActionsModel::Pack(builder, &model));
- const ActionsModel* compressed_model = GetActionsModel(
- reinterpret_cast<const char*>(builder.GetBufferPointer()));
- ASSERT_TRUE(compressed_model != nullptr);
-
- // Decompress the fields again and check that they match the original.
- std::unique_ptr<ZlibDecompressor> decompressor = ZlibDecompressor::Instance();
- ASSERT_TRUE(decompressor != nullptr);
- std::string uncompressed_pattern;
- EXPECT_TRUE(decompressor->MaybeDecompress(
- compressed_model->rules()->regex_rule()->Get(0)->compressed_pattern(),
- &uncompressed_pattern));
- EXPECT_EQ(uncompressed_pattern, kTestPattern1);
- EXPECT_TRUE(decompressor->MaybeDecompress(
- compressed_model->rules()->regex_rule()->Get(1)->compressed_pattern(),
- &uncompressed_pattern));
- EXPECT_EQ(uncompressed_pattern, kTestPattern2);
- EXPECT_TRUE(DecompressActionsModel(&model));
- EXPECT_EQ(model.rules->regex_rule[0]->pattern, kTestPattern1);
- EXPECT_EQ(model.rules->regex_rule[1]->pattern, kTestPattern2);
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/annotator/annotator.cc b/native/annotator/annotator.cc
index d992e7a..d9c69a2 100644
--- a/native/annotator/annotator.cc
+++ b/native/annotator/annotator.cc
@@ -17,7 +17,6 @@
#include "annotator/annotator.h"
#include <algorithm>
-#include <cctype>
#include <cmath>
#include <iterator>
#include <numeric>
@@ -38,6 +37,7 @@
#include "utils/strings/numbers.h"
#include "utils/strings/split.h"
#include "utils/utf8/unicodetext.h"
+#include "utils/utf8/unilib-common.h"
#include "utils/zlib/zlib_regex.h"
namespace libtextclassifier3 {
@@ -412,7 +412,9 @@
/*tokenizer_options=*/
model_->grammar_datetime_model()->grammar_tokenizer_options(),
*calendarlib_,
- /*datetime_rules=*/model_->grammar_datetime_model()->datetime_rules()));
+ /*datetime_rules=*/model_->grammar_datetime_model()->datetime_rules(),
+ model_->grammar_datetime_model()->target_classification_score(),
+ model_->grammar_datetime_model()->priority_score()));
if (!cfg_datetime_parser_) {
TC3_LOG(ERROR) << "Could not initialize context free grammar based "
"datetime parser.";
@@ -594,17 +596,15 @@
}
}
-bool Annotator::InitializePersonNameEngineFromFileDescriptor(int fd, int offset,
- int size) {
- std::unique_ptr<ScopedMmap> mmap(new ScopedMmap(fd, offset, size));
-
- if (!mmap->handle().ok()) {
+bool Annotator::InitializePersonNameEngineFromScopedMmap(
+ const ScopedMmap& mmap) {
+ if (!mmap.handle().ok()) {
TC3_LOG(ERROR) << "Mmap for person name model failed.";
return false;
}
const PersonNameModel* person_name_model = LoadAndVerifyPersonNameModel(
- mmap->handle().start(), mmap->handle().num_bytes());
+ mmap.handle().start(), mmap.handle().num_bytes());
if (person_name_model == nullptr) {
TC3_LOG(ERROR) << "Person name model verification failed.";
@@ -616,7 +616,7 @@
}
std::unique_ptr<PersonNameEngine> person_name_engine(
- new PersonNameEngine(unilib_));
+ new PersonNameEngine(selection_feature_processor_.get(), unilib_));
if (!person_name_engine->Initialize(person_name_model)) {
TC3_LOG(ERROR) << "Failed to initialize the person name engine.";
return false;
@@ -625,6 +625,17 @@
return true;
}
+bool Annotator::InitializePersonNameEngineFromPath(const std::string& path) {
+ std::unique_ptr<ScopedMmap> mmap(new ScopedMmap(path));
+ return InitializePersonNameEngineFromScopedMmap(*mmap);
+}
+
+bool Annotator::InitializePersonNameEngineFromFileDescriptor(int fd, int offset,
+ int size) {
+ std::unique_ptr<ScopedMmap> mmap(new ScopedMmap(fd, offset, size));
+ return InitializePersonNameEngineFromScopedMmap(*mmap);
+}
+
namespace {
int CountDigits(const std::string& str, CodepointSpan selection_indices) {
@@ -633,7 +644,7 @@
const UnicodeText unicode_str = UTF8ToUnicodeText(str, /*do_copy=*/false);
for (auto it = unicode_str.begin(); it != unicode_str.end(); ++it, ++i) {
if (i >= selection_indices.first && i < selection_indices.second &&
- isdigit(*it)) {
+ IsDigit(*it)) {
++count;
}
}
@@ -2259,14 +2270,14 @@
std::unique_ptr<EntityDataT> data =
LoadAndVerifyMutableFlatbuffer<libtextclassifier3::EntityData>(
*serialized_entity_data);
- if (data == nullptr) {
+ if (data == nullptr || data->money->unnormalized_amount.empty()) {
return false;
}
UnicodeText amount =
UTF8ToUnicodeText(data->money->unnormalized_amount, /*do_copy=*/false);
int separator_back_index = 0;
- auto it_decimal_separator = amount.end();
+ auto it_decimal_separator = --amount.end();
for (; it_decimal_separator != amount.begin();
--it_decimal_separator, ++separator_back_index) {
if (std::find(money_separators_.begin(), money_separators_.end(),
@@ -2279,7 +2290,7 @@
// If there are 3 digits after the last separator, we consider that a
// thousands separator => the number is an int (e.g. 1.234 is considered int).
// If there is no separator in number, also that number is an int.
- if (separator_back_index == 4 || it_decimal_separator == amount.begin()) {
+ if (separator_back_index == 3 || it_decimal_separator == amount.begin()) {
it_decimal_separator = amount.end();
}
@@ -2295,7 +2306,7 @@
const int amount_codepoints_size = amount.size_codepoints();
if (!unilib_->ParseInt32(
UnicodeText::Substring(
- amount, amount_codepoints_size - separator_back_index + 1,
+ amount, amount_codepoints_size - separator_back_index,
amount_codepoints_size, /*do_copy=*/false),
&data->money->amount_decimal_part)) {
TC3_LOG(ERROR) << "Could not parse the money decimal part as int32.";
diff --git a/native/annotator/annotator.h b/native/annotator/annotator.h
index 67ae92c..c20eb9e 100644
--- a/native/annotator/annotator.h
+++ b/native/annotator/annotator.h
@@ -139,6 +139,14 @@
// Initializes the installed app engine with the given config.
bool InitializeInstalledAppEngine(const std::string& serialized_config);
+ // Initializes the person name engine with the given person name model from
+ // the provided mmap.
+ bool InitializePersonNameEngineFromScopedMmap(const ScopedMmap& mmap);
+
+ // Initializes the person name engine with the given person name model in the
+ // provided file path.
+ bool InitializePersonNameEngineFromPath(const std::string& path);
+
// Initializes the person name engine with the given person name model in the
// provided file descriptor.
bool InitializePersonNameEngineFromFileDescriptor(int fd, int offset,
diff --git a/native/annotator/annotator_jni_test.cc b/native/annotator/annotator_jni_test.cc
deleted file mode 100644
index 929fb59..0000000
--- a/native/annotator/annotator_jni_test.cc
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/annotator_jni.h"
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(Annotator, ConvertIndicesBMPUTF8) {
- // Test boundary cases.
- EXPECT_EQ(ConvertIndicesBMPToUTF8("hello", {0, 5}), std::make_pair(0, 5));
- EXPECT_EQ(ConvertIndicesUTF8ToBMP("hello", {0, 5}), std::make_pair(0, 5));
-
- EXPECT_EQ(ConvertIndicesBMPToUTF8("hello world", {0, 5}),
- std::make_pair(0, 5));
- EXPECT_EQ(ConvertIndicesUTF8ToBMP("hello world", {0, 5}),
- std::make_pair(0, 5));
- EXPECT_EQ(ConvertIndicesBMPToUTF8("😁ello world", {0, 6}),
- std::make_pair(0, 5));
- EXPECT_EQ(ConvertIndicesUTF8ToBMP("😁ello world", {0, 5}),
- std::make_pair(0, 6));
-
- EXPECT_EQ(ConvertIndicesBMPToUTF8("hello world", {6, 11}),
- std::make_pair(6, 11));
- EXPECT_EQ(ConvertIndicesUTF8ToBMP("hello world", {6, 11}),
- std::make_pair(6, 11));
- EXPECT_EQ(ConvertIndicesBMPToUTF8("hello worl😁", {6, 12}),
- std::make_pair(6, 11));
- EXPECT_EQ(ConvertIndicesUTF8ToBMP("hello worl😁", {6, 11}),
- std::make_pair(6, 12));
-
- // Simple example where the longer character is before the selection.
- // character 😁 is 0x1f601
- EXPECT_EQ(ConvertIndicesBMPToUTF8("😁 Hello World.", {3, 8}),
- std::make_pair(2, 7));
-
- EXPECT_EQ(ConvertIndicesUTF8ToBMP("😁 Hello World.", {2, 7}),
- std::make_pair(3, 8));
-
- // Longer character is before and in selection.
- EXPECT_EQ(ConvertIndicesBMPToUTF8("😁 Hell😁 World.", {3, 9}),
- std::make_pair(2, 7));
-
- EXPECT_EQ(ConvertIndicesUTF8ToBMP("😁 Hell😁 World.", {2, 7}),
- std::make_pair(3, 9));
-
- // Longer character is before and after selection.
- EXPECT_EQ(ConvertIndicesBMPToUTF8("😁 Hello😁World.", {3, 8}),
- std::make_pair(2, 7));
-
- EXPECT_EQ(ConvertIndicesUTF8ToBMP("😁 Hello😁World.", {2, 7}),
- std::make_pair(3, 8));
-
- // Longer character is before in after selection.
- EXPECT_EQ(ConvertIndicesBMPToUTF8("😁 Hell😁😁World.", {3, 9}),
- std::make_pair(2, 7));
-
- EXPECT_EQ(ConvertIndicesUTF8ToBMP("😁 Hell😁😁World.", {2, 7}),
- std::make_pair(3, 9));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/annotator/cached-features_test.cc b/native/annotator/cached-features_test.cc
deleted file mode 100644
index 702f3ca..0000000
--- a/native/annotator/cached-features_test.cc
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/cached-features.h"
-
-#include "annotator/model-executor.h"
-#include "utils/tensor-view.h"
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-using testing::ElementsAreArray;
-using testing::FloatEq;
-using testing::Matcher;
-
-namespace libtextclassifier3 {
-namespace {
-
-Matcher<std::vector<float>> ElementsAreFloat(const std::vector<float>& values) {
- std::vector<Matcher<float>> matchers;
- for (const float value : values) {
- matchers.push_back(FloatEq(value));
- }
- return ElementsAreArray(matchers);
-}
-
-std::unique_ptr<std::vector<float>> MakeFeatures(int num_tokens) {
- std::unique_ptr<std::vector<float>> features(new std::vector<float>());
- for (int i = 1; i <= num_tokens; ++i) {
- features->push_back(i * 11.0f);
- features->push_back(-i * 11.0f);
- features->push_back(i * 0.1f);
- }
- return features;
-}
-
-std::vector<float> GetCachedClickContextFeatures(
- const CachedFeatures& cached_features, int click_pos) {
- std::vector<float> output_features;
- cached_features.AppendClickContextFeaturesForClick(click_pos,
- &output_features);
- return output_features;
-}
-
-std::vector<float> GetCachedBoundsSensitiveFeatures(
- const CachedFeatures& cached_features, TokenSpan selected_span) {
- std::vector<float> output_features;
- cached_features.AppendBoundsSensitiveFeaturesForSpan(selected_span,
- &output_features);
- return output_features;
-}
-
-TEST(CachedFeaturesTest, ClickContext) {
- FeatureProcessorOptionsT options;
- options.context_size = 2;
- options.feature_version = 1;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(CreateFeatureProcessorOptions(builder, &options));
- flatbuffers::DetachedBuffer options_fb = builder.Release();
-
- std::unique_ptr<std::vector<float>> features = MakeFeatures(9);
- std::unique_ptr<std::vector<float>> padding_features(
- new std::vector<float>{112233.0, -112233.0, 321.0});
-
- const std::unique_ptr<CachedFeatures> cached_features =
- CachedFeatures::Create(
- {3, 10}, std::move(features), std::move(padding_features),
- flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
- /*feature_vector_size=*/3);
- ASSERT_TRUE(cached_features);
-
- EXPECT_THAT(GetCachedClickContextFeatures(*cached_features, 5),
- ElementsAreFloat({11.0, -11.0, 0.1, 22.0, -22.0, 0.2, 33.0, -33.0,
- 0.3, 44.0, -44.0, 0.4, 55.0, -55.0, 0.5}));
-
- EXPECT_THAT(GetCachedClickContextFeatures(*cached_features, 6),
- ElementsAreFloat({22.0, -22.0, 0.2, 33.0, -33.0, 0.3, 44.0, -44.0,
- 0.4, 55.0, -55.0, 0.5, 66.0, -66.0, 0.6}));
-
- EXPECT_THAT(GetCachedClickContextFeatures(*cached_features, 7),
- ElementsAreFloat({33.0, -33.0, 0.3, 44.0, -44.0, 0.4, 55.0, -55.0,
- 0.5, 66.0, -66.0, 0.6, 77.0, -77.0, 0.7}));
-}
-
-TEST(CachedFeaturesTest, BoundsSensitive) {
- std::unique_ptr<FeatureProcessorOptions_::BoundsSensitiveFeaturesT> config(
- new FeatureProcessorOptions_::BoundsSensitiveFeaturesT());
- config->enabled = true;
- config->num_tokens_before = 2;
- config->num_tokens_inside_left = 2;
- config->num_tokens_inside_right = 2;
- config->num_tokens_after = 2;
- config->include_inside_bag = true;
- config->include_inside_length = true;
- FeatureProcessorOptionsT options;
- options.bounds_sensitive_features = std::move(config);
- options.feature_version = 2;
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(CreateFeatureProcessorOptions(builder, &options));
- flatbuffers::DetachedBuffer options_fb = builder.Release();
-
- std::unique_ptr<std::vector<float>> features = MakeFeatures(9);
- std::unique_ptr<std::vector<float>> padding_features(
- new std::vector<float>{112233.0, -112233.0, 321.0});
-
- const std::unique_ptr<CachedFeatures> cached_features =
- CachedFeatures::Create(
- {3, 9}, std::move(features), std::move(padding_features),
- flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()),
- /*feature_vector_size=*/3);
- ASSERT_TRUE(cached_features);
-
- EXPECT_THAT(
- GetCachedBoundsSensitiveFeatures(*cached_features, {5, 8}),
- ElementsAreFloat({11.0, -11.0, 0.1, 22.0, -22.0, 0.2, 33.0,
- -33.0, 0.3, 44.0, -44.0, 0.4, 44.0, -44.0,
- 0.4, 55.0, -55.0, 0.5, 66.0, -66.0, 0.6,
- 112233.0, -112233.0, 321.0, 44.0, -44.0, 0.4, 3.0}));
-
- EXPECT_THAT(
- GetCachedBoundsSensitiveFeatures(*cached_features, {5, 7}),
- ElementsAreFloat({11.0, -11.0, 0.1, 22.0, -22.0, 0.2, 33.0,
- -33.0, 0.3, 44.0, -44.0, 0.4, 33.0, -33.0,
- 0.3, 44.0, -44.0, 0.4, 55.0, -55.0, 0.5,
- 66.0, -66.0, 0.6, 38.5, -38.5, 0.35, 2.0}));
-
- EXPECT_THAT(
- GetCachedBoundsSensitiveFeatures(*cached_features, {6, 8}),
- ElementsAreFloat({22.0, -22.0, 0.2, 33.0, -33.0, 0.3, 44.0,
- -44.0, 0.4, 55.0, -55.0, 0.5, 44.0, -44.0,
- 0.4, 55.0, -55.0, 0.5, 66.0, -66.0, 0.6,
- 112233.0, -112233.0, 321.0, 49.5, -49.5, 0.45, 2.0}));
-
- EXPECT_THAT(
- GetCachedBoundsSensitiveFeatures(*cached_features, {6, 7}),
- ElementsAreFloat({22.0, -22.0, 0.2, 33.0, -33.0, 0.3,
- 44.0, -44.0, 0.4, 112233.0, -112233.0, 321.0,
- 112233.0, -112233.0, 321.0, 44.0, -44.0, 0.4,
- 55.0, -55.0, 0.5, 66.0, -66.0, 0.6,
- 44.0, -44.0, 0.4, 1.0}));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/annotator/datetime/parser_test.cc b/native/annotator/datetime/parser_test.cc
deleted file mode 100644
index 1ddcf50..0000000
--- a/native/annotator/datetime/parser_test.cc
+++ /dev/null
@@ -1,1439 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/datetime/parser.h"
-
-#include <time.h>
-
-#include <fstream>
-#include <iostream>
-#include <memory>
-#include <string>
-
-#include "annotator/annotator.h"
-#include "annotator/model_generated.h"
-#include "annotator/types-test-util.h"
-#include "utils/testing/annotator.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-using std::vector;
-using testing::ElementsAreArray;
-
-namespace libtextclassifier3 {
-namespace {
-// Builder class to construct the DatetimeComponents and make the test readable.
-class DatetimeComponentsBuilder {
- public:
- DatetimeComponentsBuilder Add(DatetimeComponent::ComponentType type,
- int value) {
- DatetimeComponent component;
- component.component_type = type;
- component.value = value;
- return AddComponent(component);
- }
-
- DatetimeComponentsBuilder Add(
- DatetimeComponent::ComponentType type, int value,
- DatetimeComponent::RelativeQualifier relative_qualifier,
- int relative_count) {
- DatetimeComponent component;
- component.component_type = type;
- component.value = value;
- component.relative_qualifier = relative_qualifier;
- component.relative_count = relative_count;
- return AddComponent(component);
- }
-
- std::vector<DatetimeComponent> Build() {
- std::vector<DatetimeComponent> result(datetime_components_);
- datetime_components_.clear();
- return result;
- }
-
- private:
- DatetimeComponentsBuilder AddComponent(
- const DatetimeComponent& datetime_component) {
- datetime_components_.push_back(datetime_component);
- return *this;
- }
- std::vector<DatetimeComponent> datetime_components_;
-};
-
-std::string GetModelPath() {
- return TC3_TEST_DATA_DIR;
-}
-
-std::string ReadFile(const std::string& file_name) {
- std::ifstream file_stream(file_name);
- return std::string(std::istreambuf_iterator<char>(file_stream), {});
-}
-
-class ParserTest : public testing::Test {
- public:
- void SetUp() override {
- // Loads default unmodified model. Individual tests can call LoadModel to
- // make changes.
- LoadModel([](ModelT* model) {});
- }
-
- template <typename Fn>
- void LoadModel(Fn model_visitor_fn) {
- std::string model_buffer = ReadFile(GetModelPath() + "test_model.fb");
- model_buffer_ = ModifyAnnotatorModel(model_buffer, model_visitor_fn);
- classifier_ = Annotator::FromUnownedBuffer(model_buffer_.data(),
- model_buffer_.size(), &unilib_);
- TC3_CHECK(classifier_);
- parser_ = classifier_->DatetimeParserForTests();
- TC3_CHECK(parser_);
- }
-
- bool HasNoResult(const std::string& text, bool anchor_start_end = false,
- const std::string& timezone = "Europe/Zurich",
- AnnotationUsecase annotation_usecase =
- AnnotationUsecase_ANNOTATION_USECASE_SMART) {
- std::vector<DatetimeParseResultSpan> results;
- if (!parser_->Parse(text, 0, timezone, /*locales=*/"", ModeFlag_ANNOTATION,
- annotation_usecase, anchor_start_end, &results)) {
- TC3_LOG(ERROR) << text;
- TC3_CHECK(false);
- }
- return results.empty();
- }
-
- bool ParsesCorrectly(const std::string& marked_text,
- const vector<int64>& expected_ms_utcs,
- DatetimeGranularity expected_granularity,
- vector<vector<DatetimeComponent>> datetime_components,
- bool anchor_start_end = false,
- const std::string& timezone = "Europe/Zurich",
- const std::string& locales = "en-US",
- AnnotationUsecase annotation_usecase =
- AnnotationUsecase_ANNOTATION_USECASE_SMART) {
- const UnicodeText marked_text_unicode =
- UTF8ToUnicodeText(marked_text, /*do_copy=*/false);
- auto brace_open_it =
- std::find(marked_text_unicode.begin(), marked_text_unicode.end(), '{');
- auto brace_end_it =
- std::find(marked_text_unicode.begin(), marked_text_unicode.end(), '}');
- TC3_CHECK(brace_open_it != marked_text_unicode.end());
- TC3_CHECK(brace_end_it != marked_text_unicode.end());
-
- std::string text;
- text +=
- UnicodeText::UTF8Substring(marked_text_unicode.begin(), brace_open_it);
- text += UnicodeText::UTF8Substring(std::next(brace_open_it), brace_end_it);
- text += UnicodeText::UTF8Substring(std::next(brace_end_it),
- marked_text_unicode.end());
-
- std::vector<DatetimeParseResultSpan> results;
-
- if (!parser_->Parse(text, 0, timezone, locales, ModeFlag_ANNOTATION,
- annotation_usecase, anchor_start_end, &results)) {
- TC3_LOG(ERROR) << text;
- TC3_CHECK(false);
- }
- if (results.empty()) {
- TC3_LOG(ERROR) << "No results.";
- return false;
- }
-
- const int expected_start_index =
- std::distance(marked_text_unicode.begin(), brace_open_it);
- // The -1 below is to account for the opening bracket character.
- const int expected_end_index =
- std::distance(marked_text_unicode.begin(), brace_end_it) - 1;
-
- std::vector<DatetimeParseResultSpan> filtered_results;
- for (const DatetimeParseResultSpan& result : results) {
- if (SpansOverlap(result.span,
- {expected_start_index, expected_end_index})) {
- filtered_results.push_back(result);
- }
- }
- std::vector<DatetimeParseResultSpan> expected{
- {{expected_start_index, expected_end_index},
- {},
- /*target_classification_score=*/1.0,
- /*priority_score=*/0.1}};
- expected[0].data.resize(expected_ms_utcs.size());
- for (int i = 0; i < expected_ms_utcs.size(); i++) {
- expected[0].data[i] = {expected_ms_utcs[i], expected_granularity,
- datetime_components[i]};
- }
-
- const bool matches =
- testing::Matches(ElementsAreArray(expected))(filtered_results);
- if (!matches) {
- TC3_LOG(ERROR) << "Expected: " << expected[0];
- if (filtered_results.empty()) {
- TC3_LOG(ERROR) << "But got no results.";
- }
- TC3_LOG(ERROR) << "Actual: " << filtered_results[0];
- }
-
- return matches;
- }
-
- bool ParsesCorrectly(const std::string& marked_text,
- const int64 expected_ms_utc,
- DatetimeGranularity expected_granularity,
- vector<vector<DatetimeComponent>> datetime_components,
- bool anchor_start_end = false,
- const std::string& timezone = "Europe/Zurich",
- const std::string& locales = "en-US",
- AnnotationUsecase annotation_usecase =
- AnnotationUsecase_ANNOTATION_USECASE_SMART) {
- return ParsesCorrectly(marked_text, vector<int64>{expected_ms_utc},
- expected_granularity, datetime_components,
- anchor_start_end, timezone, locales,
- annotation_usecase);
- }
-
- bool ParsesCorrectlyGerman(
- const std::string& marked_text, const vector<int64>& expected_ms_utcs,
- DatetimeGranularity expected_granularity,
- vector<vector<DatetimeComponent>> datetime_components) {
- return ParsesCorrectly(marked_text, expected_ms_utcs, expected_granularity,
- datetime_components,
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich", /*locales=*/"de");
- }
-
- bool ParsesCorrectlyGerman(
- const std::string& marked_text, const int64 expected_ms_utc,
- DatetimeGranularity expected_granularity,
- vector<vector<DatetimeComponent>> datetime_components) {
- return ParsesCorrectly(marked_text, expected_ms_utc, expected_granularity,
- datetime_components,
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich", /*locales=*/"de");
- }
-
- protected:
- std::string model_buffer_;
- std::unique_ptr<Annotator> classifier_;
- const DatetimeParser* parser_;
- UniLib unilib_;
-};
-
-// Test with just a few cases to make debugging of general failures easier.
-TEST_F(ParserTest, ParseShort) {
- EXPECT_TRUE(ParsesCorrectly(
- "{January 1, 1988}", 567990000000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 1988)
- .Build()}));
-}
-
-TEST_F(ParserTest, Parse) {
- EXPECT_TRUE(ParsesCorrectly(
- "{January 1, 1988}", 567990000000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 1988)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{january 31 2018}", 1517353200000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 31)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "lorem {1 january 2018} ipsum", 1514761200000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{09/Mar/2004 22:02:40}", 1078866160000, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::SECOND, 40)
- .Add(DatetimeComponent::ComponentType::MINUTE, 02)
- .Add(DatetimeComponent::ComponentType::HOUR, 22)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 9)
- .Add(DatetimeComponent::ComponentType::MONTH, 3)
- .Add(DatetimeComponent::ComponentType::YEAR, 2004)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{Dec 2, 2010 2:39:58 AM}", 1291253998000, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 58)
- .Add(DatetimeComponent::ComponentType::MINUTE, 39)
- .Add(DatetimeComponent::ComponentType::HOUR, 2)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 2)
- .Add(DatetimeComponent::ComponentType::MONTH, 12)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{Jun 09 2011 15:28:14}", 1307626094000, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::SECOND, 14)
- .Add(DatetimeComponent::ComponentType::MINUTE, 28)
- .Add(DatetimeComponent::ComponentType::HOUR, 15)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 9)
- .Add(DatetimeComponent::ComponentType::MONTH, 6)
- .Add(DatetimeComponent::ComponentType::YEAR, 2011)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{Mar 16 08:12:04}", {6419524000, 6462724000}, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 4)
- .Add(DatetimeComponent::ComponentType::MINUTE, 12)
- .Add(DatetimeComponent::ComponentType::HOUR, 8)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 16)
- .Add(DatetimeComponent::ComponentType::MONTH, 3)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 4)
- .Add(DatetimeComponent::ComponentType::MINUTE, 12)
- .Add(DatetimeComponent::ComponentType::HOUR, 8)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 16)
- .Add(DatetimeComponent::ComponentType::MONTH, 3)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{2010-06-26 02:31:29}", {1277512289000, 1277555489000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 29)
- .Add(DatetimeComponent::ComponentType::MINUTE, 31)
- .Add(DatetimeComponent::ComponentType::HOUR, 2)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 26)
- .Add(DatetimeComponent::ComponentType::MONTH, 6)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 29)
- .Add(DatetimeComponent::ComponentType::MINUTE, 31)
- .Add(DatetimeComponent::ComponentType::HOUR, 2)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 26)
- .Add(DatetimeComponent::ComponentType::MONTH, 6)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{2006/01/22 04:11:05}", {1137899465000, 1137942665000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 5)
- .Add(DatetimeComponent::ComponentType::MINUTE, 11)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 22)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2006)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 5)
- .Add(DatetimeComponent::ComponentType::MINUTE, 11)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 22)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2006)
- .Build()}));
- EXPECT_TRUE(
- ParsesCorrectly("{11:42:35}", {38555000, 81755000}, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{23/Apr 11:42:35}", {9715355000, 9758555000}, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{23/Apr/2015 11:42:35}", {1429782155000, 1429825355000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{23-Apr-2015 11:42:35}", {1429782155000, 1429825355000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{23 Apr 2015 11:42:35}", {1429782155000, 1429825355000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{04/23/15 11:42:35}", {1429782155000, 1429825355000}, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{04/23/2015 11:42:35}", {1429782155000, 1429825355000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{9/28/2011 2:23:15 PM}", 1317212595000, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 15)
- .Add(DatetimeComponent::ComponentType::MINUTE, 23)
- .Add(DatetimeComponent::ComponentType::HOUR, 2)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 28)
- .Add(DatetimeComponent::ComponentType::MONTH, 9)
- .Add(DatetimeComponent::ComponentType::YEAR, 2011)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "Are sentiments apartments decisively the especially alteration. "
- "Thrown shy denote ten ladies though ask saw. Or by to he going "
- "think order event music. Incommode so intention defective at "
- "convinced. Led income months itself and houses you. After nor "
- "you leave might share court balls. {19/apr/2010 06:36:15} Are "
- "sentiments apartments decisively the especially alteration. "
- "Thrown shy denote ten ladies though ask saw. Or by to he going "
- "think order event music. Incommode so intention defective at "
- "convinced. Led income months itself and houses you. After nor "
- "you leave might share court balls. ",
- {1271651775000, 1271694975000}, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 15)
- .Add(DatetimeComponent::ComponentType::MINUTE, 36)
- .Add(DatetimeComponent::ComponentType::HOUR, 6)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 19)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 15)
- .Add(DatetimeComponent::ComponentType::MINUTE, 36)
- .Add(DatetimeComponent::ComponentType::HOUR, 6)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 19)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{january 1 2018 at 4:30}", {1514777400000, 1514820600000},
- GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{january 1 2018 at 4:30 am}", 1514777400000, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{january 1 2018 at 4pm}", 1514818800000, GRANULARITY_HOUR,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
-
- EXPECT_TRUE(ParsesCorrectly(
- "{today at 0:00}", {-3600000, 39600000}, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 0)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::NOW, 0)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 0)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::NOW, 0)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{today at 0:00}", {-57600000, -14400000}, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 0)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::NOW, 0)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 0)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::NOW, 0)
- .Build()},
- /*anchor_start_end=*/false, "America/Los_Angeles"));
- EXPECT_TRUE(ParsesCorrectly(
- "{tomorrow at 4:00}", {97200000, 140400000}, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{tomorrow at 4am}", 97200000, GRANULARITY_HOUR,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{wednesday at 4am}", 529200000, GRANULARITY_HOUR,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_WEEK, 4,
- DatetimeComponent::RelativeQualifier::THIS, 0)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "last seen {today at 9:01 PM}", 72060000, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 1)
- .Add(DatetimeComponent::ComponentType::HOUR, 9)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::NOW, 0)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "set an alarm for {7am tomorrow}", 108000000, GRANULARITY_HOUR,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 7)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build()}));
- EXPECT_TRUE(
- ParsesCorrectly("set an alarm for {7 a.m}", 21600000, GRANULARITY_HOUR,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 7)
- .Build()}));
-}
-
-TEST_F(ParserTest, ParseWithAnchor) {
- EXPECT_TRUE(ParsesCorrectly(
- "{January 1, 1988}", 567990000000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 1988)
- .Build()},
- /*anchor_start_end=*/false));
- EXPECT_TRUE(ParsesCorrectly(
- "{January 1, 1988}", 567990000000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 1988)
- .Build()},
- /*anchor_start_end=*/true));
- EXPECT_TRUE(ParsesCorrectly(
- "lorem {1 january 2018} ipsum", 1514761200000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()},
- /*anchor_start_end=*/false));
- EXPECT_TRUE(HasNoResult("lorem 1 january 2018 ipsum",
- /*anchor_start_end=*/true));
-}
-
-TEST_F(ParserTest, ParseWithRawUsecase) {
- // Annotated for RAW usecase.
- EXPECT_TRUE(ParsesCorrectly(
- "{tomorrow}", 82800000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build()},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich", /*locales=*/"en-US",
- /*annotation_usecase=*/AnnotationUsecase_ANNOTATION_USECASE_RAW));
-
- EXPECT_TRUE(ParsesCorrectly(
- "call me {in two hours}", 7200000, GRANULARITY_HOUR,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::HOUR, 0,
- DatetimeComponent::RelativeQualifier::FUTURE, 2)
- .Build()},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich", /*locales=*/"en-US",
- /*annotation_usecase=*/AnnotationUsecase_ANNOTATION_USECASE_RAW));
-
- EXPECT_TRUE(ParsesCorrectly(
- "call me {next month}", 2674800000, GRANULARITY_MONTH,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MONTH, 0,
- DatetimeComponent::RelativeQualifier::NEXT, 1)
- .Build()},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich", /*locales=*/"en-US",
- /*annotation_usecase=*/AnnotationUsecase_ANNOTATION_USECASE_RAW));
- EXPECT_TRUE(ParsesCorrectly(
- "what's the time {now}", -3600000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::NOW, 0)
- .Build()},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich", /*locales=*/"en-US",
- /*annotation_usecase=*/AnnotationUsecase_ANNOTATION_USECASE_RAW));
-
- EXPECT_TRUE(ParsesCorrectly(
- "call me on {Saturday}", 169200000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_WEEK, 7,
- DatetimeComponent::RelativeQualifier::THIS, 0)
- .Build()},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich", /*locales=*/"en-US",
- /*annotation_usecase=*/AnnotationUsecase_ANNOTATION_USECASE_RAW));
-
- // Not annotated for Smart usecase.
- EXPECT_TRUE(HasNoResult(
- "{tomorrow}", /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich",
- /*annotation_usecase=*/AnnotationUsecase_ANNOTATION_USECASE_SMART));
-}
-
-TEST_F(ParserTest, AddsADayWhenTimeInThePastAndDayNotSpecified) {
- // ParsesCorrectly uses 0 as the reference time, which corresponds to:
- // "Thu Jan 01 1970 01:00:00" Zurich time. So if we pass "0:30" here, it means
- // it is in the past, and so the parser should move this to the next day ->
- // "Fri Jan 02 1970 00:30:00" Zurich time (b/139112907).
- EXPECT_TRUE(ParsesCorrectly(
- "{0:30am}", 84600000L /* 23.5 hours from reference time */,
- GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 0)
- .Build()}));
-}
-
-TEST_F(ParserTest, DoesNotAddADayWhenTimeInThePastAndDayNotSpecifiedDisabled) {
- // ParsesCorrectly uses 0 as the reference time, which corresponds to:
- // "Thu Jan 01 1970 01:00:00" Zurich time. So if we pass "0:30" here, it means
- // it is in the past. The parameter prefer_future_when_unspecified_day is
- // disabled, so the parser should annotate this to the same day: "Thu Jan 01
- // 1970 00:30:00" Zurich time.
- LoadModel([](ModelT* model) {
- // In the test model, the prefer_future_for_unspecified_date is true; make
- // it false only for this test.
- model->datetime_model->prefer_future_for_unspecified_date = false;
- });
-
- EXPECT_TRUE(ParsesCorrectly(
- "{0:30am}", -1800000L /* -30 minutes from reference time */,
- GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 0)
- .Build()}));
-}
-
-TEST_F(ParserTest, ParsesNoonAndMidnightCorrectly) {
- EXPECT_TRUE(ParsesCorrectly(
- "{January 1, 1988 12:30am}", 567991800000, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 12)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 1988)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{January 1, 1988 12:30pm}", 568035000000, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 12)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 1988)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{tomorrow at 12:00 am}", 82800000, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 12)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build()}));
-}
-
-TEST_F(ParserTest, ParseGerman) {
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{Januar 1 2018}", 1514761200000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{1 2 2018}", 1517439600000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 2)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "lorem {1 Januar 2018} ipsum", 1514761200000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{19/Apr/2010:06:36:15}", {1271651775000, 1271694975000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 15)
- .Add(DatetimeComponent::ComponentType::MINUTE, 36)
- .Add(DatetimeComponent::ComponentType::HOUR, 6)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 19)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 15)
- .Add(DatetimeComponent::ComponentType::MINUTE, 36)
- .Add(DatetimeComponent::ComponentType::HOUR, 6)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 19)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{09/März/2004 22:02:40}", 1078866160000, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::SECOND, 40)
- .Add(DatetimeComponent::ComponentType::MINUTE, 02)
- .Add(DatetimeComponent::ComponentType::HOUR, 22)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 9)
- .Add(DatetimeComponent::ComponentType::MONTH, 3)
- .Add(DatetimeComponent::ComponentType::YEAR, 2004)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{Dez 2, 2010 2:39:58}", {1291253998000, 1291297198000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 58)
- .Add(DatetimeComponent::ComponentType::MINUTE, 39)
- .Add(DatetimeComponent::ComponentType::HOUR, 2)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 2)
- .Add(DatetimeComponent::ComponentType::MONTH, 12)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 58)
- .Add(DatetimeComponent::ComponentType::MINUTE, 39)
- .Add(DatetimeComponent::ComponentType::HOUR, 2)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 2)
- .Add(DatetimeComponent::ComponentType::MONTH, 12)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{Juni 09 2011 15:28:14}", 1307626094000, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::SECOND, 14)
- .Add(DatetimeComponent::ComponentType::MINUTE, 28)
- .Add(DatetimeComponent::ComponentType::HOUR, 15)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 9)
- .Add(DatetimeComponent::ComponentType::MONTH, 6)
- .Add(DatetimeComponent::ComponentType::YEAR, 2011)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{März 16 08:12:04}", {6419524000, 6462724000}, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 4)
- .Add(DatetimeComponent::ComponentType::MINUTE, 12)
- .Add(DatetimeComponent::ComponentType::HOUR, 8)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 16)
- .Add(DatetimeComponent::ComponentType::MONTH, 3)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 4)
- .Add(DatetimeComponent::ComponentType::MINUTE, 12)
- .Add(DatetimeComponent::ComponentType::HOUR, 8)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 16)
- .Add(DatetimeComponent::ComponentType::MONTH, 3)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{2010-06-26 02:31:29}", {1277512289000, 1277555489000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 29)
- .Add(DatetimeComponent::ComponentType::MINUTE, 31)
- .Add(DatetimeComponent::ComponentType::HOUR, 2)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 26)
- .Add(DatetimeComponent::ComponentType::MONTH, 6)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 29)
- .Add(DatetimeComponent::ComponentType::MINUTE, 31)
- .Add(DatetimeComponent::ComponentType::HOUR, 2)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 26)
- .Add(DatetimeComponent::ComponentType::MONTH, 6)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{2006/01/22 04:11:05}", {1137899465000, 1137942665000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 5)
- .Add(DatetimeComponent::ComponentType::MINUTE, 11)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 22)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2006)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 5)
- .Add(DatetimeComponent::ComponentType::MINUTE, 11)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 22)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2006)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{11:42:35}", {38555000, 81755000}, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{23/Apr 11:42:35}", {9715355000, 9758555000}, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{23/Apr/2015:11:42:35}", {1429782155000, 1429825355000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{23/Apr/2015 11:42:35}", {1429782155000, 1429825355000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{23-Apr-2015 11:42:35}", {1429782155000, 1429825355000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{23 Apr 2015 11:42:35}", {1429782155000, 1429825355000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{04/23/15 11:42:35}", {1429782155000, 1429825355000}, GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{04/23/2015 11:42:35}", {1429782155000, 1429825355000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 35)
- .Add(DatetimeComponent::ComponentType::MINUTE, 42)
- .Add(DatetimeComponent::ComponentType::HOUR, 11)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 23)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{19/apr/2010:06:36:15}", {1271651775000, 1271694975000},
- GRANULARITY_SECOND,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::SECOND, 15)
- .Add(DatetimeComponent::ComponentType::MINUTE, 36)
- .Add(DatetimeComponent::ComponentType::HOUR, 6)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 19)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::SECOND, 15)
- .Add(DatetimeComponent::ComponentType::MINUTE, 36)
- .Add(DatetimeComponent::ComponentType::HOUR, 6)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 19)
- .Add(DatetimeComponent::ComponentType::MONTH, 4)
- .Add(DatetimeComponent::ComponentType::YEAR, 2010)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{januar 1 2018 um 4:30}", {1514777400000, 1514820600000},
- GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{januar 1 2018 um 4:30 nachm}", 1514820600000, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{januar 1 2018 um 4 nachm}", 1514818800000, GRANULARITY_HOUR,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{14.03.2017}", 1489446000000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 14)
- .Add(DatetimeComponent::ComponentType::MONTH, 3)
- .Add(DatetimeComponent::ComponentType::YEAR, 2017)
- .Build()}));
-
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{morgen 0:00}", {82800000, 126000000}, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 0)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 0)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{morgen um 4:00}", {97200000, 140400000}, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectlyGerman(
- "{morgen um 4 vorm}", 97200000, GRANULARITY_HOUR,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 0,
- DatetimeComponent::RelativeQualifier::TOMORROW, 1)
- .Build()}));
-}
-
-TEST_F(ParserTest, ParseNonUs) {
- auto first_may_2015 =
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 5)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build();
-
- EXPECT_TRUE(ParsesCorrectly("{1/5/15}", 1430431200000, GRANULARITY_DAY,
- {first_may_2015},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich",
- /*locales=*/"en-GB"));
- EXPECT_TRUE(ParsesCorrectly("{1/5/15}", 1430431200000, GRANULARITY_DAY,
- {first_may_2015},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich", /*locales=*/"en"));
-}
-
-TEST_F(ParserTest, ParseUs) {
- auto five_january_2015 =
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 5)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build();
-
- EXPECT_TRUE(ParsesCorrectly("{1/5/15}", 1420412400000, GRANULARITY_DAY,
- {five_january_2015},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich",
- /*locales=*/"en-US"));
- EXPECT_TRUE(ParsesCorrectly("{1/5/15}", 1420412400000, GRANULARITY_DAY,
- {five_january_2015},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich",
- /*locales=*/"es-US"));
-}
-
-TEST_F(ParserTest, ParseUnknownLanguage) {
- EXPECT_TRUE(ParsesCorrectly(
- "bylo to {31. 12. 2015} v 6 hodin", 1451516400000, GRANULARITY_DAY,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 31)
- .Add(DatetimeComponent::ComponentType::MONTH, 12)
- .Add(DatetimeComponent::ComponentType::YEAR, 2015)
- .Build()},
- /*anchor_start_end=*/false,
- /*timezone=*/"Europe/Zurich", /*locales=*/"xx"));
-}
-
-TEST_F(ParserTest, WhenAlternativesEnabledGeneratesAlternatives) {
- LoadModel([](ModelT* model) {
- model->datetime_model->generate_alternative_interpretations_when_ambiguous =
- true;
- });
-
- EXPECT_TRUE(ParsesCorrectly(
- "{january 1 2018 at 4:30}", {1514777400000, 1514820600000},
- GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{monday 3pm}", 396000000, GRANULARITY_HOUR,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::HOUR, 3)
- .Add(DatetimeComponent::ComponentType::DAY_OF_WEEK, 2,
- DatetimeComponent::RelativeQualifier::THIS, 0)
- .Build()}));
- EXPECT_TRUE(ParsesCorrectly(
- "{monday 3:00}", {352800000, 396000000}, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 0)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 3)
- .Add(DatetimeComponent::ComponentType::DAY_OF_WEEK, 2,
- DatetimeComponent::RelativeQualifier::THIS, 0)
- .Build(),
- DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MERIDIEM, 1)
- .Add(DatetimeComponent::ComponentType::MINUTE, 0)
- .Add(DatetimeComponent::ComponentType::HOUR, 3)
- .Add(DatetimeComponent::ComponentType::DAY_OF_WEEK, 2,
- DatetimeComponent::RelativeQualifier::THIS, 0)
- .Build()}));
-}
-
-TEST_F(ParserTest, WhenAlternativesDisabledDoesNotGenerateAlternatives) {
- LoadModel([](ModelT* model) {
- model->datetime_model->generate_alternative_interpretations_when_ambiguous =
- false;
- });
-
- EXPECT_TRUE(ParsesCorrectly(
- "{january 1 2018 at 4:30}", 1514777400000, GRANULARITY_MINUTE,
- {DatetimeComponentsBuilder()
- .Add(DatetimeComponent::ComponentType::MINUTE, 30)
- .Add(DatetimeComponent::ComponentType::HOUR, 4)
- .Add(DatetimeComponent::ComponentType::DAY_OF_MONTH, 1)
- .Add(DatetimeComponent::ComponentType::MONTH, 1)
- .Add(DatetimeComponent::ComponentType::YEAR, 2018)
- .Build()}));
-}
-
-class ParserLocaleTest : public testing::Test {
- public:
- void SetUp() override;
- bool HasResult(const std::string& input, const std::string& locales);
-
- protected:
- UniLib unilib_;
- CalendarLib calendarlib_;
- flatbuffers::FlatBufferBuilder builder_;
- std::unique_ptr<DatetimeParser> parser_;
-};
-
-void AddPattern(const std::string& regex, int locale,
- std::vector<std::unique_ptr<DatetimeModelPatternT>>* patterns) {
- patterns->emplace_back(new DatetimeModelPatternT);
- patterns->back()->regexes.emplace_back(new DatetimeModelPattern_::RegexT);
- patterns->back()->regexes.back()->pattern = regex;
- patterns->back()->regexes.back()->groups.push_back(
- DatetimeGroupType_GROUP_UNUSED);
- patterns->back()->locales.push_back(locale);
-}
-
-void ParserLocaleTest::SetUp() {
- DatetimeModelT model;
- model.use_extractors_for_locating = false;
- model.locales.clear();
- model.locales.push_back("en-US");
- model.locales.push_back("en-CH");
- model.locales.push_back("zh-Hant");
- model.locales.push_back("en-*");
- model.locales.push_back("zh-Hant-*");
- model.locales.push_back("*-CH");
- model.locales.push_back("default");
- model.default_locales.push_back(6);
-
- AddPattern(/*regex=*/"en-US", /*locale=*/0, &model.patterns);
- AddPattern(/*regex=*/"en-CH", /*locale=*/1, &model.patterns);
- AddPattern(/*regex=*/"zh-Hant", /*locale=*/2, &model.patterns);
- AddPattern(/*regex=*/"en-all", /*locale=*/3, &model.patterns);
- AddPattern(/*regex=*/"zh-Hant-all", /*locale=*/4, &model.patterns);
- AddPattern(/*regex=*/"all-CH", /*locale=*/5, &model.patterns);
- AddPattern(/*regex=*/"default", /*locale=*/6, &model.patterns);
-
- builder_.Finish(DatetimeModel::Pack(builder_, &model));
- const DatetimeModel* model_fb =
- flatbuffers::GetRoot<DatetimeModel>(builder_.GetBufferPointer());
- ASSERT_TRUE(model_fb);
-
- parser_ = DatetimeParser::Instance(model_fb, unilib_, calendarlib_,
- /*decompressor=*/nullptr);
- ASSERT_TRUE(parser_);
-}
-
-bool ParserLocaleTest::HasResult(const std::string& input,
- const std::string& locales) {
- std::vector<DatetimeParseResultSpan> results;
- EXPECT_TRUE(parser_->Parse(
- input, /*reference_time_ms_utc=*/0,
- /*reference_timezone=*/"", locales, ModeFlag_ANNOTATION,
- AnnotationUsecase_ANNOTATION_USECASE_SMART, false, &results));
- return results.size() == 1;
-}
-
-TEST_F(ParserLocaleTest, English) {
- EXPECT_TRUE(HasResult("en-US", /*locales=*/"en-US"));
- EXPECT_FALSE(HasResult("en-CH", /*locales=*/"en-US"));
- EXPECT_FALSE(HasResult("en-US", /*locales=*/"en-CH"));
- EXPECT_TRUE(HasResult("en-CH", /*locales=*/"en-CH"));
- EXPECT_TRUE(HasResult("default", /*locales=*/"en-CH"));
-}
-
-TEST_F(ParserLocaleTest, TraditionalChinese) {
- EXPECT_TRUE(HasResult("zh-Hant-all", /*locales=*/"zh-Hant"));
- EXPECT_TRUE(HasResult("zh-Hant-all", /*locales=*/"zh-Hant-TW"));
- EXPECT_TRUE(HasResult("zh-Hant-all", /*locales=*/"zh-Hant-SG"));
- EXPECT_FALSE(HasResult("zh-Hant-all", /*locales=*/"zh-SG"));
- EXPECT_FALSE(HasResult("zh-Hant-all", /*locales=*/"zh"));
- EXPECT_TRUE(HasResult("default", /*locales=*/"zh"));
- EXPECT_TRUE(HasResult("default", /*locales=*/"zh-Hant-SG"));
-}
-
-TEST_F(ParserLocaleTest, SwissEnglish) {
- EXPECT_TRUE(HasResult("all-CH", /*locales=*/"de-CH"));
- EXPECT_TRUE(HasResult("all-CH", /*locales=*/"en-CH"));
- EXPECT_TRUE(HasResult("en-all", /*locales=*/"en-CH"));
- EXPECT_FALSE(HasResult("all-CH", /*locales=*/"de-DE"));
- EXPECT_TRUE(HasResult("default", /*locales=*/"de-CH"));
- EXPECT_TRUE(HasResult("default", /*locales=*/"en-CH"));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/annotator/duration/duration_test.cc b/native/annotator/duration/duration_test.cc
deleted file mode 100644
index 1afd701..0000000
--- a/native/annotator/duration/duration_test.cc
+++ /dev/null
@@ -1,567 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/duration/duration.h"
-
-#include <string>
-#include <vector>
-
-#include "annotator/collections.h"
-#include "annotator/model_generated.h"
-#include "annotator/types-test-util.h"
-#include "annotator/types.h"
-#include "utils/test-utils.h"
-#include "utils/utf8/unicodetext.h"
-#include "utils/utf8/unilib.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::AllOf;
-using testing::ElementsAre;
-using testing::Field;
-using testing::IsEmpty;
-
-const DurationAnnotatorOptions* TestingDurationAnnotatorOptions() {
- static const flatbuffers::DetachedBuffer* options_data = []() {
- DurationAnnotatorOptionsT options;
- options.enabled = true;
-
- options.week_expressions.push_back("week");
- options.week_expressions.push_back("weeks");
-
- options.day_expressions.push_back("day");
- options.day_expressions.push_back("days");
-
- options.hour_expressions.push_back("hour");
- options.hour_expressions.push_back("hours");
-
- options.minute_expressions.push_back("minute");
- options.minute_expressions.push_back("minutes");
-
- options.second_expressions.push_back("second");
- options.second_expressions.push_back("seconds");
-
- options.filler_expressions.push_back("and");
- options.filler_expressions.push_back("a");
- options.filler_expressions.push_back("an");
- options.filler_expressions.push_back("one");
-
- options.half_expressions.push_back("half");
-
- options.sub_token_separator_codepoints.push_back('-');
-
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(DurationAnnotatorOptions::Pack(builder, &options));
- return new flatbuffers::DetachedBuffer(builder.Release());
- }();
-
- return flatbuffers::GetRoot<DurationAnnotatorOptions>(options_data->data());
-}
-
-std::unique_ptr<FeatureProcessor> BuildFeatureProcessor(const UniLib* unilib) {
- static const flatbuffers::DetachedBuffer* options_data = []() {
- FeatureProcessorOptionsT options;
- options.context_size = 1;
- options.max_selection_span = 1;
- options.snap_label_span_boundaries_to_containing_tokens = false;
- options.ignored_span_boundary_codepoints.push_back(',');
-
- options.tokenization_codepoint_config.emplace_back(
- new TokenizationCodepointRangeT());
- auto& config = options.tokenization_codepoint_config.back();
- config->start = 32;
- config->end = 33;
- config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
-
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(FeatureProcessorOptions::Pack(builder, &options));
- return new flatbuffers::DetachedBuffer(builder.Release());
- }();
-
- const FeatureProcessorOptions* feature_processor_options =
- flatbuffers::GetRoot<FeatureProcessorOptions>(options_data->data());
-
- return std::unique_ptr<FeatureProcessor>(
- new FeatureProcessor(feature_processor_options, unilib));
-}
-
-class DurationAnnotatorTest : public ::testing::Test {
- protected:
- DurationAnnotatorTest()
- : INIT_UNILIB_FOR_TESTING(unilib_),
- feature_processor_(BuildFeatureProcessor(&unilib_)),
- duration_annotator_(TestingDurationAnnotatorOptions(),
- feature_processor_.get(), &unilib_) {}
-
- std::vector<Token> Tokenize(const UnicodeText& text) {
- return feature_processor_->Tokenize(text);
- }
-
- UniLib unilib_;
- std::unique_ptr<FeatureProcessor> feature_processor_;
- DurationAnnotator duration_annotator_;
-};
-
-TEST_F(DurationAnnotatorTest, ClassifiesSimpleDuration) {
- ClassificationResult classification;
- EXPECT_TRUE(duration_annotator_.ClassifyText(
- UTF8ToUnicodeText("Wake me up in 15 minutes ok?"), {14, 24},
- AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification));
-
- EXPECT_THAT(classification,
- AllOf(Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms, 15 * 60 * 1000)));
-}
-
-TEST_F(DurationAnnotatorTest, ClassifiesWhenTokensDontAlignWithSelection) {
- ClassificationResult classification;
- EXPECT_TRUE(duration_annotator_.ClassifyText(
- UTF8ToUnicodeText("Wake me up in15 minutesok?"), {13, 23},
- AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification));
-
- EXPECT_THAT(classification,
- AllOf(Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms, 15 * 60 * 1000)));
-}
-
-TEST_F(DurationAnnotatorTest, DoNotClassifyWhenInputIsInvalid) {
- ClassificationResult classification;
- EXPECT_FALSE(duration_annotator_.ClassifyText(
- UTF8ToUnicodeText("Weird space"), {5, 6},
- AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification));
-}
-
-TEST_F(DurationAnnotatorTest, FindsSimpleDuration) {
- const UnicodeText text = UTF8ToUnicodeText("Wake me up in 15 minutes ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(14, 24)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 15 * 60 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, FindsDurationWithHalfExpression) {
- const UnicodeText text =
- UTF8ToUnicodeText("Set a timer for 3 and half minutes ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 34)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 3.5 * 60 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, FindsComposedDuration) {
- const UnicodeText text =
- UTF8ToUnicodeText("Wake me up in 3 hours and 5 seconds ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(14, 35)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 3 * 60 * 60 * 1000 + 5 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, AllUnitsAreCovered) {
- const UnicodeText text = UTF8ToUnicodeText(
- "See you in a week and a day and an hour and a minute and a second");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(13, 65)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 7 * 24 * 60 * 60 * 1000 + 24 * 60 * 60 * 1000 +
- 60 * 60 * 1000 + 60 * 1000 + 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, FindsHalfAnHour) {
- const UnicodeText text = UTF8ToUnicodeText("Set a timer for half an hour");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 28)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 0.5 * 60 * 60 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, FindsWhenHalfIsAfterGranularitySpecification) {
- const UnicodeText text =
- UTF8ToUnicodeText("Set a timer for 1 hour and a half");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 33)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 1.5 * 60 * 60 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, FindsAnHourAndAHalf) {
- const UnicodeText text =
- UTF8ToUnicodeText("Set a timer for an hour and a half");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(19, 34)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 1.5 * 60 * 60 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest,
- FindsCorrectlyWhenSecondsComeSecondAndDontHaveNumber) {
- const UnicodeText text =
- UTF8ToUnicodeText("Set a timer for 10 minutes and a second ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 39)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 10 * 60 * 1000 + 1 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, DoesNotGreedilyTakeFillerWords) {
- const UnicodeText text = UTF8ToUnicodeText(
- "Set a timer for a a a 10 minutes and 2 seconds an and an ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(22, 46)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 10 * 60 * 1000 + 2 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, DoesNotCrashWhenJustHalfIsSaid) {
- const UnicodeText text = UTF8ToUnicodeText("Set a timer for half ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- ASSERT_EQ(result.size(), 0);
-}
-
-TEST_F(DurationAnnotatorTest, StripsPunctuationFromTokens) {
- const UnicodeText text =
- UTF8ToUnicodeText("Set a timer for 10 ,minutes, ,and, ,2, seconds, ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 46)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 10 * 60 * 1000 + 2 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, FindsCorrectlyWithCombinedQuantityUnitToken) {
- const UnicodeText text = UTF8ToUnicodeText("Show 5-minute timer.");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(5, 13)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 5 * 60 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest,
- DoesNotIntOverflowWithDurationThatHasMoreThanInt32Millis) {
- ClassificationResult classification;
- EXPECT_TRUE(duration_annotator_.ClassifyText(
- UTF8ToUnicodeText("1400 hours"), {0, 10},
- AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification));
-
- EXPECT_THAT(classification,
- AllOf(Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 1400L * 60L * 60L * 1000L)));
-}
-
-TEST_F(DurationAnnotatorTest, FindsSimpleDurationIgnoringCase) {
- const UnicodeText text = UTF8ToUnicodeText("Wake me up in 15 MiNuTeS ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(14, 24)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 15 * 60 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, FindsDurationWithHalfExpressionIgnoringCase) {
- const UnicodeText text =
- UTF8ToUnicodeText("Set a timer for 3 and HaLf minutes ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 34)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 3.5 * 60 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest,
- FindsDurationWithHalfExpressionIgnoringFillerWordCase) {
- const UnicodeText text =
- UTF8ToUnicodeText("Set a timer for 3 AnD half minutes ok?");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 34)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 3.5 * 60 * 1000)))))));
-}
-
-TEST_F(DurationAnnotatorTest, CorrectlyAnnotatesSpanWithDanglingQuantity) {
- const UnicodeText text = UTF8ToUnicodeText("20 minutes 10");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- // TODO(b/144752747) Include test for duration_ms.
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(0, 13)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(Field(&ClassificationResult::collection,
- "duration")))))));
-}
-
-const DurationAnnotatorOptions* TestingJapaneseDurationAnnotatorOptions() {
- static const flatbuffers::DetachedBuffer* options_data = []() {
- DurationAnnotatorOptionsT options;
- options.enabled = true;
-
- options.week_expressions.push_back("週間");
-
- options.day_expressions.push_back("日間");
-
- options.hour_expressions.push_back("時間");
-
- options.minute_expressions.push_back("分");
- options.minute_expressions.push_back("分間");
-
- options.second_expressions.push_back("秒");
- options.second_expressions.push_back("秒間");
-
- options.half_expressions.push_back("半");
-
- options.require_quantity = true;
- options.enable_dangling_quantity_interpretation = false;
-
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(DurationAnnotatorOptions::Pack(builder, &options));
- return new flatbuffers::DetachedBuffer(builder.Release());
- }();
-
- return flatbuffers::GetRoot<DurationAnnotatorOptions>(options_data->data());
-}
-
-class JapaneseDurationAnnotatorTest : public ::testing::Test {
- protected:
- JapaneseDurationAnnotatorTest()
- : INIT_UNILIB_FOR_TESTING(unilib_),
- feature_processor_(BuildFeatureProcessor(&unilib_)),
- duration_annotator_(TestingJapaneseDurationAnnotatorOptions(),
- feature_processor_.get(), &unilib_) {}
-
- std::vector<Token> Tokenize(const UnicodeText& text) {
- return feature_processor_->Tokenize(text);
- }
-
- UniLib unilib_;
- std::unique_ptr<FeatureProcessor> feature_processor_;
- DurationAnnotator duration_annotator_;
-};
-
-TEST_F(JapaneseDurationAnnotatorTest, FindsDuration) {
- const UnicodeText text = UTF8ToUnicodeText("10 分 の アラーム");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(0, 4)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 10 * 60 * 1000)))))));
-}
-
-TEST_F(JapaneseDurationAnnotatorTest, FindsDurationWithHalfExpression) {
- const UnicodeText text = UTF8ToUnicodeText("2 分 半 の アラーム");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(0, 5)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 2.5 * 60 * 1000)))))));
-}
-
-TEST_F(JapaneseDurationAnnotatorTest, IgnoresDurationWithoutQuantity) {
- const UnicodeText text = UTF8ToUnicodeText("分 の アラーム");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(result, IsEmpty());
-}
-
-TEST_F(JapaneseDurationAnnotatorTest, IgnoresDanglingQuantity) {
- const UnicodeText text = UTF8ToUnicodeText("2 分 10 の アラーム");
- std::vector<Token> tokens = Tokenize(text);
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(duration_annotator_.FindAll(
- text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result));
-
- EXPECT_THAT(
- result,
- ElementsAre(
- AllOf(Field(&AnnotatedSpan::span, CodepointSpan(0, 3)),
- Field(&AnnotatedSpan::classification,
- ElementsAre(AllOf(
- Field(&ClassificationResult::collection, "duration"),
- Field(&ClassificationResult::duration_ms,
- 2 * 60 * 1000)))))));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/annotator/grammar/dates/cfg-datetime-annotator.cc b/native/annotator/grammar/dates/cfg-datetime-annotator.cc
index 5b24b31..78b8dad 100644
--- a/native/annotator/grammar/dates/cfg-datetime-annotator.cc
+++ b/native/annotator/grammar/dates/cfg-datetime-annotator.cc
@@ -256,17 +256,23 @@
options->tokenization_codepoint_config() != nullptr &&
options->tokenize_on_script_change();
return Tokenizer(options->tokenization_type(), unilib, codepoint_config,
- internal_codepoint_config, tokenize_on_script_change, true);
+ internal_codepoint_config, tokenize_on_script_change,
+ /*icu_preserve_whitespace_tokens=*/false);
}
} // namespace
CfgDatetimeAnnotator::CfgDatetimeAnnotator(
const UniLib& unilib, const GrammarTokenizerOptions* tokenizer_options,
- const CalendarLib& calendar_lib, const DatetimeRules* datetime_rules)
+ const CalendarLib& calendar_lib, const DatetimeRules* datetime_rules,
+ const float annotator_target_classification_score,
+ const float annotator_priority_score)
: calendar_lib_(calendar_lib),
tokenizer_(BuildTokenizer(&unilib, tokenizer_options)),
- parser_(unilib, datetime_rules) {}
+ parser_(unilib, datetime_rules),
+ annotator_target_classification_score_(
+ annotator_target_classification_score),
+ annotator_priority_score_(annotator_priority_score) {}
// Helper method to convert the Thing into DatetimeParseResult.
// Thing constains the annotation instance i.e. type of the annotation and its
@@ -306,8 +312,10 @@
DatetimeParseResultSpan datetime_parse_result_span;
datetime_parse_result_span.span =
CodepointSpan{annotation.begin, annotation.end};
- datetime_parse_result_span.priority_score = 0.1;
- datetime_parse_result_span.target_classification_score = 1.0;
+ datetime_parse_result_span.target_classification_score =
+ annotator_target_classification_score_;
+ datetime_parse_result_span.priority_score = annotator_priority_score_;
+
// Though the datastructre allow multiple DatetimeParseResult per span
// but for annotator based on grammar there will just one.
DatetimeParseResult datetime_parse_result;
diff --git a/native/annotator/grammar/dates/cfg-datetime-annotator.h b/native/annotator/grammar/dates/cfg-datetime-annotator.h
index 1980aae..00ed447 100644
--- a/native/annotator/grammar/dates/cfg-datetime-annotator.h
+++ b/native/annotator/grammar/dates/cfg-datetime-annotator.h
@@ -36,7 +36,9 @@
CfgDatetimeAnnotator(const UniLib& unilib,
const GrammarTokenizerOptions* tokenizer_options,
const CalendarLib& calendar_lib,
- const DatetimeRules* datetime_rules);
+ const DatetimeRules* datetime_rules,
+ const float annotator_target_classification_score,
+ const float annotator_priority_score);
// CfgDatetimeAnnotator is neither copyable nor movable.
CfgDatetimeAnnotator(const CfgDatetimeAnnotator&) = delete;
@@ -71,6 +73,8 @@
const CalendarLib& calendar_lib_;
const Tokenizer tokenizer_;
DateParser parser_;
+ const float annotator_target_classification_score_;
+ const float annotator_priority_score_;
};
} // namespace libtextclassifier3::dates
diff --git a/native/annotator/grammar/dates/parser.cc b/native/annotator/grammar/dates/parser.cc
index e745e2e..bf6e428 100644
--- a/native/annotator/grammar/dates/parser.cc
+++ b/native/annotator/grammar/dates/parser.cc
@@ -928,7 +928,7 @@
}
}
grammar::Matcher matcher(unilib_, datetime_rules_->rules(), locale_rules);
- lexer_.Process(tokens, &matcher);
+ lexer_.Process(tokens, /*matches=*/{}, &matcher);
return GetOutputAsAnnotationList(unilib_, extractor, codepoint_offsets,
options);
}
diff --git a/native/annotator/grammar/dates/testing/equals-proto.h b/native/annotator/grammar/dates/testing/equals-proto.h
deleted file mode 100644
index fba4de4..0000000
--- a/native/annotator/grammar/dates/testing/equals-proto.h
+++ /dev/null
@@ -1,28 +0,0 @@
-
-#ifndef LIBTEXTCLASSIFIER_ANNOTATOR_GRAMMAR_DATES_TESTING_EQUALS_PROTO_H_
-#define LIBTEXTCLASSIFIER_ANNOTATOR_GRAMMAR_DATES_TESTING_EQUALS_PROTO_H_
-
-#include "net/proto2/compat/public/message_lite.h" // IWYU pragma: export
-#include "gmock/gmock.h" // IWYU pragma: export
-
-#if defined(__ANDROID__) || defined(__APPLE__)
-namespace libtextclassifier3 {
-namespace portable_equals_proto {
-MATCHER_P(EqualsProto, other, "Compare MessageLite by serialized string") {
- return ::testing::ExplainMatchResult(::testing::Eq(other.SerializeAsString()),
- arg.SerializeAsString(),
- result_listener);
-} // MATCHER_P
-} // namespace portable_equals_proto
-} // namespace libtextclassifier3
-#else
-namespace libtextclassifier3 {
-namespace portable_equals_proto {
-// Leverage the powerful matcher when available, for human readable
-// differences.
-using ::testing::EqualsProto;
-} // namespace portable_equals_proto
-} // namespace libtextclassifier3
-#endif // defined(__ANDROID__) || defined(__APPLE__)
-
-#endif // LIBTEXTCLASSIFIER_ANNOTATOR_GRAMMAR_DATES_TESTING_EQUALS_PROTO_H_
diff --git a/native/annotator/grammar/dates/utils/date-match_test.cc b/native/annotator/grammar/dates/utils/date-match_test.cc
deleted file mode 100644
index d9c0af7..0000000
--- a/native/annotator/grammar/dates/utils/date-match_test.cc
+++ /dev/null
@@ -1,421 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/grammar/dates/utils/date-match.h"
-
-#include <stdint.h>
-
-#include <string>
-
-#include "annotator/grammar/dates/dates_generated.h"
-#include "annotator/grammar/dates/timezone-code_generated.h"
-#include "annotator/grammar/dates/utils/date-utils.h"
-#include "utils/strings/append.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace dates {
-namespace {
-
-class DateMatchTest : public ::testing::Test {
- protected:
- enum {
- X = NO_VAL,
- };
-
- static DayOfWeek DOW_X() { return DayOfWeek_DOW_NONE; }
- static DayOfWeek SUN() { return DayOfWeek_SUNDAY; }
-
- static BCAD BCAD_X() { return BCAD_BCAD_NONE; }
- static BCAD BC() { return BCAD_BC; }
-
- DateMatch& SetDate(DateMatch* date, int year, int8 month, int8 day,
- DayOfWeek day_of_week = DOW_X(), BCAD bc_ad = BCAD_X()) {
- date->year = year;
- date->month = month;
- date->day = day;
- date->day_of_week = day_of_week;
- date->bc_ad = bc_ad;
- return *date;
- }
-
- DateMatch& SetTimeValue(DateMatch* date, int8 hour, int8 minute = X,
- int8 second = X, double fraction_second = X) {
- date->hour = hour;
- date->minute = minute;
- date->second = second;
- date->fraction_second = fraction_second;
- return *date;
- }
-
- DateMatch& SetTimeSpan(DateMatch* date, TimespanCode time_span_code) {
- date->time_span_code = time_span_code;
- return *date;
- }
-
- DateMatch& SetTimeZone(DateMatch* date, TimezoneCode time_zone_code,
- int16 time_zone_offset = INT16_MIN) {
- date->time_zone_code = time_zone_code;
- date->time_zone_offset = time_zone_offset;
- return *date;
- }
-
- bool SameDate(const DateMatch& a, const DateMatch& b) {
- return (a.day == b.day && a.month == b.month && a.year == b.year &&
- a.day_of_week == b.day_of_week);
- }
-
- DateMatch& SetDayOfWeek(DateMatch* date, DayOfWeek dow) {
- date->day_of_week = dow;
- return *date;
- }
-};
-
-TEST_F(DateMatchTest, BitFieldWidth) {
- // For DateMatch::day_of_week (:8).
- EXPECT_GE(DayOfWeek_MIN, INT8_MIN);
- EXPECT_LE(DayOfWeek_MAX, INT8_MAX);
-
- // For DateMatch::bc_ad (:8).
- EXPECT_GE(BCAD_MIN, INT8_MIN);
- EXPECT_LE(BCAD_MAX, INT8_MAX);
-
- // For DateMatch::time_span_code (:16).
- EXPECT_GE(TimespanCode_MIN, INT16_MIN);
- EXPECT_LE(TimespanCode_MAX, INT16_MAX);
-}
-
-TEST_F(DateMatchTest, IsValid) {
- // Valid: dates.
- {
- DateMatch d;
- SetDate(&d, 2014, 1, 26);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, 2014, 1, X);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, 2014, X, X);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, X, 1, 26);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, X, 1, X);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, X, X, 26);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, 2014, 1, 26, SUN());
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, X, 1, 26, SUN());
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, X, X, 26, SUN());
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, 2014, 1, 26, DOW_X(), BC());
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- // Valid: times.
- {
- DateMatch d;
- SetTimeValue(&d, 12, 30, 59, 0.99);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetTimeValue(&d, 12, 30, 59);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetTimeValue(&d, 12, 30);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetTimeValue(&d, 12);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- // Valid: mixed.
- {
- DateMatch d;
- SetDate(&d, 2014, 1, 26);
- SetTimeValue(&d, 12, 30, 59, 0.99);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, X, 1, 26);
- SetTimeValue(&d, 12, 30, 59);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, X, X, X, SUN());
- SetTimeValue(&d, 12, 30);
- EXPECT_TRUE(d.IsValid()) << d.DebugString();
- }
- // Invalid: dates.
- {
- DateMatch d;
- SetDate(&d, X, 1, 26, DOW_X(), BC());
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, 2014, X, 26);
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, 2014, X, X, SUN());
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetDate(&d, X, 1, X, SUN());
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
- // Invalid: times.
- {
- DateMatch d;
- SetTimeValue(&d, 12, X, 59);
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetTimeValue(&d, 12, X, X, 0.99);
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetTimeValue(&d, 12, 30, X, 0.99);
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
- {
- DateMatch d;
- SetTimeValue(&d, X, 30);
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
- // Invalid: mixed.
- {
- DateMatch d;
- SetDate(&d, 2014, 1, X);
- SetTimeValue(&d, 12);
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
- // Invalid: empty.
- {
- DateMatch d;
- EXPECT_FALSE(d.IsValid()) << d.DebugString();
- }
-}
-
-std::string DebugStrings(const std::vector<DateMatch>& instances) {
- std::string res;
- for (int i = 0; i < instances.size(); ++i) {
- ::libtextclassifier3::strings::SStringAppendF(
- &res, 0, "[%d] == %s\n", i, instances[i].DebugString().c_str());
- }
- return res;
-}
-
-TEST_F(DateMatchTest, IncrementOneDay) {
- {
- DateMatch date;
- SetDate(&date, 2014, 2, 2);
- DateMatch expected;
- SetDate(&expected, 2014, 2, 3);
- IncrementOneDay(&date);
- EXPECT_TRUE(SameDate(date, expected));
- }
- {
- DateMatch date;
- SetDate(&date, 2015, 2, 28);
- DateMatch expected;
- SetDate(&expected, 2015, 3, 1);
- IncrementOneDay(&date);
- EXPECT_TRUE(SameDate(date, expected));
- }
- {
- DateMatch date;
- SetDate(&date, 2016, 2, 28);
- DateMatch expected;
- SetDate(&expected, 2016, 2, 29);
- IncrementOneDay(&date);
- EXPECT_TRUE(SameDate(date, expected));
- }
- {
- DateMatch date;
- SetDate(&date, 2017, 7, 16);
- SetDayOfWeek(&date, DayOfWeek_SUNDAY);
- DateMatch expected;
- SetDate(&expected, 2017, 7, 17);
- SetDayOfWeek(&expected, DayOfWeek_MONDAY);
- IncrementOneDay(&date);
- EXPECT_TRUE(SameDate(date, expected));
- }
- {
- DateMatch date;
- SetDate(&date, X, 7, 16);
- SetDayOfWeek(&date, DayOfWeek_MONDAY);
- DateMatch expected;
- SetDate(&expected, X, 7, 17);
- SetDayOfWeek(&expected, DayOfWeek_TUESDAY);
- IncrementOneDay(&date);
- EXPECT_TRUE(SameDate(date, expected));
- }
-}
-
-TEST_F(DateMatchTest, IsRefinement) {
- {
- DateMatch a;
- SetDate(&a, 2014, 2, X);
- DateMatch b;
- SetDate(&b, 2014, X, X);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- DateMatch b;
- SetDate(&b, 2014, 2, X);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- DateMatch b;
- SetDate(&b, X, 2, 24);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- SetTimeValue(&a, 9, X, X);
- DateMatch b;
- SetDate(&b, 2014, 2, 24);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- SetTimeValue(&a, 9, 0, X);
- DateMatch b;
- SetDate(&b, 2014, 2, 24);
- SetTimeValue(&b, 9, X, X);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- SetTimeValue(&a, 9, 0, 0);
- DateMatch b;
- SetDate(&b, 2014, 2, 24);
- SetTimeValue(&b, 9, 0, X);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- SetTimeValue(&a, 9, X, X);
- SetTimeSpan(&a, TimespanCode_AM);
- DateMatch b;
- SetDate(&b, 2014, 2, 24);
- SetTimeValue(&b, 9, X, X);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- SetTimeValue(&a, 9, X, X);
- SetTimeZone(&a, TimezoneCode_PST8PDT);
- DateMatch b;
- SetDate(&b, 2014, 2, 24);
- SetTimeValue(&b, 9, X, X);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- SetTimeValue(&a, 9, X, X);
- a.priority += 10;
- DateMatch b;
- SetDate(&b, 2014, 2, 24);
- SetTimeValue(&b, 9, X, X);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- SetTimeValue(&a, 9, X, X);
- DateMatch b;
- SetDate(&b, 2014, 2, 24);
- SetTimeValue(&b, 9, X, X);
- EXPECT_TRUE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, 2014, 2, 24);
- SetTimeValue(&a, 9, X, X);
- DateMatch b;
- SetDate(&b, X, 2, 24);
- SetTimeValue(&b, 9, 0, X);
- EXPECT_FALSE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetDate(&a, X, 2, 24);
- SetTimeValue(&a, 9, X, X);
- DateMatch b;
- SetDate(&b, 2014, 2, 24);
- EXPECT_FALSE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
- {
- DateMatch a;
- SetTimeValue(&a, 9, 0, 0);
- DateMatch b;
- SetTimeValue(&b, 9, X, X);
- SetTimeSpan(&b, TimespanCode_AM);
- EXPECT_FALSE(IsRefinement(a, b)) << DebugStrings({a, b});
- }
-}
-
-} // namespace
-} // namespace dates
-} // namespace libtextclassifier3
diff --git a/native/annotator/grammar/grammar-annotator.cc b/native/annotator/grammar/grammar-annotator.cc
index d39e33c..71d0b43 100644
--- a/native/annotator/grammar/grammar-annotator.cc
+++ b/native/annotator/grammar/grammar-annotator.cc
@@ -172,7 +172,7 @@
// Run the grammar.
grammar::Matcher matcher(*unilib_, model_->rules(), locale_rules);
- lexer_.Process(tokens, &matcher);
+ lexer_.Process(tokens, /*matches=*/{}, &matcher);
// Populate results.
return callback_handler.GetAnnotations(result);
diff --git a/native/annotator/grammar/grammar-annotator_test.cc b/native/annotator/grammar/grammar-annotator_test.cc
deleted file mode 100644
index 15dcc70..0000000
--- a/native/annotator/grammar/grammar-annotator_test.cc
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/grammar/grammar-annotator.h"
-
-#include "annotator/model_generated.h"
-#include "utils/grammar/utils/rules.h"
-#include "utils/tokenizer.h"
-#include "utils/utf8/unilib.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::ElementsAre;
-using testing::Pair;
-using testing::Value;
-
-MATCHER_P3(IsAnnotatedSpan, start, end, collection, "") {
- return Value(arg.span, Pair(start, end)) &&
- Value(arg.classification.front().collection, collection);
-}
-
-class GrammarAnnotatorTest : public testing::Test {
- protected:
- GrammarAnnotatorTest()
- : INIT_UNILIB_FOR_TESTING(unilib_),
- tokenizer_(TokenizationType_ICU, &unilib_, {}, {},
- /*split_on_script_change=*/true,
- /*icu_preserve_whitespace_tokens=*/true) {}
-
- flatbuffers::DetachedBuffer PackModel(const GrammarModelT& model) const {
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(GrammarModel::Pack(builder, &model));
- return builder.Release();
- }
-
- int AddRuleClassificationResult(const std::string& collection,
- GrammarModelT* model) const {
- const int result_id = model->rule_classification_result.size();
- model->rule_classification_result.emplace_back(
- new GrammarModel_::RuleClassificationResultT);
- GrammarModel_::RuleClassificationResultT* result =
- model->rule_classification_result.back().get();
- result->collection_name = collection;
- return result_id;
- }
-
- UniLib unilib_;
- Tokenizer tokenizer_;
-};
-
-TEST_F(GrammarAnnotatorTest, AnnotesWithGrammarRules) {
- // Create test rules.
- GrammarModelT grammar_model;
- grammar_model.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
- rules.Add("<carrier>", {"lx"});
- rules.Add("<carrier>", {"aa"});
- rules.Add("<flight_code>", {"<2_digits>"});
- rules.Add("<flight_code>", {"<3_digits>"});
- rules.Add("<flight_code>", {"<4_digits>"});
- rules.Add(
- "<flight>", {"<carrier>", "<flight_code>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(GrammarAnnotator::Callback::kRuleMatch),
- /*callback_param=*/
- AddRuleClassificationResult("flight", &grammar_model));
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- grammar_model.rules.get());
- flatbuffers::DetachedBuffer serialized_model = PackModel(grammar_model);
- GrammarAnnotator annotator(
- &unilib_, flatbuffers::GetRoot<GrammarModel>(serialized_model.data()));
-
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(annotator.Annotate(
- {Locale::FromBCP47("en")},
- tokenizer_.Tokenize(
- "My flight: LX 38 arriving at 4pm, I'll fly back on AA2014"),
- &result));
-
- EXPECT_THAT(result, ElementsAre(IsAnnotatedSpan(11, 16, "flight"),
- IsAnnotatedSpan(51, 57, "flight")));
-}
-
-TEST_F(GrammarAnnotatorTest, HandlesAssertions) {
- // Create test rules.
- GrammarModelT grammar_model;
- grammar_model.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
- rules.Add("<carrier>", {"lx"});
- rules.Add("<carrier>", {"aa"});
- rules.Add("<flight_code>", {"<2_digits>"});
- rules.Add("<flight_code>", {"<3_digits>"});
- rules.Add("<flight_code>", {"<4_digits>"});
-
- // Flight: carrier + flight code and check right context.
- rules.Add(
- "<flight>", {"<carrier>", "<flight_code>", "<context_assertion>?"},
- /*callback=*/
- static_cast<grammar::CallbackId>(GrammarAnnotator::Callback::kRuleMatch),
- /*callback_param=*/
- AddRuleClassificationResult("flight", &grammar_model));
-
- // Exclude matches like: LX 38.00 etc.
- rules.Add("<context_assertion>", {".?", "<digits>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarAnnotator::Callback::kAssertionMatch),
- /*callback_param=*/true /*negative*/);
-
- // Assertion matches will create their own match objects.
- // We declare the handler as a filter so that the grammar system knows that we
- // handle this ourselves.
- rules.DefineFilter(static_cast<grammar::CallbackId>(
- GrammarAnnotator::Callback::kAssertionMatch));
-
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- grammar_model.rules.get());
- flatbuffers::DetachedBuffer serialized_model = PackModel(grammar_model);
- GrammarAnnotator annotator(
- &unilib_, flatbuffers::GetRoot<GrammarModel>(serialized_model.data()));
-
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(annotator.Annotate(
- {Locale::FromBCP47("en")},
- tokenizer_.Tokenize("My flight: LX 38 arriving at 4pm, I'll fly back on "
- "AA2014 on LX 38.00"),
- &result));
-
- EXPECT_THAT(result, ElementsAre(IsAnnotatedSpan(11, 16, "flight"),
- IsAnnotatedSpan(51, 57, "flight")));
-}
-
-TEST_F(GrammarAnnotatorTest, HandlesCapturingGroups) {
- // Create test rules.
- GrammarModelT grammar_model;
- grammar_model.rules.reset(new grammar::RulesSetT);
- grammar::Rules rules;
- rules.Add("<low_confidence_phone>", {"<digits>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(
- GrammarAnnotator::Callback::kCapturingMatch),
- /*callback_param=*/0);
-
- // Create rule result.
- const int classification_result_id =
- AddRuleClassificationResult("phone", &grammar_model);
- grammar_model.rule_classification_result[classification_result_id]
- ->capturing_group.emplace_back(new CapturingGroupT);
- grammar_model.rule_classification_result[classification_result_id]
- ->capturing_group.back()
- ->extend_selection = true;
-
- rules.Add(
- "<phone>", {"please", "call", "<low_confidence_phone>"},
- /*callback=*/
- static_cast<grammar::CallbackId>(GrammarAnnotator::Callback::kRuleMatch),
- /*callback_param=*/classification_result_id);
-
- // Capturing matches will create their own match objects to keep track of
- // match ids, so we declare the handler as a filter so that the grammar system
- // knows that we handle this ourselves.
- rules.DefineFilter(static_cast<grammar::CallbackId>(
- GrammarAnnotator::Callback::kCapturingMatch));
-
- rules.Finalize().Serialize(/*include_debug_information=*/false,
- grammar_model.rules.get());
- flatbuffers::DetachedBuffer serialized_model = PackModel(grammar_model);
- GrammarAnnotator annotator(
- &unilib_, flatbuffers::GetRoot<GrammarModel>(serialized_model.data()));
-
- std::vector<AnnotatedSpan> result;
- EXPECT_TRUE(annotator.Annotate(
- {Locale::FromBCP47("en")},
- tokenizer_.Tokenize("Please call 911 before 10 am!"), &result));
- EXPECT_THAT(result, ElementsAre(IsAnnotatedSpan(12, 15, "phone")));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/annotator/model.fbs b/native/annotator/model.fbs
index aa99b21..3e616b6 100755
--- a/native/annotator/model.fbs
+++ b/native/annotator/model.fbs
@@ -380,6 +380,12 @@
// The datetime grammar rules.
datetime_rules:dates.DatetimeRules;
+
+ // The final score to assign to the results of grammar model
+ target_classification_score:float = 1;
+
+ // The priority score used for conflict resolution with the other models.
+ priority_score:float = 1;
}
namespace libtextclassifier3.DatetimeModelLibrary_;
diff --git a/native/annotator/number/number.cc b/native/annotator/number/number.cc
index f01ba3f..fe986ae 100644
--- a/native/annotator/number/number.cc
+++ b/native/annotator/number/number.cc
@@ -285,7 +285,6 @@
void NumberAnnotator::FindPercentages(
const UnicodeText& context, std::vector<AnnotatedSpan>* result) const {
- std::vector<AnnotatedSpan> percentage_annotations;
const int initial_result_size = result->size();
for (int i = 0; i < initial_result_size; ++i) {
AnnotatedSpan annotated_span = (*result)[i];
diff --git a/native/annotator/person_name/person-name-engine-dummy.h b/native/annotator/person_name/person-name-engine-dummy.h
index 91ae2e5..9c83241 100644
--- a/native/annotator/person_name/person-name-engine-dummy.h
+++ b/native/annotator/person_name/person-name-engine-dummy.h
@@ -32,7 +32,8 @@
// A dummy implementation of the person name engine.
class PersonNameEngine {
public:
- explicit PersonNameEngine(const UniLib* unilib) {}
+ explicit PersonNameEngine(const FeatureProcessor* feature_processor,
+ const UniLib* unilib) {}
bool Initialize(const PersonNameModel* model) {
TC3_LOG(ERROR) << "No person name engine to initialize.";
diff --git a/native/annotator/person_name/person_name_model.fbs b/native/annotator/person_name/person_name_model.fbs
index 6421341..091ad31 100755
--- a/native/annotator/person_name/person_name_model.fbs
+++ b/native/annotator/person_name/person_name_model.fbs
@@ -26,7 +26,7 @@
person_name:string (shared);
}
-// Next ID: 3
+// Next ID: 5
namespace libtextclassifier3;
table PersonNameModel {
// Decides if the person name annotator is enabled.
@@ -35,6 +35,15 @@
// List of all person names which are considered by the person name annotator.
person_names:[PersonNameModel_.PersonName];
+
+ // Decides if the English genitive ending 's is stripped, e.g., if Peter's is
+ // stripped to Peter before looking for the name in the dictionary. required
+ strip_english_genitive_ending:bool;
+
+ // List of codepoints that are considered as 'end of person name' indicator in
+ // the heuristic to find the longest person name match.
+ // required
+ end_of_person_name_indicators:[int];
}
root_type libtextclassifier3.PersonNameModel;
diff --git a/native/annotator/quantization_test.cc b/native/annotator/quantization_test.cc
deleted file mode 100644
index b995096..0000000
--- a/native/annotator/quantization_test.cc
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/quantization.h"
-
-#include <vector>
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-using testing::ElementsAreArray;
-using testing::FloatEq;
-using testing::Matcher;
-
-namespace libtextclassifier3 {
-namespace {
-
-Matcher<std::vector<float>> ElementsAreFloat(const std::vector<float>& values) {
- std::vector<Matcher<float>> matchers;
- for (const float value : values) {
- matchers.push_back(FloatEq(value));
- }
- return ElementsAreArray(matchers);
-}
-
-TEST(QuantizationTest, DequantizeAdd8bit) {
- std::vector<float> scales{{0.1, 9.0, -7.0}};
- std::vector<uint8> embeddings{{/*0: */ 0x00, 0xFF, 0x09, 0x00,
- /*1: */ 0xFF, 0x09, 0x00, 0xFF,
- /*2: */ 0x09, 0x00, 0xFF, 0x09}};
-
- const int quantization_bits = 8;
- const int bytes_per_embedding = 4;
- const int num_sparse_features = 7;
- {
- const int bucket_id = 0;
- std::vector<float> dest(4, 0.0);
- DequantizeAdd(scales.data(), embeddings.data(), bytes_per_embedding,
- num_sparse_features, quantization_bits, bucket_id,
- dest.data(), dest.size());
-
- EXPECT_THAT(dest,
- ElementsAreFloat(std::vector<float>{
- // clang-format off
- {1.0 / 7 * 0.1 * (0x00 - 128),
- 1.0 / 7 * 0.1 * (0xFF - 128),
- 1.0 / 7 * 0.1 * (0x09 - 128),
- 1.0 / 7 * 0.1 * (0x00 - 128)}
- // clang-format on
- }));
- }
-
- {
- const int bucket_id = 1;
- std::vector<float> dest(4, 0.0);
- DequantizeAdd(scales.data(), embeddings.data(), bytes_per_embedding,
- num_sparse_features, quantization_bits, bucket_id,
- dest.data(), dest.size());
-
- EXPECT_THAT(dest,
- ElementsAreFloat(std::vector<float>{
- // clang-format off
- {1.0 / 7 * 9.0 * (0xFF - 128),
- 1.0 / 7 * 9.0 * (0x09 - 128),
- 1.0 / 7 * 9.0 * (0x00 - 128),
- 1.0 / 7 * 9.0 * (0xFF - 128)}
- // clang-format on
- }));
- }
-}
-
-TEST(QuantizationTest, DequantizeAdd1bitZeros) {
- const int bytes_per_embedding = 4;
- const int num_buckets = 3;
- const int num_sparse_features = 7;
- const int quantization_bits = 1;
- const int bucket_id = 1;
-
- std::vector<float> scales(num_buckets);
- std::vector<uint8> embeddings(bytes_per_embedding * num_buckets);
- std::fill(scales.begin(), scales.end(), 1);
- std::fill(embeddings.begin(), embeddings.end(), 0);
-
- std::vector<float> dest(32);
- DequantizeAdd(scales.data(), embeddings.data(), bytes_per_embedding,
- num_sparse_features, quantization_bits, bucket_id, dest.data(),
- dest.size());
-
- std::vector<float> expected(32);
- std::fill(expected.begin(), expected.end(),
- 1.0 / num_sparse_features * (0 - 1));
- EXPECT_THAT(dest, ElementsAreFloat(expected));
-}
-
-TEST(QuantizationTest, DequantizeAdd1bitOnes) {
- const int bytes_per_embedding = 4;
- const int num_buckets = 3;
- const int num_sparse_features = 7;
- const int quantization_bits = 1;
- const int bucket_id = 1;
-
- std::vector<float> scales(num_buckets, 1.0);
- std::vector<uint8> embeddings(bytes_per_embedding * num_buckets, 0xFF);
-
- std::vector<float> dest(32);
- DequantizeAdd(scales.data(), embeddings.data(), bytes_per_embedding,
- num_sparse_features, quantization_bits, bucket_id, dest.data(),
- dest.size());
- std::vector<float> expected(32);
- std::fill(expected.begin(), expected.end(),
- 1.0 / num_sparse_features * (1 - 1));
- EXPECT_THAT(dest, ElementsAreFloat(expected));
-}
-
-TEST(QuantizationTest, DequantizeAdd3bit) {
- const int bytes_per_embedding = 4;
- const int num_buckets = 3;
- const int num_sparse_features = 7;
- const int quantization_bits = 3;
- const int bucket_id = 1;
-
- std::vector<float> scales(num_buckets, 1.0);
- scales[1] = 9.0;
- std::vector<uint8> embeddings(bytes_per_embedding * num_buckets, 0);
- // For bucket_id=1, the embedding has values 0..9 for indices 0..9:
- embeddings[4] = (1 << 7) | (1 << 6) | (1 << 4) | 1;
- embeddings[5] = (1 << 6) | (1 << 4) | (1 << 3);
- embeddings[6] = (1 << 4) | (1 << 3) | (1 << 2) | (1 << 1) | 1;
-
- std::vector<float> dest(10);
- DequantizeAdd(scales.data(), embeddings.data(), bytes_per_embedding,
- num_sparse_features, quantization_bits, bucket_id, dest.data(),
- dest.size());
-
- std::vector<float> expected;
- expected.push_back(1.0 / num_sparse_features * (1 - 4) * scales[bucket_id]);
- expected.push_back(1.0 / num_sparse_features * (2 - 4) * scales[bucket_id]);
- expected.push_back(1.0 / num_sparse_features * (3 - 4) * scales[bucket_id]);
- expected.push_back(1.0 / num_sparse_features * (4 - 4) * scales[bucket_id]);
- expected.push_back(1.0 / num_sparse_features * (5 - 4) * scales[bucket_id]);
- expected.push_back(1.0 / num_sparse_features * (6 - 4) * scales[bucket_id]);
- expected.push_back(1.0 / num_sparse_features * (7 - 4) * scales[bucket_id]);
- expected.push_back(1.0 / num_sparse_features * (0 - 4) * scales[bucket_id]);
- expected.push_back(1.0 / num_sparse_features * (0 - 4) * scales[bucket_id]);
- expected.push_back(1.0 / num_sparse_features * (0 - 4) * scales[bucket_id]);
- EXPECT_THAT(dest, ElementsAreFloat(expected));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/annotator/strip-unpaired-brackets_test.cc b/native/annotator/strip-unpaired-brackets_test.cc
deleted file mode 100644
index 32585ce..0000000
--- a/native/annotator/strip-unpaired-brackets_test.cc
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/strip-unpaired-brackets.h"
-
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-class StripUnpairedBracketsTest : public ::testing::Test {
- protected:
- StripUnpairedBracketsTest() : INIT_UNILIB_FOR_TESTING(unilib_) {}
- UniLib unilib_;
-};
-
-TEST_F(StripUnpairedBracketsTest, StripUnpairedBrackets) {
- // If the brackets match, nothing gets stripped.
- EXPECT_EQ(StripUnpairedBrackets("call me (123) 456 today", {8, 17}, unilib_),
- std::make_pair(8, 17));
- EXPECT_EQ(StripUnpairedBrackets("call me (123 456) today", {8, 17}, unilib_),
- std::make_pair(8, 17));
-
- // If the brackets don't match, they get stripped.
- EXPECT_EQ(StripUnpairedBrackets("call me (123 456 today", {8, 16}, unilib_),
- std::make_pair(9, 16));
- EXPECT_EQ(StripUnpairedBrackets("call me )123 456 today", {8, 16}, unilib_),
- std::make_pair(9, 16));
- EXPECT_EQ(StripUnpairedBrackets("call me 123 456) today", {8, 16}, unilib_),
- std::make_pair(8, 15));
- EXPECT_EQ(StripUnpairedBrackets("call me 123 456( today", {8, 16}, unilib_),
- std::make_pair(8, 15));
-
- // Strips brackets correctly from length-1 selections that consist of
- // a bracket only.
- EXPECT_EQ(StripUnpairedBrackets("call me at ) today", {11, 12}, unilib_),
- std::make_pair(12, 12));
- EXPECT_EQ(StripUnpairedBrackets("call me at ( today", {11, 12}, unilib_),
- std::make_pair(12, 12));
-
- // Handles invalid spans gracefully.
- EXPECT_EQ(StripUnpairedBrackets("call me at today", {11, 11}, unilib_),
- std::make_pair(11, 11));
- EXPECT_EQ(StripUnpairedBrackets("hello world", {0, 0}, unilib_),
- std::make_pair(0, 0));
- EXPECT_EQ(StripUnpairedBrackets("hello world", {11, 11}, unilib_),
- std::make_pair(11, 11));
- EXPECT_EQ(StripUnpairedBrackets("hello world", {-1, -1}, unilib_),
- std::make_pair(-1, -1));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/annotator/translate/translate_test.cc b/native/annotator/translate/translate_test.cc
deleted file mode 100644
index 8b2a8ef..0000000
--- a/native/annotator/translate/translate_test.cc
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/translate/translate.h"
-
-#include <memory>
-
-#include "annotator/model_generated.h"
-#include "lang_id/fb_model/lang-id-from-fb.h"
-#include "lang_id/lang-id.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::AllOf;
-using testing::Field;
-
-const TranslateAnnotatorOptions* TestingTranslateAnnotatorOptions() {
- static const flatbuffers::DetachedBuffer* options_data = []() {
- TranslateAnnotatorOptionsT options;
- options.enabled = true;
- options.algorithm = TranslateAnnotatorOptions_::Algorithm_BACKOFF;
- options.backoff_options.reset(
- new TranslateAnnotatorOptions_::BackoffOptionsT());
-
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(TranslateAnnotatorOptions::Pack(builder, &options));
- return new flatbuffers::DetachedBuffer(builder.Release());
- }();
-
- return flatbuffers::GetRoot<TranslateAnnotatorOptions>(options_data->data());
-}
-
-class TestingTranslateAnnotator : public TranslateAnnotator {
- public:
- // Make these protected members public for tests.
- using TranslateAnnotator::BackoffDetectLanguages;
- using TranslateAnnotator::FindIndexOfNextWhitespaceOrPunctuation;
- using TranslateAnnotator::TokenAlignedSubstringAroundSpan;
- using TranslateAnnotator::TranslateAnnotator;
-};
-
-std::string GetModelPath() {
- return "";
-}
-
-class TranslateAnnotatorTest : public ::testing::Test {
- protected:
- TranslateAnnotatorTest()
- : INIT_UNILIB_FOR_TESTING(unilib_),
- langid_model_(libtextclassifier3::mobile::lang_id::GetLangIdFromFlatbufferFile(
- GetModelPath() + "lang_id.smfb")),
- translate_annotator_(TestingTranslateAnnotatorOptions(),
- langid_model_.get(), &unilib_) {}
-
- UniLib unilib_;
- std::unique_ptr<libtextclassifier3::mobile::lang_id::LangId> langid_model_;
- TestingTranslateAnnotator translate_annotator_;
-};
-
-TEST_F(TranslateAnnotatorTest, WhenSpeaksEnglishGetsTranslateActionForCzech) {
- ClassificationResult classification;
- EXPECT_TRUE(translate_annotator_.ClassifyText(
- UTF8ToUnicodeText("Třista třicet tři stříbrných stříkaček."), {18, 28},
- "en", &classification));
-
- EXPECT_THAT(classification,
- AllOf(Field(&ClassificationResult::collection, "translate")));
-}
-
-TEST_F(TranslateAnnotatorTest,
- WhenSpeaksEnglishDoesntGetTranslateActionForEnglish) {
- ClassificationResult classification;
- EXPECT_FALSE(translate_annotator_.ClassifyText(
- UTF8ToUnicodeText("This is utterly unutterable."), {8, 15}, "en",
- &classification));
-}
-
-TEST_F(TranslateAnnotatorTest,
- WhenSpeaksMultipleAndNotCzechGetsTranslateActionForCzech) {
- ClassificationResult classification;
- EXPECT_TRUE(translate_annotator_.ClassifyText(
- UTF8ToUnicodeText("Třista třicet tři stříbrných stříkaček."), {8, 15},
- "de,en,ja", &classification));
-
- EXPECT_THAT(classification,
- AllOf(Field(&ClassificationResult::collection, "translate")));
-}
-
-TEST_F(TranslateAnnotatorTest,
- WhenSpeaksMultipleAndEnglishDoesntGetTranslateActionForEnglish) {
- ClassificationResult classification;
- EXPECT_FALSE(translate_annotator_.ClassifyText(
- UTF8ToUnicodeText("This is utterly unutterable."), {8, 15}, "cs,en,de,ja",
- &classification));
-}
-
-TEST_F(TranslateAnnotatorTest, FindIndexOfNextWhitespaceOrPunctuation) {
- const UnicodeText text =
- UTF8ToUnicodeText("Třista třicet, tři stříbrných stříkaček");
-
- EXPECT_EQ(
- translate_annotator_.FindIndexOfNextWhitespaceOrPunctuation(text, 0, -1),
- text.begin());
- EXPECT_EQ(
- translate_annotator_.FindIndexOfNextWhitespaceOrPunctuation(text, 35, 1),
- text.end());
- EXPECT_EQ(
- translate_annotator_.FindIndexOfNextWhitespaceOrPunctuation(text, 10, -1),
- std::next(text.begin(), 6));
- EXPECT_EQ(
- translate_annotator_.FindIndexOfNextWhitespaceOrPunctuation(text, 10, 1),
- std::next(text.begin(), 13));
-}
-
-TEST_F(TranslateAnnotatorTest, TokenAlignedSubstringAroundSpan) {
- const UnicodeText text =
- UTF8ToUnicodeText("Třista třicet, tři stříbrných stříkaček");
-
- EXPECT_EQ(translate_annotator_.TokenAlignedSubstringAroundSpan(
- text, {35, 37}, /*minimum_length=*/100),
- text);
- EXPECT_EQ(translate_annotator_.TokenAlignedSubstringAroundSpan(
- text, {35, 37}, /*minimum_length=*/0),
- UTF8ToUnicodeText("ač"));
- EXPECT_EQ(translate_annotator_.TokenAlignedSubstringAroundSpan(
- text, {35, 37}, /*minimum_length=*/3),
- UTF8ToUnicodeText("stříkaček"));
- EXPECT_EQ(translate_annotator_.TokenAlignedSubstringAroundSpan(
- text, {35, 37}, /*minimum_length=*/10),
- UTF8ToUnicodeText("stříkaček"));
- EXPECT_EQ(translate_annotator_.TokenAlignedSubstringAroundSpan(
- text, {35, 37}, /*minimum_length=*/11),
- UTF8ToUnicodeText("stříbrných stříkaček"));
-
- const UnicodeText text_no_whitespace =
- UTF8ToUnicodeText("reallyreallylongstring");
- EXPECT_EQ(translate_annotator_.TokenAlignedSubstringAroundSpan(
- text_no_whitespace, {10, 11}, /*minimum_length=*/2),
- UTF8ToUnicodeText("reallyreallylongstring"));
-}
-
-TEST_F(TranslateAnnotatorTest, TokenAlignedSubstringWhitespaceText) {
- const UnicodeText text = UTF8ToUnicodeText(" ");
-
- // Shouldn't modify the selection in case it's all whitespace.
- EXPECT_EQ(translate_annotator_.TokenAlignedSubstringAroundSpan(
- text, {5, 7}, /*minimum_length=*/3),
- UTF8ToUnicodeText(" "));
- EXPECT_EQ(translate_annotator_.TokenAlignedSubstringAroundSpan(
- text, {5, 5}, /*minimum_length=*/1),
- UTF8ToUnicodeText(""));
-}
-
-TEST_F(TranslateAnnotatorTest, TokenAlignedSubstringMostlyWhitespaceText) {
- const UnicodeText text = UTF8ToUnicodeText("a a");
-
- // Should still select the whole text even if pointing to whitespace
- // initially.
- EXPECT_EQ(translate_annotator_.TokenAlignedSubstringAroundSpan(
- text, {5, 7}, /*minimum_length=*/11),
- UTF8ToUnicodeText("a a"));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/annotator/types-test-util.h b/native/annotator/types-test-util.h
deleted file mode 100644
index 1d018a1..0000000
--- a/native/annotator/types-test-util.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LIBTEXTCLASSIFIER_ANNOTATOR_TYPES_TEST_UTIL_H_
-#define LIBTEXTCLASSIFIER_ANNOTATOR_TYPES_TEST_UTIL_H_
-
-#include <ostream>
-
-#include "annotator/types.h"
-#include "utils/base/logging.h"
-
-namespace libtextclassifier3 {
-
-#define TC3_DECLARE_PRINT_OPERATOR(TYPE_NAME) \
- inline std::ostream& operator<<(std::ostream& stream, \
- const TYPE_NAME& value) { \
- logging::LoggingStringStream tmp_stream; \
- tmp_stream << value; \
- return stream << tmp_stream.message; \
- }
-
-TC3_DECLARE_PRINT_OPERATOR(AnnotatedSpan)
-TC3_DECLARE_PRINT_OPERATOR(ClassificationResult)
-TC3_DECLARE_PRINT_OPERATOR(DatetimeParsedData)
-TC3_DECLARE_PRINT_OPERATOR(DatetimeParseResultSpan)
-TC3_DECLARE_PRINT_OPERATOR(Token)
-
-#undef TC3_DECLARE_PRINT_OPERATOR
-
-} // namespace libtextclassifier3
-
-#endif // LIBTEXTCLASSIFIER_ANNOTATOR_TYPES_TEST_UTIL_H_
diff --git a/native/annotator/zlib-utils_test.cc b/native/annotator/zlib-utils_test.cc
deleted file mode 100644
index 363c155..0000000
--- a/native/annotator/zlib-utils_test.cc
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "annotator/zlib-utils.h"
-
-#include <memory>
-
-#include "annotator/model_generated.h"
-#include "utils/zlib/zlib.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-
-TEST(ZlibUtilsTest, CompressModel) {
- ModelT model;
- model.regex_model.reset(new RegexModelT);
- model.regex_model->patterns.emplace_back(new RegexModel_::PatternT);
- model.regex_model->patterns.back()->pattern = "this is a test pattern";
- model.regex_model->patterns.emplace_back(new RegexModel_::PatternT);
- model.regex_model->patterns.back()->pattern = "this is a second test pattern";
-
- model.datetime_model.reset(new DatetimeModelT);
- model.datetime_model->patterns.emplace_back(new DatetimeModelPatternT);
- model.datetime_model->patterns.back()->regexes.emplace_back(
- new DatetimeModelPattern_::RegexT);
- model.datetime_model->patterns.back()->regexes.back()->pattern =
- "an example datetime pattern";
- model.datetime_model->extractors.emplace_back(new DatetimeModelExtractorT);
- model.datetime_model->extractors.back()->pattern =
- "an example datetime extractor";
-
- model.intent_options.reset(new IntentFactoryModelT);
- model.intent_options->generator.emplace_back(
- new IntentFactoryModel_::IntentGeneratorT);
- const std::string intent_generator1 = "lua generator 1";
- model.intent_options->generator.back()->lua_template_generator =
- std::vector<uint8_t>(intent_generator1.begin(), intent_generator1.end());
- model.intent_options->generator.emplace_back(
- new IntentFactoryModel_::IntentGeneratorT);
- const std::string intent_generator2 = "lua generator 2";
- model.intent_options->generator.back()->lua_template_generator =
- std::vector<uint8_t>(intent_generator2.begin(), intent_generator2.end());
-
- // NOTE: The resource strings contain some repetition, so that the compressed
- // version is smaller than the uncompressed one. Because the compression code
- // looks at that as well.
- model.resources.reset(new ResourcePoolT);
- model.resources->resource_entry.emplace_back(new ResourceEntryT);
- model.resources->resource_entry.back()->resource.emplace_back(new ResourceT);
- model.resources->resource_entry.back()->resource.back()->content =
- "rrrrrrrrrrrrr1.1";
- model.resources->resource_entry.back()->resource.emplace_back(new ResourceT);
- model.resources->resource_entry.back()->resource.back()->content =
- "rrrrrrrrrrrrr1.2";
- model.resources->resource_entry.emplace_back(new ResourceEntryT);
- model.resources->resource_entry.back()->resource.emplace_back(new ResourceT);
- model.resources->resource_entry.back()->resource.back()->content =
- "rrrrrrrrrrrrr2.1";
- model.resources->resource_entry.back()->resource.emplace_back(new ResourceT);
- model.resources->resource_entry.back()->resource.back()->content =
- "rrrrrrrrrrrrr2.2";
-
- // Compress the model.
- EXPECT_TRUE(CompressModel(&model));
-
- // Sanity check that uncompressed field is removed.
- EXPECT_TRUE(model.regex_model->patterns[0]->pattern.empty());
- EXPECT_TRUE(model.regex_model->patterns[1]->pattern.empty());
- EXPECT_TRUE(model.datetime_model->patterns[0]->regexes[0]->pattern.empty());
- EXPECT_TRUE(model.datetime_model->extractors[0]->pattern.empty());
- EXPECT_TRUE(
- model.intent_options->generator[0]->lua_template_generator.empty());
- EXPECT_TRUE(
- model.intent_options->generator[1]->lua_template_generator.empty());
- EXPECT_TRUE(model.resources->resource_entry[0]->resource[0]->content.empty());
- EXPECT_TRUE(model.resources->resource_entry[0]->resource[1]->content.empty());
- EXPECT_TRUE(model.resources->resource_entry[1]->resource[0]->content.empty());
- EXPECT_TRUE(model.resources->resource_entry[1]->resource[1]->content.empty());
-
- // Pack and load the model.
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(Model::Pack(builder, &model));
- const Model* compressed_model =
- GetModel(reinterpret_cast<const char*>(builder.GetBufferPointer()));
- ASSERT_TRUE(compressed_model != nullptr);
-
- // Decompress the fields again and check that they match the original.
- std::unique_ptr<ZlibDecompressor> decompressor = ZlibDecompressor::Instance();
- ASSERT_TRUE(decompressor != nullptr);
- std::string uncompressed_pattern;
- EXPECT_TRUE(decompressor->MaybeDecompress(
- compressed_model->regex_model()->patterns()->Get(0)->compressed_pattern(),
- &uncompressed_pattern));
- EXPECT_EQ(uncompressed_pattern, "this is a test pattern");
- EXPECT_TRUE(decompressor->MaybeDecompress(
- compressed_model->regex_model()->patterns()->Get(1)->compressed_pattern(),
- &uncompressed_pattern));
- EXPECT_EQ(uncompressed_pattern, "this is a second test pattern");
- EXPECT_TRUE(decompressor->MaybeDecompress(compressed_model->datetime_model()
- ->patterns()
- ->Get(0)
- ->regexes()
- ->Get(0)
- ->compressed_pattern(),
- &uncompressed_pattern));
- EXPECT_EQ(uncompressed_pattern, "an example datetime pattern");
- EXPECT_TRUE(decompressor->MaybeDecompress(compressed_model->datetime_model()
- ->extractors()
- ->Get(0)
- ->compressed_pattern(),
- &uncompressed_pattern));
- EXPECT_EQ(uncompressed_pattern, "an example datetime extractor");
-
- EXPECT_TRUE(DecompressModel(&model));
- EXPECT_EQ(model.regex_model->patterns[0]->pattern, "this is a test pattern");
- EXPECT_EQ(model.regex_model->patterns[1]->pattern,
- "this is a second test pattern");
- EXPECT_EQ(model.datetime_model->patterns[0]->regexes[0]->pattern,
- "an example datetime pattern");
- EXPECT_EQ(model.datetime_model->extractors[0]->pattern,
- "an example datetime extractor");
- EXPECT_EQ(
- model.intent_options->generator[0]->lua_template_generator,
- std::vector<uint8_t>(intent_generator1.begin(), intent_generator1.end()));
- EXPECT_EQ(
- model.intent_options->generator[1]->lua_template_generator,
- std::vector<uint8_t>(intent_generator2.begin(), intent_generator2.end()));
- EXPECT_EQ(model.resources->resource_entry[0]->resource[0]->content,
- "rrrrrrrrrrrrr1.1");
- EXPECT_EQ(model.resources->resource_entry[0]->resource[1]->content,
- "rrrrrrrrrrrrr1.2");
- EXPECT_EQ(model.resources->resource_entry[1]->resource[0]->content,
- "rrrrrrrrrrrrr2.1");
- EXPECT_EQ(model.resources->resource_entry[1]->resource[1]->content,
- "rrrrrrrrrrrrr2.2");
-}
-
-} // namespace libtextclassifier3
diff --git a/native/utils/base/logging.h b/native/utils/base/logging.h
index b465230..4939b7e 100644
--- a/native/utils/base/logging.h
+++ b/native/utils/base/logging.h
@@ -144,7 +144,7 @@
// Debug checks: a TC3_DCHECK<suffix> macro should behave like TC3_CHECK<suffix>
// in debug mode an don't check / don't print anything in non-debug mode.
-#ifdef NDEBUG
+#if defined(NDEBUG) && !defined(TC3_DEBUG_LOGGING)
#define TC3_DCHECK(x) TC3_NULLSTREAM
#define TC3_DCHECK_EQ(x, y) TC3_NULLSTREAM
diff --git a/native/utils/base/logging_raw.cc b/native/utils/base/logging_raw.cc
index ccaef22..e3a73e2 100644
--- a/native/utils/base/logging_raw.cc
+++ b/native/utils/base/logging_raw.cc
@@ -17,8 +17,14 @@
#include "utils/base/logging_raw.h"
#include <stdio.h>
+
#include <string>
+#define TC3_RETURN_IF_NOT_ERROR_OR_FATAL \
+ if (severity != ERROR && severity != FATAL) { \
+ return; \
+ }
+
// NOTE: this file contains two implementations: one for Android, one for all
// other cases. We always build exactly one implementation.
#if defined(__ANDROID__)
@@ -49,13 +55,10 @@
void LowLevelLogging(LogSeverity severity, const std::string& tag,
const std::string& message) {
- const int android_log_level = GetAndroidLogLevel(severity);
#if !defined(TC3_DEBUG_LOGGING)
- if (android_log_level != ANDROID_LOG_ERROR &&
- android_log_level != ANDROID_LOG_FATAL) {
- return;
- }
+ TC3_RETURN_IF_NOT_ERROR_OR_FATAL
#endif
+ const int android_log_level = GetAndroidLogLevel(severity);
__android_log_write(android_log_level, tag.c_str(), message.c_str());
}
@@ -88,6 +91,9 @@
void LowLevelLogging(LogSeverity severity, const std::string &tag,
const std::string &message) {
+#if !defined(TC3_DEBUG_LOGGING)
+ TC3_RETURN_IF_NOT_ERROR_OR_FATAL
+#endif
fprintf(stderr, "[%s] %s : %s\n", LogSeverityToString(severity), tag.c_str(),
message.c_str());
fflush(stderr);
diff --git a/native/utils/base/status_test.cc b/native/utils/base/status_test.cc
deleted file mode 100644
index 82d5aad..0000000
--- a/native/utils/base/status_test.cc
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/base/status.h"
-
-#include "utils/base/logging.h"
-#include "utils/base/status_macros.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(StatusTest, PrintsAbortedStatus) {
- logging::LoggingStringStream stream;
- stream << Status::UNKNOWN;
- EXPECT_EQ(Status::UNKNOWN.error_code(), 2);
- EXPECT_EQ(Status::UNKNOWN.CanonicalCode(), StatusCode::UNKNOWN);
- EXPECT_EQ(Status::UNKNOWN.error_message(), "");
- EXPECT_EQ(stream.message, "2");
-}
-
-TEST(StatusTest, PrintsOKStatus) {
- logging::LoggingStringStream stream;
- stream << Status::OK;
- EXPECT_EQ(Status::OK.error_code(), 0);
- EXPECT_EQ(Status::OK.CanonicalCode(), StatusCode::OK);
- EXPECT_EQ(Status::OK.error_message(), "");
- EXPECT_EQ(stream.message, "0");
-}
-
-TEST(StatusTest, UnknownStatusHasRightAttributes) {
- EXPECT_EQ(Status::UNKNOWN.error_code(), 2);
- EXPECT_EQ(Status::UNKNOWN.CanonicalCode(), StatusCode::UNKNOWN);
- EXPECT_EQ(Status::UNKNOWN.error_message(), "");
-}
-
-TEST(StatusTest, OkStatusHasRightAttributes) {
- EXPECT_EQ(Status::OK.error_code(), 0);
- EXPECT_EQ(Status::OK.CanonicalCode(), StatusCode::OK);
- EXPECT_EQ(Status::OK.error_message(), "");
-}
-
-TEST(StatusTest, CustomStatusHasRightAttributes) {
- Status status(StatusCode::INVALID_ARGUMENT, "You can't put this here!");
- EXPECT_EQ(status.error_code(), 3);
- EXPECT_EQ(status.CanonicalCode(), StatusCode::INVALID_ARGUMENT);
- EXPECT_EQ(status.error_message(), "You can't put this here!");
-}
-
-TEST(StatusTest, AssignmentPreservesMembers) {
- Status status(StatusCode::INVALID_ARGUMENT, "You can't put this here!");
-
- Status status2 = status;
-
- EXPECT_EQ(status2.error_code(), 3);
- EXPECT_EQ(status2.CanonicalCode(), StatusCode::INVALID_ARGUMENT);
- EXPECT_EQ(status2.error_message(), "You can't put this here!");
-}
-
-TEST(StatusTest, ReturnIfErrorOkStatus) {
- bool returned_due_to_error = true;
- auto lambda = [&returned_due_to_error](const Status& s) {
- TC3_RETURN_IF_ERROR(s);
- returned_due_to_error = false;
- return Status::OK;
- };
-
- // OK should allow execution to continue and the returned status should also
- // be OK.
- Status status = lambda(Status());
- EXPECT_EQ(status.error_code(), 0);
- EXPECT_EQ(status.CanonicalCode(), StatusCode::OK);
- EXPECT_EQ(status.error_message(), "");
- EXPECT_FALSE(returned_due_to_error);
-}
-
-TEST(StatusTest, ReturnIfErrorInvalidArgumentStatus) {
- bool returned_due_to_error = true;
- auto lambda = [&returned_due_to_error](const Status& s) {
- TC3_RETURN_IF_ERROR(s);
- returned_due_to_error = false;
- return Status::OK;
- };
-
- // INVALID_ARGUMENT should cause an early return.
- Status invalid_arg_status(StatusCode::INVALID_ARGUMENT, "You can't do that!");
- Status status = lambda(invalid_arg_status);
- EXPECT_EQ(status.error_code(), 3);
- EXPECT_EQ(status.CanonicalCode(), StatusCode::INVALID_ARGUMENT);
- EXPECT_EQ(status.error_message(), "You can't do that!");
- EXPECT_TRUE(returned_due_to_error);
-}
-
-TEST(StatusTest, ReturnIfErrorUnknownStatus) {
- bool returned_due_to_error = true;
- auto lambda = [&returned_due_to_error](const Status& s) {
- TC3_RETURN_IF_ERROR(s);
- returned_due_to_error = false;
- return Status::OK;
- };
-
- // UNKNOWN should cause an early return.
- Status unknown_status(StatusCode::UNKNOWN,
- "We also know there are known unknowns.");
- libtextclassifier3::Status status = lambda(unknown_status);
- EXPECT_EQ(status.error_code(), 2);
- EXPECT_EQ(status.CanonicalCode(), StatusCode::UNKNOWN);
- EXPECT_EQ(status.error_message(), "We also know there are known unknowns.");
- EXPECT_TRUE(returned_due_to_error);
-}
-
-TEST(StatusTest, ReturnIfErrorOnlyInvokesExpressionOnce) {
- int num_invocations = 0;
- auto ok_internal_expr = [&num_invocations]() {
- ++num_invocations;
- return Status::OK;
- };
- auto ok_lambda = [&ok_internal_expr]() {
- TC3_RETURN_IF_ERROR(ok_internal_expr());
- return Status::OK;
- };
-
- libtextclassifier3::Status status = ok_lambda();
- EXPECT_EQ(status.CanonicalCode(), StatusCode::OK);
- EXPECT_EQ(num_invocations, 1);
-
- num_invocations = 0;
- auto error_internal_expr = [&num_invocations]() {
- ++num_invocations;
- return Status::UNKNOWN;
- };
- auto error_lambda = [&error_internal_expr]() {
- TC3_RETURN_IF_ERROR(error_internal_expr());
- return Status::OK;
- };
-
- status = error_lambda();
- EXPECT_EQ(status.CanonicalCode(), StatusCode::UNKNOWN);
- EXPECT_EQ(num_invocations, 1);
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/base/statusor_test.cc b/native/utils/base/statusor_test.cc
deleted file mode 100644
index 6e8afb1..0000000
--- a/native/utils/base/statusor_test.cc
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/base/statusor.h"
-
-#include "utils/base/logging.h"
-#include "utils/base/status.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(StatusOrTest, DoesntDieWhenOK) {
- StatusOr<std::string> status_or_string = std::string("Hello World");
- EXPECT_TRUE(status_or_string.ok());
- EXPECT_EQ(status_or_string.ValueOrDie(), "Hello World");
-}
-
-TEST(StatusOrTest, DiesWhenNotOK) {
- StatusOr<std::string> status_or_string = {Status::UNKNOWN};
- EXPECT_FALSE(status_or_string.ok());
- EXPECT_DEATH(status_or_string.ValueOrDie(),
- "Attempting to fetch value of non-OK StatusOr: 2");
-}
-
-// Foo is NOT default constructible and can be implicitly converted to from int.
-class Foo {
- public:
- // Copy value conversion
- Foo(int i) : i_(i) {} // NOLINT
- int i() const { return i_; }
-
- private:
- int i_;
-};
-
-TEST(StatusOrTest, HandlesNonDefaultConstructibleValues) {
- StatusOr<Foo> foo_or(Foo(7));
- EXPECT_TRUE(foo_or.ok());
- EXPECT_EQ(foo_or.ValueOrDie().i(), 7);
-
- StatusOr<Foo> error_or(Status::UNKNOWN);
- EXPECT_FALSE(error_or.ok());
- EXPECT_EQ(error_or.status().CanonicalCode(), StatusCode::UNKNOWN);
-}
-
-class Bar {
- public:
- // Move value conversion
- Bar(Foo&& f) : i_(2 * f.i()) {} // NOLINT
-
- // Movable, but not copyable.
- Bar(const Bar& other) = delete;
- Bar& operator=(const Bar& rhs) = delete;
- Bar(Bar&& other) = default;
- Bar& operator=(Bar&& rhs) = default;
-
- int i() const { return i_; }
-
- private:
- int i_;
-};
-
-TEST(StatusOrTest, HandlesValueConversion) {
- // Copy value conversion constructor : StatusOr<Foo>(const int&)
- StatusOr<Foo> foo_status(19);
- EXPECT_TRUE(foo_status.ok());
- EXPECT_EQ(foo_status.ValueOrDie().i(), 19);
-
- // Move value conversion constructor : StatusOr<Bar>(Foo&&)
- StatusOr<Bar> bar_status(std::move(foo_status));
- EXPECT_TRUE(bar_status.ok());
- EXPECT_EQ(bar_status.ValueOrDie().i(), 38);
-
- StatusOr<int> int_status(19);
- // Copy conversion constructor : StatusOr<Foo>(const StatusOr<int>&)
- StatusOr<Foo> copied_status(int_status);
- EXPECT_TRUE(copied_status.ok());
- EXPECT_EQ(copied_status.ValueOrDie().i(), 19);
-
- // Move conversion constructor : StatusOr<Bar>(StatusOr<Foo>&&)
- StatusOr<Bar> moved_status(std::move(copied_status));
- EXPECT_TRUE(moved_status.ok());
- EXPECT_EQ(moved_status.ValueOrDie().i(), 38);
-
- // Move conversion constructor with error : StatusOr<Bar>(StatusOr<Foo>&&)
- StatusOr<Foo> error_status(Status::UNKNOWN);
- StatusOr<Bar> moved_error_status(std::move(error_status));
- EXPECT_FALSE(moved_error_status.ok());
-}
-
-struct OkFn {
- StatusOr<int> operator()() { return 42; }
-};
-TEST(StatusOrTest, AssignOrReturnValOk) {
- auto lambda = []() {
- TC3_ASSIGN_OR_RETURN(int i, OkFn()(), -1);
- return i;
- };
-
- // OkFn() should return a valid integer, so lambda should return that integer.
- EXPECT_EQ(lambda(), 42);
-}
-
-struct FailFn {
- StatusOr<int> operator()() { return Status::UNKNOWN; }
-};
-TEST(StatusOrTest, AssignOrReturnValError) {
- auto lambda = []() {
- TC3_ASSIGN_OR_RETURN(int i, FailFn()(), -1);
- return i;
- };
-
- // FailFn() should return an error, so lambda should return -1.
- EXPECT_EQ(lambda(), -1);
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/calendar/calendar_test-include.cc b/native/utils/calendar/calendar_test-include.cc
deleted file mode 100644
index 7fe6f53..0000000
--- a/native/utils/calendar/calendar_test-include.cc
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/calendar/calendar_test-include.h"
-
-namespace libtextclassifier3 {
-namespace test_internal {
-
-static constexpr int kWednesday = 4;
-
-TEST_F(CalendarTest, Interface) {
- int64 time;
- DatetimeGranularity granularity;
- std::string timezone;
- DatetimeParsedData data;
- bool result = calendarlib_.InterpretParseData(
- data, /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity);
- TC3_LOG(INFO) << result;
-}
-
-TEST_F(CalendarTest, SetsZeroTimeWhenNotRelative) {
- int64 time;
- DatetimeGranularity granularity;
- DatetimeParsedData data;
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::YEAR, 2018);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1514761200000L /* Jan 01 2018 00:00:00 */);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/1L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1514761200000L /* Jan 01 2018 00:00:00 */);
-}
-
-TEST_F(CalendarTest, RoundingToGranularityBasic) {
- int64 time;
- DatetimeGranularity granularity;
- DatetimeParsedData data;
-
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::YEAR, 2018);
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1514761200000L /* Jan 01 2018 00:00:00 */);
-
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::MONTH, 4);
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1522533600000L /* Apr 01 2018 00:00:00 */);
-
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::DAY_OF_MONTH, 25);
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1524607200000L /* Apr 25 2018 00:00:00 */);
-
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::HOUR, 9);
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1524639600000L /* Apr 25 2018 09:00:00 */);
-
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::MINUTE, 33);
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1524641580000 /* Apr 25 2018 09:33:00 */);
-
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::SECOND, 59);
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1524641639000 /* Apr 25 2018 09:33:59 */);
-}
-
-TEST_F(CalendarTest, RoundingToGranularityWeek) {
- int64 time;
- DatetimeGranularity granularity;
- // Prepare data structure that means: "next week"
- DatetimeParsedData data;
- data.SetRelativeValue(DatetimeComponent::ComponentType::WEEK,
- DatetimeComponent::RelativeQualifier::NEXT);
- data.SetRelativeCount(DatetimeComponent::ComponentType::WEEK, 1);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"de-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 342000000L /* Mon Jan 05 1970 00:00:00 */);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/0L, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-US",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 255600000L /* Sun Jan 04 1970 00:00:00 */);
-}
-
-TEST_F(CalendarTest, RelativeTime) {
- const int64 ref_time = 1524648839000L; /* 25 April 2018 09:33:59 */
- int64 time;
- DatetimeGranularity granularity;
-
- // Two Weds from now.
- DatetimeParsedData future_wed_parse;
- future_wed_parse.SetRelativeValue(
- DatetimeComponent::ComponentType::DAY_OF_WEEK,
- DatetimeComponent::RelativeQualifier::FUTURE);
- future_wed_parse.SetRelativeCount(
- DatetimeComponent::ComponentType::DAY_OF_WEEK, 2);
- future_wed_parse.SetAbsoluteValue(
- DatetimeComponent::ComponentType::DAY_OF_WEEK, kWednesday);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- future_wed_parse, ref_time, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-US",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1525858439000L /* Wed May 09 2018 11:33:59 */);
- EXPECT_EQ(granularity, GRANULARITY_DAY);
-
- // Next Wed.
- DatetimeParsedData next_wed_parse;
- next_wed_parse.SetAbsoluteValue(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- kWednesday);
- next_wed_parse.SetRelativeValue(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- DatetimeComponent::RelativeQualifier::NEXT);
- next_wed_parse.SetRelativeCount(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- 1);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- next_wed_parse, ref_time, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-US",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1525212000000L /* Wed May 02 2018 00:00:00 */);
- EXPECT_EQ(granularity, GRANULARITY_DAY);
-
- // Same Wed.
- DatetimeParsedData same_wed_parse;
- same_wed_parse.SetRelativeValue(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- DatetimeComponent::RelativeQualifier::THIS);
- same_wed_parse.SetAbsoluteValue(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- kWednesday);
- same_wed_parse.SetRelativeCount(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- 1);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- same_wed_parse, ref_time, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-US",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1524607200000L /* Wed Apr 25 2018 00:00:00 */);
- EXPECT_EQ(granularity, GRANULARITY_DAY);
-
- // Previous Wed.
- DatetimeParsedData last_wed_parse;
- last_wed_parse.SetRelativeValue(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- DatetimeComponent::RelativeQualifier::LAST);
- last_wed_parse.SetAbsoluteValue(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- kWednesday);
- last_wed_parse.SetRelativeCount(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- 1);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- last_wed_parse, ref_time, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-US",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1524002400000L /* Wed Apr 18 2018 00:00:00 */);
- EXPECT_EQ(granularity, GRANULARITY_DAY);
-
- // Two Weds ago.
- DatetimeParsedData past_wed_parse;
- past_wed_parse.SetRelativeValue(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- DatetimeComponent::RelativeQualifier::PAST);
- past_wed_parse.SetAbsoluteValue(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- kWednesday);
- past_wed_parse.SetRelativeCount(DatetimeComponent::ComponentType::DAY_OF_WEEK,
- 2);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- past_wed_parse, ref_time, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-US",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1523439239000L /* Wed Apr 11 2018 11:33:59 */);
- EXPECT_EQ(granularity, GRANULARITY_DAY);
-
- // In 3 hours.
- DatetimeParsedData in_3_hours_parse;
- in_3_hours_parse.SetRelativeValue(
- DatetimeComponent::ComponentType::HOUR,
- DatetimeComponent::RelativeQualifier::FUTURE);
- in_3_hours_parse.SetRelativeCount(DatetimeComponent::ComponentType::HOUR, 3);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- in_3_hours_parse, ref_time, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-US",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1524659639000L /* Wed Apr 25 2018 14:33:59 */);
- EXPECT_EQ(granularity, GRANULARITY_HOUR);
-
- // In 5 minutes.
- DatetimeParsedData in_5_minutes_parse;
- in_5_minutes_parse.SetRelativeValue(
- DatetimeComponent::ComponentType::MINUTE,
- DatetimeComponent::RelativeQualifier::FUTURE);
- in_5_minutes_parse.SetRelativeCount(DatetimeComponent::ComponentType::MINUTE,
- 5);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- in_5_minutes_parse, ref_time, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-US",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1524649139000L /* Wed Apr 25 2018 14:33:59 */);
- EXPECT_EQ(granularity, GRANULARITY_MINUTE);
-
- // In 10 seconds.
- DatetimeParsedData in_10_seconds_parse;
- in_10_seconds_parse.SetRelativeValue(
- DatetimeComponent::ComponentType::SECOND,
- DatetimeComponent::RelativeQualifier::FUTURE);
- in_10_seconds_parse.SetRelativeCount(DatetimeComponent::ComponentType::SECOND,
- 10);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- in_10_seconds_parse, ref_time, /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-US",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1524648849000L /* Wed Apr 25 2018 14:33:59 */);
- EXPECT_EQ(granularity, GRANULARITY_SECOND);
-}
-
-TEST_F(CalendarTest, AddsADayWhenTimeInThePastAndDayNotSpecified) {
- int64 time;
- DatetimeGranularity granularity;
- DatetimeParsedData data;
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::HOUR, 7);
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::MINUTE, 10);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/1567317600000L /* Sept 01 2019 00:00:00 */,
- /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH", /*prefer_future_for_unspecified_date=*/true,
- &time, &granularity));
- EXPECT_EQ(time, 1567401000000L /* Sept 02 2019 07:10:00 */);
-}
-
-TEST_F(CalendarTest,
- DoesntAddADayWhenTimeInThePastAndDayNotSpecifiedAndDisabled) {
- int64 time;
- DatetimeGranularity granularity;
- DatetimeParsedData data;
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::HOUR, 7);
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::MINUTE, 10);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/1567317600000L /* Sept 01 2019 00:00:00 */,
- /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1567314600000L /* Sept 01 2019 07:10:00 */);
-}
-
-TEST_F(CalendarTest, DoesntAddADayWhenTimeInTheFutureAndDayNotSpecified) {
- int64 time;
- DatetimeGranularity granularity;
- DatetimeParsedData data;
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::HOUR, 9);
- data.SetAbsoluteValue(DatetimeComponent::ComponentType::MINUTE, 10);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/1567317600000L /* Sept 01 2019 00:00:00 */,
- /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH", /*prefer_future_for_unspecified_date=*/true,
- &time, &granularity));
- EXPECT_EQ(time, 1567321800000L /* Sept 01 2019 09:10:00 */);
-
- ASSERT_TRUE(calendarlib_.InterpretParseData(
- data,
- /*reference_time_ms_utc=*/1567317600000L /* Sept 01 2019 00:00:00 */,
- /*reference_timezone=*/"Europe/Zurich",
- /*reference_locale=*/"en-CH",
- /*prefer_future_for_unspecified_date=*/false, &time, &granularity));
- EXPECT_EQ(time, 1567321800000L /* Sept 01 2019 09:10:00 */);
-}
-
-} // namespace test_internal
-} // namespace libtextclassifier3
diff --git a/native/utils/calendar/calendar_test-include.h b/native/utils/calendar/calendar_test-include.h
deleted file mode 100644
index 58ad6e0..0000000
--- a/native/utils/calendar/calendar_test-include.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This is a shared test between icu and javaicu calendar implementations.
-// It is meant to be #include'd.
-
-#ifndef LIBTEXTCLASSIFIER_UTILS_CALENDAR_CALENDAR_TEST_INCLUDE_H_
-#define LIBTEXTCLASSIFIER_UTILS_CALENDAR_CALENDAR_TEST_INCLUDE_H_
-
-#include "gtest/gtest.h"
-
-#if defined TC3_CALENDAR_ICU
-#include "utils/calendar/calendar-icu.h"
-#define TC3_TESTING_CREATE_CALENDARLIB_INSTANCE(VAR) VAR()
-#elif defined TC3_CALENDAR_APPLE
-#include "utils/calendar/calendar-apple.h"
-#define TC3_TESTING_CREATE_CALENDARLIB_INSTANCE(VAR) VAR()
-#elif defined TC3_CALENDAR_JAVAICU
-#include <jni.h>
-extern JNIEnv* g_jenv;
-#define TC3_TESTING_CREATE_CALENDARLIB_INSTANCE(VAR) \
- VAR(JniCache::Create(g_jenv))
-#include "utils/calendar/calendar-javaicu.h"
-#else
-#error Unsupported calendar implementation.
-#endif
-
-// This can get overridden in the javaicu version which needs to pass an JNIEnv*
-// argument to the constructor.
-#ifndef TC3_TESTING_CREATE_CALENDARLIB_INSTANCE
-
-#endif
-
-namespace libtextclassifier3 {
-namespace test_internal {
-
-class CalendarTest : public ::testing::Test {
- protected:
- CalendarTest() : TC3_TESTING_CREATE_CALENDARLIB_INSTANCE(calendarlib_) {}
- CalendarLib calendarlib_;
-};
-
-} // namespace test_internal
-} // namespace libtextclassifier3
-
-#endif // LIBTEXTCLASSIFIER_UTILS_CALENDAR_CALENDAR_TEST_INCLUDE_H_
diff --git a/native/utils/calendar/calendar_test.cc b/native/utils/calendar/calendar_test.cc
deleted file mode 100644
index 54ed2a0..0000000
--- a/native/utils/calendar/calendar_test.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "gtest/gtest.h"
-
-// The actual code of the test is in the following include:
-#include "utils/calendar/calendar_test-include.h"
diff --git a/native/utils/checksum_test.cc b/native/utils/checksum_test.cc
deleted file mode 100644
index dd04956..0000000
--- a/native/utils/checksum_test.cc
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/checksum.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(LuhnTest, CorrectlyHandlesSimpleCases) {
- EXPECT_TRUE(VerifyLuhnChecksum("3782 8224 6310 005"));
- EXPECT_FALSE(VerifyLuhnChecksum("0"));
- EXPECT_FALSE(VerifyLuhnChecksum("1"));
- EXPECT_FALSE(VerifyLuhnChecksum("0A"));
-}
-
-TEST(LuhnTest, CorrectlyVerifiesPaymentCardNumbers) {
- // Fake test numbers.
- EXPECT_TRUE(VerifyLuhnChecksum("3782 8224 6310 005"));
- EXPECT_TRUE(VerifyLuhnChecksum("371449635398431"));
- EXPECT_TRUE(VerifyLuhnChecksum("5610591081018250"));
- EXPECT_TRUE(VerifyLuhnChecksum("38520000023237"));
- EXPECT_TRUE(VerifyLuhnChecksum("6011000990139424"));
- EXPECT_TRUE(VerifyLuhnChecksum("3566002020360505"));
- EXPECT_TRUE(VerifyLuhnChecksum("5105105105105100"));
- EXPECT_TRUE(VerifyLuhnChecksum("4012 8888 8888 1881"));
-}
-
-TEST(LuhnTest, HandlesWhitespace) {
- EXPECT_TRUE(
- VerifyLuhnChecksum("3782 8224 6310 005 ", /*ignore_whitespace=*/true));
- EXPECT_FALSE(
- VerifyLuhnChecksum("3782 8224 6310 005 ", /*ignore_whitespace=*/false));
-}
-
-TEST(LuhnTest, HandlesEdgeCases) {
- EXPECT_FALSE(VerifyLuhnChecksum(" ", /*ignore_whitespace=*/true));
- EXPECT_FALSE(VerifyLuhnChecksum(" ", /*ignore_whitespace=*/false));
- EXPECT_FALSE(VerifyLuhnChecksum("", /*ignore_whitespace=*/true));
- EXPECT_FALSE(VerifyLuhnChecksum("", /*ignore_whitespace=*/false));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/container/double-array-trie.h b/native/utils/container/double-array-trie.h
index e1b43da..39c8822 100644
--- a/native/utils/container/double-array-trie.h
+++ b/native/utils/container/double-array-trie.h
@@ -30,7 +30,7 @@
// A trie node specifies a node in the tree, either an intermediate node or
// a leaf node.
// A leaf node contains the id as an int of the string match. This id is encoded
-// in the lower 30 bits, thus the number of distinct ids is 2^30.
+// in the lower 31 bits, thus the number of distinct ids is 2^31.
// An intermediate node has an associated label and an offset to it's children.
// The label is encoded in the least significant byte and must match the input
// character during matching.
diff --git a/native/utils/container/double-array-trie_test.cc b/native/utils/container/double-array-trie_test.cc
deleted file mode 100644
index 8ceec00..0000000
--- a/native/utils/container/double-array-trie_test.cc
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/container/double-array-trie.h"
-
-#include <fstream>
-#include <string>
-#include <vector>
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-std::string GetTestConfigPath() {
- return "";
-}
-
-TEST(DoubleArrayTest, Lookup) {
- // Test trie that contains pieces "hell", "hello", "o", "there".
- std::ifstream test_config_stream(GetTestConfigPath());
- std::string config((std::istreambuf_iterator<char>(test_config_stream)),
- (std::istreambuf_iterator<char>()));
- DoubleArrayTrie trie(reinterpret_cast<const TrieNode*>(config.data()),
- config.size() / sizeof(TrieNode));
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(trie.FindAllPrefixMatches("hello there", &matches));
- EXPECT_EQ(matches.size(), 2);
- EXPECT_EQ(matches[0].id, 0 /*hell*/);
- EXPECT_EQ(matches[0].match_length, 4 /*hell*/);
- EXPECT_EQ(matches[1].id, 1 /*hello*/);
- EXPECT_EQ(matches[1].match_length, 5 /*hello*/);
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(trie.FindAllPrefixMatches("he", &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(trie.FindAllPrefixMatches("abcd", &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(trie.FindAllPrefixMatches("", &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(trie.FindAllPrefixMatches("hi there", &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(trie.FindAllPrefixMatches(StringPiece("\0", 1), &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(
- trie.FindAllPrefixMatches(StringPiece("\xff, \xfe", 2), &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- StringSet::Match match;
- EXPECT_TRUE(trie.LongestPrefixMatch("hella there", &match));
- EXPECT_EQ(match.id, 0 /*hell*/);
- }
-
- {
- StringSet::Match match;
- EXPECT_TRUE(trie.LongestPrefixMatch("hello there", &match));
- EXPECT_EQ(match.id, 1 /*hello*/);
- }
-
- {
- StringSet::Match match;
- EXPECT_TRUE(trie.LongestPrefixMatch("abcd", &match));
- EXPECT_EQ(match.id, -1);
- }
-
- {
- StringSet::Match match;
- EXPECT_TRUE(trie.LongestPrefixMatch("", &match));
- EXPECT_EQ(match.id, -1);
- }
-
- {
- int value;
- EXPECT_TRUE(trie.Find("hell", &value));
- EXPECT_EQ(value, 0);
- }
-
- {
- int value;
- EXPECT_FALSE(trie.Find("hella", &value));
- }
-
- {
- int value;
- EXPECT_TRUE(trie.Find("hello", &value));
- EXPECT_EQ(value, 1 /*hello*/);
- }
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/container/sorted-strings-table_test.cc b/native/utils/container/sorted-strings-table_test.cc
deleted file mode 100644
index a93b197..0000000
--- a/native/utils/container/sorted-strings-table_test.cc
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/container/sorted-strings-table.h"
-
-#include <vector>
-
-#include "utils/base/integral_types.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(SortedStringsTest, Lookup) {
- const char pieces[] = "hell\0hello\0o\0there\0";
- const uint32 offsets[] = {0, 5, 11, 13};
-
- SortedStringsTable table(/*num_pieces=*/4, offsets, StringPiece(pieces, 18),
- /*use_linear_scan_threshold=*/1);
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(table.FindAllPrefixMatches("hello there", &matches));
- EXPECT_EQ(matches.size(), 2);
- EXPECT_EQ(matches[0].id, 0 /*hell*/);
- EXPECT_EQ(matches[0].match_length, 4 /*hell*/);
- EXPECT_EQ(matches[1].id, 1 /*hello*/);
- EXPECT_EQ(matches[1].match_length, 5 /*hello*/);
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(table.FindAllPrefixMatches("he", &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(table.FindAllPrefixMatches("he", &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(table.FindAllPrefixMatches("abcd", &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(table.FindAllPrefixMatches("", &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(table.FindAllPrefixMatches("hi there", &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(table.FindAllPrefixMatches(StringPiece("\0", 1), &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- std::vector<StringSet::Match> matches;
- EXPECT_TRUE(
- table.FindAllPrefixMatches(StringPiece("\xff, \xfe", 2), &matches));
- EXPECT_THAT(matches, testing::IsEmpty());
- }
-
- {
- StringSet::Match match;
- EXPECT_TRUE(table.LongestPrefixMatch("hella there", &match));
- EXPECT_EQ(match.id, 0 /*hell*/);
- }
-
- {
- StringSet::Match match;
- EXPECT_TRUE(table.LongestPrefixMatch("hello there", &match));
- EXPECT_EQ(match.id, 1 /*hello*/);
- }
-
- {
- StringSet::Match match;
- EXPECT_TRUE(table.LongestPrefixMatch("abcd", &match));
- EXPECT_EQ(match.id, -1);
- }
-
- {
- StringSet::Match match;
- EXPECT_TRUE(table.LongestPrefixMatch("", &match));
- EXPECT_EQ(match.id, -1);
- }
-
- {
- int value;
- EXPECT_TRUE(table.Find("hell", &value));
- EXPECT_EQ(value, 0);
- }
-
- {
- int value;
- EXPECT_FALSE(table.Find("hella", &value));
- }
-
- {
- int value;
- EXPECT_TRUE(table.Find("hello", &value));
- EXPECT_EQ(value, 1 /*hello*/);
- }
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/flatbuffers.cc b/native/utils/flatbuffers.cc
index 53805e0..bb3bbb7 100644
--- a/native/utils/flatbuffers.cc
+++ b/native/utils/flatbuffers.cc
@@ -183,34 +183,6 @@
return libtextclassifier3::GetFieldByOffsetOrNull(type_, field_offset);
}
-bool ReflectiveFlatbuffer::IsMatchingType(const reflection::Field* field,
- const Variant& value) const {
- switch (field->type()->base_type()) {
- case reflection::Bool:
- return value.HasBool();
- case reflection::Byte:
- return value.HasInt8();
- case reflection::UByte:
- return value.HasUInt8();
- case reflection::Int:
- return value.HasInt();
- case reflection::UInt:
- return value.HasUInt();
- case reflection::Long:
- return value.HasInt64();
- case reflection::ULong:
- return value.HasUInt64();
- case reflection::Float:
- return value.HasFloat();
- case reflection::Double:
- return value.HasDouble();
- case reflection::String:
- return value.HasString();
- default:
- return false;
- }
-}
-
bool ReflectiveFlatbuffer::ParseAndSet(const reflection::Field* field,
const std::string& value) {
switch (field->type()->base_type()) {
@@ -518,7 +490,7 @@
case reflection::Vector:
switch (field->type()->element()) {
case reflection::Int:
- AppendFromVector<int>(from, field);
+ AppendFromVector<int32>(from, field);
break;
case reflection::UInt:
AppendFromVector<uint>(from, field);
diff --git a/native/utils/flatbuffers.h b/native/utils/flatbuffers.h
index e40c1bc..d987609 100644
--- a/native/utils/flatbuffers.h
+++ b/native/utils/flatbuffers.h
@@ -130,8 +130,37 @@
reflection::Field const** field);
// Checks whether a variant value type agrees with a field type.
- bool IsMatchingType(const reflection::Field* field,
- const Variant& value) const;
+ template <typename T>
+ bool IsMatchingType(const reflection::BaseType type) const {
+ switch (type) {
+ case reflection::Bool:
+ return std::is_same<T, bool>::value;
+ case reflection::Byte:
+ return std::is_same<T, int8>::value;
+ case reflection::UByte:
+ return std::is_same<T, uint8>::value;
+ case reflection::Int:
+ return std::is_same<T, int32>::value;
+ case reflection::UInt:
+ return std::is_same<T, uint32>::value;
+ case reflection::Long:
+ return std::is_same<T, int64>::value;
+ case reflection::ULong:
+ return std::is_same<T, uint64>::value;
+ case reflection::Float:
+ return std::is_same<T, float>::value;
+ case reflection::Double:
+ return std::is_same<T, double>::value;
+ case reflection::String:
+ return std::is_same<T, std::string>::value ||
+ std::is_same<T, StringPiece>::value ||
+ std::is_same<T, const char*>::value;
+ case reflection::Obj:
+ return std::is_same<T, ReflectiveFlatbuffer>::value;
+ default:
+ return false;
+ }
+ }
// Sets a (primitive) field to a specific value.
// Returns true if successful, and false if the field was not found or the
@@ -154,7 +183,7 @@
return false;
}
Variant variant_value(value);
- if (!IsMatchingType(field, variant_value)) {
+ if (!IsMatchingType<T>(field->type()->base_type())) {
TC3_LOG(ERROR) << "Type mismatch for field `" << field->name()->str()
<< "`, expected: " << field->type()->base_type()
<< ", got: " << variant_value.GetType();
@@ -193,6 +222,11 @@
template <typename T>
TypedRepeatedField<T>* Repeated(const reflection::Field* field) {
+ if (!IsMatchingType<T>(field->type()->element())) {
+ TC3_LOG(ERROR) << "Type mismatch for field `" << field->name()->str()
+ << "`";
+ return nullptr;
+ }
return static_cast<TypedRepeatedField<T>*>(Repeated(field));
}
diff --git a/native/utils/flatbuffers_test.cc b/native/utils/flatbuffers_test.cc
deleted file mode 100644
index e3aaa6c..0000000
--- a/native/utils/flatbuffers_test.cc
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/flatbuffers.h"
-
-#include <fstream>
-#include <map>
-#include <memory>
-#include <string>
-
-#include "utils/flatbuffers_generated.h"
-#include "utils/flatbuffers_test_generated.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-#include "flatbuffers/flatbuffers.h"
-#include "flatbuffers/reflection.h"
-#include "flatbuffers/reflection_generated.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-std::string GetTestMetadataPath() {
- return "flatbuffers_test.bfbs";
-}
-
-std::string LoadTestMetadata() {
- std::ifstream test_config_stream(GetTestMetadataPath());
- return std::string((std::istreambuf_iterator<char>(test_config_stream)),
- (std::istreambuf_iterator<char>()));
-}
-
-TEST(FlatbuffersTest, PrimitiveFieldsAreCorrectlySet) {
- std::string metadata_buffer = LoadTestMetadata();
- ReflectiveFlatbufferBuilder reflective_builder(
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data()));
-
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
- EXPECT_TRUE(buffer != nullptr);
- EXPECT_TRUE(buffer->Set("an_int_field", 42));
- EXPECT_TRUE(buffer->Set("a_long_field", 84ll));
- EXPECT_TRUE(buffer->Set("a_bool_field", true));
- EXPECT_TRUE(buffer->Set("a_float_field", 1.f));
- EXPECT_TRUE(buffer->Set("a_double_field", 1.0));
-
- // Try to parse with the generated code.
- std::string serialized_entity_data = buffer->Serialize();
- std::unique_ptr<test::EntityDataT> entity_data =
- LoadAndVerifyMutableFlatbuffer<test::EntityData>(
- serialized_entity_data.data(), serialized_entity_data.size());
- EXPECT_TRUE(entity_data != nullptr);
- EXPECT_EQ(entity_data->an_int_field, 42);
- EXPECT_EQ(entity_data->a_long_field, 84);
- EXPECT_EQ(entity_data->a_bool_field, true);
- EXPECT_NEAR(entity_data->a_float_field, 1.f, 1e-4);
- EXPECT_NEAR(entity_data->a_double_field, 1.f, 1e-4);
-}
-
-TEST(FlatbuffersTest, HandlesUnknownFields) {
- std::string metadata_buffer = LoadTestMetadata();
- const reflection::Schema* schema =
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data());
- ReflectiveFlatbufferBuilder reflective_builder(schema);
-
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
- EXPECT_TRUE(buffer != nullptr);
-
- // Add a field that is not known to the (statically generated) code.
- EXPECT_TRUE(buffer->Set("mystic", "this is an unknown field."));
-
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(flatbuffers::Offset<void>(buffer->Serialize(&builder)));
-
- // Try to read the field again.
- const flatbuffers::Table* extra =
- flatbuffers::GetAnyRoot(builder.GetBufferPointer());
- EXPECT_EQ(extra
- ->GetPointer<const flatbuffers::String*>(
- buffer->GetFieldOrNull("mystic")->offset())
- ->str(),
- "this is an unknown field.");
-}
-
-TEST(FlatbuffersTest, HandlesNestedFields) {
- std::string metadata_buffer = LoadTestMetadata();
- const reflection::Schema* schema =
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data());
- ReflectiveFlatbufferBuilder reflective_builder(schema);
-
- FlatbufferFieldPathT path;
- path.field.emplace_back(new FlatbufferFieldT);
- path.field.back()->field_name = "flight_number";
- path.field.emplace_back(new FlatbufferFieldT);
- path.field.back()->field_name = "carrier_code";
- flatbuffers::FlatBufferBuilder path_builder;
- path_builder.Finish(FlatbufferFieldPath::Pack(path_builder, &path));
-
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
-
- ReflectiveFlatbuffer* parent = nullptr;
- reflection::Field const* field = nullptr;
- EXPECT_TRUE(
- buffer->GetFieldWithParent(flatbuffers::GetRoot<FlatbufferFieldPath>(
- path_builder.GetBufferPointer()),
- &parent, &field));
- EXPECT_EQ(parent, buffer->Mutable("flight_number"));
- EXPECT_EQ(field,
- buffer->Mutable("flight_number")->GetFieldOrNull("carrier_code"));
-}
-
-TEST(FlatbuffersTest, HandlesMultipleNestedFields) {
- std::string metadata_buffer = LoadTestMetadata();
- ReflectiveFlatbufferBuilder reflective_builder(
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data()));
-
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
- ReflectiveFlatbuffer* flight_info = buffer->Mutable("flight_number");
- flight_info->Set("carrier_code", "LX");
- flight_info->Set("flight_code", 38);
-
- ReflectiveFlatbuffer* contact_info = buffer->Mutable("contact_info");
- EXPECT_TRUE(contact_info->Set("first_name", "Barack"));
- EXPECT_TRUE(contact_info->Set("last_name", "Obama"));
- EXPECT_TRUE(contact_info->Set("phone_number", "1-800-TEST"));
- EXPECT_TRUE(contact_info->Set("score", 1.f));
-
- // Try to parse with the generated code.
- std::string serialized_entity_data = buffer->Serialize();
- std::unique_ptr<test::EntityDataT> entity_data =
- LoadAndVerifyMutableFlatbuffer<test::EntityData>(
- serialized_entity_data.data(), serialized_entity_data.size());
- EXPECT_TRUE(entity_data != nullptr);
- EXPECT_EQ(entity_data->flight_number->carrier_code, "LX");
- EXPECT_EQ(entity_data->flight_number->flight_code, 38);
- EXPECT_EQ(entity_data->contact_info->first_name, "Barack");
- EXPECT_EQ(entity_data->contact_info->last_name, "Obama");
- EXPECT_EQ(entity_data->contact_info->phone_number, "1-800-TEST");
- EXPECT_NEAR(entity_data->contact_info->score, 1.f, 1e-4);
-}
-
-TEST(FlatbuffersTest, HandlesFieldsSetWithNamePath) {
- std::string metadata_buffer = LoadTestMetadata();
- ReflectiveFlatbufferBuilder reflective_builder(
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data()));
-
- FlatbufferFieldPathT path;
- path.field.emplace_back(new FlatbufferFieldT);
- path.field.back()->field_name = "flight_number";
- path.field.emplace_back(new FlatbufferFieldT);
- path.field.back()->field_name = "carrier_code";
- flatbuffers::FlatBufferBuilder path_builder;
- path_builder.Finish(FlatbufferFieldPath::Pack(path_builder, &path));
-
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
- // Test setting value using Set function.
- buffer->Mutable("flight_number")->Set("flight_code", 38);
- // Test setting value using FlatbufferFieldPath.
- buffer->Set(flatbuffers::GetRoot<FlatbufferFieldPath>(
- path_builder.GetBufferPointer()),
- "LX");
-
- // Try to parse with the generated code.
- std::string serialized_entity_data = buffer->Serialize();
- std::unique_ptr<test::EntityDataT> entity_data =
- LoadAndVerifyMutableFlatbuffer<test::EntityData>(
- serialized_entity_data.data(), serialized_entity_data.size());
- EXPECT_TRUE(entity_data != nullptr);
- EXPECT_EQ(entity_data->flight_number->carrier_code, "LX");
- EXPECT_EQ(entity_data->flight_number->flight_code, 38);
-}
-
-TEST(FlatbuffersTest, HandlesFieldsSetWithOffsetPath) {
- std::string metadata_buffer = LoadTestMetadata();
- ReflectiveFlatbufferBuilder reflective_builder(
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data()));
-
- FlatbufferFieldPathT path;
- path.field.emplace_back(new FlatbufferFieldT);
- path.field.back()->field_offset = 14;
- path.field.emplace_back(new FlatbufferFieldT);
- path.field.back()->field_offset = 4;
- flatbuffers::FlatBufferBuilder path_builder;
- path_builder.Finish(FlatbufferFieldPath::Pack(path_builder, &path));
-
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
- // Test setting value using Set function.
- buffer->Mutable("flight_number")->Set("flight_code", 38);
- // Test setting value using FlatbufferFieldPath.
- buffer->Set(flatbuffers::GetRoot<FlatbufferFieldPath>(
- path_builder.GetBufferPointer()),
- "LX");
-
- // Try to parse with the generated code.
- std::string serialized_entity_data = buffer->Serialize();
- std::unique_ptr<test::EntityDataT> entity_data =
- LoadAndVerifyMutableFlatbuffer<test::EntityData>(
- serialized_entity_data.data(), serialized_entity_data.size());
- EXPECT_TRUE(entity_data != nullptr);
- EXPECT_EQ(entity_data->flight_number->carrier_code, "LX");
- EXPECT_EQ(entity_data->flight_number->flight_code, 38);
-}
-
-TEST(FlatbuffersTest, PartialBuffersAreCorrectlyMerged) {
- std::string metadata_buffer = LoadTestMetadata();
- ReflectiveFlatbufferBuilder reflective_builder(
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data()));
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
- buffer->Set("an_int_field", 42);
- buffer->Set("a_long_field", 84ll);
- ReflectiveFlatbuffer* flight_info = buffer->Mutable("flight_number");
- flight_info->Set("carrier_code", "LX");
- flight_info->Set("flight_code", 38);
- auto* reminders = buffer->Repeated<ReflectiveFlatbuffer>("reminders");
- ReflectiveFlatbuffer* reminder1 = reminders->Add();
- reminder1->Set("title", "reminder1");
- auto* reminder1_notes = reminder1->Repeated<std::string>("notes");
- reminder1_notes->Add("note1");
- reminder1_notes->Add("note2");
-
- // Create message to merge.
- test::EntityDataT additional_entity_data;
- additional_entity_data.an_int_field = 43;
- additional_entity_data.flight_number.reset(new test::FlightNumberInfoT);
- additional_entity_data.flight_number->flight_code = 39;
- additional_entity_data.contact_info.reset(new test::ContactInfoT);
- additional_entity_data.contact_info->first_name = "Barack";
- additional_entity_data.reminders.push_back(
- std::unique_ptr<test::ReminderT>(new test::ReminderT));
- additional_entity_data.reminders[0]->notes.push_back("additional note1");
- additional_entity_data.reminders[0]->notes.push_back("additional note2");
- additional_entity_data.numbers.push_back(9);
- additional_entity_data.numbers.push_back(10);
- additional_entity_data.strings.push_back("str1");
- additional_entity_data.strings.push_back("str2");
-
- flatbuffers::FlatBufferBuilder to_merge_builder;
- to_merge_builder.Finish(
- test::EntityData::Pack(to_merge_builder, &additional_entity_data));
-
- // Merge it.
- EXPECT_TRUE(buffer->MergeFrom(
- flatbuffers::GetAnyRoot(to_merge_builder.GetBufferPointer())));
-
- // Try to parse it with the generated code.
- std::string serialized_entity_data = buffer->Serialize();
- std::unique_ptr<test::EntityDataT> entity_data =
- LoadAndVerifyMutableFlatbuffer<test::EntityData>(
- serialized_entity_data.data(), serialized_entity_data.size());
- EXPECT_TRUE(entity_data != nullptr);
- EXPECT_EQ(entity_data->an_int_field, 43);
- EXPECT_EQ(entity_data->a_long_field, 84);
- EXPECT_EQ(entity_data->flight_number->carrier_code, "LX");
- EXPECT_EQ(entity_data->flight_number->flight_code, 39);
- EXPECT_EQ(entity_data->contact_info->first_name, "Barack");
- ASSERT_EQ(entity_data->reminders.size(), 2);
- EXPECT_EQ(entity_data->reminders[1]->notes[0], "additional note1");
- EXPECT_EQ(entity_data->reminders[1]->notes[1], "additional note2");
- ASSERT_EQ(entity_data->numbers.size(), 2);
- EXPECT_EQ(entity_data->numbers[0], 9);
- EXPECT_EQ(entity_data->numbers[1], 10);
- ASSERT_EQ(entity_data->strings.size(), 2);
- EXPECT_EQ(entity_data->strings[0], "str1");
- EXPECT_EQ(entity_data->strings[1], "str2");
-}
-
-TEST(FlatbuffersTest, PrimitiveAndNestedFieldsAreCorrectlyFlattened) {
- std::string metadata_buffer = LoadTestMetadata();
- ReflectiveFlatbufferBuilder reflective_builder(
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data()));
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
- buffer->Set("an_int_field", 42);
- buffer->Set("a_long_field", 84ll);
- ReflectiveFlatbuffer* flight_info = buffer->Mutable("flight_number");
- flight_info->Set("carrier_code", "LX");
- flight_info->Set("flight_code", 38);
-
- std::map<std::string, Variant> entity_data_map = buffer->AsFlatMap();
- EXPECT_EQ(4, entity_data_map.size());
- EXPECT_EQ(42, entity_data_map["an_int_field"].IntValue());
- EXPECT_EQ(84, entity_data_map["a_long_field"].Int64Value());
- EXPECT_EQ("LX", entity_data_map["flight_number.carrier_code"].StringValue());
- EXPECT_EQ(38, entity_data_map["flight_number.flight_code"].IntValue());
-}
-
-TEST(FlatbuffersTest, ToTextProtoWorks) {
- std::string metadata_buffer = LoadTestMetadata();
- ReflectiveFlatbufferBuilder reflective_builder(
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data()));
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
- buffer->Set("an_int_field", 42);
- buffer->Set("a_long_field", 84ll);
- ReflectiveFlatbuffer* flight_info = buffer->Mutable("flight_number");
- flight_info->Set("carrier_code", "LX");
- flight_info->Set("flight_code", 38);
-
- EXPECT_EQ(buffer->ToTextProto(),
- "a_long_field: 84, an_int_field: 42, flight_number "
- "{flight_code: 38, carrier_code: 'LX'}");
-}
-
-TEST(FlatbuffersTest, RepeatedFieldSetThroughReflectionCanBeRead) {
- std::string metadata_buffer = LoadTestMetadata();
- const reflection::Schema* schema =
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data());
- ReflectiveFlatbufferBuilder reflective_builder(schema);
- std::unique_ptr<ReflectiveFlatbuffer> buffer = reflective_builder.NewRoot();
-
- auto reminders = buffer->Repeated<ReflectiveFlatbuffer>("reminders");
- {
- auto reminder = reminders->Add();
- reminder->Set("title", "test reminder");
- auto notes = reminder->Repeated<std::string>("notes");
- notes->Add("note A");
- notes->Add("note B");
- }
- {
- auto reminder = reminders->Add();
- reminder->Set("title", "test reminder 2");
- auto notes = reminder->Repeated<std::string>("notes");
- notes->Add("note i");
- notes->Add("note ii");
- notes->Add("note iii");
- }
- const std::string serialized_entity_data = buffer->Serialize();
-
- std::unique_ptr<test::EntityDataT> entity_data =
- LoadAndVerifyMutableFlatbuffer<test::EntityData>(
- serialized_entity_data.data(), serialized_entity_data.size());
- EXPECT_TRUE(entity_data != nullptr);
- EXPECT_EQ(2, entity_data->reminders.size());
- EXPECT_EQ("test reminder", entity_data->reminders[0]->title);
- EXPECT_THAT(entity_data->reminders[0]->notes,
- testing::ElementsAreArray({"note A", "note B"}));
- EXPECT_EQ("test reminder 2", entity_data->reminders[1]->title);
- EXPECT_THAT(entity_data->reminders[1]->notes,
- testing::ElementsAreArray({"note i", "note ii", "note iii"}));
-}
-
-TEST(FlatbuffersTest, ResolvesFieldOffsets) {
- std::string metadata_buffer = LoadTestMetadata();
- const reflection::Schema* schema =
- flatbuffers::GetRoot<reflection::Schema>(metadata_buffer.data());
- FlatbufferFieldPathT path;
- path.field.emplace_back(new FlatbufferFieldT);
- path.field.back()->field_name = "flight_number";
- path.field.emplace_back(new FlatbufferFieldT);
- path.field.back()->field_name = "carrier_code";
-
- EXPECT_TRUE(SwapFieldNamesForOffsetsInPath(schema, &path));
-
- EXPECT_THAT(path.field[0]->field_name, testing::IsEmpty());
- EXPECT_EQ(14, path.field[0]->field_offset);
- EXPECT_THAT(path.field[1]->field_name, testing::IsEmpty());
- EXPECT_EQ(4, path.field[1]->field_offset);
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/flatbuffers_test.fbs b/native/utils/flatbuffers_test.fbs
deleted file mode 100644
index f208ff4..0000000
--- a/native/utils/flatbuffers_test.fbs
+++ /dev/null
@@ -1,49 +0,0 @@
-//
-// Copyright (C) 2018 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-namespace libtextclassifier3.test;
-
-table FlightNumberInfo {
- carrier_code: string;
- flight_code: int;
-}
-
-table ContactInfo {
- first_name: string;
- last_name: string;
- phone_number: string;
- score: float;
-}
-
-table Reminder {
- title: string;
- notes: [string];
-}
-
-table EntityData {
- an_int_field: int;
- a_long_field: int64;
- a_bool_field: bool;
- a_float_field: float;
- a_double_field: double;
- flight_number: FlightNumberInfo;
- contact_info: ContactInfo;
- reminders: [Reminder];
- numbers: [int];
- strings: [string];
-}
-
-root_type libtextclassifier3.test.EntityData;
diff --git a/native/utils/grammar/lexer.cc b/native/utils/grammar/lexer.cc
index 960f6c4..190545a 100644
--- a/native/utils/grammar/lexer.cc
+++ b/native/utils/grammar/lexer.cc
@@ -16,6 +16,8 @@
#include "utils/grammar/lexer.h"
+#include <unordered_map>
+
namespace libtextclassifier3::grammar {
namespace {
@@ -25,9 +27,20 @@
return matcher->ArenaSize() <= kMaxMemoryUsage;
}
+Match* CheckedAddMatch(const Nonterm nonterm,
+ const CodepointSpan codepoint_span,
+ const int match_offset, const int16 type,
+ Matcher* matcher) {
+ if (nonterm == kUnassignedNonterm || !CheckMemoryUsage(matcher)) {
+ return nullptr;
+ }
+ return matcher->AllocateAndInitMatch<Match>(nonterm, codepoint_span,
+ match_offset, type);
+}
+
void CheckedEmit(const Nonterm nonterm, const CodepointSpan codepoint_span,
const int match_offset, int16 type, Matcher* matcher) {
- if (CheckMemoryUsage(matcher)) {
+ if (nonterm != kUnassignedNonterm && CheckMemoryUsage(matcher)) {
matcher->AddMatch(matcher->AllocateAndInitMatch<Match>(
nonterm, codepoint_span, match_offset, type));
}
@@ -35,104 +48,95 @@
} // namespace
-void Lexer::Emit(const StringPiece value, const CodepointSpan codepoint_span,
- const int match_offset, const TokenType type,
- const RulesSet_::Nonterminals* nonterms,
+void Lexer::Emit(const Symbol& symbol, const RulesSet_::Nonterminals* nonterms,
Matcher* matcher) const {
- // Emit the token as terminal.
- if (CheckMemoryUsage(matcher)) {
- matcher->AddTerminal(codepoint_span, match_offset, value);
- }
-
- // Emit <token> if used by rules.
- if (nonterms->token_nt() != kUnassignedNonterm) {
- CheckedEmit(nonterms->token_nt(), codepoint_span, match_offset,
- Match::kTokenType, matcher);
- }
-
- // Emit token type specific non-terminals.
- if (type == TOKEN_TYPE_DIGITS) {
- // Emit <digits> if used by the rules.
- if (nonterms->digits_nt() != kUnassignedNonterm) {
- CheckedEmit(nonterms->digits_nt(), codepoint_span, match_offset,
- Match::kDigitsType, matcher);
+ switch (symbol.type) {
+ case Symbol::Type::TYPE_MATCH: {
+ // Just emit the match.
+ matcher->AddMatch(symbol.match);
+ return;
}
+ case Symbol::Type::TYPE_DIGITS: {
+ // Emit <digits> if used by the rules.
+ CheckedEmit(nonterms->digits_nt(), symbol.codepoint_span,
+ symbol.match_offset, Match::kDigitsType, matcher);
- // Emit <n_digits> if used by the rules.
- if (nonterms->n_digits_nt() != nullptr) {
- const int num_digits = codepoint_span.second - codepoint_span.first;
- if (num_digits <= nonterms->n_digits_nt()->size()) {
- if (const Nonterm n_digits =
- nonterms->n_digits_nt()->Get(num_digits - 1)) {
- CheckedEmit(n_digits, codepoint_span, match_offset,
+ // Emit <n_digits> if used by the rules.
+ if (nonterms->n_digits_nt() != nullptr) {
+ const int num_digits =
+ symbol.codepoint_span.second - symbol.codepoint_span.first;
+ if (num_digits <= nonterms->n_digits_nt()->size()) {
+ CheckedEmit(nonterms->n_digits_nt()->Get(num_digits - 1),
+ symbol.codepoint_span, symbol.match_offset,
Match::kDigitsType, matcher);
}
}
+ break;
}
+ default:
+ break;
}
+
+ // Emit the token as terminal.
+ if (CheckMemoryUsage(matcher)) {
+ matcher->AddTerminal(symbol.codepoint_span, symbol.match_offset,
+ symbol.lexeme);
+ }
+
+ // Emit <token> if used by rules.
+ CheckedEmit(nonterms->token_nt(), symbol.codepoint_span, symbol.match_offset,
+ Match::kTokenType, matcher);
}
-Lexer::TokenType Lexer::GetTokenType(
+Lexer::Symbol::Type Lexer::GetSymbolType(
const UnicodeText::const_iterator& it) const {
if (unilib_.IsPunctuation(*it)) {
- return TOKEN_TYPE_PUNCTUATION;
+ return Symbol::Type::TYPE_PUNCTUATION;
} else if (unilib_.IsDigit(*it)) {
- return TOKEN_TYPE_DIGITS;
- } else if (unilib_.IsWhitespace(*it)) {
- return TOKEN_TYPE_WHITESPACE;
+ return Symbol::Type::TYPE_DIGITS;
} else {
- return TOKEN_TYPE_TERM;
+ return Symbol::Type::TYPE_TERM;
}
}
-int Lexer::ProcessToken(const StringPiece value, const int prev_token_end,
- const CodepointSpan codepoint_span,
- const RulesSet_::Nonterminals* nonterms,
- Matcher* matcher) const {
+void Lexer::ProcessToken(const StringPiece value, const int prev_token_end,
+ const CodepointSpan codepoint_span,
+ std::vector<Lexer::Symbol>* symbols) const {
+ // Possibly split token.
UnicodeText token_unicode = UTF8ToUnicodeText(value.data(), value.size(),
/*do_copy=*/false);
-
- if (unilib_.IsWhitespace(*token_unicode.begin())) {
- // Ignore whitespace tokens.
- return prev_token_end;
- }
-
- // Possibly split token.
int last_end = prev_token_end;
auto token_end = token_unicode.end();
auto it = token_unicode.begin();
- TokenType type = GetTokenType(it);
+ Symbol::Type type = GetSymbolType(it);
CodepointIndex sub_token_start = codepoint_span.first;
while (it != token_end) {
auto next = std::next(it);
int num_codepoints = 1;
- TokenType next_type;
+ Symbol::Type next_type;
while (next != token_end) {
- next_type = GetTokenType(next);
- if (type == TOKEN_TYPE_PUNCTUATION || next_type != type) {
+ next_type = GetSymbolType(next);
+ if (type == Symbol::Type::TYPE_PUNCTUATION || next_type != type) {
break;
}
++next;
++num_codepoints;
}
-
- // Emit token.
- StringPiece sub_token =
- StringPiece(it.utf8_data(), next.utf8_data() - it.utf8_data());
- if (type != TOKEN_TYPE_WHITESPACE) {
- Emit(sub_token,
- CodepointSpan{sub_token_start, sub_token_start + num_codepoints},
- /*match_offset=*/last_end, type, nonterms, matcher);
- last_end = sub_token_start + num_codepoints;
- }
+ symbols->push_back(Symbol{
+ type, CodepointSpan{sub_token_start, sub_token_start + num_codepoints},
+ /*match_offset=*/last_end,
+ /*lexeme=*/
+ StringPiece(it.utf8_data(), next.utf8_data() - it.utf8_data())});
+ last_end = sub_token_start + num_codepoints;
it = next;
type = next_type;
sub_token_start = last_end;
}
- return last_end;
}
-void Lexer::Process(const std::vector<Token>& tokens, Matcher* matcher) const {
+void Lexer::Process(const std::vector<Token>& tokens,
+ const std::vector<Match*>& matches,
+ Matcher* matcher) const {
if (tokens.empty()) {
return;
}
@@ -140,28 +144,84 @@
const RulesSet_::Nonterminals* nonterminals = matcher->nonterminals();
// Initialize processing of new text.
- int prev_token_end = 0;
+ CodepointIndex prev_token_end = 0;
+ std::vector<Symbol> symbols;
matcher->Reset();
- // Emit start symbol if used by the grammar.
- if (nonterminals->start_nt() != kUnassignedNonterm) {
- matcher->AddMatch(matcher->AllocateAndInitMatch<Match>(
- nonterminals->start_nt(), CodepointSpan{0, 0},
- /*match_offset=*/0));
+ // The matcher expects the terminals and non-terminals it received to be in
+ // non-decreasing end-position order. The sorting above makes sure the
+ // pre-defined matches adhere to that order.
+ // Ideally, we would just have to emit a predefined match whenever we see that
+ // the next token we feed would be ending later.
+ // But as we implicitly ignore whitespace, we have to merge preceding
+ // whitespace to the match start so that tokens and non-terminals fed appear
+ // as next to each other without whitespace.
+ // We keep track of real token starts and precending whitespace in
+ // `token_match_start`, so that we can extend a predefined match's start to
+ // include the preceding whitespace.
+ std::unordered_map<CodepointIndex, CodepointIndex> token_match_start;
+
+ // Add start symbols.
+ if (Match* match =
+ CheckedAddMatch(nonterminals->start_nt(), CodepointSpan{0, 0},
+ /*match_offset=*/0, Match::kBreakType, matcher)) {
+ symbols.push_back(Symbol(match));
+ }
+ if (Match* match =
+ CheckedAddMatch(nonterminals->wordbreak_nt(), CodepointSpan{0, 0},
+ /*match_offset=*/0, Match::kBreakType, matcher)) {
+ symbols.push_back(Symbol(match));
}
for (const Token& token : tokens) {
- prev_token_end = ProcessToken(token.value,
- /*prev_token_end=*/prev_token_end,
- CodepointSpan{token.start, token.end},
- nonterminals, matcher);
+ // Record match starts for token boundaries, so that we can snap pre-defined
+ // matches to it.
+ if (prev_token_end != token.start) {
+ token_match_start[token.start] = prev_token_end;
+ }
+
+ ProcessToken(token.value,
+ /*prev_token_end=*/prev_token_end,
+ CodepointSpan{token.start, token.end}, &symbols);
+ prev_token_end = token.end;
+
+ // Add word break symbol if used by the grammar.
+ if (Match* match = CheckedAddMatch(
+ nonterminals->wordbreak_nt(), CodepointSpan{token.end, token.end},
+ /*match_offset=*/token.end, Match::kBreakType, matcher)) {
+ symbols.push_back(Symbol(match));
+ }
}
- // Emit end symbol if used by the grammar.
- if (nonterminals->end_nt() != kUnassignedNonterm) {
- matcher->AddMatch(matcher->AllocateAndInitMatch<Match>(
- nonterminals->end_nt(), CodepointSpan{prev_token_end, prev_token_end},
- /*match_offset=*/prev_token_end));
+ // Add end symbol if used by the grammar.
+ if (Match* match = CheckedAddMatch(
+ nonterminals->end_nt(), CodepointSpan{prev_token_end, prev_token_end},
+ /*match_offset=*/prev_token_end, Match::kBreakType, matcher)) {
+ symbols.push_back(Symbol(match));
+ }
+
+ // Add predefined matches.
+ for (Match* match : matches) {
+ // Decrease match offset to include preceding whitespace.
+ auto token_match_start_it = token_match_start.find(match->match_offset);
+ if (token_match_start_it != token_match_start.end()) {
+ match->match_offset = token_match_start_it->second;
+ }
+ symbols.push_back(Symbol(match));
+ }
+
+ std::sort(symbols.begin(), symbols.end(),
+ [](const Symbol& a, const Symbol& b) {
+ // Sort by increasing (end, start) position to guarantee the
+ // matcher requirement that the tokens are fed in non-decreasing
+ // end position order.
+ return std::tie(a.codepoint_span.second, a.codepoint_span.first) <
+ std::tie(b.codepoint_span.second, b.codepoint_span.first);
+ });
+
+ // Emit symbols to matcher.
+ for (const Symbol& symbol : symbols) {
+ Emit(symbol, nonterminals, matcher);
}
}
diff --git a/native/utils/grammar/lexer.h b/native/utils/grammar/lexer.h
index fd1a99d..87fd504 100644
--- a/native/utils/grammar/lexer.h
+++ b/native/utils/grammar/lexer.h
@@ -57,9 +57,8 @@
// This makes it appear to the Matcher as if the tokens are adjacent -- so
// whitespace is simply ignored.
//
-// A minor optimization: We don't bother to output <token> unless
-// the grammar includes a nonterminal with that name. Similarly, we don't
-// output <digits> unless the grammar uses them.
+// A minor optimization: We don't bother to output nonterminals if the grammar
+// rules don't reference them.
#ifndef LIBTEXTCLASSIFIER_UTILS_GRAMMAR_LEXER_H_
#define LIBTEXTCLASSIFIER_UTILS_GRAMMAR_LEXER_H_
@@ -78,30 +77,77 @@
public:
explicit Lexer(const UniLib& unilib) : unilib_(unilib) {}
- void Process(const std::vector<Token>& tokens, Matcher* matcher) const;
+ // Processes a tokenized text. Classifies the tokens and feeds them to the
+ // matcher. Predefined existing matches `matches` will be fed to the matcher
+ // alongside the tokens.
+ void Process(const std::vector<Token>& tokens,
+ const std::vector<Match*>& matches, Matcher* matcher) const;
private:
- enum TokenType {
- TOKEN_TYPE_TERM,
- TOKEN_TYPE_WHITESPACE,
- TOKEN_TYPE_DIGITS,
- TOKEN_TYPE_PUNCTUATION
+ // A lexical symbol with an identified meaning that represents raw tokens,
+ // token categories or predefined text matches.
+ // It is the unit fed to the grammar matcher.
+ struct Symbol {
+ // The type of the lexical symbol.
+ enum class Type {
+ // A raw token.
+ TYPE_TERM,
+
+ // A symbol representing a string of digits.
+ TYPE_DIGITS,
+
+ // Punctuation characters.
+ TYPE_PUNCTUATION,
+
+ // A predefined match.
+ TYPE_MATCH
+ };
+
+ explicit Symbol() = default;
+
+ // Constructs a symbol of a given type with an anchor in the text.
+ Symbol(const Type type, const CodepointSpan codepoint_span,
+ const int match_offset, StringPiece lexeme)
+ : type(type),
+ codepoint_span(codepoint_span),
+ match_offset(match_offset),
+ lexeme(lexeme) {}
+
+ // Constructs a symbol from a pre-defined match.
+ explicit Symbol(Match* match)
+ : type(Type::TYPE_MATCH),
+ codepoint_span(match->codepoint_span),
+ match_offset(match->match_offset),
+ match(match) {}
+
+ // The type of the symbole.
+ Type type;
+
+ // The span in the text as codepoint offsets.
+ CodepointSpan codepoint_span;
+
+ // The match start offset (including preceding whitespace) as codepoint
+ // offset.
+ int match_offset;
+
+ // The symbol text value.
+ StringPiece lexeme;
+
+ // The predefined match.
+ Match* match;
};
- // Processes a single token: the token is split, classified and passed to the
- // matcher. Returns the end of the last part emitted.
- int ProcessToken(const StringPiece value, const int prev_token_end,
- const CodepointSpan codepoint_span,
- const RulesSet_::Nonterminals* nonterms,
- Matcher* matcher) const;
+ // Processes a single token: the token is split and classified into symbols.
+ void ProcessToken(const StringPiece value, const int prev_token_end,
+ const CodepointSpan codepoint_span,
+ std::vector<Symbol>* symbols) const;
// Emits a token to the matcher.
- void Emit(const StringPiece value, const CodepointSpan codepoint_span,
- int match_offset, const TokenType type,
- const RulesSet_::Nonterminals* nonterms, Matcher* matcher) const;
+ void Emit(const Symbol& symbol, const RulesSet_::Nonterminals* nonterms,
+ Matcher* matcher) const;
// Gets the type of a character.
- TokenType GetTokenType(const UnicodeText::const_iterator& it) const;
+ Symbol::Type GetSymbolType(const UnicodeText::const_iterator& it) const;
private:
const UniLib& unilib_;
diff --git a/native/utils/grammar/lexer_test.cc b/native/utils/grammar/lexer_test.cc
deleted file mode 100644
index bcf2309..0000000
--- a/native/utils/grammar/lexer_test.cc
+++ /dev/null
@@ -1,437 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Unit tests for the lexer.
-
-#include "utils/grammar/lexer.h"
-
-#include <string>
-#include <vector>
-
-#include "utils/grammar/callback-delegate.h"
-#include "utils/grammar/matcher.h"
-#include "utils/grammar/rules_generated.h"
-#include "utils/grammar/utils/ir.h"
-#include "utils/tokenizer.h"
-#include "utils/utf8/unilib.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3::grammar {
-namespace {
-
-using testing::ElementsAre;
-using testing::Eq;
-using testing::Value;
-
-// Superclass of all tests here.
-class LexerTest : public testing::Test {
- protected:
- LexerTest()
- : INIT_UNILIB_FOR_TESTING(unilib_),
- tokenizer_(TokenizationType_ICU, &unilib_,
- /*codepoint_ranges=*/{},
- /*internal_tokenizer_codepoint_ranges=*/{},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/true),
- lexer_(unilib_) {}
-
- // Creates a grammar just checking the specified terminals.
- std::string GrammarForTerminals(const std::vector<std::string>& terminals) {
- const CallbackId callback = 1;
- Ir ir;
- for (const std::string& terminal : terminals) {
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, terminal);
- }
- return ir.SerializeAsFlatbuffer();
- }
-
- UniLib unilib_;
- Tokenizer tokenizer_;
- Lexer lexer_;
-};
-
-struct TestMatchResult {
- int begin;
- int end;
- std::string terminal;
- std::string nonterminal;
-};
-
-MATCHER_P3(IsTerminal, begin, end, terminal, "") {
- return Value(arg.begin, begin) && Value(arg.end, end) &&
- Value(arg.terminal, terminal);
-}
-
-MATCHER_P3(IsNonterminal, begin, end, name, "") {
- return Value(arg.begin, begin) && Value(arg.end, end) &&
- Value(arg.nonterminal, name);
-}
-
-// This is a simple callback for testing purposes.
-class TestCallbackDelegate : public CallbackDelegate {
- public:
- explicit TestCallbackDelegate(
- const RulesSet_::DebugInformation* debug_information)
- : debug_information_(debug_information) {}
-
- void MatchFound(const Match* match, const CallbackId, const int64,
- Matcher*) override {
- TestMatchResult result;
- result.begin = match->match_offset;
- result.end = match->codepoint_span.second;
- if (match->IsTerminalRule()) {
- result.terminal = match->terminal;
- } else if (match->IsUnaryRule()) {
- // We use callbacks on unary rules to attach a callback to a
- // predefined lhs.
- result.nonterminal = GetNonterminalName(match->unary_rule_rhs()->lhs);
- } else {
- result.nonterminal = GetNonterminalName(match->lhs);
- }
- log_.push_back(result);
- }
-
- void Clear() { log_.clear(); }
-
- const std::vector<TestMatchResult>& log() const { return log_; }
-
- private:
- std::string GetNonterminalName(const Nonterm nonterminal) const {
- if (const RulesSet_::DebugInformation_::NonterminalNamesEntry* entry =
- debug_information_->nonterminal_names()->LookupByKey(nonterminal)) {
- return entry->value()->str();
- }
- // Unnamed Nonterm.
- return "()";
- }
-
- const RulesSet_::DebugInformation* debug_information_;
- std::vector<TestMatchResult> log_;
-};
-
-TEST_F(LexerTest, HandlesSimpleLetters) {
- std::string rules_buffer = GrammarForTerminals({"a", "is", "this", "word"});
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(tokenizer_.Tokenize("This is a word"), &matcher);
-
- EXPECT_THAT(test_logger.log().size(), Eq(4));
-}
-
-TEST_F(LexerTest, HandlesConcatedLettersAndDigit) {
- std::string rules_buffer =
- GrammarForTerminals({"1234", "4321", "a", "cde", "this"});
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(tokenizer_.Tokenize("1234This a4321cde"), &matcher);
-
- EXPECT_THAT(test_logger.log().size(), Eq(5));
-}
-
-TEST_F(LexerTest, HandlesPunctuation) {
- std::string rules_buffer = GrammarForTerminals({"10", "18", "2014", "/"});
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(tokenizer_.Tokenize("10/18/2014"), &matcher);
-
- EXPECT_THAT(test_logger.log().size(), Eq(5));
-}
-
-TEST_F(LexerTest, HandlesUTF8Punctuation) {
- std::string rules_buffer =
- GrammarForTerminals({"电话", "：", "0871", "—", "6857", "（", "曹"});
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(tokenizer_.Tokenize("电话：0871—6857（曹"), &matcher);
-
- EXPECT_THAT(test_logger.log().size(), Eq(7));
-}
-
-TEST_F(LexerTest, HandlesMixedPunctuation) {
- std::string rules_buffer =
- GrammarForTerminals({"电话", "：", "0871", "—", "6857", "（", "曹"});
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(tokenizer_.Tokenize("电话 ：0871—6857（曹"), &matcher);
-
- EXPECT_THAT(test_logger.log().size(), Eq(7));
-}
-
-// Tests that the tokenizer adds the correct tokens, including <digits>, to
-// the Matcher.
-TEST_F(LexerTest, CorrectTokenOutputWithDigits) {
- const CallbackId callback = 1;
- Ir ir;
- const Nonterm digits = ir.AddUnshareableNonterminal("<digits>");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, digits);
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "the");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "quick");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "brown");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "fox");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "2345");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "88");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, ".");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "<");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "&");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "\xE2\x80\x94");
- std::string rules_buffer =
- ir.SerializeAsFlatbuffer(/*include_debug_information=*/true);
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(
- tokenizer_.Tokenize("The.qUIck\n brown2345fox88 \xE2\x80\x94 the"),
- &matcher);
-
- EXPECT_THAT(
- test_logger.log(),
- ElementsAre(IsTerminal(0, 3, "the"), IsTerminal(3, 4, "."),
- IsTerminal(4, 9, "quick"), IsTerminal(9, 16, "brown"),
-
- // Lexer automatically creates a digits nonterminal.
- IsTerminal(16, 20, "2345"), IsNonterminal(16, 20, "<digits>"),
-
- IsTerminal(20, 23, "fox"),
-
- // Lexer automatically creates a digits nonterminal.
- IsTerminal(23, 25, "88"), IsNonterminal(23, 25, "<digits>"),
-
- IsTerminal(25, 27, "\xE2\x80\x94"),
- IsTerminal(27, 31, "the")));
-}
-
-// Tests that the tokenizer adds the correct tokens, including the
-// special <token> tokens, to the Matcher.
-TEST_F(LexerTest, CorrectTokenOutputWithGenericTokens) {
- const CallbackId callback = 1;
- Ir ir;
- const Nonterm token = ir.AddUnshareableNonterminal("<token>");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, token);
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "the");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "quick");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "brown");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "fox");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "2345");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "88");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, ".");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "<");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "&");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "\xE2\x80\x94");
- std::string rules_buffer =
- ir.SerializeAsFlatbuffer(/*include_debug_information=*/true);
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(
- tokenizer_.Tokenize("The.qUIck\n brown2345fox88 \xE2\x80\x94 the"),
- &matcher);
-
- EXPECT_THAT(
- test_logger.log(),
- // Lexer will create a <token> nonterminal for each token.
- ElementsAre(IsTerminal(0, 3, "the"), IsNonterminal(0, 3, "<token>"),
- IsTerminal(3, 4, "."), IsNonterminal(3, 4, "<token>"),
- IsTerminal(4, 9, "quick"), IsNonterminal(4, 9, "<token>"),
- IsTerminal(9, 16, "brown"), IsNonterminal(9, 16, "<token>"),
- IsTerminal(16, 20, "2345"), IsNonterminal(16, 20, "<token>"),
- IsTerminal(20, 23, "fox"), IsNonterminal(20, 23, "<token>"),
- IsTerminal(23, 25, "88"), IsNonterminal(23, 25, "<token>"),
- IsTerminal(25, 27, "\xE2\x80\x94"),
- IsNonterminal(25, 27, "<token>"), IsTerminal(27, 31, "the"),
- IsNonterminal(27, 31, "<token>")));
-}
-
-// Tests that the tokenizer adds the correct tokens, including <digits>, to
-// the Matcher. This test includes UTF8 letters.
-TEST_F(LexerTest, CorrectTokenOutputWithDigitsAndUTF8) {
- const CallbackId callback = 1;
- Ir ir;
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}},
- ir.AddUnshareableNonterminal("<digits>"));
-
- const std::string em_dash = "\xE2\x80\x94"; // "Em dash" in UTF-8
- const std::string capital_i_acute = "\xC3\x8D";
- const std::string lower_i_acute = "\xC3\xAD";
-
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "the");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, ".");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}},
- "quick" + lower_i_acute + lower_i_acute + lower_i_acute + "h");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, em_dash);
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, lower_i_acute + "h");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "22");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "brown");
- std::string rules_buffer =
- ir.SerializeAsFlatbuffer(/*include_debug_information=*/true);
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(tokenizer_.Tokenize("The.qUIck" + lower_i_acute +
- capital_i_acute + lower_i_acute + "h" +
- em_dash + capital_i_acute + "H22brown"),
- &matcher);
-
- EXPECT_THAT(
- test_logger.log(),
- ElementsAre(IsTerminal(0, 3, "the"), IsTerminal(3, 4, "."),
- IsTerminal(4, 13, "quickíííh"), IsTerminal(13, 14, "—"),
- IsTerminal(14, 16, "íh"), IsTerminal(16, 18, "22"),
- IsNonterminal(16, 18, "<digits>"),
- IsTerminal(18, 23, "brown")));
-}
-
-// Tests that the tokenizer adds the correct tokens to the Matcher.
-// For this test, there's no <digits> nonterminal in the Rules, and some
-// tokens aren't in any grammar rules.
-TEST_F(LexerTest, CorrectTokenOutputWithoutDigits) {
- const CallbackId callback = 1;
- Ir ir;
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "the");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "2345");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "88");
- std::string rules_buffer =
- ir.SerializeAsFlatbuffer(/*include_debug_information=*/true);
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(tokenizer_.Tokenize("The.qUIck\n brown2345fox88"), &matcher);
-
- EXPECT_THAT(test_logger.log(),
- ElementsAre(IsTerminal(0, 3, "the"), IsTerminal(16, 20, "2345"),
- IsTerminal(23, 25, "88")));
-}
-
-// Tests that the tokenizer adds the correct <n_digits> tokens to the Matcher.
-TEST_F(LexerTest, CorrectTokenOutputWithNDigits) {
- const CallbackId callback = 1;
- Ir ir;
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}},
- ir.AddUnshareableNonterminal("<digits>"));
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}},
- ir.AddUnshareableNonterminal("<2_digits>"));
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}},
- ir.AddUnshareableNonterminal("<4_digits>"));
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "the");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "2345");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "88");
- std::string rules_buffer =
- ir.SerializeAsFlatbuffer(/*include_debug_information=*/true);
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(tokenizer_.Tokenize("The.qUIck\n brown2345fox88"), &matcher);
-
- EXPECT_THAT(
- test_logger.log(),
- ElementsAre(IsTerminal(0, 3, "the"),
- // Lexer should generate <digits> and <4_digits> for 2345.
- IsTerminal(16, 20, "2345"), IsNonterminal(16, 20, "<digits>"),
- IsNonterminal(16, 20, "<4_digits>"),
-
- // Lexer should generate <digits> and <2_digits> for 88.
- IsTerminal(23, 25, "88"), IsNonterminal(23, 25, "<digits>"),
- IsNonterminal(23, 25, "<2_digits>")));
-}
-
-// Tests that the tokenizer splits "million+" into separate tokens.
-TEST_F(LexerTest, CorrectTokenOutputWithPlusSigns) {
- const CallbackId callback = 1;
- Ir ir;
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "the");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "2345");
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}}, "+");
- const std::string lower_i_acute = "\xC3\xAD";
- ir.Add(Ir::Lhs{kUnassignedNonterm, {callback}},
- lower_i_acute + lower_i_acute);
- std::string rules_buffer =
- ir.SerializeAsFlatbuffer(/*include_debug_information=*/true);
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- lexer_.Process(tokenizer_.Tokenize("The+2345++the +" + lower_i_acute +
- lower_i_acute + "+"),
- &matcher);
-
- EXPECT_THAT(test_logger.log(),
- ElementsAre(IsTerminal(0, 3, "the"), IsTerminal(3, 4, "+"),
- IsTerminal(4, 8, "2345"), IsTerminal(8, 9, "+"),
- IsTerminal(9, 10, "+"), IsTerminal(10, 13, "the"),
- IsTerminal(13, 15, "+"), IsTerminal(15, 17, "íí"),
- IsTerminal(17, 18, "+")));
-}
-
-// Tests that the tokenizer correctly uses the anchor tokens.
-TEST_F(LexerTest, HandlesStartAnchor) {
- const CallbackId log_callback = 1;
- Ir ir;
- // <test> ::= <^> the test <$>
- ir.Add(Ir::Lhs{ir.AddNonterminal("<test>"), {log_callback}},
- std::vector<Nonterm>{
- ir.AddUnshareableNonterminal(kStartNonterm),
- ir.Add(Ir::Lhs{kUnassignedNonterm, {log_callback}}, "the"),
- ir.Add(Ir::Lhs{kUnassignedNonterm, {log_callback}}, "test"),
- ir.AddUnshareableNonterminal(kEndNonterm),
- });
- std::string rules_buffer =
- ir.SerializeAsFlatbuffer(/*include_debug_information=*/true);
- const RulesSet* rules = flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules->debug_information());
- Matcher matcher(unilib_, rules, &test_logger);
-
- // Make sure the grammar recognizes "the test".
- lexer_.Process(tokenizer_.Tokenize("the test"), &matcher);
- EXPECT_THAT(test_logger.log(),
- // Expect logging of the two terminals and then matching of the
- // nonterminal.
- ElementsAre(IsTerminal(0, 3, "the"), IsTerminal(3, 8, "test"),
- IsNonterminal(0, 8, "<test>")));
-
- // Make sure that only left anchored matches are propagated.
- test_logger.Clear();
- lexer_.Process(tokenizer_.Tokenize("the the test"), &matcher); // NOTYPO
- EXPECT_THAT(test_logger.log(),
- // Expect that "<test>" nonterminal is not matched.
- ElementsAre(IsTerminal(0, 3, "the"), IsTerminal(3, 7, "the"),
- IsTerminal(7, 12, "test")));
-
- // Make sure that only right anchored matches are propagated.
- test_logger.Clear();
- lexer_.Process(tokenizer_.Tokenize("the test test"), &matcher);
- EXPECT_THAT(test_logger.log(),
- // Expect that "<test>" nonterminal is not matched.
- ElementsAre(IsTerminal(0, 3, "the"), IsTerminal(3, 8, "test"),
- IsTerminal(8, 13, "test")));
-}
-
-} // namespace
-} // namespace libtextclassifier3::grammar
diff --git a/native/utils/grammar/match.h b/native/utils/grammar/match.h
index 2e578d8..c1c7022 100644
--- a/native/utils/grammar/match.h
+++ b/native/utils/grammar/match.h
@@ -33,8 +33,9 @@
static const int16 kUnknownType = 0;
static const int16 kTokenType = -1;
static const int16 kDigitsType = -2;
- static const int16 kCapturingMatch = -3;
- static const int16 kAssertionMatch = -4;
+ static const int16 kBreakType = -3;
+ static const int16 kCapturingMatch = -4;
+ static const int16 kAssertionMatch = -5;
void Init(const Nonterm arg_lhs, const CodepointSpan arg_codepoint_span,
const int arg_match_offset, const int arg_type = kUnknownType) {
diff --git a/native/utils/grammar/matcher.h b/native/utils/grammar/matcher.h
index 64db80f..662ff9c 100644
--- a/native/utils/grammar/matcher.h
+++ b/native/utils/grammar/matcher.h
@@ -19,9 +19,8 @@
// A lexer passes token to the matcher: literal terminal strings and token
// types. It passes tokens to the matcher by calling AddTerminal() and
// AddMatch() for literal terminals and token types, respectively.
-// The lexer passes
-// each token along with the [begin, end) position range in which it
-// occurs. So for an input string "Groundhog February 2, 2007", the
+// The lexer passes each token along with the [begin, end) position range
+// in which it occurs. So for an input string "Groundhog February 2, 2007", the
// lexer would tell the matcher that:
//
// "Groundhog" occurs at [0, 9)
@@ -47,7 +46,7 @@
// to the matcher in left-to-right order, strictly speaking, their "end"
// positions must be nondecreasing. (This constraint allows a more
// efficient matching algorithm.) The "begin" positions can be in any
-// order. Zero-width tokens (begin==end) are not allowed.
+// order.
//
// There are two kinds of supported callbacks:
// (1) OUTPUT: Callbacks are the only output mechanism a matcher has. For each
diff --git a/native/utils/grammar/matcher_test.cc b/native/utils/grammar/matcher_test.cc
deleted file mode 100644
index e5c0ab9..0000000
--- a/native/utils/grammar/matcher_test.cc
+++ /dev/null
@@ -1,520 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/grammar/matcher.h"
-
-#include <iostream>
-#include <string>
-#include <vector>
-
-#include "utils/grammar/callback-delegate.h"
-#include "utils/grammar/rules_generated.h"
-#include "utils/grammar/types.h"
-#include "utils/grammar/utils/rules.h"
-#include "utils/strings/append.h"
-#include "utils/utf8/unilib.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3::grammar {
-namespace {
-
-using testing::ElementsAre;
-using testing::IsEmpty;
-using testing::Value;
-
-struct TestMatchResult {
- int begin;
- int end;
- std::string terminal;
- std::string nonterminal;
- int callback_id;
- int callback_param;
-};
-
-MATCHER_P3(IsTerminal, begin, end, terminal, "") {
- return Value(arg.begin, begin) && Value(arg.end, end) &&
- Value(arg.terminal, terminal);
-}
-
-MATCHER_P4(IsTerminalWithCallback, begin, end, terminal, callback, "") {
- return (ExplainMatchResult(IsTerminal(begin, end, terminal), arg,
- result_listener) &&
- Value(arg.callback_id, callback));
-}
-
-MATCHER_P3(IsNonterminal, begin, end, name, "") {
- return Value(arg.begin, begin) && Value(arg.end, end) &&
- Value(arg.nonterminal, name);
-}
-
-MATCHER_P4(IsNonterminalWithCallback, begin, end, name, callback, "") {
- return (ExplainMatchResult(IsNonterminal(begin, end, name), arg,
- result_listener) &&
- Value(arg.callback_id, callback));
-}
-
-MATCHER_P5(IsNonterminalWithCallbackAndParam, begin, end, name, callback, param,
- "") {
- return (
- ExplainMatchResult(IsNonterminalWithCallback(begin, end, name, callback),
- arg, result_listener) &&
- Value(arg.callback_param, param));
-}
-
-// Superclass of all tests.
-class MatcherTest : public testing::Test {
- protected:
- MatcherTest() : INIT_UNILIB_FOR_TESTING(unilib_) {}
-
- UniLib unilib_;
-};
-
-// This is a simple delegate implementation for testing purposes.
-// All it does is produce a record of all matches that were added.
-class TestCallbackDelegate : public CallbackDelegate {
- public:
- explicit TestCallbackDelegate(
- const RulesSet_::DebugInformation* debug_information)
- : debug_information_(debug_information) {}
-
- void MatchFound(const Match* match, const CallbackId callback_id,
- const int64 callback_param, Matcher*) override {
- TestMatchResult result;
- result.begin = match->codepoint_span.first;
- result.end = match->codepoint_span.second;
- result.callback_id = callback_id;
- result.callback_param = static_cast<int>(callback_param);
- result.nonterminal = GetNonterminalName(match->lhs);
- if (match->IsTerminalRule()) {
- result.terminal = match->terminal;
- }
- log_.push_back(result);
- }
-
- void ClearLog() { log_.clear(); }
-
- const std::vector<TestMatchResult> GetLogAndReset() {
- const auto result = log_;
- ClearLog();
- return result;
- }
-
- protected:
- std::string GetNonterminalName(const Nonterm nonterminal) const {
- if (const RulesSet_::DebugInformation_::NonterminalNamesEntry* entry =
- debug_information_->nonterminal_names()->LookupByKey(nonterminal)) {
- return entry->value()->str();
- }
- // Unnamed Nonterm.
- return "()";
- }
-
- std::vector<TestMatchResult> log_;
- const RulesSet_::DebugInformation* debug_information_;
-};
-
-TEST_F(MatcherTest, HandlesBasicOperations) {
- // Create an example grammar.
- Rules rules;
- const CallbackId callback = 1;
- rules.Add("<test>", {"the", "quick", "brown", "fox"}, callback);
- rules.Add("<action>", {"<test>"}, callback);
- const std::string rules_buffer = rules.Finalize().SerializeAsFlatbuffer(
- /*include_debug_information=*/true);
- const RulesSet* rules_set =
- flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules_set->debug_information());
- Matcher matcher(unilib_, rules_set, &test_logger);
-
- matcher.AddTerminal(0, 1, "the");
- matcher.AddTerminal(1, 2, "quick");
- matcher.AddTerminal(2, 3, "brown");
- matcher.AddTerminal(3, 4, "fox");
-
- EXPECT_THAT(test_logger.GetLogAndReset(),
- ElementsAre(IsNonterminal(0, 4, "<test>"),
- IsNonterminal(0, 4, "<action>")));
-}
-
-std::string CreateTestGrammar() {
- // Create an example grammar.
- Rules rules;
-
- // Callbacks on terminal rules.
- rules.Add("<output_5>", {"quick"}, 6);
- rules.Add("<output_0>", {"the"}, 1);
-
- // Callbacks on non-terminal rules.
- rules.Add("<output_1>", {"the", "quick", "brown", "fox"}, 2);
- rules.Add("<output_2>", {"the", "quick"}, 3, static_cast<int64>(-1));
- rules.Add("<output_3>", {"brown", "fox"}, 4);
- // Now a complex thing: "the* brown fox".
- rules.Add("<thestarbrownfox>", {"brown", "fox"}, 5);
- rules.Add("<thestarbrownfox>", {"the", "<thestarbrownfox>"}, 5);
-
- return rules.Finalize().SerializeAsFlatbuffer(
- /*include_debug_information=*/true);
-}
-
-std::string CreateTestGrammarWithOptionalElements() {
- // Create an example grammar.
- Rules rules;
- rules.Add("<output_0>", {"a?", "b?", "c?", "d?", "e"}, 1);
- rules.Add("<output_1>", {"a", "b?", "c", "d?", "e"}, 2);
- rules.Add("<output_2>", {"a", "b?", "c", "d", "e?"}, 3);
-
- return rules.Finalize().SerializeAsFlatbuffer(
- /*include_debug_information=*/true);
-}
-
-Nonterm FindNontermForName(const RulesSet* rules,
- const std::string& nonterminal_name) {
- for (const RulesSet_::DebugInformation_::NonterminalNamesEntry* entry :
- *rules->debug_information()->nonterminal_names()) {
- if (entry->value()->str() == nonterminal_name) {
- return entry->key();
- }
- }
- return kUnassignedNonterm;
-}
-
-TEST_F(MatcherTest, HandlesBasicOperationsWithCallbacks) {
- const std::string rules_buffer = CreateTestGrammar();
- const RulesSet* rules_set =
- flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules_set->debug_information());
- Matcher matcher(unilib_, rules_set, &test_logger);
-
- matcher.AddTerminal(0, 1, "the");
- EXPECT_THAT(test_logger.GetLogAndReset(),
- ElementsAre(IsTerminalWithCallback(/*begin=*/0, /*end=*/1, "the",
- /*callback=*/1)));
- matcher.AddTerminal(1, 2, "quick");
- EXPECT_THAT(
- test_logger.GetLogAndReset(),
- ElementsAre(IsTerminalWithCallback(/*begin=*/1, /*end=*/2, "quick",
- /*callback=*/6),
- IsNonterminalWithCallbackAndParam(
- /*begin=*/0, /*end=*/2, "<output_2>",
- /*callback=*/3, -1)));
-
- matcher.AddTerminal(2, 3, "brown");
- EXPECT_THAT(test_logger.GetLogAndReset(), IsEmpty());
-
- matcher.AddTerminal(3, 4, "fox");
- EXPECT_THAT(
- test_logger.GetLogAndReset(),
- ElementsAre(
- IsNonterminalWithCallback(/*begin=*/0, /*end=*/4, "<output_1>",
- /*callback=*/2),
- IsNonterminalWithCallback(/*begin=*/2, /*end=*/4, "<output_3>",
- /*callback=*/4),
- IsNonterminalWithCallback(/*begin=*/2, /*end=*/4, "<thestarbrownfox>",
- /*callback=*/5)));
-
- matcher.AddTerminal(3, 5, "fox");
- EXPECT_THAT(
- test_logger.GetLogAndReset(),
- ElementsAre(
- IsNonterminalWithCallback(/*begin=*/0, /*end=*/5, "<output_1>",
- /*callback=*/2),
- IsNonterminalWithCallback(/*begin=*/2, /*end=*/5, "<output_3>",
- /*callback=*/4),
- IsNonterminalWithCallback(/*begin=*/2, /*end=*/5, "<thestarbrownfox>",
- /*callback=*/5)));
-
- matcher.AddTerminal(4, 6, "fox"); // Not adjacent to "brown".
- EXPECT_THAT(test_logger.GetLogAndReset(), IsEmpty());
-}
-
-TEST_F(MatcherTest, HandlesRecursiveRules) {
- const std::string rules_buffer = CreateTestGrammar();
- const RulesSet* rules_set =
- flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules_set->debug_information());
- Matcher matcher(unilib_, rules_set, &test_logger);
-
- matcher.AddTerminal(0, 1, "the");
- matcher.AddTerminal(1, 2, "the");
- matcher.AddTerminal(2, 4, "the");
- matcher.AddTerminal(3, 4, "the");
- matcher.AddTerminal(4, 5, "brown");
- matcher.AddTerminal(5, 6, "fox"); // Generates 5 of <thestarbrownfox>
-
- EXPECT_THAT(
- test_logger.GetLogAndReset(),
- ElementsAre(IsTerminal(/*begin=*/0, /*end=*/1, "the"),
- IsTerminal(/*begin=*/1, /*end=*/2, "the"),
- IsTerminal(/*begin=*/2, /*end=*/4, "the"),
- IsTerminal(/*begin=*/3, /*end=*/4, "the"),
- IsNonterminal(/*begin=*/4, /*end=*/6, "<output_3>"),
- IsNonterminal(/*begin=*/4, /*end=*/6, "<thestarbrownfox>"),
- IsNonterminal(/*begin=*/3, /*end=*/6, "<thestarbrownfox>"),
- IsNonterminal(/*begin=*/2, /*end=*/6, "<thestarbrownfox>"),
- IsNonterminal(/*begin=*/1, /*end=*/6, "<thestarbrownfox>"),
- IsNonterminal(/*begin=*/0, /*end=*/6, "<thestarbrownfox>")));
-}
-
-TEST_F(MatcherTest, HandlesManualAddMatchCalls) {
- const std::string rules_buffer = CreateTestGrammar();
- const RulesSet* rules_set =
- flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules_set->debug_information());
- Matcher matcher(unilib_, rules_set, &test_logger);
-
- // Test having the lexer call AddMatch() instead of AddTerminal()
- matcher.AddTerminal(-4, 37, "the");
- Match* match = matcher.AllocateMatch(sizeof(Match));
- match->codepoint_span = {37, 42};
- match->match_offset = 37;
- match->lhs = FindNontermForName(rules_set, "<thestarbrownfox>");
- matcher.AddMatch(match);
-
- EXPECT_THAT(test_logger.GetLogAndReset(),
- ElementsAre(IsTerminal(/*begin=*/-4, /*end=*/37, "the"),
- IsNonterminal(/*begin=*/-4, /*end=*/42,
- "<thestarbrownfox>")));
-}
-
-TEST_F(MatcherTest, HandlesOptionalRuleElements) {
- const std::string rules_buffer = CreateTestGrammarWithOptionalElements();
- const RulesSet* rules_set =
- flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules_set->debug_information());
- Matcher matcher(unilib_, rules_set, &test_logger);
-
- // Run the matcher on "a b c d e".
- matcher.AddTerminal(0, 1, "a");
- matcher.AddTerminal(1, 2, "b");
- matcher.AddTerminal(2, 3, "c");
- matcher.AddTerminal(3, 4, "d");
- matcher.AddTerminal(4, 5, "e");
-
- EXPECT_THAT(test_logger.GetLogAndReset(),
- ElementsAre(IsNonterminal(/*begin=*/0, /*end=*/4, "<output_2>"),
- IsTerminal(/*begin=*/4, /*end=*/5, "e"),
- IsNonterminal(/*begin=*/0, /*end=*/5, "<output_0>"),
- IsNonterminal(/*begin=*/0,
- /*end=*/5, "<output_1>"),
- IsNonterminal(/*begin=*/0, /*end=*/5, "<output_2>"),
- IsNonterminal(/*begin=*/1,
- /*end=*/5, "<output_0>"),
- IsNonterminal(/*begin=*/2, /*end=*/5, "<output_0>"),
- IsNonterminal(/*begin=*/3,
- /*end=*/5, "<output_0>")));
-}
-
-class FilterTestCallbackDelegate : public TestCallbackDelegate {
- public:
- FilterTestCallbackDelegate(
- const RulesSet_::DebugInformation* debug_information,
- const std::string& filter)
- : TestCallbackDelegate(debug_information), filter_(filter) {}
-
- void MatchFound(const Match* candidate, const CallbackId callback_id,
- const int64 callback_param, Matcher* matcher) override {
- TestCallbackDelegate::MatchFound(candidate, callback_id, callback_param,
- matcher);
- // Filter callback.
- if (callback_id == 1) {
- if (candidate->IsTerminalRule() && filter_ != candidate->terminal) {
- return;
- } else {
- std::vector<const Match*> terminals = SelectTerminals(candidate);
- if (terminals.empty() || terminals.front()->terminal != filter_) {
- return;
- }
- }
- Match* match = matcher->AllocateAndInitMatch<Match>(*candidate);
- matcher->AddMatch(match);
- matcher->AddTerminal(match->codepoint_span, match->match_offset,
- "faketerminal");
- }
- }
-
- protected:
- const std::string filter_;
-};
-
-std::string CreateExampleGrammarWithFilters() {
- // Create an example grammar.
- Rules rules;
- const CallbackId filter_callback = 1;
- rules.DefineFilter(filter_callback);
-
- rules.Add("<term_pass>", {"hello"}, filter_callback);
- rules.Add("<nonterm_pass>", {"hello", "there"}, filter_callback);
- rules.Add("<term_fail>", {"there"}, filter_callback);
- rules.Add("<nonterm_fail>", {"there", "world"}, filter_callback);
-
- // We use this to test whether AddTerminal() worked from inside a filter
- // callback.
- const CallbackId output_callback = 2;
- rules.Add("<output_faketerminal>", {"faketerminal"}, output_callback);
-
- // We use this to test whether AddMatch() worked from inside a filter
- // callback.
- rules.Add("<output_term_pass>", {"<term_pass>"}, output_callback);
- rules.Add("<output_nonterm_pass>", {"<nonterm_pass>"}, output_callback);
- rules.Add("<output_term_fail>", {"<term_fail>"}, output_callback);
- rules.Add("<output_nonterm_fail>", {"<term_nonterm_fail>"}, output_callback);
-
- // We use this to make sure rules with output callbacks are always adding
- // their lhs to the chart. This is to check for off-by-one errors in the
- // callback numbering, make sure we don't mistakenly treat the output
- // callback as a filter callback.
- rules.Add("<output>", {"<output_faketerminal>"}, output_callback);
- rules.Add("<output>", {"<output_term_pass>"}, output_callback);
- rules.Add("<output>", {"<output_nonterm_pass>"}, output_callback);
-
- return rules.Finalize().SerializeAsFlatbuffer(
- /*include_debug_information=*/true);
-}
-
-TEST_F(MatcherTest, HandlesTerminalFilters) {
- const std::string rules_buffer = CreateExampleGrammarWithFilters();
- const RulesSet* rules_set =
- flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- FilterTestCallbackDelegate test_logger(rules_set->debug_information(),
- "hello");
- Matcher matcher(unilib_, rules_set, &test_logger);
- matcher.AddTerminal(0, 1, "hello");
-
- EXPECT_THAT(
- test_logger.GetLogAndReset(),
- ElementsAre(
- // Bubbling up of:
- // "hello" -> "<term_pass>" -> "<output_term_pass>" -> "<output>"
- IsNonterminal(0, 1, "<term_pass>"),
- IsNonterminal(0, 1, "<output_term_pass>"),
- IsNonterminal(0, 1, "<output>"),
-
- // Bubbling up of:
- // "faketerminal" -> "<output_faketerminal>" -> "<output>"
- IsNonterminal(0, 1, "<output_faketerminal>"),
- IsNonterminal(0, 1, "<output>")));
-}
-
-TEST_F(MatcherTest, HandlesNonterminalFilters) {
- const std::string rules_buffer = CreateExampleGrammarWithFilters();
- const RulesSet* rules_set =
- flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- FilterTestCallbackDelegate test_logger(rules_set->debug_information(),
- "hello");
- Matcher matcher(unilib_, rules_set, &test_logger);
-
- matcher.AddTerminal(0, 1, "hello");
- test_logger.ClearLog();
- matcher.AddTerminal(1, 2, "there");
-
- EXPECT_THAT(test_logger.GetLogAndReset(),
- ElementsAre(
- // "<term_fail>" is filtered, and doesn't bubble up.
- IsNonterminal(1, 2, "<term_fail>"),
-
- // <nonterm_pass> ::= hello there
- IsNonterminal(0, 2, "<nonterm_pass>"),
- IsNonterminal(0, 2, "<output_faketerminal>"),
- IsNonterminal(0, 2, "<output>"),
- IsNonterminal(0, 2, "<output_nonterm_pass>"),
- IsNonterminal(0, 2, "<output>")));
-}
-
-TEST_F(MatcherTest, HandlesWhitespaceGapLimits) {
- Rules rules;
- rules.Add("<iata>", {"lx"});
- rules.Add("<iata>", {"aa"});
- // Require no whitespace between code and flight number.
- rules.Add("<flight_number>", {"<iata>", "<4_digits>"}, /*callback=*/1, 0,
- /*max_whitespace_gap=*/0);
- const std::string rules_buffer = rules.Finalize().SerializeAsFlatbuffer(
- /*include_debug_information=*/true);
- const RulesSet* rules_set =
- flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules_set->debug_information());
- Matcher matcher(unilib_, rules_set, &test_logger);
-
- // Check that the grammar triggers on LX1138.
- {
- matcher.AddTerminal(0, 2, "LX");
- matcher.AddMatch(matcher.AllocateAndInitMatch<Match>(
- rules_set->nonterminals()->n_digits_nt()->Get(4 - 1),
- CodepointSpan{2, 6}, /*match_offset=*/2));
- EXPECT_THAT(test_logger.GetLogAndReset(),
- ElementsAre(IsNonterminal(0, 6, "<flight_number>")));
- }
-
- // Check that the grammar doesn't trigger on LX 1138.
- {
- matcher.AddTerminal(6, 8, "LX");
- matcher.AddMatch(matcher.AllocateAndInitMatch<Match>(
- rules_set->nonterminals()->n_digits_nt()->Get(4 - 1),
- CodepointSpan{9, 13}, /*match_offset=*/8));
- EXPECT_THAT(test_logger.GetLogAndReset(), IsEmpty());
- }
-}
-
-TEST_F(MatcherTest, HandlesCaseSensitiveTerminals) {
- Rules rules;
- rules.Add("<iata>", {"LX"}, /*callback=*/kNoCallback, 0,
- /*max_whitespace_gap*/ -1, /*case_sensitive=*/true);
- rules.Add("<iata>", {"AA"}, /*callback=*/kNoCallback, 0,
- /*max_whitespace_gap*/ -1, /*case_sensitive=*/true);
- rules.Add("<iata>", {"dl"}, /*callback=*/kNoCallback, 0,
- /*max_whitespace_gap*/ -1, /*case_sensitive=*/false);
- // Require no whitespace between code and flight number.
- rules.Add("<flight_number>", {"<iata>", "<4_digits>"}, /*callback=*/1, 0,
- /*max_whitespace_gap=*/0);
- const std::string rules_buffer = rules.Finalize().SerializeAsFlatbuffer(
- /*include_debug_information=*/true);
- const RulesSet* rules_set =
- flatbuffers::GetRoot<RulesSet>(rules_buffer.data());
- TestCallbackDelegate test_logger(rules_set->debug_information());
- Matcher matcher(unilib_, rules_set, &test_logger);
-
- // Check that the grammar triggers on LX1138.
- {
- matcher.AddTerminal(0, 2, "LX");
- matcher.AddMatch(matcher.AllocateAndInitMatch<Match>(
- rules_set->nonterminals()->n_digits_nt()->Get(4 - 1),
- CodepointSpan{2, 6}, /*match_offset=*/2));
- EXPECT_THAT(test_logger.GetLogAndReset(),
- ElementsAre(IsNonterminal(0, 6, "<flight_number>")));
- }
-
- // Check that the grammar doesn't trigger on lx1138.
- {
- matcher.AddTerminal(6, 8, "lx");
- matcher.AddMatch(matcher.AllocateAndInitMatch<Match>(
- rules_set->nonterminals()->n_digits_nt()->Get(4 - 1),
- CodepointSpan{8, 12}, /*match_offset=*/8));
- EXPECT_THAT(test_logger.GetLogAndReset(), IsEmpty());
- }
-
- // Check that the grammar does trigger on dl1138.
- {
- matcher.AddTerminal(12, 14, "dl");
- matcher.AddMatch(matcher.AllocateAndInitMatch<Match>(
- rules_set->nonterminals()->n_digits_nt()->Get(4 - 1),
- CodepointSpan{14, 18}, /*match_offset=*/14));
- EXPECT_THAT(test_logger.GetLogAndReset(),
- ElementsAre(IsNonterminal(12, 18, "<flight_number>")));
- }
-}
-
-} // namespace
-} // namespace libtextclassifier3::grammar
diff --git a/native/utils/grammar/rules-utils_test.cc b/native/utils/grammar/rules-utils_test.cc
deleted file mode 100644
index e562508..0000000
--- a/native/utils/grammar/rules-utils_test.cc
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/grammar/rules-utils.h"
-
-#include <vector>
-
-#include "utils/grammar/match.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3::grammar {
-namespace {
-
-using testing::ElementsAre;
-using testing::Value;
-
-// Create test match object.
-Match CreateMatch(const CodepointIndex begin, const CodepointIndex end) {
- Match match;
- match.Init(0, CodepointSpan{begin, end},
- /*arg_match_offset=*/begin);
- return match;
-}
-
-MATCHER_P(IsRuleMatch, candidate, "") {
- return Value(arg.rule_id, candidate.rule_id) &&
- Value(arg.match, candidate.match);
-}
-
-TEST(UtilsTest, DeduplicatesMatches) {
- // Overlapping matches from the same rule.
- Match matches[] = {CreateMatch(0, 1), CreateMatch(1, 2), CreateMatch(0, 2)};
- const std::vector<RuleMatch> candidates = {{&matches[0], /*rule_id=*/0},
- {&matches[1], /*rule_id=*/0},
- {&matches[2], /*rule_id=*/0}};
-
- // Keep longest.
- EXPECT_THAT(DeduplicateMatches(candidates),
- ElementsAre(IsRuleMatch(candidates[2])));
-}
-
-TEST(UtilsTest, DeduplicatesMatchesPerRule) {
- // Overlapping matches from different rules.
- Match matches[] = {CreateMatch(0, 1), CreateMatch(1, 2), CreateMatch(0, 2)};
- const std::vector<RuleMatch> candidates = {{&matches[0], /*rule_id=*/0},
- {&matches[1], /*rule_id=*/0},
- {&matches[2], /*rule_id=*/0},
- {&matches[0], /*rule_id=*/1}};
-
- // Keep longest for rule 0, but also keep match from rule 1.
- EXPECT_THAT(
- DeduplicateMatches(candidates),
- ElementsAre(IsRuleMatch(candidates[2]), IsRuleMatch(candidates[3])));
-}
-
-TEST(UtilsTest, KeepNonoverlapping) {
- // Non-overlapping matches.
- Match matches[] = {CreateMatch(0, 1), CreateMatch(1, 2), CreateMatch(2, 3)};
- const std::vector<RuleMatch> candidates = {{&matches[0], /*rule_id=*/0},
- {&matches[1], /*rule_id=*/0},
- {&matches[2], /*rule_id=*/0}};
-
- // Keep all matches.
- EXPECT_THAT(
- DeduplicateMatches(candidates),
- ElementsAre(IsRuleMatch(candidates[0]), IsRuleMatch(candidates[1]),
- IsRuleMatch(candidates[2])));
-}
-
-} // namespace
-} // namespace libtextclassifier3::grammar
diff --git a/native/utils/grammar/rules.fbs b/native/utils/grammar/rules.fbs
index 2de7a95..41c2c86 100755
--- a/native/utils/grammar/rules.fbs
+++ b/native/utils/grammar/rules.fbs
@@ -128,6 +128,9 @@
// `n_digits_nt[k]` is the id of the nonterminal indicating a string of
// `k` digits.
n_digits_nt:[int];
+
+ // Id of the nonterminal indicating a word or token boundary.
+ wordbreak_nt:int;
}
// Callback information.
diff --git a/native/utils/grammar/utils/ir.cc b/native/utils/grammar/utils/ir.cc
index 5184c6b..6be1128 100644
--- a/native/utils/grammar/utils/ir.cc
+++ b/native/utils/grammar/utils/ir.cc
@@ -406,6 +406,7 @@
output->nonterminals.reset(new RulesSet_::NonterminalsT);
output->nonterminals->start_nt = GetNonterminalForName(kStartNonterm);
output->nonterminals->end_nt = GetNonterminalForName(kEndNonterm);
+ output->nonterminals->wordbreak_nt = GetNonterminalForName(kWordBreakNonterm);
output->nonterminals->token_nt = GetNonterminalForName(kTokenNonterm);
output->nonterminals->digits_nt = GetNonterminalForName(kDigitsNonterm);
for (int i = 1; i <= kMaxNDigitsNontermLength; i++) {
diff --git a/native/utils/grammar/utils/ir.h b/native/utils/grammar/utils/ir.h
index 7e5c984..89087c6 100644
--- a/native/utils/grammar/utils/ir.h
+++ b/native/utils/grammar/utils/ir.h
@@ -31,6 +31,7 @@
// Pre-defined nonterminal classes that the lexer can handle.
constexpr const char* kStartNonterm = "<^>";
constexpr const char* kEndNonterm = "<$>";
+constexpr const char* kWordBreakNonterm = "<\b>";
constexpr const char* kTokenNonterm = "<token>";
constexpr const char* kDigitsNonterm = "<digits>";
constexpr const char* kNDigitsNonterm = "<%d_digits>";
diff --git a/native/utils/grammar/utils/ir_test.cc b/native/utils/grammar/utils/ir_test.cc
deleted file mode 100644
index b3f0417..0000000
--- a/native/utils/grammar/utils/ir_test.cc
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/grammar/utils/ir.h"
-
-#include "utils/grammar/rules_generated.h"
-#include "utils/grammar/types.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3::grammar {
-namespace {
-
-using testing::Eq;
-using testing::IsEmpty;
-using testing::Ne;
-
-TEST(IrTest, HandlesSharingWithTerminalRules) {
- Ir ir;
-
- // <t1> ::= the
- const Nonterm t1 = ir.Add(kUnassignedNonterm, "the");
-
- // <t2> ::= quick
- const Nonterm t2 = ir.Add(kUnassignedNonterm, "quick");
-
- // <t3> ::= quick -- should share with <t2>
- const Nonterm t3 = ir.Add(kUnassignedNonterm, "quick");
-
- // <t4> ::= quick -- specify unshareable <t4>
- // <t4> ::= brown
- const Nonterm t4_unshareable = ir.AddUnshareableNonterminal();
- ir.Add(t4_unshareable, "quick");
- ir.Add(t4_unshareable, "brown");
-
- // <t5> ::= brown -- should not be shared with <t4>
- const Nonterm t5 = ir.Add(kUnassignedNonterm, "brown");
-
- // <t6> ::= brown -- specify unshareable <t6>
- const Nonterm t6_unshareable = ir.AddUnshareableNonterminal();
- ir.Add(t6_unshareable, "brown");
-
- // <t7> ::= brown -- should share with <t5>
- const Nonterm t7 = ir.Add(kUnassignedNonterm, "brown");
-
- EXPECT_THAT(t1, Ne(kUnassignedNonterm));
- EXPECT_THAT(t2, Ne(kUnassignedNonterm));
- EXPECT_THAT(t1, Ne(t2));
- EXPECT_THAT(t2, Eq(t3));
- EXPECT_THAT(t4_unshareable, Ne(kUnassignedNonterm));
- EXPECT_THAT(t4_unshareable, Ne(t3));
- EXPECT_THAT(t4_unshareable, Ne(t5));
- EXPECT_THAT(t6_unshareable, Ne(kUnassignedNonterm));
- EXPECT_THAT(t6_unshareable, Ne(t4_unshareable));
- EXPECT_THAT(t6_unshareable, Ne(t5));
- EXPECT_THAT(t7, Eq(t5));
-}
-
-TEST(IrTest, HandlesSharingWithNonterminalRules) {
- Ir ir;
-
- // Setup a few terminal rules.
- const std::vector<Nonterm> rhs = {
- ir.Add(kUnassignedNonterm, "the"), ir.Add(kUnassignedNonterm, "quick"),
- ir.Add(kUnassignedNonterm, "brown"), ir.Add(kUnassignedNonterm, "fox")};
-
- // Check for proper sharing using nonterminal rules.
- for (int rhs_length = 1; rhs_length <= rhs.size(); rhs_length++) {
- std::vector<Nonterm> rhs_truncated = rhs;
- rhs_truncated.resize(rhs_length);
- const Nonterm nt_u = ir.AddUnshareableNonterminal();
- ir.Add(nt_u, rhs_truncated);
- const Nonterm nt_1 = ir.Add(kUnassignedNonterm, rhs_truncated);
- const Nonterm nt_2 = ir.Add(kUnassignedNonterm, rhs_truncated);
-
- EXPECT_THAT(nt_1, Eq(nt_2));
- EXPECT_THAT(nt_1, Ne(nt_u));
- }
-}
-
-TEST(IrTest, HandlesSharingWithCallbacksWithSameParameters) {
- // Test sharing in the presence of callbacks.
- constexpr CallbackId kOutput1 = 1;
- constexpr CallbackId kOutput2 = 2;
- constexpr CallbackId kFilter1 = 3;
- constexpr CallbackId kFilter2 = 4;
- Ir ir(/*filters=*/{kFilter1, kFilter2});
-
- const Nonterm x1 = ir.Add(kUnassignedNonterm, "hello");
- const Nonterm x2 =
- ir.Add(Ir::Lhs{kUnassignedNonterm, {kOutput1, 0}}, "hello");
- const Nonterm x3 =
- ir.Add(Ir::Lhs{kUnassignedNonterm, {kFilter1, 0}}, "hello");
- const Nonterm x4 =
- ir.Add(Ir::Lhs{kUnassignedNonterm, {kOutput2, 0}}, "hello");
- const Nonterm x5 =
- ir.Add(Ir::Lhs{kUnassignedNonterm, {kFilter2, 0}}, "hello");
-
- // Duplicate entry.
- const Nonterm x6 =
- ir.Add(Ir::Lhs{kUnassignedNonterm, {kOutput2, 0}}, "hello");
-
- EXPECT_THAT(x2, Eq(x1));
- EXPECT_THAT(x3, Ne(x1));
- EXPECT_THAT(x4, Eq(x1));
- EXPECT_THAT(x5, Ne(x1));
- EXPECT_THAT(x5, Ne(x3));
- EXPECT_THAT(x6, Ne(x3));
-}
-
-TEST(IrTest, HandlesSharingWithCallbacksWithDifferentParameters) {
- // Test sharing in the presence of callbacks.
- constexpr CallbackId kOutput = 1;
- constexpr CallbackId kFilter = 2;
- Ir ir(/*filters=*/{kFilter});
-
- const Nonterm x1 = ir.Add(Ir::Lhs{kUnassignedNonterm, {kOutput, 0}}, "world");
- const Nonterm x2 = ir.Add(Ir::Lhs{kUnassignedNonterm, {kOutput, 1}}, "world");
- const Nonterm x3 = ir.Add(Ir::Lhs{kUnassignedNonterm, {kFilter, 0}}, "world");
- const Nonterm x4 = ir.Add(Ir::Lhs{kUnassignedNonterm, {kFilter, 1}}, "world");
-
- EXPECT_THAT(x2, Eq(x1));
- EXPECT_THAT(x3, Ne(x1));
- EXPECT_THAT(x4, Ne(x1));
- EXPECT_THAT(x4, Ne(x3));
-}
-
-TEST(IrTest, SerializesRulesToFlatbufferFormat) {
- constexpr CallbackId kOutput = 1;
- Ir ir;
- const Nonterm verb = ir.AddUnshareableNonterminal();
- ir.Add(verb, "buy");
- ir.Add(Ir::Lhs{verb, {kOutput}}, "bring");
- ir.Add(verb, "upbring");
- ir.Add(verb, "remind");
- const Nonterm set_reminder = ir.AddUnshareableNonterminal();
- ir.Add(set_reminder,
- std::vector<Nonterm>{ir.Add(kUnassignedNonterm, "remind"),
- ir.Add(kUnassignedNonterm, "me"),
- ir.Add(kUnassignedNonterm, "to"), verb});
- const Nonterm action = ir.AddUnshareableNonterminal();
- ir.Add(action, set_reminder);
- RulesSetT rules;
- ir.Serialize(/*include_debug_information=*/false, &rules);
-
- EXPECT_THAT(rules.rules.size(), Eq(1));
-
- // Only one rule uses a callback, the rest will be encoded directly.
- EXPECT_THAT(rules.lhs.size(), Eq(1));
- EXPECT_THAT(rules.lhs.front().callback_id(), kOutput);
-
- // 6 distinct terminals: "buy", "upbring", "bring", "remind", "me" and "to".
- EXPECT_THAT(
- rules.rules.front()->lowercase_terminal_rules->terminal_offsets.size(),
- Eq(6));
- EXPECT_THAT(rules.rules.front()->terminal_rules->terminal_offsets, IsEmpty());
-
- // As "bring" is a suffix of "upbring" it is expected to be suffix merged in
- // the string pool
- EXPECT_THAT(rules.terminals,
- Eq(std::string("buy\0me\0remind\0to\0upbring\0", 25)));
-
- EXPECT_THAT(rules.rules.front()->binary_rules.size(), Eq(3));
-
- // One unary rule: <action> ::= <set_reminder>
- EXPECT_THAT(rules.rules.front()->unary_rules.size(), Eq(1));
-}
-
-TEST(IrTest, HandlesRulesSharding) {
- Ir ir(/*filters=*/{}, /*num_shards=*/2);
- const Nonterm verb = ir.AddUnshareableNonterminal();
- const Nonterm set_reminder = ir.AddUnshareableNonterminal();
-
- // Shard 0: en
- ir.Add(verb, "buy");
- ir.Add(verb, "bring");
- ir.Add(verb, "remind");
- ir.Add(set_reminder,
- std::vector<Nonterm>{ir.Add(kUnassignedNonterm, "remind"),
- ir.Add(kUnassignedNonterm, "me"),
- ir.Add(kUnassignedNonterm, "to"), verb});
-
- // Shard 1: de
- ir.Add(verb, "kaufen", /*case_sensitive=*/false, /*shard=*/1);
- ir.Add(verb, "bringen", /*case_sensitive=*/false, /*shard=*/1);
- ir.Add(verb, "erinnern", /*case_sensitive=*/false, /*shard=*/1);
- ir.Add(set_reminder,
- std::vector<Nonterm>{ir.Add(kUnassignedNonterm, "erinnere",
- /*case_sensitive=*/false, /*shard=*/1),
- ir.Add(kUnassignedNonterm, "mich",
- /*case_sensitive=*/false, /*shard=*/1),
- ir.Add(kUnassignedNonterm, "zu",
- /*case_sensitive=*/false, /*shard=*/1),
- verb},
- /*shard=*/1);
-
- // Test that terminal strings are correctly merged into the shared
- // string pool.
- RulesSetT rules;
- ir.Serialize(/*include_debug_information=*/false, &rules);
-
- EXPECT_THAT(rules.rules.size(), Eq(2));
-
- // 5 distinct terminals: "buy", "bring", "remind", "me" and "to".
- EXPECT_THAT(rules.rules[0]->lowercase_terminal_rules->terminal_offsets.size(),
- Eq(5));
- EXPECT_THAT(rules.rules[0]->terminal_rules->terminal_offsets, IsEmpty());
-
- // 6 distinct terminals: "kaufen", "bringen", "erinnern", "erinnere", "mich"
- // and "zu".
- EXPECT_THAT(rules.rules[1]->lowercase_terminal_rules->terminal_offsets.size(),
- Eq(6));
- EXPECT_THAT(rules.rules[1]->terminal_rules->terminal_offsets, IsEmpty());
-
- EXPECT_THAT(rules.terminals,
- Eq(std::string("bring\0bringen\0buy\0erinnere\0erinnern\0kaufen\0"
- "me\0mich\0remind\0to\0zu\0",
- 64)));
-
- EXPECT_THAT(rules.rules[0]->binary_rules.size(), Eq(3));
- EXPECT_THAT(rules.rules[1]->binary_rules.size(), Eq(3));
-}
-
-} // namespace
-} // namespace libtextclassifier3::grammar
diff --git a/native/utils/grammar/utils/rules.cc b/native/utils/grammar/utils/rules.cc
index af7a56f..4dbab61 100644
--- a/native/utils/grammar/utils/rules.cc
+++ b/native/utils/grammar/utils/rules.cc
@@ -18,6 +18,7 @@
#include <set>
+#include "utils/grammar/utils/ir.h"
#include "utils/strings/append.h"
namespace libtextclassifier3::grammar {
@@ -26,7 +27,8 @@
// Returns whether a nonterminal is a pre-defined one.
bool IsPredefinedNonterminal(const std::string& nonterminal_name) {
if (nonterminal_name == kStartNonterm || nonterminal_name == kEndNonterm ||
- nonterminal_name == kTokenNonterm || nonterminal_name == kDigitsNonterm) {
+ nonterminal_name == kTokenNonterm || nonterminal_name == kDigitsNonterm ||
+ nonterminal_name == kWordBreakNonterm) {
return true;
}
for (int digits = 1; digits <= kMaxNDigitsNontermLength; digits++) {
@@ -99,6 +101,22 @@
rhs_nonterms, rule.shard);
}
+// Check whether this component is a non-terminal.
+bool IsNonterminal(StringPiece rhs_component) {
+ return rhs_component[0] == '<' &&
+ rhs_component[rhs_component.size() - 1] == '>';
+}
+
+// Sanity check for common typos -- '<' or '>' in a terminal.
+void ValidateTerminal(StringPiece rhs_component) {
+ TC3_CHECK_EQ(rhs_component.find('<'), std::string::npos)
+ << "Rhs terminal `" << rhs_component << "` contains an angle bracket.";
+ TC3_CHECK_EQ(rhs_component.find('>'), std::string::npos)
+ << "Rhs terminal `" << rhs_component << "` contains an angle bracket.";
+ TC3_CHECK_EQ(rhs_component.find('?'), std::string::npos)
+ << "Rhs terminal `" << rhs_component << "` contains a question mark.";
+}
+
} // namespace
int Rules::AddNonterminal(StringPiece nonterminal_name) {
@@ -175,20 +193,12 @@
}
// Check whether this component is a non-terminal.
- if (rhs_component[0] == '<' &&
- rhs_component[rhs_component.size() - 1] == '>') {
+ if (IsNonterminal(rhs_component)) {
rhs_elements.push_back(RhsElement(AddNonterminal(rhs_component)));
} else {
// A terminal.
// Sanity check for common typos -- '<' or '>' in a terminal.
- TC3_CHECK_EQ(rhs_component.find('<'), std::string::npos)
- << "Rhs terminal `" << rhs_component
- << "` contains an angle bracket.";
- TC3_CHECK_EQ(rhs_component.find('>'), std::string::npos)
- << "Rhs terminal `" << rhs_component
- << "` contains an angle bracket.";
- TC3_CHECK_EQ(rhs_component.find('?'), std::string::npos)
- << "Rhs terminal `" << rhs_component << "` contains a question mark.";
+ ValidateTerminal(rhs_component);
rhs_elements.push_back(RhsElement(rhs_component.ToString()));
}
}
@@ -203,7 +213,7 @@
optional_element_indices.end(), &omit_these);
}
-Ir Rules::Finalize() const {
+Ir Rules::Finalize(const std::set<std::string>& predefined_nonterminals) const {
Ir rules(filters_, num_shards_);
std::unordered_map<int, Nonterm> nonterminal_ids;
@@ -212,7 +222,9 @@
// Define all used predefined nonterminals.
for (const auto it : nonterminal_names_) {
- if (IsPredefinedNonterminal(it.first)) {
+ if (IsPredefinedNonterminal(it.first) ||
+ predefined_nonterminals.find(it.first) !=
+ predefined_nonterminals.end()) {
nonterminal_ids[it.second] = rules.AddUnshareableNonterminal(it.first);
}
}
diff --git a/native/utils/grammar/utils/rules.h b/native/utils/grammar/utils/rules.h
index 8a5257c..42fa7cd 100644
--- a/native/utils/grammar/utils/rules.h
+++ b/native/utils/grammar/utils/rules.h
@@ -110,7 +110,11 @@
void DefineFilter(const CallbackId filter_id) { filters_.insert(filter_id); }
// Lowers the rule set into the intermediate representation.
- Ir Finalize() const;
+ // Treats nonterminals given by the argument `predefined_nonterminals` as
+ // defined externally. This allows to define rules that are dependent on
+ // non-terminals produced by e.g. existing text annotations and that will be
+ // fed to the matcher by the lexer.
+ Ir Finalize(const std::set<std::string>& predefined_nonterminals = {}) const;
private:
// Expands optional components in rules.
diff --git a/native/utils/grammar/utils/rules_test.cc b/native/utils/grammar/utils/rules_test.cc
deleted file mode 100644
index 84183f4..0000000
--- a/native/utils/grammar/utils/rules_test.cc
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/grammar/utils/rules.h"
-
-#include "utils/grammar/rules_generated.h"
-#include "utils/grammar/utils/ir.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3::grammar {
-namespace {
-
-using testing::Eq;
-using testing::IsEmpty;
-
-TEST(SerializeRulesTest, HandlesSimpleRuleSet) {
- Rules rules;
-
- rules.Add("<verb>", {"buy"});
- rules.Add("<verb>", {"bring"});
- rules.Add("<verb>", {"remind"});
- rules.Add("<reminder>", {"remind", "me", "to", "<verb>"});
- rules.Add("<action>", {"<reminder>"});
-
- const Ir ir = rules.Finalize();
- RulesSetT frozen_rules;
- ir.Serialize(/*include_debug_information=*/false, &frozen_rules);
-
- EXPECT_THAT(frozen_rules.rules.size(), Eq(1));
- EXPECT_THAT(frozen_rules.lhs, IsEmpty());
- EXPECT_THAT(frozen_rules.terminals,
- Eq(std::string("bring\0buy\0me\0remind\0to\0", 23)));
- EXPECT_THAT(frozen_rules.rules.front()->binary_rules.size(), Eq(3));
- EXPECT_THAT(frozen_rules.rules.front()->unary_rules.size(), Eq(1));
-}
-
-TEST(SerializeRulesTest, HandlesRulesSetWithCallbacks) {
- Rules rules;
- const CallbackId output = 1;
- const CallbackId filter = 2;
- rules.DefineFilter(filter);
-
- rules.Add("<verb>", {"buy"});
- rules.Add("<verb>", {"bring"}, output, 0);
- rules.Add("<verb>", {"remind"}, output, 0);
- rules.Add("<reminder>", {"remind", "me", "to", "<verb>"});
- rules.Add("<action>", {"<reminder>"}, filter, 0);
-
- const Ir ir = rules.Finalize();
- RulesSetT frozen_rules;
- ir.Serialize(/*include_debug_information=*/false, &frozen_rules);
-
- EXPECT_THAT(frozen_rules.rules.size(), Eq(1));
- EXPECT_THAT(frozen_rules.terminals,
- Eq(std::string("bring\0buy\0me\0remind\0to\0", 23)));
-
- // We have two identical output calls and one filter call in the rule set
- // definition above.
- EXPECT_THAT(frozen_rules.lhs.size(), Eq(2));
-
- EXPECT_THAT(frozen_rules.rules.front()->binary_rules.size(), Eq(3));
- EXPECT_THAT(frozen_rules.rules.front()->unary_rules.size(), Eq(1));
-}
-
-TEST(SerializeRulesTest, HandlesRulesWithWhitespaceGapLimits) {
- Rules rules;
- rules.Add("<iata>", {"lx"});
- rules.Add("<iata>", {"aa"});
- rules.Add("<flight>", {"<iata>", "<4_digits>"}, kNoCallback, 0,
- /*max_whitespace_gap=*/0);
-
- const Ir ir = rules.Finalize();
- RulesSetT frozen_rules;
- ir.Serialize(/*include_debug_information=*/false, &frozen_rules);
-
- EXPECT_THAT(frozen_rules.rules.size(), Eq(1));
- EXPECT_THAT(frozen_rules.terminals, Eq(std::string("aa\0lx\0", 6)));
- EXPECT_THAT(frozen_rules.lhs.size(), Eq(1));
-}
-
-TEST(SerializeRulesTest, HandlesCaseSensitiveTerminals) {
- Rules rules;
- rules.Add("<iata>", {"LX"}, kNoCallback, 0, /*max_whitespace_gap=*/-1,
- /*case_sensitive=*/true);
- rules.Add("<iata>", {"AA"}, kNoCallback, 0, /*max_whitespace_gap=*/-1,
- /*case_sensitive=*/true);
- rules.Add("<iata>", {"dl"}, kNoCallback, 0, /*max_whitespace_gap=*/-1,
- /*case_sensitive=*/false);
- rules.Add("<flight>", {"<iata>", "<4_digits>"}, kNoCallback, 0,
- /*max_whitespace_gap=*/0);
-
- const Ir ir = rules.Finalize();
- RulesSetT frozen_rules;
- ir.Serialize(/*include_debug_information=*/false, &frozen_rules);
-
- EXPECT_THAT(frozen_rules.rules.size(), Eq(1));
- EXPECT_THAT(frozen_rules.terminals, Eq(std::string("AA\0LX\0dl\0", 9)));
- EXPECT_THAT(frozen_rules.lhs.size(), Eq(1));
-}
-
-TEST(SerializeRulesTest, HandlesMultipleShards) {
- Rules rules(/*num_shards=*/2);
- rules.Add("<iata>", {"LX"}, kNoCallback, 0, /*max_whitespace_gap=*/-1,
- /*case_sensitive=*/true, /*shard=*/0);
- rules.Add("<iata>", {"aa"}, kNoCallback, 0, /*max_whitespace_gap=*/-1,
- /*case_sensitive=*/false, /*shard=*/1);
-
- const Ir ir = rules.Finalize();
- RulesSetT frozen_rules;
- ir.Serialize(/*include_debug_information=*/false, &frozen_rules);
-
- EXPECT_THAT(frozen_rules.rules.size(), Eq(2));
- EXPECT_THAT(frozen_rules.terminals, Eq(std::string("LX\0aa\0", 6)));
-}
-
-} // namespace
-} // namespace libtextclassifier3::grammar
diff --git a/native/utils/i18n/locale_test.cc b/native/utils/i18n/locale_test.cc
deleted file mode 100644
index faea4f6..0000000
--- a/native/utils/i18n/locale_test.cc
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/i18n/locale.h"
-
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(LocaleTest, ParseUnknown) {
- Locale locale = Locale::Invalid();
- EXPECT_FALSE(locale.IsValid());
-}
-
-TEST(LocaleTest, ParseSwissEnglish) {
- Locale locale = Locale::FromBCP47("en-CH");
- EXPECT_TRUE(locale.IsValid());
- EXPECT_EQ(locale.Language(), "en");
- EXPECT_EQ(locale.Script(), "");
- EXPECT_EQ(locale.Region(), "CH");
-}
-
-TEST(LocaleTest, ParseChineseChina) {
- Locale locale = Locale::FromBCP47("zh-CN");
- EXPECT_TRUE(locale.IsValid());
- EXPECT_EQ(locale.Language(), "zh");
- EXPECT_EQ(locale.Script(), "");
- EXPECT_EQ(locale.Region(), "CN");
-}
-
-TEST(LocaleTest, ParseChineseTaiwan) {
- Locale locale = Locale::FromBCP47("zh-Hant-TW");
- EXPECT_TRUE(locale.IsValid());
- EXPECT_EQ(locale.Language(), "zh");
- EXPECT_EQ(locale.Script(), "Hant");
- EXPECT_EQ(locale.Region(), "TW");
-}
-
-TEST(LocaleTest, ParseEnglish) {
- Locale locale = Locale::FromBCP47("en");
- EXPECT_TRUE(locale.IsValid());
- EXPECT_EQ(locale.Language(), "en");
- EXPECT_EQ(locale.Script(), "");
- EXPECT_EQ(locale.Region(), "");
-}
-
-TEST(LocaleTest, ParseCineseTraditional) {
- Locale locale = Locale::FromBCP47("zh-Hant");
- EXPECT_TRUE(locale.IsValid());
- EXPECT_EQ(locale.Language(), "zh");
- EXPECT_EQ(locale.Script(), "Hant");
- EXPECT_EQ(locale.Region(), "");
-}
-
-TEST(LocaleTest, IsAnyLocaleSupportedMatch) {
- std::vector<Locale> locales = {Locale::FromBCP47("zh-HK"),
- Locale::FromBCP47("en-UK")};
- std::vector<Locale> supported_locales = {Locale::FromBCP47("en")};
-
- EXPECT_TRUE(Locale::IsAnyLocaleSupported(locales, supported_locales,
- /*default_value=*/false));
-}
-
-TEST(LocaleTest, IsAnyLocaleSupportedNotMatch) {
- std::vector<Locale> locales = {Locale::FromBCP47("zh-tw")};
- std::vector<Locale> supported_locales = {Locale::FromBCP47("en"),
- Locale::FromBCP47("fr")};
-
- EXPECT_FALSE(Locale::IsAnyLocaleSupported(locales, supported_locales,
- /*default_value=*/false));
-}
-
-TEST(LocaleTest, IsAnyLocaleSupportedAnyLocale) {
- std::vector<Locale> locales = {Locale::FromBCP47("zh-tw")};
- std::vector<Locale> supported_locales = {Locale::FromBCP47("*")};
-
- EXPECT_TRUE(Locale::IsAnyLocaleSupported(locales, supported_locales,
- /*default_value=*/false));
-}
-
-TEST(LocaleTest, IsAnyLocaleSupportedEmptyLocales) {
- std::vector<Locale> supported_locales = {Locale::FromBCP47("en")};
-
- EXPECT_TRUE(Locale::IsAnyLocaleSupported({}, supported_locales,
- /*default_value=*/true));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/intents/intent-generator.cc b/native/utils/intents/intent-generator.cc
index 7dc24e4..36453bb 100644
--- a/native/utils/intents/intent-generator.cc
+++ b/native/utils/intents/intent-generator.cc
@@ -56,6 +56,8 @@
static constexpr const char* kDeviceLocaleKey = "device_locales";
static constexpr const char* kFormatKey = "format";
+static constexpr const int kIndexStackTop = -1;
+
// An Android specific Lua environment with JNI backed callbacks.
class JniLuaEnvironment : public LuaEnvironment {
public:
@@ -91,8 +93,19 @@
// Reads the extras from the Lua result.
void ReadExtras(std::map<std::string, Variant>* extra);
- // Reads the intent categories array from a Lua result.
- void ReadCategories(std::vector<std::string>* category);
+ // Reads a vector from a Lua result. read_element_func is the function to read
+ // a single element from the lua side.
+ template <class T>
+ std::vector<T> ReadVector(const std::function<T()> read_element_func) const;
+
+ // Reads a string vector from a Lua result.
+ std::vector<std::string> ReadStringVector() const;
+
+ // Reads a float vector from a Lua result.
+ std::vector<float> ReadFloatVector() const;
+
+ // Reads a int vector from a Lua result.
+ std::vector<int> ReadIntVector() const;
// Retrieves user manager if not previously done.
bool RetrieveUserManager();
@@ -193,7 +206,7 @@
}
int JniLuaEnvironment::HandleExternalCallback() {
- const StringPiece key = ReadString(/*index=*/-1);
+ const StringPiece key = ReadString(kIndexStackTop);
if (key.Equals(kHashKey)) {
Bind<JniLuaEnvironment, &JniLuaEnvironment::HandleHash>();
return 1;
@@ -208,7 +221,7 @@
}
int JniLuaEnvironment::HandleAndroidCallback() {
- const StringPiece key = ReadString(/*index=*/-1);
+ const StringPiece key = ReadString(kIndexStackTop);
if (key.Equals(kDeviceLocaleKey)) {
// Provide the locale as table with the individual fields set.
lua_newtable(state_);
@@ -292,7 +305,7 @@
return 0;
}
- const StringPiece key_str = ReadString(/*index=*/-1);
+ const StringPiece key_str = ReadString(kIndexStackTop);
if (key_str.empty()) {
TC3_LOG(ERROR) << "Expected string, got null.";
lua_error(state_);
@@ -410,7 +423,7 @@
}
int JniLuaEnvironment::HandleUrlHost() {
- const StringPiece url = ReadString(/*index=*/-1);
+ const StringPiece url = ReadString(kIndexStackTop);
const StatusOr<ScopedLocalRef<jobject>> status_or_parsed_uri = ParseUri(url);
if (!status_or_parsed_uri.ok()) {
@@ -443,7 +456,7 @@
}
int JniLuaEnvironment::HandleHash() {
- const StringPiece input = ReadString(/*index=*/-1);
+ const StringPiece input = ReadString(kIndexStackTop);
lua_pushinteger(state_, tc3farmhash::Hash32(input.data(), input.length()));
return 1;
}
@@ -464,7 +477,7 @@
return false;
}
- const StringPiece resource_name = ReadString(/*index=*/-1);
+ const StringPiece resource_name = ReadString(kIndexStackTop);
std::string resource_content;
if (!resources_.GetResourceContent(device_locales_, resource_name,
&resource_content)) {
@@ -492,10 +505,10 @@
int resource_id;
switch (lua_type(state_, -1)) {
case LUA_TNUMBER:
- resource_id = static_cast<int>(lua_tonumber(state_, /*idx=*/-1));
+ resource_id = static_cast<int>(lua_tonumber(state_, kIndexStackTop));
break;
case LUA_TSTRING: {
- const StringPiece resource_name_str = ReadString(/*index=*/-1);
+ const StringPiece resource_name_str = ReadString(kIndexStackTop);
if (resource_name_str.empty()) {
TC3_LOG(ERROR) << "No resource name provided.";
lua_error(state_);
@@ -595,27 +608,28 @@
while (lua_next(state_, /*idx=*/-2)) {
const StringPiece key = ReadString(/*index=*/-2);
if (key.Equals("title_without_entity")) {
- result.title_without_entity = ReadString(/*index=*/-1).ToString();
+ result.title_without_entity = ReadString(kIndexStackTop).ToString();
} else if (key.Equals("title_with_entity")) {
- result.title_with_entity = ReadString(/*index=*/-1).ToString();
+ result.title_with_entity = ReadString(kIndexStackTop).ToString();
} else if (key.Equals("description")) {
- result.description = ReadString(/*index=*/-1).ToString();
+ result.description = ReadString(kIndexStackTop).ToString();
} else if (key.Equals("description_with_app_name")) {
- result.description_with_app_name = ReadString(/*index=*/-1).ToString();
+ result.description_with_app_name = ReadString(kIndexStackTop).ToString();
} else if (key.Equals("action")) {
- result.action = ReadString(/*index=*/-1).ToString();
+ result.action = ReadString(kIndexStackTop).ToString();
} else if (key.Equals("data")) {
- result.data = ReadString(/*index=*/-1).ToString();
+ result.data = ReadString(kIndexStackTop).ToString();
} else if (key.Equals("type")) {
- result.type = ReadString(/*index=*/-1).ToString();
+ result.type = ReadString(kIndexStackTop).ToString();
} else if (key.Equals("flags")) {
- result.flags = static_cast<int>(lua_tointeger(state_, /*idx=*/-1));
+ result.flags = static_cast<int>(lua_tointeger(state_, kIndexStackTop));
} else if (key.Equals("package_name")) {
- result.package_name = ReadString(/*index=*/-1).ToString();
+ result.package_name = ReadString(kIndexStackTop).ToString();
} else if (key.Equals("request_code")) {
- result.request_code = static_cast<int>(lua_tointeger(state_, /*idx=*/-1));
+ result.request_code =
+ static_cast<int>(lua_tointeger(state_, kIndexStackTop));
} else if (key.Equals("category")) {
- ReadCategories(&result.category);
+ result.category = ReadStringVector();
} else if (key.Equals("extra")) {
ReadExtras(&result.extra);
} else {
@@ -627,25 +641,43 @@
return result;
}
-void JniLuaEnvironment::ReadCategories(std::vector<std::string>* category) {
- // Read category array.
- if (lua_type(state_, /*idx=*/-1) != LUA_TTABLE) {
- TC3_LOG(ERROR) << "Expected categories table, got: "
- << lua_type(state_, /*idx=*/-1);
+template <class T>
+std::vector<T> JniLuaEnvironment::ReadVector(
+ const std::function<T()> read_element_func) const {
+ std::vector<T> vector;
+ if (lua_type(state_, kIndexStackTop) != LUA_TTABLE) {
+ TC3_LOG(ERROR) << "Expected a table, got: "
+ << lua_type(state_, kIndexStackTop);
lua_pop(state_, 1);
- return;
+ return {};
}
lua_pushnil(state_);
while (lua_next(state_, /*idx=*/-2)) {
- category->push_back(ReadString(/*index=*/-1).ToString());
+ vector.push_back(read_element_func());
lua_pop(state_, 1);
}
+ return vector;
+}
+
+std::vector<std::string> JniLuaEnvironment::ReadStringVector() const {
+ return ReadVector<std::string>(
+ [this]() { return this->ReadString(kIndexStackTop).ToString(); });
+}
+
+std::vector<float> JniLuaEnvironment::ReadFloatVector() const {
+ return ReadVector<float>(
+ [this]() { return lua_tonumber(state_, kIndexStackTop); });
+}
+
+std::vector<int> JniLuaEnvironment::ReadIntVector() const {
+ return ReadVector<int>(
+ [this]() { return lua_tonumber(state_, kIndexStackTop); });
}
void JniLuaEnvironment::ReadExtras(std::map<std::string, Variant>* extra) {
- if (lua_type(state_, /*idx=*/-1) != LUA_TTABLE) {
+ if (lua_type(state_, kIndexStackTop) != LUA_TTABLE) {
TC3_LOG(ERROR) << "Expected extras table, got: "
- << lua_type(state_, /*idx=*/-1);
+ << lua_type(state_, kIndexStackTop);
lua_pop(state_, 1);
return;
}
@@ -654,9 +686,9 @@
// Each entry is a table specifying name and value.
// The value is specified via a type specific field as Lua doesn't allow
// to easily distinguish between different number types.
- if (lua_type(state_, /*idx=*/-1) != LUA_TTABLE) {
+ if (lua_type(state_, kIndexStackTop) != LUA_TTABLE) {
TC3_LOG(ERROR) << "Expected a table for an extra, got: "
- << lua_type(state_, /*idx=*/-1);
+ << lua_type(state_, kIndexStackTop);
lua_pop(state_, 1);
return;
}
@@ -667,17 +699,26 @@
while (lua_next(state_, /*idx=*/-2)) {
const StringPiece key = ReadString(/*index=*/-2);
if (key.Equals("name")) {
- name = ReadString(/*index=*/-1).ToString();
+ name = ReadString(kIndexStackTop).ToString();
} else if (key.Equals("int_value")) {
- value = Variant(static_cast<int>(lua_tonumber(state_, /*idx=*/-1)));
+ value = Variant(static_cast<int>(lua_tonumber(state_, kIndexStackTop)));
} else if (key.Equals("long_value")) {
- value = Variant(static_cast<int64>(lua_tonumber(state_, /*idx=*/-1)));
+ value =
+ Variant(static_cast<int64>(lua_tonumber(state_, kIndexStackTop)));
} else if (key.Equals("float_value")) {
- value = Variant(static_cast<float>(lua_tonumber(state_, /*idx=*/-1)));
+ value =
+ Variant(static_cast<float>(lua_tonumber(state_, kIndexStackTop)));
} else if (key.Equals("bool_value")) {
- value = Variant(static_cast<bool>(lua_toboolean(state_, /*idx=*/-1)));
+ value =
+ Variant(static_cast<bool>(lua_toboolean(state_, kIndexStackTop)));
} else if (key.Equals("string_value")) {
- value = Variant(ReadString(/*index=*/-1).ToString());
+ value = Variant(ReadString(kIndexStackTop).ToString());
+ } else if (key.Equals("string_array_value")) {
+ value = Variant(ReadStringVector());
+ } else if (key.Equals("float_array_value")) {
+ value = Variant(ReadFloatVector());
+ } else if (key.Equals("int_array_value")) {
+ value = Variant(ReadIntVector());
} else {
TC3_LOG(INFO) << "Unknown extra field: " << key.ToString();
}
@@ -695,7 +736,7 @@
int JniLuaEnvironment::ReadRemoteActionTemplates(
std::vector<RemoteActionTemplate>* result) {
// Read result.
- if (lua_type(state_, /*idx=*/-1) != LUA_TTABLE) {
+ if (lua_type(state_, kIndexStackTop) != LUA_TTABLE) {
TC3_LOG(ERROR) << "Unexpected result for snippet: " << lua_type(state_, -1);
lua_error(state_);
return LUA_ERRRUN;
@@ -704,9 +745,9 @@
// Read remote action templates array.
lua_pushnil(state_);
while (lua_next(state_, /*idx=*/-2)) {
- if (lua_type(state_, /*idx=*/-1) != LUA_TTABLE) {
+ if (lua_type(state_, kIndexStackTop) != LUA_TTABLE) {
TC3_LOG(ERROR) << "Expected intent table, got: "
- << lua_type(state_, /*idx=*/-1);
+ << lua_type(state_, kIndexStackTop);
lua_pop(state_, 1);
continue;
}
@@ -897,6 +938,7 @@
// Retrieve generator for specified entity.
auto it = generators_.find(classification.collection);
if (it == generators_.end()) {
+ TC3_VLOG(INFO) << "Cannot find a generator for the specified collection.";
return true;
}
diff --git a/native/utils/intents/jni.cc b/native/utils/intents/jni.cc
index ed7737e..bd0fc7d 100644
--- a/native/utils/intents/jni.cc
+++ b/native/utils/intents/jni.cc
@@ -77,7 +77,12 @@
"(Ljava/lang/String;Z)V");
TC3_GET_METHOD(named_variant_class_, named_variant_from_string_, "<init>",
"(Ljava/lang/String;Ljava/lang/String;)V");
-
+ TC3_GET_METHOD(named_variant_class_, named_variant_from_string_array_,
+ "<init>", "(Ljava/lang/String;[Ljava/lang/String;)V");
+ TC3_GET_METHOD(named_variant_class_, named_variant_from_float_array_,
+ "<init>", "(Ljava/lang/String;[F)V");
+ TC3_GET_METHOD(named_variant_class_, named_variant_from_int_array_, "<init>",
+ "(Ljava/lang/String;[I)V");
return handler;
}
@@ -124,6 +129,38 @@
return result;
}
+StatusOr<ScopedLocalRef<jfloatArray>>
+RemoteActionTemplatesHandler::AsFloatArray(
+ const std::vector<float>& values) const {
+ if (values.empty()) {
+ return {{nullptr, jni_cache_->GetEnv()}};
+ }
+
+ TC3_ASSIGN_OR_RETURN(
+ ScopedLocalRef<jfloatArray> result,
+ JniHelper::NewFloatArray(jni_cache_->GetEnv(), values.size()));
+
+ jni_cache_->GetEnv()->SetFloatArrayRegion(result.get(), /*start=*/0,
+ /*len=*/values.size(),
+ &(values[0]));
+ return result;
+}
+
+StatusOr<ScopedLocalRef<jintArray>> RemoteActionTemplatesHandler::AsIntArray(
+ const std::vector<int>& values) const {
+ if (values.empty()) {
+ return {{nullptr, jni_cache_->GetEnv()}};
+ }
+
+ TC3_ASSIGN_OR_RETURN(
+ ScopedLocalRef<jintArray> result,
+ JniHelper::NewIntArray(jni_cache_->GetEnv(), values.size()));
+
+ jni_cache_->GetEnv()->SetIntArrayRegion(result.get(), /*start=*/0,
+ /*len=*/values.size(), &(values[0]));
+ return result;
+}
+
StatusOr<ScopedLocalRef<jobject>> RemoteActionTemplatesHandler::AsNamedVariant(
const std::string& name_str, const Variant& value) const {
TC3_ASSIGN_OR_RETURN(ScopedLocalRef<jstring> name,
@@ -165,6 +202,33 @@
value_jstring.get());
}
+ case Variant::TYPE_STRING_VECTOR_VALUE: {
+ TC3_ASSIGN_OR_RETURN(ScopedLocalRef<jobjectArray> value_jstring_array,
+ AsStringArray(value.StringVectorValue()));
+
+ return JniHelper::NewObject(env, named_variant_class_.get(),
+ named_variant_from_string_array_, name.get(),
+ value_jstring_array.get());
+ }
+
+ case Variant::TYPE_FLOAT_VECTOR_VALUE: {
+ TC3_ASSIGN_OR_RETURN(ScopedLocalRef<jfloatArray> value_jfloat_array,
+ AsFloatArray(value.FloatVectorValue()));
+
+ return JniHelper::NewObject(env, named_variant_class_.get(),
+ named_variant_from_float_array_, name.get(),
+ value_jfloat_array.get());
+ }
+
+ case Variant::TYPE_INT_VECTOR_VALUE: {
+ TC3_ASSIGN_OR_RETURN(ScopedLocalRef<jintArray> value_jint_array,
+ AsIntArray(value.IntVectorValue()));
+
+ return JniHelper::NewObject(env, named_variant_class_.get(),
+ named_variant_from_float_array_, name.get(),
+ value_jint_array.get());
+ }
+
case Variant::TYPE_EMPTY:
return {Status::UNKNOWN};
diff --git a/native/utils/intents/jni.h b/native/utils/intents/jni.h
index a185472..a2d57a0 100644
--- a/native/utils/intents/jni.h
+++ b/native/utils/intents/jni.h
@@ -60,6 +60,10 @@
const Optional<int>& optional) const;
StatusOr<ScopedLocalRef<jobjectArray>> AsStringArray(
const std::vector<std::string>& values) const;
+ StatusOr<ScopedLocalRef<jfloatArray>> AsFloatArray(
+ const std::vector<float>& values) const;
+ StatusOr<ScopedLocalRef<jintArray>> AsIntArray(
+ const std::vector<int>& values) const;
StatusOr<ScopedLocalRef<jobject>> AsNamedVariant(const std::string& name,
const Variant& value) const;
StatusOr<ScopedLocalRef<jobjectArray>> AsNamedVariantArray(
@@ -98,6 +102,9 @@
jmethodID named_variant_from_double_ = nullptr;
jmethodID named_variant_from_bool_ = nullptr;
jmethodID named_variant_from_string_ = nullptr;
+ jmethodID named_variant_from_string_array_ = nullptr;
+ jmethodID named_variant_from_float_array_ = nullptr;
+ jmethodID named_variant_from_int_array_ = nullptr;
};
} // namespace libtextclassifier3
diff --git a/native/utils/java/jni-helper.cc b/native/utils/java/jni-helper.cc
index 34e0ab8..6618cfa 100644
--- a/native/utils/java/jni-helper.cc
+++ b/native/utils/java/jni-helper.cc
@@ -120,6 +120,15 @@
return result;
}
+StatusOr<ScopedLocalRef<jfloatArray>> JniHelper::NewFloatArray(JNIEnv* env,
+ jsize length) {
+ TC3_ENSURE_LOCAL_CAPACITY_OR_RETURN;
+ ScopedLocalRef<jfloatArray> result(env->NewFloatArray(length), env);
+ TC3_NOT_NULL_OR_RETURN;
+ TC3_NO_EXCEPTION_OR_RETURN;
+ return result;
+}
+
StatusOr<ScopedLocalRef<jobjectArray>> JniHelper::NewObjectArray(
JNIEnv* env, jsize length, jclass element_class, jobject initial_element) {
TC3_ENSURE_LOCAL_CAPACITY_OR_RETURN;
diff --git a/native/utils/java/jni-helper.h b/native/utils/java/jni-helper.h
index 56a07b7..71d31cb 100644
--- a/native/utils/java/jni-helper.h
+++ b/native/utils/java/jni-helper.h
@@ -93,6 +93,8 @@
jsize length);
static StatusOr<ScopedLocalRef<jstring>> NewStringUTF(JNIEnv* env,
const char* bytes);
+ static StatusOr<ScopedLocalRef<jfloatArray>> NewFloatArray(JNIEnv* env,
+ jsize length);
// Call* methods.
TC3_DEFINE_VARIADIC_SCOPED_LOCAL_REF_ENV_METHOD(CallObjectMethod, jobject,
diff --git a/native/utils/memory/mmap.h b/native/utils/memory/mmap.h
index acce7db..974cc02 100644
--- a/native/utils/memory/mmap.h
+++ b/native/utils/memory/mmap.h
@@ -130,7 +130,7 @@
}
}
- const MmapHandle &handle() { return handle_; }
+ const MmapHandle &handle() const { return handle_; }
private:
MmapHandle handle_;
diff --git a/native/utils/normalization_test.cc b/native/utils/normalization_test.cc
deleted file mode 100644
index 1bf9fae..0000000
--- a/native/utils/normalization_test.cc
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/normalization.h"
-
-#include <string>
-
-#include "utils/base/integral_types.h"
-#include "utils/utf8/unicodetext.h"
-#include "utils/utf8/unilib.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::Eq;
-
-class NormalizationTest : public testing::Test {
- protected:
- NormalizationTest() : INIT_UNILIB_FOR_TESTING(unilib_) {}
-
- std::string NormalizeTextCodepointWise(const std::string& text,
- const int32 codepointwise_ops) {
- return libtextclassifier3::NormalizeTextCodepointWise(
- &unilib_, codepointwise_ops,
- UTF8ToUnicodeText(text, /*do_copy=*/false))
- .ToUTF8String();
- }
-
- UniLib unilib_;
-};
-
-TEST_F(NormalizationTest, ReturnsIdenticalStringWhenNoNormalization) {
- EXPECT_THAT(NormalizeTextCodepointWise(
- "Never gonna let you down.",
- NormalizationOptions_::CodepointwiseNormalizationOp_NONE),
- Eq("Never gonna let you down."));
-}
-
-#if !defined(TC3_UNILIB_DUMMY)
-TEST_F(NormalizationTest, DropsWhitespace) {
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "Never gonna let you down.",
- NormalizationOptions_::CodepointwiseNormalizationOp_DROP_WHITESPACE),
- Eq("Nevergonnaletyoudown."));
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "Never\tgonna\t\tlet\tyou\tdown.",
- NormalizationOptions_::CodepointwiseNormalizationOp_DROP_WHITESPACE),
- Eq("Nevergonnaletyoudown."));
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "Never\u2003gonna\u2003let\u2003you\u2003down.",
- NormalizationOptions_::CodepointwiseNormalizationOp_DROP_WHITESPACE),
- Eq("Nevergonnaletyoudown."));
-}
-
-TEST_F(NormalizationTest, DropsPunctuation) {
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "Never gonna let you down.",
- NormalizationOptions_::CodepointwiseNormalizationOp_DROP_PUNCTUATION),
- Eq("Never gonna let you down"));
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "αʹ. Σημεῖόν ἐστιν, οὗ μέρος οὐθέν.",
- NormalizationOptions_::CodepointwiseNormalizationOp_DROP_PUNCTUATION),
- Eq("αʹ Σημεῖόν ἐστιν οὗ μέρος οὐθέν"));
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "978—3—16—148410—0",
- NormalizationOptions_::CodepointwiseNormalizationOp_DROP_PUNCTUATION),
- Eq("9783161484100"));
-}
-
-TEST_F(NormalizationTest, LowercasesUnicodeText) {
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "αʹ. Σημεῖόν ἐστιν, οὗ μέρος οὐθέν.",
- NormalizationOptions_::CodepointwiseNormalizationOp_LOWERCASE),
- Eq("αʹ. σημεῖόν ἐστιν, οὗ μέρος οὐθέν."));
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "αʹ. Σημεῖόν ἐστιν, οὗ μέρος οὐθέν.",
- NormalizationOptions_::CodepointwiseNormalizationOp_DROP_WHITESPACE |
- NormalizationOptions_::CodepointwiseNormalizationOp_LOWERCASE),
- Eq("αʹ.σημεῖόνἐστιν,οὗμέροςοὐθέν."));
-}
-
-TEST_F(NormalizationTest, UppercasesUnicodeText) {
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "Κανένας άνθρωπος δεν ξέρει",
- NormalizationOptions_::CodepointwiseNormalizationOp_UPPERCASE),
- Eq("ΚΑΝΈΝΑΣ ΆΝΘΡΩΠΟΣ ΔΕΝ ΞΈΡΕΙ"));
- EXPECT_THAT(
- NormalizeTextCodepointWise(
- "Κανένας άνθρωπος δεν ξέρει",
- NormalizationOptions_::CodepointwiseNormalizationOp_DROP_WHITESPACE |
- NormalizationOptions_::CodepointwiseNormalizationOp_UPPERCASE),
- Eq("ΚΑΝΈΝΑΣΆΝΘΡΩΠΟΣΔΕΝΞΈΡΕΙ"));
-}
-#endif
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/regex-match_test.cc b/native/utils/regex-match_test.cc
deleted file mode 100644
index c45fb29..0000000
--- a/native/utils/regex-match_test.cc
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/regex-match.h"
-
-#include <memory>
-
-#include "utils/utf8/unicodetext.h"
-#include "utils/utf8/unilib.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-class RegexMatchTest : public testing::Test {
- protected:
- RegexMatchTest() : INIT_UNILIB_FOR_TESTING(unilib_) {}
- UniLib unilib_;
-};
-
-#ifdef TC3_UNILIB_ICU
-#ifndef TC3_DISABLE_LUA
-TEST_F(RegexMatchTest, HandlesSimpleVerification) {
- EXPECT_TRUE(VerifyMatch(/*context=*/"", /*matcher=*/nullptr, "return true;"));
-}
-#endif // TC3_DISABLE_LUA
-
-#ifndef TC3_DISABLE_LUA
-TEST_F(RegexMatchTest, HandlesCustomVerification) {
- UnicodeText pattern = UTF8ToUnicodeText("(\\d{16})",
- /*do_copy=*/true);
- UnicodeText message = UTF8ToUnicodeText("cc: 4012888888881881",
- /*do_copy=*/true);
- const std::string verifier = R"(
-function luhn(candidate)
- local sum = 0
- local num_digits = string.len(candidate)
- local parity = num_digits % 2
- for pos = 1,num_digits do
- d = tonumber(string.sub(candidate, pos, pos))
- if pos % 2 ~= parity then
- d = d * 2
- end
- if d > 9 then
- d = d - 9
- end
- sum = sum + d
- end
- return (sum % 10) == 0
-end
-return luhn(match[1].text);
- )";
- const std::unique_ptr<UniLib::RegexPattern> regex_pattern =
- unilib_.CreateRegexPattern(pattern);
- ASSERT_TRUE(regex_pattern != nullptr);
- const std::unique_ptr<UniLib::RegexMatcher> matcher =
- regex_pattern->Matcher(message);
- ASSERT_TRUE(matcher != nullptr);
- int status = UniLib::RegexMatcher::kNoError;
- ASSERT_TRUE(matcher->Find(&status) &&
- status == UniLib::RegexMatcher::kNoError);
-
- EXPECT_TRUE(VerifyMatch(message.ToUTF8String(), matcher.get(), verifier));
-}
-#endif // TC3_DISABLE_LUA
-
-TEST_F(RegexMatchTest, RetrievesMatchGroupTest) {
- UnicodeText pattern =
- UTF8ToUnicodeText("never gonna (?:give (you) up|let (you) down)",
- /*do_copy=*/true);
- const std::unique_ptr<UniLib::RegexPattern> regex_pattern =
- unilib_.CreateRegexPattern(pattern);
- ASSERT_TRUE(regex_pattern != nullptr);
- UnicodeText message =
- UTF8ToUnicodeText("never gonna give you up - never gonna let you down");
- const std::unique_ptr<UniLib::RegexMatcher> matcher =
- regex_pattern->Matcher(message);
- ASSERT_TRUE(matcher != nullptr);
- int status = UniLib::RegexMatcher::kNoError;
-
- ASSERT_TRUE(matcher->Find(&status) &&
- status == UniLib::RegexMatcher::kNoError);
- EXPECT_THAT(GetCapturingGroupText(matcher.get(), 0).value(),
- testing::Eq("never gonna give you up"));
- EXPECT_THAT(GetCapturingGroupText(matcher.get(), 1).value(),
- testing::Eq("you"));
- EXPECT_FALSE(GetCapturingGroupText(matcher.get(), 2).has_value());
-
- ASSERT_TRUE(matcher->Find(&status) &&
- status == UniLib::RegexMatcher::kNoError);
- EXPECT_THAT(GetCapturingGroupText(matcher.get(), 0).value(),
- testing::Eq("never gonna let you down"));
- EXPECT_FALSE(GetCapturingGroupText(matcher.get(), 1).has_value());
- EXPECT_THAT(GetCapturingGroupText(matcher.get(), 2).value(),
- testing::Eq("you"));
-}
-#endif
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/resources_test.cc b/native/utils/resources_test.cc
deleted file mode 100644
index c385f39..0000000
--- a/native/utils/resources_test.cc
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/resources.h"
-#include "utils/i18n/locale.h"
-#include "utils/resources_generated.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-class ResourcesTest
- : public testing::TestWithParam<testing::tuple<bool, bool>> {
- protected:
- ResourcesTest() {}
-
- std::string BuildTestResources(bool add_default_language = true) const {
- ResourcePoolT test_resources;
-
- // Test locales.
- test_resources.locale.emplace_back(new LanguageTagT);
- test_resources.locale.back()->language = "en";
- test_resources.locale.back()->region = "US";
- test_resources.locale.emplace_back(new LanguageTagT);
- test_resources.locale.back()->language = "en";
- test_resources.locale.back()->region = "GB";
- test_resources.locale.emplace_back(new LanguageTagT);
- test_resources.locale.back()->language = "de";
- test_resources.locale.back()->region = "DE";
- test_resources.locale.emplace_back(new LanguageTagT);
- test_resources.locale.back()->language = "fr";
- test_resources.locale.back()->region = "FR";
- test_resources.locale.emplace_back(new LanguageTagT);
- test_resources.locale.back()->language = "pt";
- test_resources.locale.back()->region = "PT";
- test_resources.locale.emplace_back(new LanguageTagT);
- test_resources.locale.back()->language = "pt";
- test_resources.locale.emplace_back(new LanguageTagT);
- test_resources.locale.back()->language = "zh";
- test_resources.locale.back()->script = "Hans";
- test_resources.locale.back()->region = "CN";
- test_resources.locale.emplace_back(new LanguageTagT);
- test_resources.locale.back()->language = "zh";
- test_resources.locale.emplace_back(new LanguageTagT);
- test_resources.locale.back()->language = "fr";
- test_resources.locale.back()->language = "fr-CA";
- if (add_default_language) {
- test_resources.locale.emplace_back(new LanguageTagT); // default
- }
-
- // Test entries.
- test_resources.resource_entry.emplace_back(new ResourceEntryT);
- test_resources.resource_entry.back()->name = /*resource_name=*/"A";
-
- // en-US, default
- test_resources.resource_entry.back()->resource.emplace_back(new ResourceT);
- test_resources.resource_entry.back()->resource.back()->content = "localize";
- test_resources.resource_entry.back()->resource.back()->locale.push_back(0);
- if (add_default_language) {
- test_resources.resource_entry.back()->resource.back()->locale.push_back(
- 9);
- }
-
- // en-GB
- test_resources.resource_entry.back()->resource.emplace_back(new ResourceT);
- test_resources.resource_entry.back()->resource.back()->content = "localise";
- test_resources.resource_entry.back()->resource.back()->locale.push_back(1);
-
- // de-DE
- test_resources.resource_entry.back()->resource.emplace_back(new ResourceT);
- test_resources.resource_entry.back()->resource.back()->content =
- "lokalisieren";
- test_resources.resource_entry.back()->resource.back()->locale.push_back(2);
-
- // fr-FR, fr-CA
- test_resources.resource_entry.back()->resource.emplace_back(new ResourceT);
- test_resources.resource_entry.back()->resource.back()->content =
- "localiser";
- test_resources.resource_entry.back()->resource.back()->locale.push_back(3);
- test_resources.resource_entry.back()->resource.back()->locale.push_back(8);
-
- // pt-PT
- test_resources.resource_entry.back()->resource.emplace_back(new ResourceT);
- test_resources.resource_entry.back()->resource.back()->content =
- "localizar";
- test_resources.resource_entry.back()->resource.back()->locale.push_back(4);
-
- // pt
- test_resources.resource_entry.back()->resource.emplace_back(new ResourceT);
- test_resources.resource_entry.back()->resource.back()->content =
- "concentrar";
- test_resources.resource_entry.back()->resource.back()->locale.push_back(5);
-
- // zh-Hans-CN
- test_resources.resource_entry.back()->resource.emplace_back(new ResourceT);
- test_resources.resource_entry.back()->resource.back()->content = "龙";
- test_resources.resource_entry.back()->resource.back()->locale.push_back(6);
-
- // zh
- test_resources.resource_entry.back()->resource.emplace_back(new ResourceT);
- test_resources.resource_entry.back()->resource.back()->content = "龍";
- test_resources.resource_entry.back()->resource.back()->locale.push_back(7);
-
- if (compress()) {
- EXPECT_TRUE(CompressResources(
- &test_resources,
- /*build_compression_dictionary=*/build_dictionary()));
- }
-
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(ResourcePool::Pack(builder, &test_resources));
-
- return std::string(
- reinterpret_cast<const char*>(builder.GetBufferPointer()),
- builder.GetSize());
- }
-
- bool compress() const { return testing::get<0>(GetParam()); }
-
- bool build_dictionary() const { return testing::get<1>(GetParam()); }
-};
-
-INSTANTIATE_TEST_SUITE_P(Compression, ResourcesTest,
- testing::Combine(testing::Bool(), testing::Bool()));
-
-TEST_P(ResourcesTest, CorrectlyHandlesExactMatch) {
- std::string test_resources = BuildTestResources();
- Resources resources(
- flatbuffers::GetRoot<ResourcePool>(test_resources.data()));
- std::string content;
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("en-US")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("localize", content);
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("en-GB")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("localise", content);
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("pt-PT")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("localizar", content);
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("zh-Hans-CN")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("龙", content);
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("zh")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("龍", content);
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("fr-CA")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("localiser", content);
-}
-
-TEST_P(ResourcesTest, CorrectlyHandlesTie) {
- std::string test_resources = BuildTestResources();
- Resources resources(
- flatbuffers::GetRoot<ResourcePool>(test_resources.data()));
- // Uses first best match in case of a tie.
- std::string content;
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("en-CA")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("localize", content);
-}
-
-TEST_P(ResourcesTest, RequiresLanguageMatch) {
- {
- std::string test_resources =
- BuildTestResources(/*add_default_language=*/false);
- Resources resources(
- flatbuffers::GetRoot<ResourcePool>(test_resources.data()));
- EXPECT_FALSE(resources.GetResourceContent({Locale::FromBCP47("es-US")},
- /*resource_name=*/"A",
- /*result=*/nullptr));
- }
- {
- std::string test_resources =
- BuildTestResources(/*add_default_language=*/true);
- Resources resources(
- flatbuffers::GetRoot<ResourcePool>(test_resources.data()));
- std::string content;
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("es-US")},
- /*resource_name=*/"A",
- /*result=*/&content));
- EXPECT_EQ("localize", content);
- }
-}
-
-TEST_P(ResourcesTest, HandlesFallback) {
- std::string test_resources = BuildTestResources();
- Resources resources(
- flatbuffers::GetRoot<ResourcePool>(test_resources.data()));
- std::string content;
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("fr-CH")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("localiser", content);
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("zh-Hans")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("龙", content);
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("zh-Hans-ZZ")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("龙", content);
-
- // Fallback to default, en-US.
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("ru")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("localize", content);
-}
-
-TEST_P(ResourcesTest, HandlesFallbackMultipleLocales) {
- std::string test_resources = BuildTestResources();
- Resources resources(
- flatbuffers::GetRoot<ResourcePool>(test_resources.data()));
- std::string content;
-
- // Still use inexact match with primary locale if language matches,
- // even though secondary locale would match exactly.
- EXPECT_TRUE(resources.GetResourceContent(
- {Locale::FromBCP47("fr-CH"), Locale::FromBCP47("en-US")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("localiser", content);
-
- // Use secondary language instead of default fallback if that is an exact
- // language match.
- EXPECT_TRUE(resources.GetResourceContent(
- {Locale::FromBCP47("ru"), Locale::FromBCP47("de")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("lokalisieren", content);
-
- // Use tertiary language.
- EXPECT_TRUE(resources.GetResourceContent(
- {Locale::FromBCP47("ru"), Locale::FromBCP47("it-IT"),
- Locale::FromBCP47("de")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("lokalisieren", content);
-
- // Default fallback if no locale matches.
- EXPECT_TRUE(resources.GetResourceContent(
- {Locale::FromBCP47("ru"), Locale::FromBCP47("it-IT"),
- Locale::FromBCP47("es")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("localize", content);
-}
-
-TEST_P(ResourcesTest, PreferGenericCallback) {
- std::string test_resources = BuildTestResources();
- Resources resources(
- flatbuffers::GetRoot<ResourcePool>(test_resources.data()));
- std::string content;
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("pt-BR")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("concentrar", content); // Falls back to pt, not pt-PT.
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("zh-Hant")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("龍", content); // Falls back to zh, not zh-Hans-CN.
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("zh-Hant-CN")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("龍", content); // Falls back to zh, not zh-Hans-CN.
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("zh-CN")},
- /*resource_name=*/"A", &content));
- EXPECT_EQ("龍", content); // Falls back to zh, not zh-Hans-CN.
-}
-
-TEST_P(ResourcesTest, PreferGenericWhenGeneric) {
- std::string test_resources = BuildTestResources();
- Resources resources(
- flatbuffers::GetRoot<ResourcePool>(test_resources.data()));
- std::string content;
- EXPECT_TRUE(resources.GetResourceContent({Locale::FromBCP47("pt")},
- /*resource_name=*/"A", &content));
-
- // Uses pt, not pt-PT.
- EXPECT_EQ("concentrar", content);
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/sentencepiece/encoder_test.cc b/native/utils/sentencepiece/encoder_test.cc
deleted file mode 100644
index 740db35..0000000
--- a/native/utils/sentencepiece/encoder_test.cc
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/sentencepiece/encoder.h"
-
-#include <memory>
-#include <vector>
-
-#include "utils/base/integral_types.h"
-#include "utils/container/sorted-strings-table.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::ElementsAre;
-
-TEST(EncoderTest, SimpleTokenization) {
- const char pieces_table[] = "hell\0hello\0o\0there\0";
- const uint32 offsets[] = {0, 5, 11, 13};
- float scores[] = {-0.5, -1.0, -10.0, -1.0};
- std::unique_ptr<StringSet> pieces(new SortedStringsTable(
- /*num_pieces=*/4, offsets, StringPiece(pieces_table, 18)));
- const Encoder encoder(pieces.get(),
- /*num_pieces=*/4, scores);
-
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("hellothere", &encoded_text));
- EXPECT_THAT(encoded_text, ElementsAre(0, 3, 5, 1));
- }
-
- // Make probability of hello very low:
- // hello gets now tokenized as hell + o.
- scores[1] = -100.0;
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("hellothere", &encoded_text));
- EXPECT_THAT(encoded_text, ElementsAre(0, 2, 4, 5, 1));
- }
-}
-
-TEST(EncoderTest, HandlesEdgeCases) {
- const char pieces_table[] = "hell\0hello\0o\0there\0";
- const uint32 offsets[] = {0, 5, 11, 13};
- float scores[] = {-0.5, -1.0, -10.0, -1.0};
- std::unique_ptr<StringSet> pieces(new SortedStringsTable(
- /*num_pieces=*/4, offsets, StringPiece(pieces_table, 18)));
- const Encoder encoder(pieces.get(),
- /*num_pieces=*/4, scores);
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("hellhello", &encoded_text));
- EXPECT_THAT(encoded_text, ElementsAre(0, 2, 3, 1));
- }
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("hellohell", &encoded_text));
- EXPECT_THAT(encoded_text, ElementsAre(0, 3, 2, 1));
- }
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("", &encoded_text));
- EXPECT_THAT(encoded_text, ElementsAre(0, 1));
- }
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("hellathere", &encoded_text));
- EXPECT_THAT(encoded_text, ElementsAre(0, 1));
- }
-}
-
-TEST(EncoderTest, HandlesOutOfDictionary) {
- const char pieces_table[] = "hell\0hello\0o\0there\0";
- const uint32 offsets[] = {0, 5, 11, 13};
- float scores[] = {-0.5, -1.0, -10.0, -1.0};
- std::unique_ptr<StringSet> pieces(new SortedStringsTable(
- /*num_pieces=*/4, offsets, StringPiece(pieces_table, 18)));
- const Encoder encoder(pieces.get(),
- /*num_pieces=*/4, scores,
- /*start_code=*/0, /*end_code=*/1,
- /*encoding_offset=*/3, /*unknown_code=*/2,
- /*unknown_score=*/-100.0);
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("hellhello", &encoded_text));
- EXPECT_THAT(encoded_text, ElementsAre(0, 3, 4, 1));
- }
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("hellohell", &encoded_text));
- EXPECT_THAT(encoded_text, ElementsAre(0, 4, 3, 1));
- }
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("", &encoded_text));
- EXPECT_THAT(encoded_text, ElementsAre(0, 1));
- }
- {
- std::vector<int> encoded_text;
- EXPECT_TRUE(encoder.Encode("hellathere", &encoded_text));
- EXPECT_THAT(encoded_text,
- ElementsAre(0, /*hell*/ 3, /*unknown*/ 2, /*there*/ 6, 1));
- }
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/sentencepiece/normalizer_test.cc b/native/utils/sentencepiece/normalizer_test.cc
deleted file mode 100644
index 5010035..0000000
--- a/native/utils/sentencepiece/normalizer_test.cc
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/sentencepiece/normalizer.h"
-
-#include <fstream>
-#include <string>
-
-#include "utils/container/double-array-trie.h"
-#include "utils/sentencepiece/test_utils.h"
-#include "utils/strings/stringpiece.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-std::string GetTestConfigPath() {
- return "";
-}
-
-TEST(NormalizerTest, NormalizesAsReferenceNormalizer) {
- std::ifstream test_config_stream(GetTestConfigPath());
- std::string config((std::istreambuf_iterator<char>(test_config_stream)),
- (std::istreambuf_iterator<char>()));
- SentencePieceNormalizer normalizer =
- NormalizerFromSpec(config, /*add_dummy_prefix=*/true,
- /*remove_extra_whitespaces=*/true,
- /*escape_whitespaces=*/true);
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("hello there", &normalized));
- EXPECT_EQ(normalized, "▁hello▁there");
- }
-
- // Redundant whitespace.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("when is the world cup?", &normalized));
- EXPECT_EQ(normalized, "▁when▁is▁the▁world▁cup?");
- }
-
- // Different whitespace.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("general\tkenobi", &normalized));
- EXPECT_EQ(normalized, "▁general▁kenobi");
- }
-
- // NFKC char to multi-char normalization.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("㍿", &normalized));
- EXPECT_EQ(normalized, "▁株式会社");
- }
-
- // Half width katakana, character composition happens.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize(" ｸﾞｰｸﾞﾙ ", &normalized));
- EXPECT_EQ(normalized, "▁グーグル");
- }
-
- // NFKC char to char normalization.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("①②③", &normalized));
- EXPECT_EQ(normalized, "▁123");
- }
-}
-
-TEST(NormalizerTest, NoDummyPrefix) {
- std::ifstream test_config_stream(GetTestConfigPath());
- std::string config((std::istreambuf_iterator<char>(test_config_stream)),
- (std::istreambuf_iterator<char>()));
- SentencePieceNormalizer normalizer =
- NormalizerFromSpec(config, /*add_dummy_prefix=*/false,
- /*remove_extra_whitespaces=*/true,
- /*escape_whitespaces=*/true);
-
- // NFKC char to char normalization.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("hello there", &normalized));
- EXPECT_EQ(normalized, "hello▁there");
- }
-
- // Redundant whitespace.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("when is the world cup?", &normalized));
- EXPECT_EQ(normalized, "when▁is▁the▁world▁cup?");
- }
-
- // Different whitespace.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("general\tkenobi", &normalized));
- EXPECT_EQ(normalized, "general▁kenobi");
- }
-
- // NFKC char to multi-char normalization.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("㍿", &normalized));
- EXPECT_EQ(normalized, "株式会社");
- }
-
- // Half width katakana, character composition happens.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize(" ｸﾞｰｸﾞﾙ ", &normalized));
- EXPECT_EQ(normalized, "グーグル");
- }
-
- // NFKC char to char normalization.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("①②③", &normalized));
- EXPECT_EQ(normalized, "123");
- }
-}
-
-TEST(NormalizerTest, NoRemoveExtraWhitespace) {
- std::ifstream test_config_stream(GetTestConfigPath());
- std::string config((std::istreambuf_iterator<char>(test_config_stream)),
- (std::istreambuf_iterator<char>()));
- SentencePieceNormalizer normalizer =
- NormalizerFromSpec(config, /*add_dummy_prefix=*/false,
- /*remove_extra_whitespaces=*/false,
- /*escape_whitespaces=*/true);
-
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("hello there", &normalized));
- EXPECT_EQ(normalized, "hello▁there");
- }
-
- // Redundant whitespace.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("when is the world cup?", &normalized));
- EXPECT_EQ(normalized, "when▁is▁▁the▁▁world▁cup?");
- }
-
- // Different whitespace.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("general\tkenobi", &normalized));
- EXPECT_EQ(normalized, "general▁kenobi");
- }
-}
-
-TEST(NormalizerTest, NoEscapeWhitespaces) {
- std::ifstream test_config_stream(GetTestConfigPath());
- std::string config((std::istreambuf_iterator<char>(test_config_stream)),
- (std::istreambuf_iterator<char>()));
- SentencePieceNormalizer normalizer =
- NormalizerFromSpec(config, /*add_dummy_prefix=*/false,
- /*remove_extra_whitespaces=*/false,
- /*escape_whitespaces=*/false);
-
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("hello there", &normalized));
- EXPECT_EQ(normalized, "hello there");
- }
-
- // Redundant whitespace.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("when is the world cup?", &normalized));
- EXPECT_EQ(normalized, "when is the world cup?");
- }
-
- // Different whitespace.
- {
- std::string normalized;
- EXPECT_TRUE(normalizer.Normalize("general\tkenobi", &normalized));
- EXPECT_EQ(normalized, "general kenobi");
- }
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/strings/append_test.cc b/native/utils/strings/append_test.cc
deleted file mode 100644
index 8950761..0000000
--- a/native/utils/strings/append_test.cc
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/strings/append.h"
-
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace strings {
-
-TEST(StringUtilTest, SStringAppendF) {
- std::string str;
- SStringAppendF(&str, 5, "%d %d", 0, 1);
- EXPECT_EQ(str, "0 1");
-
- SStringAppendF(&str, 1, "%d", 9);
- EXPECT_EQ(str, "0 19");
-
- SStringAppendF(&str, 1, "%d", 10);
- EXPECT_EQ(str, "0 191");
-
- str.clear();
-
- SStringAppendF(&str, 5, "%d", 100);
- EXPECT_EQ(str, "100");
-}
-
-TEST(StringUtilTest, SStringAppendFBufCalc) {
- std::string str;
- SStringAppendF(&str, 0, "%d %s %d", 1, "hello", 2);
- EXPECT_EQ(str, "1 hello 2");
-}
-
-TEST(StringUtilTest, JoinStrings) {
- std::vector<std::string> vec;
- vec.push_back("1");
- vec.push_back("2");
- vec.push_back("3");
-
- EXPECT_EQ("1,2,3", JoinStrings(",", vec));
- EXPECT_EQ("123", JoinStrings("", vec));
- EXPECT_EQ("1, 2, 3", JoinStrings(", ", vec));
- EXPECT_EQ("", JoinStrings(",", std::vector<std::string>()));
-}
-
-} // namespace strings
-} // namespace libtextclassifier3
diff --git a/native/utils/strings/numbers_test.cc b/native/utils/strings/numbers_test.cc
deleted file mode 100644
index bf2f84a..0000000
--- a/native/utils/strings/numbers_test.cc
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/strings/numbers.h"
-
-#include "utils/base/integral_types.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-void TestParseInt32(const char *c_str, bool expected_parsing_success,
- int32 expected_parsed_value = 0) {
- int32 parsed_value = 0;
- EXPECT_EQ(expected_parsing_success, ParseInt32(c_str, &parsed_value));
- if (expected_parsing_success) {
- EXPECT_EQ(expected_parsed_value, parsed_value);
- }
-}
-
-TEST(ParseInt32Test, Normal) {
- TestParseInt32("2", true, 2);
- TestParseInt32("-357", true, -357);
- TestParseInt32("7", true, 7);
- TestParseInt32("+7", true, 7);
- TestParseInt32(" +7", true, 7);
- TestParseInt32("-23", true, -23);
- TestParseInt32(" -23", true, -23);
- TestParseInt32("04", true, 4);
- TestParseInt32("07", true, 7);
- TestParseInt32("08", true, 8);
- TestParseInt32("09", true, 9);
-}
-
-TEST(ParseInt32Test, ErrorCases) {
- TestParseInt32("", false);
- TestParseInt32(" ", false);
- TestParseInt32("not-a-number", false);
- TestParseInt32("123a", false);
-}
-
-void TestParseInt64(const char *c_str, bool expected_parsing_success,
- int64 expected_parsed_value = 0) {
- int64 parsed_value = 0;
- EXPECT_EQ(expected_parsing_success, ParseInt64(c_str, &parsed_value));
- if (expected_parsing_success) {
- EXPECT_EQ(expected_parsed_value, parsed_value);
- }
-}
-
-TEST(ParseInt64Test, Normal) {
- TestParseInt64("2", true, 2);
- TestParseInt64("-357", true, -357);
- TestParseInt64("7", true, 7);
- TestParseInt64("+7", true, 7);
- TestParseInt64(" +7", true, 7);
- TestParseInt64("-23", true, -23);
- TestParseInt64(" -23", true, -23);
- TestParseInt64("07", true, 7);
- TestParseInt64("08", true, 8);
-}
-
-TEST(ParseInt64Test, ErrorCases) {
- TestParseInt64("", false);
- TestParseInt64(" ", false);
- TestParseInt64("not-a-number", false);
- TestParseInt64("23z", false);
-}
-
-void TestParseDouble(const char *c_str, bool expected_parsing_success,
- double expected_parsed_value = 0.0) {
- double parsed_value = 0.0;
- EXPECT_EQ(expected_parsing_success, ParseDouble(c_str, &parsed_value));
- if (expected_parsing_success) {
- EXPECT_NEAR(expected_parsed_value, parsed_value, 0.00001);
- }
-}
-
-TEST(ParseDoubleTest, Normal) {
- TestParseDouble("2", true, 2.0);
- TestParseDouble("-357.023", true, -357.023);
- TestParseDouble("7.04", true, 7.04);
- TestParseDouble("+7.2", true, 7.2);
- TestParseDouble(" +7.236", true, 7.236);
- TestParseDouble("-23.4", true, -23.4);
- TestParseDouble(" -23.4", true, -23.4);
-}
-
-TEST(ParseDoubleTest, ErrorCases) {
- TestParseDouble("", false);
- TestParseDouble(" ", false);
- TestParseDouble("not-a-number", false);
- TestParseDouble("23.5a", false);
-}
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/strings/stringpiece_test.cc b/native/utils/strings/stringpiece_test.cc
deleted file mode 100644
index 64808d3..0000000
--- a/native/utils/strings/stringpiece_test.cc
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-#include "utils/strings/stringpiece.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(StringPieceTest, EndsWith) {
- EXPECT_TRUE(EndsWith("hello there!", "there!"));
- EXPECT_TRUE(EndsWith("hello there!", "!"));
- EXPECT_FALSE(EndsWith("hello there!", "there"));
- EXPECT_FALSE(EndsWith("hello there!", " hello there!"));
- EXPECT_TRUE(EndsWith("hello there!", ""));
- EXPECT_FALSE(EndsWith("", "hello there!"));
-}
-
-TEST(StringPieceTest, StartsWith) {
- EXPECT_TRUE(StartsWith("hello there!", "hello"));
- EXPECT_TRUE(StartsWith("hello there!", "hello "));
- EXPECT_FALSE(StartsWith("hello there!", "there!"));
- EXPECT_FALSE(StartsWith("hello there!", " hello there! "));
- EXPECT_TRUE(StartsWith("hello there!", ""));
- EXPECT_FALSE(StartsWith("", "hello there!"));
-}
-
-TEST(StringPieceTest, ConsumePrefix) {
- StringPiece str("hello there!");
- EXPECT_TRUE(ConsumePrefix(&str, "hello "));
- EXPECT_EQ(str.ToString(), "there!");
- EXPECT_TRUE(ConsumePrefix(&str, "there"));
- EXPECT_EQ(str.ToString(), "!");
- EXPECT_FALSE(ConsumePrefix(&str, "!!"));
- EXPECT_TRUE(ConsumePrefix(&str, ""));
- EXPECT_TRUE(ConsumePrefix(&str, "!"));
- EXPECT_EQ(str.ToString(), "");
- EXPECT_TRUE(ConsumePrefix(&str, ""));
- EXPECT_FALSE(ConsumePrefix(&str, "!"));
-}
-
-TEST(StringPieceTest, ConsumeSuffix) {
- StringPiece str("hello there!");
- EXPECT_TRUE(ConsumeSuffix(&str, "!"));
- EXPECT_EQ(str.ToString(), "hello there");
- EXPECT_TRUE(ConsumeSuffix(&str, " there"));
- EXPECT_EQ(str.ToString(), "hello");
- EXPECT_FALSE(ConsumeSuffix(&str, "!!"));
- EXPECT_TRUE(ConsumeSuffix(&str, ""));
- EXPECT_TRUE(ConsumeSuffix(&str, "hello"));
- EXPECT_EQ(str.ToString(), "");
- EXPECT_TRUE(ConsumeSuffix(&str, ""));
- EXPECT_FALSE(ConsumeSuffix(&str, "!"));
-}
-
-TEST(StringPieceTest, Find) {
- StringPiece str("<hello there!>");
- EXPECT_EQ(str.find('<'), 0);
- EXPECT_EQ(str.find('>'), str.length() - 1);
- EXPECT_EQ(str.find('?'), StringPiece::npos);
- EXPECT_EQ(str.find('<', str.length() - 1), StringPiece::npos);
- EXPECT_EQ(str.find('<', 0), 0);
- EXPECT_EQ(str.find('>', str.length() - 1), str.length() - 1);
-}
-
-TEST(StringPieceTest, FindStringPiece) {
- StringPiece str("<foo bar baz!>");
- EXPECT_EQ(str.find("foo"), 1);
- EXPECT_EQ(str.find("bar"), 5);
- EXPECT_EQ(str.find("baz"), 9);
- EXPECT_EQ(str.find("qux"), StringPiece::npos);
- EXPECT_EQ(str.find("?"), StringPiece::npos);
- EXPECT_EQ(str.find(">"), str.length() - 1);
- EXPECT_EQ(str.find("<", str.length() - 1), StringPiece::npos);
- EXPECT_EQ(str.find("<", 0), 0);
- EXPECT_EQ(str.find(">", str.length() - 1), str.length() - 1);
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/strings/substitute_test.cc b/native/utils/strings/substitute_test.cc
deleted file mode 100644
index 94b37ab..0000000
--- a/native/utils/strings/substitute_test.cc
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/strings/substitute.h"
-
-#include <vector>
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-#include "utils/strings/stringpiece.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(SubstituteTest, Substitute) {
- EXPECT_EQ("Hello, world!",
- strings::Substitute("$0, $1!", {"Hello", "world"}));
-
- // Out of order.
- EXPECT_EQ("world, Hello!",
- strings::Substitute("$1, $0!", {"Hello", "world"}));
- EXPECT_EQ("b, a, c, b",
- strings::Substitute("$1, $0, $2, $1", {"a", "b", "c"}));
-
- // Literal $
- EXPECT_EQ("$", strings::Substitute("$$", {}));
- EXPECT_EQ("$1", strings::Substitute("$$1", {}));
-
- const char* null_cstring = nullptr;
- EXPECT_EQ("Text: ''", strings::Substitute("Text: '$0'", {null_cstring}));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/strings/utf8_test.cc b/native/utils/strings/utf8_test.cc
deleted file mode 100644
index 59f3864..0000000
--- a/native/utils/strings/utf8_test.cc
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/strings/utf8.h"
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::Eq;
-
-TEST(Utf8Test, GetNumBytesForUTF8Char) {
- EXPECT_THAT(GetNumBytesForUTF8Char("\x00"), Eq(0));
- EXPECT_THAT(GetNumBytesForUTF8Char("h"), Eq(1));
- EXPECT_THAT(GetNumBytesForUTF8Char("😋"), Eq(4));
- EXPECT_THAT(GetNumBytesForUTF8Char("㍿"), Eq(3));
-}
-
-TEST(Utf8Test, IsValidUTF8) {
- EXPECT_TRUE(IsValidUTF8("1234😋hello", 13));
- EXPECT_TRUE(IsValidUTF8("\u304A\u00B0\u106B", 8));
- EXPECT_TRUE(IsValidUTF8("this is a test😋😋😋", 26));
- EXPECT_TRUE(IsValidUTF8("\xf0\x9f\x98\x8b", 4));
- // Too short (string is too short).
- EXPECT_FALSE(IsValidUTF8("\xf0\x9f", 2));
- // Too long (too many trailing bytes).
- EXPECT_FALSE(IsValidUTF8("\xf0\x9f\x98\x8b\x8b", 5));
- // Too short (too few trailing bytes).
- EXPECT_FALSE(IsValidUTF8("\xf0\x9f\x98\x61\x61", 5));
-}
-
-TEST(Utf8Test, ValidUTF8CharLength) {
- EXPECT_THAT(ValidUTF8CharLength("1234😋hello", 13), Eq(1));
- EXPECT_THAT(ValidUTF8CharLength("\u304A\u00B0\u106B", 8), Eq(3));
- EXPECT_THAT(ValidUTF8CharLength("this is a test😋😋😋", 26), Eq(1));
- EXPECT_THAT(ValidUTF8CharLength("\xf0\x9f\x98\x8b", 4), Eq(4));
- // Too short (string is too short).
- EXPECT_THAT(ValidUTF8CharLength("\xf0\x9f", 2), Eq(-1));
- // Too long (too many trailing bytes). First character is valid.
- EXPECT_THAT(ValidUTF8CharLength("\xf0\x9f\x98\x8b\x8b", 5), Eq(4));
- // Too short (too few trailing bytes).
- EXPECT_THAT(ValidUTF8CharLength("\xf0\x9f\x98\x61\x61", 5), Eq(-1));
-}
-
-TEST(Utf8Test, CorrectlyTruncatesStrings) {
- EXPECT_THAT(SafeTruncateLength("FooBar", 3), Eq(3));
- EXPECT_THAT(SafeTruncateLength("früh", 3), Eq(2));
- EXPECT_THAT(SafeTruncateLength("مَمِمّمَّمِّ", 5), Eq(4));
-}
-
-TEST(Utf8Test, CorrectlyConvertsFromUtf8) {
- EXPECT_THAT(ValidCharToRune("a"), Eq(97));
- EXPECT_THAT(ValidCharToRune("\0"), Eq(0));
- EXPECT_THAT(ValidCharToRune("\u304A"), Eq(0x304a));
- EXPECT_THAT(ValidCharToRune("\xe3\x81\x8a"), Eq(0x304a));
-}
-
-TEST(Utf8Test, CorrectlyConvertsToUtf8) {
- char utf8_encoding[4];
- EXPECT_THAT(ValidRuneToChar(97, utf8_encoding), Eq(1));
- EXPECT_THAT(ValidRuneToChar(0, utf8_encoding), Eq(1));
- EXPECT_THAT(ValidRuneToChar(0x304a, utf8_encoding), Eq(3));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/tensor-view_test.cc b/native/utils/tensor-view_test.cc
deleted file mode 100644
index 9467264..0000000
--- a/native/utils/tensor-view_test.cc
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/tensor-view.h"
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(TensorViewTest, TestSize) {
- std::vector<float> data{0.1, 0.2, 0.3, 0.4, 0.5, 0.6};
- const TensorView<float> tensor(data.data(), {3, 1, 2});
- EXPECT_TRUE(tensor.is_valid());
- EXPECT_EQ(tensor.shape(), (std::vector<int>{3, 1, 2}));
- EXPECT_EQ(tensor.data(), data.data());
- EXPECT_EQ(tensor.size(), 6);
- EXPECT_EQ(tensor.dims(), 3);
- EXPECT_EQ(tensor.dim(0), 3);
- EXPECT_EQ(tensor.dim(1), 1);
- EXPECT_EQ(tensor.dim(2), 2);
- std::vector<float> output_data(6);
- EXPECT_TRUE(tensor.copy_to(output_data.data(), output_data.size()));
- EXPECT_EQ(data, output_data);
-
- // Should not copy when the output is small.
- std::vector<float> small_output_data{-1, -1, -1};
- EXPECT_FALSE(
- tensor.copy_to(small_output_data.data(), small_output_data.size()));
- // The output buffer should not be changed.
- EXPECT_EQ(small_output_data, (std::vector<float>{-1, -1, -1}));
-
- const TensorView<float> invalid_tensor = TensorView<float>::Invalid();
- EXPECT_FALSE(invalid_tensor.is_valid());
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/test-utils.cc b/native/utils/test-utils.cc
deleted file mode 100644
index cad7be5..0000000
--- a/native/utils/test-utils.cc
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/test-utils.h"
-
-#include <iterator>
-
-#include "utils/codepoint-range.h"
-#include "utils/strings/utf8.h"
-#include "utils/utf8/unicodetext.h"
-
-namespace libtextclassifier3 {
-
-using libtextclassifier3::Token;
-
-std::vector<Token> TokenizeOnSpace(const std::string& text) {
- const char32 space_codepoint = ValidCharToRune(" ");
- const UnicodeText unicode_text = UTF8ToUnicodeText(text, /*do_copy=*/false);
-
- std::vector<Token> result;
-
- int token_start_codepoint = 0;
- auto token_start_it = unicode_text.begin();
- int codepoint_idx = 0;
-
- UnicodeText::const_iterator it;
- for (it = unicode_text.begin(); it < unicode_text.end(); it++) {
- if (*it == space_codepoint) {
- result.push_back(Token{UnicodeText::UTF8Substring(token_start_it, it),
- token_start_codepoint, codepoint_idx});
-
- token_start_codepoint = codepoint_idx + 1;
- token_start_it = it;
- token_start_it++;
- }
-
- codepoint_idx++;
- }
- result.push_back(Token{UnicodeText::UTF8Substring(token_start_it, it),
- token_start_codepoint, codepoint_idx});
-
- return result;
-}
-
-} // namespace libtextclassifier3
diff --git a/native/utils/test-utils.h b/native/utils/test-utils.h
deleted file mode 100644
index 582736f..0000000
--- a/native/utils/test-utils.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Utilities for tests.
-
-#ifndef LIBTEXTCLASSIFIER_UTILS_TEST_UTILS_H_
-#define LIBTEXTCLASSIFIER_UTILS_TEST_UTILS_H_
-
-#include <string>
-
-#include "annotator/types.h"
-
-namespace libtextclassifier3 {
-
-// Returns a list of Tokens for a given input string.
-std::vector<Token> TokenizeOnSpace(const std::string& text);
-
-} // namespace libtextclassifier3
-
-#endif // LIBTEXTCLASSIFIER_UTILS_TEST_UTILS_H_
diff --git a/native/utils/test-utils_test.cc b/native/utils/test-utils_test.cc
deleted file mode 100644
index f596de4..0000000
--- a/native/utils/test-utils_test.cc
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/test-utils.h"
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(TestUtilTest, Utf8) {
- std::vector<Token> tokens =
- TokenizeOnSpace("Where is Jörg Borg located? Maybe in Zürich ...");
-
- EXPECT_EQ(tokens.size(), 9);
-
- EXPECT_EQ(tokens[0].value, "Where");
- EXPECT_EQ(tokens[0].start, 0);
- EXPECT_EQ(tokens[0].end, 5);
-
- EXPECT_EQ(tokens[1].value, "is");
- EXPECT_EQ(tokens[1].start, 6);
- EXPECT_EQ(tokens[1].end, 8);
-
- EXPECT_EQ(tokens[2].value, "Jörg");
- EXPECT_EQ(tokens[2].start, 9);
- EXPECT_EQ(tokens[2].end, 13);
-
- EXPECT_EQ(tokens[3].value, "Borg");
- EXPECT_EQ(tokens[3].start, 14);
- EXPECT_EQ(tokens[3].end, 18);
-
- EXPECT_EQ(tokens[4].value, "located?");
- EXPECT_EQ(tokens[4].start, 19);
- EXPECT_EQ(tokens[4].end, 27);
-
- EXPECT_EQ(tokens[5].value, "Maybe");
- EXPECT_EQ(tokens[5].start, 28);
- EXPECT_EQ(tokens[5].end, 33);
-
- EXPECT_EQ(tokens[6].value, "in");
- EXPECT_EQ(tokens[6].start, 34);
- EXPECT_EQ(tokens[6].end, 36);
-
- EXPECT_EQ(tokens[7].value, "Zürich");
- EXPECT_EQ(tokens[7].start, 37);
- EXPECT_EQ(tokens[7].end, 43);
-
- EXPECT_EQ(tokens[8].value, "...");
- EXPECT_EQ(tokens[8].start, 44);
- EXPECT_EQ(tokens[8].end, 47);
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/testing/annotator.cc b/native/utils/testing/annotator.cc
deleted file mode 100644
index 43ed1df..0000000
--- a/native/utils/testing/annotator.cc
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/testing/annotator.h"
-
-namespace libtextclassifier3 {
-
-std::string FirstResult(const std::vector<ClassificationResult>& results) {
- if (results.empty()) {
- return "<INVALID RESULTS>";
- }
- return results[0].collection;
-}
-
-std::string ReadFile(const std::string& file_name) {
- std::ifstream file_stream(file_name);
- return std::string(std::istreambuf_iterator<char>(file_stream), {});
-}
-
-std::unique_ptr<RegexModel_::PatternT> MakePattern(
- const std::string& collection_name, const std::string& pattern,
- const bool enabled_for_classification, const bool enabled_for_selection,
- const bool enabled_for_annotation, const float score,
- const float priority_score) {
- std::unique_ptr<RegexModel_::PatternT> result(new RegexModel_::PatternT);
- result->collection_name = collection_name;
- result->pattern = pattern;
- // We cannot directly operate with |= on the flag, so use an int here.
- int enabled_modes = ModeFlag_NONE;
- if (enabled_for_annotation) enabled_modes |= ModeFlag_ANNOTATION;
- if (enabled_for_classification) enabled_modes |= ModeFlag_CLASSIFICATION;
- if (enabled_for_selection) enabled_modes |= ModeFlag_SELECTION;
- result->enabled_modes = static_cast<ModeFlag>(enabled_modes);
- result->target_classification_score = score;
- result->priority_score = priority_score;
- return result;
-}
-
-// Shortcut function that doesn't need to specify the priority score.
-std::unique_ptr<RegexModel_::PatternT> MakePattern(
- const std::string& collection_name, const std::string& pattern,
- const bool enabled_for_classification, const bool enabled_for_selection,
- const bool enabled_for_annotation, const float score) {
- return MakePattern(collection_name, pattern, enabled_for_classification,
- enabled_for_selection, enabled_for_annotation,
- /*score=*/score,
- /*priority_score=*/score);
-}
-
-void AddTestRegexModel(ModelT* unpacked_model) {
- // Add test regex models.
- unpacked_model->regex_model->patterns.push_back(MakePattern(
- "person_with_age", "(Barack) (?:(Obama) )?is (\\d+) years old",
- /*enabled_for_classification=*/true,
- /*enabled_for_selection=*/true, /*enabled_for_annotation=*/true, 1.0));
-
- // Use meta data to generate custom serialized entity data.
- ReflectiveFlatbufferBuilder entity_data_builder(
- flatbuffers::GetRoot<reflection::Schema>(
- unpacked_model->entity_data_schema.data()));
- RegexModel_::PatternT* pattern =
- unpacked_model->regex_model->patterns.back().get();
-
- {
- std::unique_ptr<ReflectiveFlatbuffer> entity_data =
- entity_data_builder.NewRoot();
- entity_data->Set("is_alive", true);
- pattern->serialized_entity_data = entity_data->Serialize();
- }
- pattern->capturing_group.emplace_back(new CapturingGroupT);
- pattern->capturing_group.emplace_back(new CapturingGroupT);
- pattern->capturing_group.emplace_back(new CapturingGroupT);
- pattern->capturing_group.emplace_back(new CapturingGroupT);
- // Group 0 is the full match, capturing groups starting at 1.
- pattern->capturing_group[1]->entity_field_path.reset(
- new FlatbufferFieldPathT);
- pattern->capturing_group[1]->entity_field_path->field.emplace_back(
- new FlatbufferFieldT);
- pattern->capturing_group[1]->entity_field_path->field.back()->field_name =
- "first_name";
- pattern->capturing_group[2]->entity_field_path.reset(
- new FlatbufferFieldPathT);
- pattern->capturing_group[2]->entity_field_path->field.emplace_back(
- new FlatbufferFieldT);
- pattern->capturing_group[2]->entity_field_path->field.back()->field_name =
- "last_name";
- // Set `former_us_president` field if we match Obama.
- {
- std::unique_ptr<ReflectiveFlatbuffer> entity_data =
- entity_data_builder.NewRoot();
- entity_data->Set("former_us_president", true);
- pattern->capturing_group[2]->serialized_entity_data =
- entity_data->Serialize();
- }
- pattern->capturing_group[3]->entity_field_path.reset(
- new FlatbufferFieldPathT);
- pattern->capturing_group[3]->entity_field_path->field.emplace_back(
- new FlatbufferFieldT);
- pattern->capturing_group[3]->entity_field_path->field.back()->field_name =
- "age";
-}
-
-std::string CreateEmptyModel() {
- ModelT model;
- flatbuffers::FlatBufferBuilder builder;
- FinishModelBuffer(builder, Model::Pack(builder, &model));
- return std::string(reinterpret_cast<const char*>(builder.GetBufferPointer()),
- builder.GetSize());
-}
-
-// Create fake entity data schema meta data.
-void AddTestEntitySchemaData(ModelT* unpacked_model) {
- // Cannot use object oriented API here as that is not available for the
- // reflection schema.
- flatbuffers::FlatBufferBuilder schema_builder;
- std::vector<flatbuffers::Offset<reflection::Field>> fields = {
- reflection::CreateField(
- schema_builder,
- /*name=*/schema_builder.CreateString("first_name"),
- /*type=*/
- reflection::CreateType(schema_builder,
- /*base_type=*/reflection::String),
- /*id=*/0,
- /*offset=*/4),
- reflection::CreateField(
- schema_builder,
- /*name=*/schema_builder.CreateString("is_alive"),
- /*type=*/
- reflection::CreateType(schema_builder,
- /*base_type=*/reflection::Bool),
- /*id=*/1,
- /*offset=*/6),
- reflection::CreateField(
- schema_builder,
- /*name=*/schema_builder.CreateString("last_name"),
- /*type=*/
- reflection::CreateType(schema_builder,
- /*base_type=*/reflection::String),
- /*id=*/2,
- /*offset=*/8),
- reflection::CreateField(
- schema_builder,
- /*name=*/schema_builder.CreateString("age"),
- /*type=*/
- reflection::CreateType(schema_builder,
- /*base_type=*/reflection::Int),
- /*id=*/3,
- /*offset=*/10),
- reflection::CreateField(
- schema_builder,
- /*name=*/schema_builder.CreateString("former_us_president"),
- /*type=*/
- reflection::CreateType(schema_builder,
- /*base_type=*/reflection::Bool),
- /*id=*/4,
- /*offset=*/12)};
- std::vector<flatbuffers::Offset<reflection::Enum>> enums;
- std::vector<flatbuffers::Offset<reflection::Object>> objects = {
- reflection::CreateObject(
- schema_builder,
- /*name=*/schema_builder.CreateString("EntityData"),
- /*fields=*/
- schema_builder.CreateVectorOfSortedTables(&fields))};
- schema_builder.Finish(reflection::CreateSchema(
- schema_builder, schema_builder.CreateVectorOfSortedTables(&objects),
- schema_builder.CreateVectorOfSortedTables(&enums),
- /*(unused) file_ident=*/0,
- /*(unused) file_ext=*/0,
- /*root_table*/ objects[0]));
-
- unpacked_model->entity_data_schema.assign(
- schema_builder.GetBufferPointer(),
- schema_builder.GetBufferPointer() + schema_builder.GetSize());
-}
-
-AnnotatedSpan MakeAnnotatedSpan(CodepointSpan span,
- const std::string& collection,
- const float score,
- AnnotatedSpan::Source source) {
- AnnotatedSpan result;
- result.span = span;
- result.classification.push_back({collection, score});
- result.source = source;
- return result;
-}
-
-} // namespace libtextclassifier3
diff --git a/native/utils/testing/annotator.h b/native/utils/testing/annotator.h
deleted file mode 100644
index 1565e0d..0000000
--- a/native/utils/testing/annotator.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Helper utilities for testing Annotator.
-
-#ifndef LIBTEXTCLASSIFIER_UTILS_TESTING_ANNOTATOR_H_
-#define LIBTEXTCLASSIFIER_UTILS_TESTING_ANNOTATOR_H_
-
-#include <fstream>
-#include <memory>
-#include <string>
-
-#include "annotator/model_generated.h"
-#include "annotator/types.h"
-#include "flatbuffers/flatbuffers.h"
-
-namespace libtextclassifier3 {
-
-// Loads FlatBuffer model, unpacks it and passes it to the visitor_fn so that it
-// can modify it. Afterwards the modified unpacked model is serialized back to a
-// flatbuffer.
-template <typename Fn>
-std::string ModifyAnnotatorModel(const std::string& model_flatbuffer,
- Fn visitor_fn) {
- std::unique_ptr<ModelT> unpacked_model =
- UnPackModel(model_flatbuffer.c_str());
-
- visitor_fn(unpacked_model.get());
-
- flatbuffers::FlatBufferBuilder builder;
- FinishModelBuffer(builder, Model::Pack(builder, unpacked_model.get()));
-
- return std::string(reinterpret_cast<char*>(builder.GetBufferPointer()),
- builder.GetSize());
-}
-
-std::string FirstResult(const std::vector<ClassificationResult>& results);
-
-std::string ReadFile(const std::string& file_name);
-
-std::unique_ptr<RegexModel_::PatternT> MakePattern(
- const std::string& collection_name, const std::string& pattern,
- const bool enabled_for_classification, const bool enabled_for_selection,
- const bool enabled_for_annotation, const float score,
- const float priority_score);
-
-// Shortcut function that doesn't need to specify the priority score.
-std::unique_ptr<RegexModel_::PatternT> MakePattern(
- const std::string& collection_name, const std::string& pattern,
- const bool enabled_for_classification, const bool enabled_for_selection,
- const bool enabled_for_annotation, const float score);
-
-void AddTestRegexModel(ModelT* unpacked_model);
-
-std::string CreateEmptyModel();
-
-// Create fake entity data schema meta data.
-void AddTestEntitySchemaData(ModelT* unpacked_model);
-
-AnnotatedSpan MakeAnnotatedSpan(
- CodepointSpan span, const std::string& collection, const float score,
- AnnotatedSpan::Source source = AnnotatedSpan::Source::OTHER);
-
-} // namespace libtextclassifier3
-
-#endif // LIBTEXTCLASSIFIER_UTILS_TESTING_ANNOTATOR_H_
diff --git a/native/utils/testing/logging_event_listener.h b/native/utils/testing/logging_event_listener.h
deleted file mode 100644
index 2663a9c..0000000
--- a/native/utils/testing/logging_event_listener.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LIBTEXTCLASSIFIER_UTILS_TESTING_LOGGING_EVENT_LISTENER_H_
-#define LIBTEXTCLASSIFIER_UTILS_TESTING_LOGGING_EVENT_LISTENER_H_
-
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-
-// TestEventListener that writes test results to the log so that they will be
-// visible in the logcat output in Sponge.
-// The formatting of the output is patterend after the output produced by the
-// standard PrettyUnitTestResultPrinter.
-class LoggingEventListener : public ::testing::TestEventListener {
- public:
- void OnTestProgramStart(const testing::UnitTest& unit_test) override;
-
- void OnTestIterationStart(const testing::UnitTest& unit_test,
- int iteration) override;
-
- void OnEnvironmentsSetUpStart(const testing::UnitTest& unit_test) override;
-
- void OnEnvironmentsSetUpEnd(const testing::UnitTest& unit_test) override;
-
- void OnTestCaseStart(const testing::TestCase& test_case) override;
-
- void OnTestStart(const testing::TestInfo& test_info) override;
-
- void OnTestPartResult(
- const testing::TestPartResult& test_part_result) override;
-
- void OnTestEnd(const testing::TestInfo& test_info) override;
-
- void OnTestCaseEnd(const testing::TestCase& test_case) override;
-
- void OnEnvironmentsTearDownStart(const testing::UnitTest& unit_test) override;
-
- void OnEnvironmentsTearDownEnd(const testing::UnitTest& unit_test) override;
-
- void OnTestIterationEnd(const testing::UnitTest& unit_test,
- int iteration) override;
-
- void OnTestProgramEnd(const testing::UnitTest& unit_test) override;
-};
-
-} // namespace libtextclassifier3
-
-#endif // LIBTEXTCLASSIFIER_UTILS_TESTING_LOGGING_EVENT_LISTENER_H_
diff --git a/native/utils/tflite/dist_diversification_test.cc b/native/utils/tflite/dist_diversification_test.cc
deleted file mode 100644
index 2380116..0000000
--- a/native/utils/tflite/dist_diversification_test.cc
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/tflite/dist_diversification.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-#include "tensorflow/lite/interpreter.h"
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/kernels/test_util.h"
-#include "tensorflow/lite/model.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-class DistanceDiversificationOpModel : public tflite::SingleOpModel {
- public:
- explicit DistanceDiversificationOpModel(int matrix_rows);
- void SetDistanceMatrix(const std::initializer_list<float>& values) {
- PopulateTensor(distance_matrix_, values);
- }
- void SetNumOutput(int length) { PopulateTensor(num_results_, {length}); }
- void SetMinDistance(float min_distance) {
- PopulateTensor(min_distance_, {min_distance});
- }
- int GetOutputLen() { return ExtractVector<int>(output_len_).front(); }
- std::vector<int> GetOutputIndexes(int output_length) {
- auto res = ExtractVector<int>(output_indexes_);
- res.resize(output_length);
- return res;
- }
-
- private:
- int distance_matrix_;
- int num_results_;
- int min_distance_;
-
- int output_len_;
- int output_indexes_;
-};
-
-DistanceDiversificationOpModel::DistanceDiversificationOpModel(
- int matrix_rows) {
- distance_matrix_ = AddInput(tflite::TensorType_FLOAT32);
- min_distance_ = AddInput(tflite::TensorType_FLOAT32);
- num_results_ = AddInput(tflite::TensorType_INT32);
-
- output_indexes_ = AddOutput(tflite::TensorType_INT32);
- output_len_ = AddOutput(tflite::TensorType_INT32);
- SetCustomOp("DistanceDiversification", {},
- tflite::ops::custom::Register_DISTANCE_DIVERSIFICATION);
- BuildInterpreter({{matrix_rows, matrix_rows}, {1}, {1}});
-}
-
-// Tests
-TEST(DistanceDiversificationOp, Simple) {
- DistanceDiversificationOpModel m(5);
- m.SetDistanceMatrix({0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.0, 0.1, 0.2,
- 0.3, 0.2, 0.1, 0.0, 0.1, 0.2, 0.3, 0.2, 0.1,
- 0.0, 0.1, 0.4, 0.3, 0.2, 0.1, 0.0});
- m.SetMinDistance(0.21);
- m.SetNumOutput(3);
- m.Invoke();
- const int output_length = m.GetOutputLen();
- EXPECT_EQ(output_length, 2);
- EXPECT_THAT(m.GetOutputIndexes(output_length), testing::ElementsAre(0, 3));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/tflite/encoder_common_test.cc b/native/utils/tflite/encoder_common_test.cc
deleted file mode 100644
index 247689f..0000000
--- a/native/utils/tflite/encoder_common_test.cc
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/tflite/encoder_common.h"
-
-#include "gtest/gtest.h"
-#include "tensorflow/lite/model.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(EncoderUtilsTest, CreateIntArray) {
- TfLiteIntArray* a = CreateIntArray({1, 2, 3});
- EXPECT_EQ(a->data[0], 1);
- EXPECT_EQ(a->data[1], 2);
- EXPECT_EQ(a->data[2], 3);
- TfLiteIntArrayFree(a);
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/tflite/text_encoder_test.cc b/native/utils/tflite/text_encoder_test.cc
deleted file mode 100644
index 6386432..0000000
--- a/native/utils/tflite/text_encoder_test.cc
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <fstream>
-#include <string>
-#include <vector>
-
-#include "utils/tflite/text_encoder.h"
-#include "gtest/gtest.h"
-#include "third_party/absl/flags/flag.h"
-#include "flatbuffers/flexbuffers.h"
-#include "tensorflow/lite/interpreter.h"
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/kernels/test_util.h"
-#include "tensorflow/lite/model.h"
-#include "tensorflow/lite/string_util.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-std::string GetTestConfigPath() {
- return "";
-}
-
-class TextEncoderOpModel : public tflite::SingleOpModel {
- public:
- TextEncoderOpModel(std::initializer_list<int> input_strings_shape,
- std::initializer_list<int> attribute_shape);
- void SetInputText(const std::initializer_list<std::string>& strings) {
- PopulateStringTensor(input_string_, strings);
- PopulateTensor(input_length_, {static_cast<int32_t>(strings.size())});
- }
- void SetMaxOutputLength(int length) {
- PopulateTensor(input_output_maxlength_, {length});
- }
- void SetInt32Attribute(const std::initializer_list<int>& attribute) {
- PopulateTensor(input_attributes_int32_, attribute);
- }
- void SetFloatAttribute(const std::initializer_list<float>& attribute) {
- PopulateTensor(input_attributes_float_, attribute);
- }
-
- std::vector<int> GetOutputEncoding() {
- return ExtractVector<int>(output_encoding_);
- }
- std::vector<int> GetOutputPositions() {
- return ExtractVector<int>(output_positions_);
- }
- std::vector<int> GetOutputAttributeInt32() {
- return ExtractVector<int>(output_attributes_int32_);
- }
- std::vector<float> GetOutputAttributeFloat() {
- return ExtractVector<float>(output_attributes_float_);
- }
- int GetEncodedLength() { return ExtractVector<int>(output_length_)[0]; }
-
- private:
- int input_string_;
- int input_length_;
- int input_output_maxlength_;
- int input_attributes_int32_;
- int input_attributes_float_;
-
- int output_encoding_;
- int output_positions_;
- int output_length_;
- int output_attributes_int32_;
- int output_attributes_float_;
-};
-
-TextEncoderOpModel::TextEncoderOpModel(
- std::initializer_list<int> input_strings_shape,
- std::initializer_list<int> attribute_shape) {
- input_string_ = AddInput(tflite::TensorType_STRING);
- input_length_ = AddInput(tflite::TensorType_INT32);
- input_output_maxlength_ = AddInput(tflite::TensorType_INT32);
- input_attributes_int32_ = AddInput(tflite::TensorType_INT32);
- input_attributes_float_ = AddInput(tflite::TensorType_FLOAT32);
-
- output_encoding_ = AddOutput(tflite::TensorType_INT32);
- output_positions_ = AddOutput(tflite::TensorType_INT32);
- output_length_ = AddOutput(tflite::TensorType_INT32);
- output_attributes_int32_ = AddOutput(tflite::TensorType_INT32);
- output_attributes_float_ = AddOutput(tflite::TensorType_FLOAT32);
-
- std::ifstream test_config_stream(GetTestConfigPath());
- std::string config((std::istreambuf_iterator<char>(test_config_stream)),
- (std::istreambuf_iterator<char>()));
- flexbuffers::Builder builder;
- builder.Map([&]() { builder.String("text_encoder_config", config); });
- builder.Finish();
- SetCustomOp("TextEncoder", builder.GetBuffer(),
- tflite::ops::custom::Register_TEXT_ENCODER);
- BuildInterpreter(
- {input_strings_shape, {1}, {1}, attribute_shape, attribute_shape});
-}
-
-// Tests
-TEST(TextEncoderTest, SimpleEncoder) {
- TextEncoderOpModel m({1, 1}, {1, 1});
- m.SetInputText({"Hello"});
- m.SetMaxOutputLength(10);
- m.SetInt32Attribute({7});
- m.SetFloatAttribute({3.f});
-
- m.Invoke();
-
- EXPECT_EQ(m.GetEncodedLength(), 5);
- EXPECT_THAT(m.GetOutputEncoding(),
- testing::ElementsAre(1, 90, 547, 58, 2, 2, 2, 2, 2, 2));
- EXPECT_THAT(m.GetOutputPositions(),
- testing::ElementsAre(0, 1, 2, 3, 4, 10, 10, 10, 10, 10));
- EXPECT_THAT(m.GetOutputAttributeInt32(),
- testing::ElementsAre(7, 7, 7, 7, 7, 7, 7, 7, 7, 7));
- EXPECT_THAT(
- m.GetOutputAttributeFloat(),
- testing::ElementsAre(3.f, 3.f, 3.f, 3.f, 3.f, 3.f, 3.f, 3.f, 3.f, 3.f));
-}
-
-TEST(TextEncoderTest, ManyStrings) {
- TextEncoderOpModel m({1, 3}, {1, 3});
- m.SetInt32Attribute({1, 2, 3});
- m.SetFloatAttribute({5.f, 4.f, 3.f});
- m.SetInputText({"Hello", "Hi", "Bye"});
- m.SetMaxOutputLength(10);
-
- m.Invoke();
-
- EXPECT_EQ(m.GetEncodedLength(), 10);
- EXPECT_THAT(m.GetOutputEncoding(),
- testing::ElementsAre(547, 58, 2, 1, 862, 2, 1, 1919, 19, 2));
- EXPECT_THAT(m.GetOutputPositions(),
- testing::ElementsAre(2, 3, 4, 0, 1, 2, 0, 1, 2, 3));
- EXPECT_THAT(m.GetOutputAttributeInt32(),
- testing::ElementsAre(1, 1, 1, 2, 2, 2, 3, 3, 3, 3));
- EXPECT_THAT(
- m.GetOutputAttributeFloat(),
- testing::ElementsAre(5.f, 5.f, 5.f, 4.f, 4.f, 4.f, 3.f, 3.f, 3.f, 3.f));
-}
-
-TEST(TextEncoderTest, LongStrings) {
- TextEncoderOpModel m({1, 4}, {1, 4});
- m.SetInt32Attribute({1, 2, 3, 4});
- m.SetFloatAttribute({5.f, 4.f, 3.f, 2.f});
- m.SetInputText({"Hello", "Hi", "Bye", "Hi"});
- m.SetMaxOutputLength(9);
-
- m.Invoke();
-
- EXPECT_EQ(m.GetEncodedLength(), 9);
- EXPECT_THAT(m.GetOutputEncoding(),
- testing::ElementsAre(862, 2, 1, 1919, 19, 2, 1, 862, 2));
- EXPECT_THAT(m.GetOutputPositions(),
- testing::ElementsAre(1, 2, 0, 1, 2, 3, 0, 1, 2));
- EXPECT_THAT(m.GetOutputAttributeInt32(),
- testing::ElementsAre(2, 2, 3, 3, 3, 3, 4, 4, 4));
- EXPECT_THAT(
- m.GetOutputAttributeFloat(),
- testing::ElementsAre(4.f, 4.f, 3.f, 3.f, 3.f, 3.f, 2.f, 2.f, 2.f));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/tflite/token_encoder_test.cc b/native/utils/tflite/token_encoder_test.cc
deleted file mode 100644
index c7f51a1..0000000
--- a/native/utils/tflite/token_encoder_test.cc
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <vector>
-
-#include "utils/tflite/token_encoder.h"
-#include "gtest/gtest.h"
-#include "third_party/absl/flags/flag.h"
-#include "tensorflow/lite/interpreter.h"
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/kernels/test_util.h"
-#include "tensorflow/lite/model.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-class TokenEncoderOpModel : public tflite::SingleOpModel {
- public:
- TokenEncoderOpModel(std::initializer_list<int> input_shape,
- std::initializer_list<int> attribute_shape);
- void SetNumTokens(const std::initializer_list<int>& num_tokens) {
- PopulateTensor(input_num_tokens_, num_tokens);
- PopulateTensor(input_length_, {static_cast<int32_t>(num_tokens.size())});
- }
- void SetMaxOutputLength(int length) {
- PopulateTensor(input_output_maxlength_, {length});
- }
- void SetInt32Attribute(const std::initializer_list<int>& attribute) {
- PopulateTensor(input_attributes_int32_, attribute);
- }
- void SetFloatAttribute(const std::initializer_list<float>& attribute) {
- PopulateTensor(input_attributes_float_, attribute);
- }
- std::vector<int> GetOutputPositions() {
- return ExtractVector<int>(output_positions_);
- }
- std::vector<int> GetOutputAttributeInt32() {
- return ExtractVector<int>(output_attributes_int32_);
- }
- std::vector<float> GetOutputAttributeFloat() {
- return ExtractVector<float>(output_attributes_float_);
- }
- int GetOutputLength() { return ExtractVector<int>(output_length_)[0]; }
-
- private:
- int input_num_tokens_;
- int input_length_;
- int input_output_maxlength_;
- int input_attributes_int32_;
- int input_attributes_float_;
-
- int output_positions_;
- int output_length_;
- int output_attributes_int32_;
- int output_attributes_float_;
-};
-
-TokenEncoderOpModel::TokenEncoderOpModel(
- std::initializer_list<int> input_shape,
- std::initializer_list<int> attribute_shape) {
- input_num_tokens_ = AddInput(tflite::TensorType_INT32);
- input_length_ = AddInput(tflite::TensorType_INT32);
- input_output_maxlength_ = AddInput(tflite::TensorType_INT32);
- input_attributes_int32_ = AddInput(tflite::TensorType_INT32);
- input_attributes_float_ = AddInput(tflite::TensorType_FLOAT32);
-
- output_positions_ = AddOutput(tflite::TensorType_INT32);
- output_length_ = AddOutput(tflite::TensorType_INT32);
- output_attributes_int32_ = AddOutput(tflite::TensorType_INT32);
- output_attributes_float_ = AddOutput(tflite::TensorType_FLOAT32);
-
- SetCustomOp("TokenEncoder", {}, tflite::ops::custom::Register_TOKEN_ENCODER);
- BuildInterpreter({input_shape, {1}, {1}, attribute_shape, attribute_shape});
-}
-
-// Tests
-TEST(TokenEncoderTest, SimpleEncoder) {
- TokenEncoderOpModel m({1, 1}, {1, 1});
- m.SetNumTokens({1});
- m.SetMaxOutputLength(10);
- m.SetInt32Attribute({7});
- m.SetFloatAttribute({3.f});
-
- m.Invoke();
-
- EXPECT_EQ(m.GetOutputLength(), 3);
- EXPECT_THAT(m.GetOutputPositions(),
- testing::ElementsAre(0, 1, 2, 10, 10, 10, 10, 10, 10, 10));
- EXPECT_THAT(m.GetOutputAttributeInt32(),
- testing::ElementsAre(7, 7, 7, 7, 7, 7, 7, 7, 7, 7));
- EXPECT_THAT(
- m.GetOutputAttributeFloat(),
- testing::ElementsAre(3.f, 3.f, 3.f, 3.f, 3.f, 3.f, 3.f, 3.f, 3.f, 3.f));
-}
-
-TEST(TokenEncoderTest, ManyMessages) {
- TokenEncoderOpModel m({1, 3}, {1, 3});
- m.SetInt32Attribute({1, 2, 3});
- m.SetFloatAttribute({5.f, 4.f, 3.f});
- m.SetNumTokens({1, 1, 1});
- m.SetMaxOutputLength(10);
-
- m.Invoke();
-
- EXPECT_EQ(m.GetOutputLength(), 9);
- EXPECT_THAT(m.GetOutputPositions(),
- testing::ElementsAre(0, 1, 2, 0, 1, 2, 0, 1, 2, 10));
- EXPECT_THAT(m.GetOutputAttributeInt32(),
- testing::ElementsAre(1, 1, 1, 2, 2, 2, 3, 3, 3, 3));
- EXPECT_THAT(
- m.GetOutputAttributeFloat(),
- testing::ElementsAre(5.f, 5.f, 5.f, 4.f, 4.f, 4.f, 3.f, 3.f, 3.f, 3.f));
-}
-
-TEST(TokenEncoderTest, ManyMessagesMultipleTokens) {
- TokenEncoderOpModel m({1, 4}, {1, 4});
- m.SetInt32Attribute({1, 2, 3, 4});
- m.SetFloatAttribute({5.f, 4.f, 3.f, 2.f});
- m.SetNumTokens({1, 2, 3, 4});
- m.SetMaxOutputLength(9);
-
- m.Invoke();
-
- EXPECT_EQ(m.GetOutputLength(), 9);
- EXPECT_THAT(m.GetOutputPositions(),
- testing::ElementsAre(2, 3, 4, 0, 1, 2, 3, 4, 5));
- EXPECT_THAT(m.GetOutputAttributeInt32(),
- testing::ElementsAre(3, 3, 3, 4, 4, 4, 4, 4, 4));
- EXPECT_THAT(
- m.GetOutputAttributeFloat(),
- testing::ElementsAre(3.f, 3.f, 3.f, 2.f, 2.f, 2.f, 2.f, 2.f, 2.f));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/token-feature-extractor.cc b/native/utils/token-feature-extractor.cc
index 9faebca..b14f96e 100644
--- a/native/utils/token-feature-extractor.cc
+++ b/native/utils/token-feature-extractor.cc
@@ -109,8 +109,7 @@
if (options_.unicode_aware_features) {
UnicodeText token_unicode =
UTF8ToUnicodeText(token.value, /*do_copy=*/false);
- const bool is_upper = unilib_.IsUpper(*token_unicode.begin());
- if (!token.value.empty() && is_upper) {
+ if (!token.value.empty() && unilib_.IsUpper(*token_unicode.begin())) {
dense_features.push_back(1.0);
} else {
dense_features.push_back(-1.0);
diff --git a/native/utils/token-feature-extractor_test.cc b/native/utils/token-feature-extractor_test.cc
deleted file mode 100644
index 9a97e42..0000000
--- a/native/utils/token-feature-extractor_test.cc
+++ /dev/null
@@ -1,556 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/token-feature-extractor.h"
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-class TokenFeatureExtractorTest : public ::testing::Test {
- protected:
- TokenFeatureExtractorTest() : INIT_UNILIB_FOR_TESTING(unilib_) {}
- UniLib unilib_;
-};
-
-class TestingTokenFeatureExtractor : public TokenFeatureExtractor {
- public:
- using TokenFeatureExtractor::HashToken;
- using TokenFeatureExtractor::TokenFeatureExtractor;
-};
-
-TEST_F(TokenFeatureExtractorTest, ExtractAscii) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2, 3};
- options.extract_case_feature = true;
- options.unicode_aware_features = false;
- options.extract_selection_mask_feature = true;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
-
- extractor.Extract(Token{"Hello", 0, 5}, true, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({
- // clang-format off
- extractor.HashToken("H"),
- extractor.HashToken("e"),
- extractor.HashToken("l"),
- extractor.HashToken("l"),
- extractor.HashToken("o"),
- extractor.HashToken("^H"),
- extractor.HashToken("He"),
- extractor.HashToken("el"),
- extractor.HashToken("ll"),
- extractor.HashToken("lo"),
- extractor.HashToken("o$"),
- extractor.HashToken("^He"),
- extractor.HashToken("Hel"),
- extractor.HashToken("ell"),
- extractor.HashToken("llo"),
- extractor.HashToken("lo$")
- // clang-format on
- }));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({1.0, 1.0}));
-
- sparse_features.clear();
- dense_features.clear();
- extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({
- // clang-format off
- extractor.HashToken("w"),
- extractor.HashToken("o"),
- extractor.HashToken("r"),
- extractor.HashToken("l"),
- extractor.HashToken("d"),
- extractor.HashToken("!"),
- extractor.HashToken("^w"),
- extractor.HashToken("wo"),
- extractor.HashToken("or"),
- extractor.HashToken("rl"),
- extractor.HashToken("ld"),
- extractor.HashToken("d!"),
- extractor.HashToken("!$"),
- extractor.HashToken("^wo"),
- extractor.HashToken("wor"),
- extractor.HashToken("orl"),
- extractor.HashToken("rld"),
- extractor.HashToken("ld!"),
- extractor.HashToken("d!$"),
- // clang-format on
- }));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, 0.0}));
-}
-
-TEST_F(TokenFeatureExtractorTest, ExtractAsciiNoChargrams) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{};
- options.extract_case_feature = true;
- options.unicode_aware_features = false;
- options.extract_selection_mask_feature = true;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
-
- extractor.Extract(Token{"Hello", 0, 5}, true, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({extractor.HashToken("^Hello$")}));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({1.0, 1.0}));
-
- sparse_features.clear();
- dense_features.clear();
- extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({extractor.HashToken("^world!$")}));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, 0.0}));
-}
-
-TEST_F(TokenFeatureExtractorTest, ExtractUnicode) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2, 3};
- options.extract_case_feature = true;
- options.unicode_aware_features = true;
- options.extract_selection_mask_feature = true;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
-
- extractor.Extract(Token{"Hělló", 0, 5}, true, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({
- // clang-format off
- extractor.HashToken("H"),
- extractor.HashToken("ě"),
- extractor.HashToken("l"),
- extractor.HashToken("l"),
- extractor.HashToken("ó"),
- extractor.HashToken("^H"),
- extractor.HashToken("Hě"),
- extractor.HashToken("ěl"),
- extractor.HashToken("ll"),
- extractor.HashToken("ló"),
- extractor.HashToken("ó$"),
- extractor.HashToken("^Hě"),
- extractor.HashToken("Hěl"),
- extractor.HashToken("ěll"),
- extractor.HashToken("lló"),
- extractor.HashToken("ló$")
- // clang-format on
- }));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({1.0, 1.0}));
-
- sparse_features.clear();
- dense_features.clear();
- extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({
- // clang-format off
- extractor.HashToken("w"),
- extractor.HashToken("o"),
- extractor.HashToken("r"),
- extractor.HashToken("l"),
- extractor.HashToken("d"),
- extractor.HashToken("!"),
- extractor.HashToken("^w"),
- extractor.HashToken("wo"),
- extractor.HashToken("or"),
- extractor.HashToken("rl"),
- extractor.HashToken("ld"),
- extractor.HashToken("d!"),
- extractor.HashToken("!$"),
- extractor.HashToken("^wo"),
- extractor.HashToken("wor"),
- extractor.HashToken("orl"),
- extractor.HashToken("rld"),
- extractor.HashToken("ld!"),
- extractor.HashToken("d!$"),
- // clang-format on
- }));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, -1.0}));
-}
-
-TEST_F(TokenFeatureExtractorTest, ExtractUnicodeNoChargrams) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{};
- options.extract_case_feature = true;
- options.unicode_aware_features = true;
- options.extract_selection_mask_feature = true;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
-
- extractor.Extract(Token{"Hělló", 0, 5}, true, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({extractor.HashToken("^Hělló$")}));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({1.0, 1.0}));
-
- sparse_features.clear();
- dense_features.clear();
- extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features, testing::ElementsAreArray({
- extractor.HashToken("^world!$"),
- }));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, -1.0}));
-}
-
-#ifdef TC3_TEST_ICU
-TEST_F(TokenFeatureExtractorTest, ICUCaseFeature) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2};
- options.extract_case_feature = true;
- options.unicode_aware_features = true;
- options.extract_selection_mask_feature = false;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
- extractor.Extract(Token{"Hělló", 0, 5}, true, &sparse_features,
- &dense_features);
- EXPECT_THAT(dense_features, testing::ElementsAreArray({1.0}));
-
- sparse_features.clear();
- dense_features.clear();
- extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features,
- &dense_features);
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0}));
-
- sparse_features.clear();
- dense_features.clear();
- extractor.Extract(Token{"Ř", 23, 29}, false, &sparse_features,
- &dense_features);
- EXPECT_THAT(dense_features, testing::ElementsAreArray({1.0}));
-
- sparse_features.clear();
- dense_features.clear();
- extractor.Extract(Token{"ř", 23, 29}, false, &sparse_features,
- &dense_features);
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0}));
-}
-#endif
-
-TEST_F(TokenFeatureExtractorTest, DigitRemapping) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2};
- options.remap_digits = true;
- options.unicode_aware_features = false;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
- extractor.Extract(Token{"9:30am", 0, 6}, true, &sparse_features,
- &dense_features);
-
- std::vector<int> sparse_features2;
- extractor.Extract(Token{"5:32am", 0, 6}, true, &sparse_features2,
- &dense_features);
- EXPECT_THAT(sparse_features, testing::ElementsAreArray(sparse_features2));
-
- extractor.Extract(Token{"10:32am", 0, 6}, true, &sparse_features2,
- &dense_features);
- EXPECT_THAT(sparse_features,
- testing::Not(testing::ElementsAreArray(sparse_features2)));
-}
-
-TEST_F(TokenFeatureExtractorTest, DigitRemappingUnicode) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2};
- options.remap_digits = true;
- options.unicode_aware_features = true;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
- extractor.Extract(Token{"9:30am", 0, 6}, true, &sparse_features,
- &dense_features);
-
- std::vector<int> sparse_features2;
- extractor.Extract(Token{"5:32am", 0, 6}, true, &sparse_features2,
- &dense_features);
- EXPECT_THAT(sparse_features, testing::ElementsAreArray(sparse_features2));
-
- extractor.Extract(Token{"10:32am", 0, 6}, true, &sparse_features2,
- &dense_features);
- EXPECT_THAT(sparse_features,
- testing::Not(testing::ElementsAreArray(sparse_features2)));
-}
-
-TEST_F(TokenFeatureExtractorTest, LowercaseAscii) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2};
- options.lowercase_tokens = true;
- options.unicode_aware_features = false;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
- extractor.Extract(Token{"AABB", 0, 6}, true, &sparse_features,
- &dense_features);
-
- std::vector<int> sparse_features2;
- extractor.Extract(Token{"aaBB", 0, 6}, true, &sparse_features2,
- &dense_features);
- EXPECT_THAT(sparse_features, testing::ElementsAreArray(sparse_features2));
-
- extractor.Extract(Token{"aAbB", 0, 6}, true, &sparse_features2,
- &dense_features);
- EXPECT_THAT(sparse_features, testing::ElementsAreArray(sparse_features2));
-}
-
-#ifdef TC3_TEST_ICU
-TEST_F(TokenFeatureExtractorTest, LowercaseUnicode) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2};
- options.lowercase_tokens = true;
- options.unicode_aware_features = true;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
- extractor.Extract(Token{"ŘŘ", 0, 6}, true, &sparse_features, &dense_features);
-
- std::vector<int> sparse_features2;
- extractor.Extract(Token{"řř", 0, 6}, true, &sparse_features2,
- &dense_features);
- EXPECT_THAT(sparse_features, testing::ElementsAreArray(sparse_features2));
-}
-#endif
-
-#ifdef TC3_TEST_ICU
-TEST_F(TokenFeatureExtractorTest, RegexFeatures) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2};
- options.remap_digits = false;
- options.unicode_aware_features = false;
- options.regexp_features.push_back("^[a-z]+$"); // all lower case.
- options.regexp_features.push_back("^[0-9]+$"); // all digits.
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
- extractor.Extract(Token{"abCde", 0, 6}, true, &sparse_features,
- &dense_features);
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, -1.0}));
-
- dense_features.clear();
- extractor.Extract(Token{"abcde", 0, 6}, true, &sparse_features,
- &dense_features);
- EXPECT_THAT(dense_features, testing::ElementsAreArray({1.0, -1.0}));
-
- dense_features.clear();
- extractor.Extract(Token{"12c45", 0, 6}, true, &sparse_features,
- &dense_features);
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, -1.0}));
-
- dense_features.clear();
- extractor.Extract(Token{"12345", 0, 6}, true, &sparse_features,
- &dense_features);
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, 1.0}));
-}
-#endif
-
-TEST_F(TokenFeatureExtractorTest, ExtractTooLongWord) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{22};
- options.extract_case_feature = true;
- options.unicode_aware_features = true;
- options.extract_selection_mask_feature = true;
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- // Test that this runs. ASAN should catch problems.
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
- extractor.Extract(Token{"abcdefghijklmnopqřstuvwxyz", 0, 0}, true,
- &sparse_features, &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({
- // clang-format off
- extractor.HashToken("^abcdefghij\1qřstuvwxyz"),
- extractor.HashToken("abcdefghij\1qřstuvwxyz$"),
- // clang-format on
- }));
-}
-
-TEST_F(TokenFeatureExtractorTest, ExtractAsciiUnicodeMatches) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2, 3, 4, 5};
- options.extract_case_feature = true;
- options.unicode_aware_features = true;
- options.extract_selection_mask_feature = true;
-
- TestingTokenFeatureExtractor extractor_unicode(options, unilib_);
-
- options.unicode_aware_features = false;
- TestingTokenFeatureExtractor extractor_ascii(options, unilib_);
-
- for (const std::string& input :
- {"https://www.abcdefgh.com/in/xxxkkkvayio",
- "https://www.fjsidofj.om/xx/abadfy/xxxx/?xfjiis=ffffiijiihil",
- "asdfhasdofjiasdofj#%()*%#*(aisdojfaosdifjiaofjdsiofjdi_fdis3w", "abcd",
- "x", "Hello", "Hey,", "Hi", ""}) {
- std::vector<int> sparse_features_unicode;
- std::vector<float> dense_features_unicode;
- extractor_unicode.Extract(Token{input, 0, 0}, true,
- &sparse_features_unicode,
- &dense_features_unicode);
-
- std::vector<int> sparse_features_ascii;
- std::vector<float> dense_features_ascii;
- extractor_ascii.Extract(Token{input, 0, 0}, true, &sparse_features_ascii,
- &dense_features_ascii);
-
- EXPECT_THAT(sparse_features_unicode, sparse_features_ascii) << input;
- EXPECT_THAT(dense_features_unicode, dense_features_ascii) << input;
- }
-}
-
-TEST_F(TokenFeatureExtractorTest, ExtractForPadToken) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2};
- options.extract_case_feature = true;
- options.unicode_aware_features = false;
- options.extract_selection_mask_feature = true;
-
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
-
- extractor.Extract(Token(), false, &sparse_features, &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({extractor.HashToken("<PAD>")}));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, 0.0}));
-}
-
-TEST_F(TokenFeatureExtractorTest, ExtractFiltered) {
- TokenFeatureExtractorOptions options;
- options.num_buckets = 1000;
- options.chargram_orders = std::vector<int>{1, 2, 3};
- options.extract_case_feature = true;
- options.unicode_aware_features = false;
- options.extract_selection_mask_feature = true;
- options.allowed_chargrams.insert("^H");
- options.allowed_chargrams.insert("ll");
- options.allowed_chargrams.insert("llo");
- options.allowed_chargrams.insert("w");
- options.allowed_chargrams.insert("!");
- options.allowed_chargrams.insert("\xc4"); // UTF8 control character.
-
- TestingTokenFeatureExtractor extractor(options, unilib_);
-
- std::vector<int> sparse_features;
- std::vector<float> dense_features;
-
- extractor.Extract(Token{"Hěllo", 0, 5}, true, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features,
- testing::ElementsAreArray({
- // clang-format off
- 0,
- extractor.HashToken("\xc4"),
- 0,
- 0,
- 0,
- 0,
- extractor.HashToken("^H"),
- 0,
- 0,
- 0,
- extractor.HashToken("ll"),
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- extractor.HashToken("llo"),
- 0
- // clang-format on
- }));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({1.0, 1.0}));
-
- sparse_features.clear();
- dense_features.clear();
- extractor.Extract(Token{"world!", 23, 29}, false, &sparse_features,
- &dense_features);
-
- EXPECT_THAT(sparse_features, testing::ElementsAreArray({
- // clang-format off
- extractor.HashToken("w"),
- 0,
- 0,
- 0,
- 0,
- extractor.HashToken("!"),
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- // clang-format on
- }));
- EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, 0.0}));
- EXPECT_EQ(extractor.HashToken("<PAD>"), 1);
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/tokenizer_test.cc b/native/utils/tokenizer_test.cc
deleted file mode 100644
index 0f5501d..0000000
--- a/native/utils/tokenizer_test.cc
+++ /dev/null
@@ -1,580 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/tokenizer.h"
-
-#include <vector>
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-using testing::ElementsAreArray;
-
-class TestingTokenizer : public Tokenizer {
- public:
- TestingTokenizer(
- const TokenizationType type, const UniLib* unilib,
- const std::vector<const TokenizationCodepointRange*>& codepoint_ranges,
- const std::vector<const CodepointRange*>&
- internal_tokenizer_codepoint_ranges,
- const bool split_on_script_change,
- const bool icu_preserve_whitespace_tokens,
- const bool preserve_floating_numbers)
- : Tokenizer(type, unilib, codepoint_ranges,
- internal_tokenizer_codepoint_ranges, split_on_script_change,
- icu_preserve_whitespace_tokens, preserve_floating_numbers) {}
-
- using Tokenizer::FindTokenizationRange;
-};
-
-class TestingTokenizerProxy {
- public:
- TestingTokenizerProxy(
- TokenizationType type,
- const std::vector<TokenizationCodepointRangeT>& codepoint_range_configs,
- const std::vector<CodepointRangeT>& internal_codepoint_range_configs,
- const bool split_on_script_change,
- const bool icu_preserve_whitespace_tokens,
- const bool preserve_floating_numbers)
- : INIT_UNILIB_FOR_TESTING(unilib_) {
- const int num_configs = codepoint_range_configs.size();
- std::vector<const TokenizationCodepointRange*> configs_fb;
- configs_fb.reserve(num_configs);
- const int num_internal_configs = internal_codepoint_range_configs.size();
- std::vector<const CodepointRange*> internal_configs_fb;
- internal_configs_fb.reserve(num_internal_configs);
- buffers_.reserve(num_configs + num_internal_configs);
- for (int i = 0; i < num_configs; i++) {
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(CreateTokenizationCodepointRange(
- builder, &codepoint_range_configs[i]));
- buffers_.push_back(builder.Release());
- configs_fb.push_back(flatbuffers::GetRoot<TokenizationCodepointRange>(
- buffers_.back().data()));
- }
- for (int i = 0; i < num_internal_configs; i++) {
- flatbuffers::FlatBufferBuilder builder;
- builder.Finish(
- CreateCodepointRange(builder, &internal_codepoint_range_configs[i]));
- buffers_.push_back(builder.Release());
- internal_configs_fb.push_back(
- flatbuffers::GetRoot<CodepointRange>(buffers_.back().data()));
- }
- tokenizer_ = std::unique_ptr<TestingTokenizer>(new TestingTokenizer(
- type, &unilib_, configs_fb, internal_configs_fb, split_on_script_change,
- icu_preserve_whitespace_tokens, preserve_floating_numbers));
- }
-
- TokenizationCodepointRange_::Role TestFindTokenizationRole(int c) const {
- const TokenizationCodepointRangeT* range =
- tokenizer_->FindTokenizationRange(c);
- if (range != nullptr) {
- return range->role;
- } else {
- return TokenizationCodepointRange_::Role_DEFAULT_ROLE;
- }
- }
-
- std::vector<Token> Tokenize(const std::string& utf8_text) const {
- return tokenizer_->Tokenize(utf8_text);
- }
-
- private:
- UniLib unilib_;
- std::vector<flatbuffers::DetachedBuffer> buffers_;
- std::unique_ptr<TestingTokenizer> tokenizer_;
-};
-
-TEST(TokenizerTest, FindTokenizationRange) {
- std::vector<TokenizationCodepointRangeT> configs;
- TokenizationCodepointRangeT* config;
-
- configs.emplace_back();
- config = &configs.back();
- config->start = 0;
- config->end = 10;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
-
- configs.emplace_back();
- config = &configs.back();
- config->start = 32;
- config->end = 33;
- config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
-
- configs.emplace_back();
- config = &configs.back();
- config->start = 1234;
- config->end = 12345;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
-
- TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, configs,
- {}, /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
-
- // Test hits to the first group.
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(0),
- TokenizationCodepointRange_::Role_TOKEN_SEPARATOR);
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(5),
- TokenizationCodepointRange_::Role_TOKEN_SEPARATOR);
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(10),
- TokenizationCodepointRange_::Role_DEFAULT_ROLE);
-
- // Test a hit to the second group.
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(31),
- TokenizationCodepointRange_::Role_DEFAULT_ROLE);
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(32),
- TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR);
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(33),
- TokenizationCodepointRange_::Role_DEFAULT_ROLE);
-
- // Test hits to the third group.
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(1233),
- TokenizationCodepointRange_::Role_DEFAULT_ROLE);
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(1234),
- TokenizationCodepointRange_::Role_TOKEN_SEPARATOR);
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(12344),
- TokenizationCodepointRange_::Role_TOKEN_SEPARATOR);
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(12345),
- TokenizationCodepointRange_::Role_DEFAULT_ROLE);
-
- // Test a hit outside.
- EXPECT_EQ(tokenizer.TestFindTokenizationRole(99),
- TokenizationCodepointRange_::Role_DEFAULT_ROLE);
-}
-
-TEST(TokenizerTest, TokenizeOnSpace) {
- std::vector<TokenizationCodepointRangeT> configs;
- TokenizationCodepointRangeT* config;
-
- configs.emplace_back();
- config = &configs.back();
- // Space character.
- config->start = 32;
- config->end = 33;
- config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
-
- TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, configs,
- {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
- std::vector<Token> tokens = tokenizer.Tokenize("Hello world!");
-
- EXPECT_THAT(tokens,
- ElementsAreArray({Token("Hello", 0, 5), Token("world!", 6, 12)}));
-}
-
-TEST(TokenizerTest, TokenizeOnSpaceAndScriptChange) {
- std::vector<TokenizationCodepointRangeT> configs;
- TokenizationCodepointRangeT* config;
-
- // Latin.
- configs.emplace_back();
- config = &configs.back();
- config->start = 0;
- config->end = 32;
- config->role = TokenizationCodepointRange_::Role_DEFAULT_ROLE;
- config->script_id = 1;
- configs.emplace_back();
- config = &configs.back();
- config->start = 32;
- config->end = 33;
- config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
- config->script_id = 1;
- configs.emplace_back();
- config = &configs.back();
- config->start = 33;
- config->end = 0x77F + 1;
- config->role = TokenizationCodepointRange_::Role_DEFAULT_ROLE;
- config->script_id = 1;
-
- TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, configs,
- {},
- /*split_on_script_change=*/true,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
- EXPECT_THAT(tokenizer.Tokenize("앨라배마 주 전화(123) 456-789웹사이트"),
- std::vector<Token>({Token("앨라배마", 0, 4), Token("주", 5, 6),
- Token("전화", 7, 10), Token("(123)", 10, 15),
- Token("456-789", 16, 23),
- Token("웹사이트", 23, 28)}));
-} // namespace
-
-TEST(TokenizerTest, TokenizeComplex) {
- std::vector<TokenizationCodepointRangeT> configs;
- TokenizationCodepointRangeT* config;
-
- // Source: http://www.unicode.org/Public/10.0.0/ucd/Blocks-10.0.0d1.txt
- // Latin - cyrilic.
- // 0000..007F; Basic Latin
- // 0080..00FF; Latin-1 Supplement
- // 0100..017F; Latin Extended-A
- // 0180..024F; Latin Extended-B
- // 0250..02AF; IPA Extensions
- // 02B0..02FF; Spacing Modifier Letters
- // 0300..036F; Combining Diacritical Marks
- // 0370..03FF; Greek and Coptic
- // 0400..04FF; Cyrillic
- // 0500..052F; Cyrillic Supplement
- // 0530..058F; Armenian
- // 0590..05FF; Hebrew
- // 0600..06FF; Arabic
- // 0700..074F; Syriac
- // 0750..077F; Arabic Supplement
- configs.emplace_back();
- config = &configs.back();
- config->start = 0;
- config->end = 32;
- config->role = TokenizationCodepointRange_::Role_DEFAULT_ROLE;
- configs.emplace_back();
- config = &configs.back();
- config->start = 32;
- config->end = 33;
- config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 33;
- config->end = 0x77F + 1;
- config->role = TokenizationCodepointRange_::Role_DEFAULT_ROLE;
-
- // CJK
- // 2E80..2EFF; CJK Radicals Supplement
- // 3000..303F; CJK Symbols and Punctuation
- // 3040..309F; Hiragana
- // 30A0..30FF; Katakana
- // 3100..312F; Bopomofo
- // 3130..318F; Hangul Compatibility Jamo
- // 3190..319F; Kanbun
- // 31A0..31BF; Bopomofo Extended
- // 31C0..31EF; CJK Strokes
- // 31F0..31FF; Katakana Phonetic Extensions
- // 3200..32FF; Enclosed CJK Letters and Months
- // 3300..33FF; CJK Compatibility
- // 3400..4DBF; CJK Unified Ideographs Extension A
- // 4DC0..4DFF; Yijing Hexagram Symbols
- // 4E00..9FFF; CJK Unified Ideographs
- // A000..A48F; Yi Syllables
- // A490..A4CF; Yi Radicals
- // A4D0..A4FF; Lisu
- // A500..A63F; Vai
- // F900..FAFF; CJK Compatibility Ideographs
- // FE30..FE4F; CJK Compatibility Forms
- // 20000..2A6DF; CJK Unified Ideographs Extension B
- // 2A700..2B73F; CJK Unified Ideographs Extension C
- // 2B740..2B81F; CJK Unified Ideographs Extension D
- // 2B820..2CEAF; CJK Unified Ideographs Extension E
- // 2CEB0..2EBEF; CJK Unified Ideographs Extension F
- // 2F800..2FA1F; CJK Compatibility Ideographs Supplement
- configs.emplace_back();
- config = &configs.back();
- config->start = 0x2E80;
- config->end = 0x2EFF + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 0x3000;
- config->end = 0xA63F + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 0xF900;
- config->end = 0xFAFF + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 0xFE30;
- config->end = 0xFE4F + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 0x20000;
- config->end = 0x2A6DF + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 0x2A700;
- config->end = 0x2B73F + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 0x2B740;
- config->end = 0x2B81F + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 0x2B820;
- config->end = 0x2CEAF + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 0x2CEB0;
- config->end = 0x2EBEF + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
- configs.emplace_back();
- config = &configs.back();
- config->start = 0x2F800;
- config->end = 0x2FA1F + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
-
- // Thai.
- // 0E00..0E7F; Thai
- configs.emplace_back();
- config = &configs.back();
- config->start = 0x0E00;
- config->end = 0x0E7F + 1;
- config->role = TokenizationCodepointRange_::Role_TOKEN_SEPARATOR;
-
- TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, configs,
- {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
- std::vector<Token> tokens;
-
- tokens = tokenizer.Tokenize(
- "問少目木輸走猶術権自京門録球変。細開括省用掲情結傍走愛明氷。");
- EXPECT_EQ(tokens.size(), 30);
-
- tokens = tokenizer.Tokenize("問少目 hello 木輸ยามきゃ");
- // clang-format off
- EXPECT_THAT(
- tokens,
- ElementsAreArray({Token("問", 0, 1),
- Token("少", 1, 2),
- Token("目", 2, 3),
- Token("hello", 4, 9),
- Token("木", 10, 11),
- Token("輸", 11, 12),
- Token("ย", 12, 13),
- Token("า", 13, 14),
- Token("ม", 14, 15),
- Token("き", 15, 16),
- Token("ゃ", 16, 17)}));
- // clang-format on
-}
-
-#ifdef TC3_TEST_ICU
-TEST(TokenizerTest, ICUTokenize) {
- TestingTokenizerProxy tokenizer(TokenizationType_ICU, {}, {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
- std::vector<Token> tokens = tokenizer.Tokenize("พระบาทสมเด็จพระปรมิ");
- ASSERT_EQ(tokens,
- // clang-format off
- std::vector<Token>({Token("พระบาท", 0, 6),
- Token("สมเด็จ", 6, 12),
- Token("พระ", 12, 15),
- Token("ปร", 15, 17),
- Token("มิ", 17, 19)}));
- // clang-format on
-}
-
-TEST(TokenizerTest, ICUTokenizeWithWhitespaces) {
- TestingTokenizerProxy tokenizer(TokenizationType_ICU, {}, {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/true,
- /*preserve_floating_numbers=*/false);
- std::vector<Token> tokens = tokenizer.Tokenize("พระบาท สมเด็จ พระ ปร มิ");
- ASSERT_EQ(tokens,
- // clang-format off
- std::vector<Token>({Token("พระบาท", 0, 6),
- Token(" ", 6, 7),
- Token("สมเด็จ", 7, 13),
- Token(" ", 13, 14),
- Token("พระ", 14, 17),
- Token(" ", 17, 18),
- Token("ปร", 18, 20),
- Token(" ", 20, 21),
- Token("มิ", 21, 23)}));
- // clang-format on
-}
-
-TEST(TokenizerTest, MixedTokenize) {
- std::vector<TokenizationCodepointRangeT> configs;
- TokenizationCodepointRangeT* config;
-
- configs.emplace_back();
- config = &configs.back();
- config->start = 32;
- config->end = 33;
- config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;
-
- std::vector<CodepointRangeT> internal_configs;
- CodepointRangeT* interal_config;
-
- internal_configs.emplace_back();
- interal_config = &internal_configs.back();
- interal_config->start = 0;
- interal_config->end = 128;
-
- internal_configs.emplace_back();
- interal_config = &internal_configs.back();
- interal_config->start = 128;
- interal_config->end = 256;
-
- internal_configs.emplace_back();
- interal_config = &internal_configs.back();
- interal_config->start = 256;
- interal_config->end = 384;
-
- internal_configs.emplace_back();
- interal_config = &internal_configs.back();
- interal_config->start = 384;
- interal_config->end = 592;
-
- TestingTokenizerProxy tokenizer(TokenizationType_MIXED, configs,
- internal_configs,
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
-
- std::vector<Token> tokens = tokenizer.Tokenize(
- "こんにちはJapanese-ląnguagę text 你好世界 http://www.google.com/");
- ASSERT_EQ(
- tokens,
- // clang-format off
- std::vector<Token>({Token("こんにちは", 0, 5),
- Token("Japanese-ląnguagę", 5, 22),
- Token("text", 23, 27),
- Token("你好", 28, 30),
- Token("世界", 30, 32),
- Token("http://www.google.com/", 33, 55)}));
- // clang-format on
-}
-
-TEST(TokenizerTest, InternalTokenizeOnScriptChange) {
- std::vector<TokenizationCodepointRangeT> configs;
- TokenizationCodepointRangeT* config;
-
- configs.emplace_back();
- config = &configs.back();
- config->start = 0;
- config->end = 256;
- config->role = TokenizationCodepointRange_::Role_DEFAULT_ROLE;
-
- {
- TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER,
- configs, {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
-
- EXPECT_EQ(tokenizer.Tokenize("앨라배마123웹사이트"),
- std::vector<Token>({Token("앨라배마123웹사이트", 0, 11)}));
- }
-
- {
- TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER,
- configs, {},
- /*split_on_script_change=*/true,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
- EXPECT_EQ(tokenizer.Tokenize("앨라배마123웹사이트"),
- std::vector<Token>({Token("앨라배마", 0, 4), Token("123", 4, 7),
- Token("웹사이트", 7, 11)}));
- }
-}
-#endif
-
-TEST(TokenizerTest, LetterDigitTokenize) {
- TestingTokenizerProxy tokenizer(TokenizationType_LETTER_DIGIT, {}, {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/true);
- std::vector<Token> tokens = tokenizer.Tokenize("7% -3.14 68.9#? 7% $99 .18.");
- ASSERT_EQ(tokens,
- std::vector<Token>(
- {Token("7", 0, 1), Token("%", 1, 2), Token(" ", 2, 3),
- Token("-", 3, 4), Token("3.14", 4, 8), Token(" ", 8, 9),
- Token("68.9", 9, 13), Token("#", 13, 14), Token("?", 14, 15),
- Token(" ", 15, 16), Token("7", 16, 17), Token("%", 17, 18),
- Token(" ", 18, 19), Token("$", 19, 20), Token("99", 20, 22),
- Token(" ", 22, 23), Token(".", 23, 24), Token("18", 24, 26),
- Token(".", 26, 27)}));
-}
-
-TEST(TokenizerTest, LetterDigitTokenizeUnicode) {
- TestingTokenizerProxy tokenizer(TokenizationType_LETTER_DIGIT, {}, {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/true);
- std::vector<Token> tokens = tokenizer.Tokenize("２ pércént ３パーセント");
- ASSERT_EQ(tokens, std::vector<Token>({Token("２", 0, 1), Token(" ", 1, 2),
- Token("pércént", 2, 9),
- Token(" ", 9, 10), Token("３", 10, 11),
- Token("パーセント", 11, 16)}));
-}
-
-TEST(TokenizerTest, LetterDigitTokenizeWithDots) {
- TestingTokenizerProxy tokenizer(TokenizationType_LETTER_DIGIT, {}, {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/true);
- std::vector<Token> tokens = tokenizer.Tokenize("3 3﹒2 3．3%");
- ASSERT_EQ(tokens,
- std::vector<Token>({Token("3", 0, 1), Token(" ", 1, 2),
- Token("3﹒2", 2, 5), Token(" ", 5, 6),
- Token("3．3", 6, 9), Token("%", 9, 10)}));
-}
-
-TEST(TokenizerTest, LetterDigitTokenizeDoNotPreserveFloatingNumbers) {
- TestingTokenizerProxy tokenizer(TokenizationType_LETTER_DIGIT, {}, {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
- std::vector<Token> tokens = tokenizer.Tokenize("15.12.2019 january's 3.2");
- ASSERT_EQ(tokens,
- std::vector<Token>(
- {Token("15", 0, 2), Token(".", 2, 3), Token("12", 3, 5),
- Token(".", 5, 6), Token("2019", 6, 10), Token(" ", 10, 11),
- Token("january", 11, 18), Token("'", 18, 19),
- Token("s", 19, 20), Token(" ", 20, 21), Token("3", 21, 22),
- Token(".", 22, 23), Token("2", 23, 24)}));
-}
-
-TEST(TokenizerTest, LetterDigitTokenizeStrangeStringFloatingNumbers) {
- TestingTokenizerProxy tokenizer(TokenizationType_LETTER_DIGIT, {}, {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
- std::vector<Token> tokens = tokenizer.Tokenize("The+2345++the +íí+");
- ASSERT_EQ(tokens,
- std::vector<Token>({Token("The", 0, 3), Token("+", 3, 4),
- Token("2345", 4, 8), Token("+", 8, 9),
- Token("+", 9, 10), Token("the", 10, 13),
- Token(" ", 13, 14), Token("+", 14, 15),
- Token("íí", 15, 17), Token("+", 17, 18)}));
-}
-
-TEST(TokenizerTest, LetterDigitTokenizeWhitespcesInSameToken) {
- TestingTokenizerProxy tokenizer(TokenizationType_LETTER_DIGIT, {}, {},
- /*split_on_script_change=*/false,
- /*icu_preserve_whitespace_tokens=*/false,
- /*preserve_floating_numbers=*/false);
- std::vector<Token> tokens = tokenizer.Tokenize("2 3 4 5");
- ASSERT_EQ(tokens, std::vector<Token>({Token("2", 0, 1), Token(" ", 1, 2),
- Token("3", 2, 3), Token(" ", 3, 5),
- Token("4", 5, 6), Token(" ", 6, 9),
- Token("5", 9, 10)}));
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/utf8/unicodetext.h b/native/utils/utf8/unicodetext.h
index cf78e47..9810480 100644
--- a/native/utils/utf8/unicodetext.h
+++ b/native/utils/utf8/unicodetext.h
@@ -223,6 +223,8 @@
// std::string, or from ::string to std::string, because if this happens it
// often results in invalid memory access to a temporary object created during
// such conversion (if do_copy == false).
+// NOTE: These methods don't check if the input string is UTF8 well formed, for
+// efficiency reasons. Use UnicodeText::is_valid() when explicitly needed.
UnicodeText UTF8ToUnicodeText(const char* utf8_buf, int len,
bool do_copy = true);
UnicodeText UTF8ToUnicodeText(const char* utf8_buf, bool do_copy = true);
diff --git a/native/utils/utf8/unicodetext_test.cc b/native/utils/utf8/unicodetext_test.cc
deleted file mode 100644
index 4e8883b..0000000
--- a/native/utils/utf8/unicodetext_test.cc
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/utf8/unicodetext.h"
-
-#include "utils/strings/stringpiece.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-class UnicodeTextTest : public testing::Test {
- protected:
- UnicodeTextTest() : empty_text_() {
- text_.push_back(0x1C0);
- text_.push_back(0x4E8C);
- text_.push_back(0xD7DB);
- text_.push_back(0x34);
- text_.push_back(0x1D11E);
- }
-
- UnicodeText empty_text_;
- UnicodeText text_;
-};
-
-TEST(UnicodeTextTest, ConstructionFromUnicodeText) {
- UnicodeText text = UTF8ToUnicodeText("1234😋hello", /*do_copy=*/false);
- EXPECT_EQ(UnicodeText(text).ToUTF8String(), "1234😋hello");
- EXPECT_EQ(UnicodeText(text, /*do_copy=*/false).ToUTF8String(), "1234😋hello");
-}
-
-// Tests for our modifications of UnicodeText.
-TEST(UnicodeTextTest, Custom) {
- UnicodeText text = UTF8ToUnicodeText("1234😋hello", /*do_copy=*/false);
- EXPECT_EQ(text.ToUTF8String(), "1234😋hello");
- EXPECT_EQ(text.size_codepoints(), 10);
- EXPECT_EQ(text.size_bytes(), 13);
-
- auto it_begin = text.begin();
- std::advance(it_begin, 4);
- auto it_end = text.begin();
- std::advance(it_end, 6);
- EXPECT_EQ(text.UTF8Substring(it_begin, it_end), "😋h");
-}
-
-TEST(UnicodeTextTest, StringPieceView) {
- std::string raw_text = "1234😋hello";
- UnicodeText text =
- UTF8ToUnicodeText(StringPiece(raw_text), /*do_copy=*/false);
- EXPECT_EQ(text.ToUTF8String(), "1234😋hello");
- EXPECT_EQ(text.size_codepoints(), 10);
- EXPECT_EQ(text.size_bytes(), 13);
-
- auto it_begin = text.begin();
- std::advance(it_begin, 4);
- auto it_end = text.begin();
- std::advance(it_end, 6);
- EXPECT_EQ(text.UTF8Substring(it_begin, it_end), "😋h");
-}
-
-TEST(UnicodeTextTest, Substring) {
- UnicodeText text = UTF8ToUnicodeText("1234😋hello", /*do_copy=*/false);
-
- EXPECT_EQ(
- UnicodeText::Substring(std::next(text.begin(), 4),
- std::next(text.begin(), 6), /*do_copy=*/true),
- UTF8ToUnicodeText("😋h"));
- EXPECT_EQ(
- UnicodeText::Substring(std::next(text.begin(), 4),
- std::next(text.begin(), 6), /*do_copy=*/false),
- UTF8ToUnicodeText("😋h"));
- EXPECT_EQ(UnicodeText::Substring(text, 4, 6, /*do_copy=*/true),
- UTF8ToUnicodeText("😋h"));
- EXPECT_EQ(UnicodeText::Substring(text, 4, 6, /*do_copy=*/false),
- UTF8ToUnicodeText("😋h"));
-}
-
-TEST(UnicodeTextTest, Ownership) {
- const std::string src = "\u304A\u00B0\u106B";
-
- UnicodeText alias;
- alias.PointToUTF8(src.data(), src.size());
- EXPECT_EQ(alias.data(), src.data());
- UnicodeText::const_iterator it = alias.begin();
- EXPECT_EQ(*it++, 0x304A);
- EXPECT_EQ(*it++, 0x00B0);
- EXPECT_EQ(*it++, 0x106B);
- EXPECT_EQ(it, alias.end());
-
- UnicodeText t = alias; // Copy initialization copies the data.
- EXPECT_NE(t.data(), alias.data());
-}
-
-TEST(UnicodeTextTest, Validation) {
- EXPECT_TRUE(UTF8ToUnicodeText("1234😋hello", /*do_copy=*/false).is_valid());
- EXPECT_TRUE(
- UTF8ToUnicodeText("\u304A\u00B0\u106B", /*do_copy=*/false).is_valid());
- EXPECT_TRUE(
- UTF8ToUnicodeText("this is a test😋😋😋", /*do_copy=*/false).is_valid());
- EXPECT_TRUE(
- UTF8ToUnicodeText("\xf0\x9f\x98\x8b", /*do_copy=*/false).is_valid());
- // Too short (string is too short).
- EXPECT_FALSE(UTF8ToUnicodeText("\xf0\x9f", /*do_copy=*/false).is_valid());
- // Too long (too many trailing bytes).
- EXPECT_FALSE(
- UTF8ToUnicodeText("\xf0\x9f\x98\x8b\x8b", /*do_copy=*/false).is_valid());
- // Too short (too few trailing bytes).
- EXPECT_FALSE(
- UTF8ToUnicodeText("\xf0\x9f\x98\x61\x61", /*do_copy=*/false).is_valid());
- // Invalid with context.
- EXPECT_FALSE(
- UTF8ToUnicodeText("hello \xf0\x9f\x98\x61\x61 world1", /*do_copy=*/false)
- .is_valid());
-}
-
-class IteratorTest : public UnicodeTextTest {};
-
-TEST_F(IteratorTest, Iterates) {
- UnicodeText::const_iterator iter = text_.begin();
- EXPECT_EQ(0x1C0, *iter);
- EXPECT_EQ(&iter, &++iter); // operator++ returns *this.
- EXPECT_EQ(0x4E8C, *iter++);
- EXPECT_EQ(0xD7DB, *iter);
- // Make sure you can dereference more than once.
- EXPECT_EQ(0xD7DB, *iter);
- EXPECT_EQ(0x34, *++iter);
- EXPECT_EQ(0x1D11E, *++iter);
- ASSERT_TRUE(iter != text_.end());
- iter++;
- EXPECT_TRUE(iter == text_.end());
-}
-
-TEST_F(IteratorTest, MultiPass) {
- // Also tests Default Constructible and Assignable.
- UnicodeText::const_iterator i1, i2;
- i1 = text_.begin();
- i2 = i1;
- EXPECT_EQ(0x4E8C, *++i1);
- EXPECT_TRUE(i1 != i2);
- EXPECT_EQ(0x1C0, *i2);
- ++i2;
- EXPECT_TRUE(i1 == i2);
- EXPECT_EQ(0x4E8C, *i2);
-}
-
-TEST_F(IteratorTest, ReverseIterates) {
- UnicodeText::const_iterator iter = text_.end();
- EXPECT_TRUE(iter == text_.end());
- iter--;
- ASSERT_TRUE(iter != text_.end());
- EXPECT_EQ(0x1D11E, *iter--);
- EXPECT_EQ(0x34, *iter);
- EXPECT_EQ(0xD7DB, *--iter);
- // Make sure you can dereference more than once.
- EXPECT_EQ(0xD7DB, *iter);
- --iter;
- EXPECT_EQ(0x4E8C, *iter--);
- EXPECT_EQ(0x1C0, *iter);
- EXPECT_TRUE(iter == text_.begin());
-}
-
-TEST_F(IteratorTest, Comparable) {
- UnicodeText::const_iterator i1, i2;
- i1 = text_.begin();
- i2 = i1;
- ++i2;
-
- EXPECT_TRUE(i1 < i2);
- EXPECT_TRUE(text_.begin() <= i1);
- EXPECT_FALSE(i1 >= i2);
- EXPECT_FALSE(i1 > text_.end());
-}
-
-TEST_F(IteratorTest, Advance) {
- UnicodeText::const_iterator iter = text_.begin();
- EXPECT_EQ(0x1C0, *iter);
- std::advance(iter, 4);
- EXPECT_EQ(0x1D11E, *iter);
- ++iter;
- EXPECT_TRUE(iter == text_.end());
-}
-
-TEST_F(IteratorTest, Distance) {
- UnicodeText::const_iterator iter = text_.begin();
- EXPECT_EQ(0, std::distance(text_.begin(), iter));
- EXPECT_EQ(5, std::distance(iter, text_.end()));
- ++iter;
- ++iter;
- EXPECT_EQ(2, std::distance(text_.begin(), iter));
- EXPECT_EQ(3, std::distance(iter, text_.end()));
- ++iter;
- ++iter;
- EXPECT_EQ(4, std::distance(text_.begin(), iter));
- ++iter;
- EXPECT_EQ(0, std::distance(iter, text_.end()));
-}
-
-class OperatorTest : public UnicodeTextTest {};
-
-TEST_F(OperatorTest, Clear) {
- UnicodeText empty_text(UTF8ToUnicodeText("", /*do_copy=*/false));
- EXPECT_FALSE(text_ == empty_text);
- text_.clear();
- EXPECT_TRUE(text_ == empty_text);
-}
-
-TEST_F(OperatorTest, Empty) {
- EXPECT_TRUE(empty_text_.empty());
- EXPECT_FALSE(text_.empty());
- text_.clear();
- EXPECT_TRUE(text_.empty());
-}
-
-} // namespace
-} // namespace libtextclassifier3
diff --git a/native/utils/utf8/unilib_test-include.cc b/native/utils/utf8/unilib_test-include.cc
deleted file mode 100644
index 7b2a179..0000000
--- a/native/utils/utf8/unilib_test-include.cc
+++ /dev/null
@@ -1,502 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/utf8/unilib_test-include.h"
-
-#include "utils/base/logging.h"
-#include "gmock/gmock.h"
-
-namespace libtextclassifier3 {
-namespace test_internal {
-
-using ::testing::ElementsAre;
-
-TEST_F(UniLibTest, CharacterClassesAscii) {
- EXPECT_TRUE(unilib_.IsOpeningBracket('('));
- EXPECT_TRUE(unilib_.IsClosingBracket(')'));
- EXPECT_FALSE(unilib_.IsWhitespace(')'));
- EXPECT_TRUE(unilib_.IsWhitespace(' '));
- EXPECT_FALSE(unilib_.IsDigit(')'));
- EXPECT_TRUE(unilib_.IsDigit('0'));
- EXPECT_TRUE(unilib_.IsDigit('9'));
- EXPECT_FALSE(unilib_.IsUpper(')'));
- EXPECT_TRUE(unilib_.IsUpper('A'));
- EXPECT_TRUE(unilib_.IsUpper('Z'));
- EXPECT_FALSE(unilib_.IsLower(')'));
- EXPECT_TRUE(unilib_.IsLower('a'));
- EXPECT_TRUE(unilib_.IsLower('z'));
- EXPECT_TRUE(unilib_.IsPunctuation('!'));
- EXPECT_TRUE(unilib_.IsPunctuation('?'));
- EXPECT_TRUE(unilib_.IsPunctuation('#'));
- EXPECT_TRUE(unilib_.IsPunctuation('('));
- EXPECT_FALSE(unilib_.IsPunctuation('0'));
- EXPECT_FALSE(unilib_.IsPunctuation('$'));
- EXPECT_TRUE(unilib_.IsPercentage('%'));
- EXPECT_TRUE(unilib_.IsPercentage(u'％'));
- EXPECT_TRUE(unilib_.IsSlash('/'));
- EXPECT_TRUE(unilib_.IsSlash(u'／'));
- EXPECT_TRUE(unilib_.IsMinus('-'));
- EXPECT_TRUE(unilib_.IsMinus(u'－'));
- EXPECT_TRUE(unilib_.IsNumberSign('#'));
- EXPECT_TRUE(unilib_.IsNumberSign(u'＃'));
- EXPECT_TRUE(unilib_.IsDot('.'));
- EXPECT_TRUE(unilib_.IsDot(u'．'));
-
- EXPECT_TRUE(unilib_.IsLatinLetter('A'));
- EXPECT_TRUE(unilib_.IsArabicLetter(u'ب')); // ARABIC LETTER BEH
- EXPECT_TRUE(
- unilib_.IsCyrillicLetter(u'ᲀ')); // CYRILLIC SMALL LETTER ROUNDED VE
- EXPECT_TRUE(unilib_.IsChineseLetter(u'豈')); // CJK COMPATIBILITY IDEOGRAPH
- EXPECT_TRUE(unilib_.IsJapaneseLetter(u'ぁ')); // HIRAGANA LETTER SMALL A
- EXPECT_TRUE(unilib_.IsKoreanLetter(u'ㄱ')); // HANGUL LETTER KIYEOK
- EXPECT_TRUE(unilib_.IsThaiLetter(u'ก')); // THAI CHARACTER KO KAI
- EXPECT_TRUE(unilib_.IsCJTletter(u'ก')); // THAI CHARACTER KO KAI
- EXPECT_FALSE(unilib_.IsCJTletter('A'));
-
- EXPECT_TRUE(unilib_.IsLetter('A'));
- EXPECT_TRUE(unilib_.IsLetter(u'Ａ'));
- EXPECT_TRUE(unilib_.IsLetter(u'ト')); // KATAKANA LETTER TO
- EXPECT_TRUE(unilib_.IsLetter(u'ﾄ')); // HALFWIDTH KATAKANA LETTER TO
- EXPECT_TRUE(unilib_.IsLetter(u'豈')); // CJK COMPATIBILITY IDEOGRAPH
-
- EXPECT_EQ(unilib_.ToLower('A'), 'a');
- EXPECT_EQ(unilib_.ToLower('Z'), 'z');
- EXPECT_EQ(unilib_.ToLower(')'), ')');
- EXPECT_EQ(unilib_.ToLowerText(UTF8ToUnicodeText("Never gonna give you up."))
- .ToUTF8String(),
- "never gonna give you up.");
- EXPECT_EQ(unilib_.ToUpper('a'), 'A');
- EXPECT_EQ(unilib_.ToUpper('z'), 'Z');
- EXPECT_EQ(unilib_.ToUpper(')'), ')');
- EXPECT_EQ(unilib_.ToUpperText(UTF8ToUnicodeText("Never gonna let you down."))
- .ToUTF8String(),
- "NEVER GONNA LET YOU DOWN.");
- EXPECT_EQ(unilib_.GetPairedBracket(')'), '(');
- EXPECT_EQ(unilib_.GetPairedBracket('}'), '{');
-}
-
-TEST_F(UniLibTest, CharacterClassesUnicode) {
- EXPECT_TRUE(unilib_.IsOpeningBracket(0x0F3C)); // TIBET ANG KHANG GYON
- EXPECT_TRUE(unilib_.IsClosingBracket(0x0F3D)); // TIBET ANG KHANG GYAS
- EXPECT_FALSE(unilib_.IsWhitespace(0x23F0)); // ALARM CLOCK
- EXPECT_TRUE(unilib_.IsWhitespace(0x2003)); // EM SPACE
- EXPECT_FALSE(unilib_.IsDigit(0xA619)); // VAI SYMBOL JONG
- EXPECT_TRUE(unilib_.IsDigit(0xA620)); // VAI DIGIT ZERO
- EXPECT_TRUE(unilib_.IsDigit(0xA629)); // VAI DIGIT NINE
- EXPECT_FALSE(unilib_.IsDigit(0xA62A)); // VAI SYLLABLE NDOLE MA
- EXPECT_FALSE(unilib_.IsUpper(0x0211)); // SMALL R WITH DOUBLE GRAVE
- EXPECT_TRUE(unilib_.IsUpper(0x0212)); // CAPITAL R WITH DOUBLE GRAVE
- EXPECT_TRUE(unilib_.IsUpper(0x0391)); // GREEK CAPITAL ALPHA
- EXPECT_TRUE(unilib_.IsUpper(0x03AB)); // GREEK CAPITAL UPSILON W DIAL
- EXPECT_FALSE(unilib_.IsUpper(0x03AC)); // GREEK SMALL ALPHA WITH TONOS
- EXPECT_TRUE(unilib_.IsLower(0x03AC)); // GREEK SMALL ALPHA WITH TONOS
- EXPECT_TRUE(unilib_.IsLower(0x03B1)); // GREEK SMALL ALPHA
- EXPECT_TRUE(unilib_.IsLower(0x03CB)); // GREEK SMALL UPSILON
- EXPECT_TRUE(unilib_.IsLower(0x0211)); // SMALL R WITH DOUBLE GRAVE
- EXPECT_TRUE(unilib_.IsLower(0x03C0)); // GREEK SMALL PI
- EXPECT_TRUE(unilib_.IsLower(0x007A)); // SMALL Z
- EXPECT_FALSE(unilib_.IsLower(0x005A)); // CAPITAL Z
- EXPECT_FALSE(unilib_.IsLower(0x0212)); // CAPITAL R WITH DOUBLE GRAVE
- EXPECT_FALSE(unilib_.IsLower(0x0391)); // GREEK CAPITAL ALPHA
- EXPECT_TRUE(unilib_.IsPunctuation(0x055E)); // ARMENIAN QUESTION MARK
- EXPECT_TRUE(unilib_.IsPunctuation(0x066C)); // ARABIC THOUSANDS SEPARATOR
- EXPECT_TRUE(unilib_.IsPunctuation(0x07F7)); // NKO SYMBOL GBAKURUNEN
- EXPECT_TRUE(unilib_.IsPunctuation(0x10AF2)); // DOUBLE DOT WITHIN DOT
- EXPECT_FALSE(unilib_.IsPunctuation(0x00A3)); // POUND SIGN
- EXPECT_FALSE(unilib_.IsPunctuation(0xA838)); // NORTH INDIC RUPEE MARK
- EXPECT_TRUE(unilib_.IsPercentage(0x0025)); // PERCENT SIGN
- EXPECT_TRUE(unilib_.IsPercentage(0xFF05)); // FULLWIDTH PERCENT SIGN
- EXPECT_TRUE(unilib_.IsSlash(0x002F)); // SOLIDUS
- EXPECT_TRUE(unilib_.IsSlash(0xFF0F)); // FULLWIDTH SOLIDUS
- EXPECT_TRUE(unilib_.IsMinus(0x002D)); // HYPHEN-MINUS
- EXPECT_TRUE(unilib_.IsMinus(0xFF0D)); // FULLWIDTH HYPHEN-MINUS
- EXPECT_TRUE(unilib_.IsNumberSign(0x0023)); // NUMBER SIGN
- EXPECT_TRUE(unilib_.IsNumberSign(0xFF03)); // FULLWIDTH NUMBER SIGN
- EXPECT_TRUE(unilib_.IsDot(0x002E)); // FULL STOP
- EXPECT_TRUE(unilib_.IsDot(0xFF0E)); // FULLWIDTH FULL STOP
-
- EXPECT_TRUE(unilib_.IsLatinLetter(0x0041)); // LATIN CAPITAL LETTER A
- EXPECT_TRUE(unilib_.IsArabicLetter(0x0628)); // ARABIC LETTER BEH
- EXPECT_TRUE(
- unilib_.IsCyrillicLetter(0x1C80)); // CYRILLIC SMALL LETTER ROUNDED VE
- EXPECT_TRUE(unilib_.IsChineseLetter(0xF900)); // CJK COMPATIBILITY IDEOGRAPH
- EXPECT_TRUE(unilib_.IsJapaneseLetter(0x3041)); // HIRAGANA LETTER SMALL A
- EXPECT_TRUE(unilib_.IsKoreanLetter(0x3131)); // HANGUL LETTER KIYEOK
- EXPECT_TRUE(unilib_.IsThaiLetter(0x0E01)); // THAI CHARACTER KO KAI
- EXPECT_TRUE(unilib_.IsCJTletter(0x0E01)); // THAI CHARACTER KO KAI
- EXPECT_FALSE(unilib_.IsCJTletter(0x0041)); // LATIN CAPITAL LETTER A
-
- EXPECT_TRUE(unilib_.IsLetter(0x0041)); // LATIN CAPITAL LETTER A
- EXPECT_TRUE(unilib_.IsLetter(0xFF21)); // FULLWIDTH LATIN CAPITAL LETTER A
- EXPECT_TRUE(unilib_.IsLetter(0x30C8)); // KATAKANA LETTER TO
- EXPECT_TRUE(unilib_.IsLetter(0xFF84)); // HALFWIDTH KATAKANA LETTER TO
- EXPECT_TRUE(unilib_.IsLetter(0xF900)); // CJK COMPATIBILITY IDEOGRAPH
-
- EXPECT_EQ(unilib_.ToLower(0x0391), 0x03B1); // GREEK ALPHA
- EXPECT_EQ(unilib_.ToLower(0x03AB), 0x03CB); // GREEK UPSILON WITH DIALYTIKA
- EXPECT_EQ(unilib_.ToLower(0x03C0), 0x03C0); // GREEK SMALL PI
- EXPECT_EQ(unilib_.ToLower(0x03A3), 0x03C3); // GREEK CAPITAL LETTER SIGMA
- EXPECT_EQ(unilib_.ToLowerText(UTF8ToUnicodeText("Κανένας άνθρωπος δεν ξέρει"))
- .ToUTF8String(),
- "κανένας άνθρωπος δεν ξέρει");
- EXPECT_EQ(unilib_.ToUpper(0x03B1), 0x0391); // GREEK ALPHA
- EXPECT_EQ(unilib_.ToUpper(0x03CB), 0x03AB); // GREEK UPSILON WITH DIALYTIKA
- EXPECT_EQ(unilib_.ToUpper(0x0391), 0x0391); // GREEK CAPITAL ALPHA
- EXPECT_EQ(unilib_.ToUpper(0x03C3), 0x03A3); // GREEK CAPITAL LETTER SIGMA
- EXPECT_EQ(unilib_.ToUpper(0x03C2), 0x03A3); // GREEK CAPITAL LETTER SIGMA
- EXPECT_EQ(unilib_.ToUpperText(UTF8ToUnicodeText("Κανένας άνθρωπος δεν ξέρει"))
- .ToUTF8String(),
- "ΚΑΝΈΝΑΣ ΆΝΘΡΩΠΟΣ ΔΕΝ ΞΈΡΕΙ");
- EXPECT_EQ(unilib_.GetPairedBracket(0x0F3C), 0x0F3D);
- EXPECT_EQ(unilib_.GetPairedBracket(0x0F3D), 0x0F3C);
-}
-
-TEST_F(UniLibTest, RegexInterface) {
- const UnicodeText regex_pattern =
- UTF8ToUnicodeText("[0-9]+", /*do_copy=*/true);
- std::unique_ptr<UniLib::RegexPattern> pattern =
- unilib_.CreateRegexPattern(regex_pattern);
- const UnicodeText input = UTF8ToUnicodeText("hello 0123", /*do_copy=*/false);
- int status;
- std::unique_ptr<UniLib::RegexMatcher> matcher = pattern->Matcher(input);
- TC3_LOG(INFO) << matcher->Matches(&status);
- TC3_LOG(INFO) << matcher->Find(&status);
- TC3_LOG(INFO) << matcher->Start(0, &status);
- TC3_LOG(INFO) << matcher->End(0, &status);
- TC3_LOG(INFO) << matcher->Group(0, &status).size_codepoints();
-}
-
-TEST_F(UniLibTest, Regex) {
- // The smiley face is a 4-byte UTF8 codepoint 0x1F60B, and it's important to
- // test the regex functionality with it to verify we are handling the indices
- // correctly.
- const UnicodeText regex_pattern =
- UTF8ToUnicodeText("[0-9]+😋", /*do_copy=*/false);
- std::unique_ptr<UniLib::RegexPattern> pattern =
- unilib_.CreateRegexPattern(regex_pattern);
- int status;
- std::unique_ptr<UniLib::RegexMatcher> matcher;
-
- matcher = pattern->Matcher(UTF8ToUnicodeText("0123😋", /*do_copy=*/false));
- EXPECT_TRUE(matcher->Matches(&status));
- EXPECT_TRUE(matcher->ApproximatelyMatches(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_TRUE(matcher->Matches(&status)); // Check that the state is reset.
- EXPECT_TRUE(matcher->ApproximatelyMatches(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
-
- matcher = pattern->Matcher(
- UTF8ToUnicodeText("hello😋😋 0123😋 world", /*do_copy=*/false));
- EXPECT_FALSE(matcher->Matches(&status));
- EXPECT_FALSE(matcher->ApproximatelyMatches(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
-
- matcher = pattern->Matcher(
- UTF8ToUnicodeText("hello😋😋 0123😋 world", /*do_copy=*/false));
- EXPECT_TRUE(matcher->Find(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Start(0, &status), 8);
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->End(0, &status), 13);
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Group(0, &status).ToUTF8String(), "0123😋");
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
-}
-
-TEST_F(UniLibTest, RegexLazy) {
- std::unique_ptr<UniLib::RegexPattern> pattern =
- unilib_.CreateLazyRegexPattern(
- UTF8ToUnicodeText("[a-z][0-9]", /*do_copy=*/false));
- int status;
- std::unique_ptr<UniLib::RegexMatcher> matcher;
-
- matcher = pattern->Matcher(UTF8ToUnicodeText("a3", /*do_copy=*/false));
- EXPECT_TRUE(matcher->Matches(&status));
- EXPECT_TRUE(matcher->ApproximatelyMatches(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_TRUE(matcher->Matches(&status)); // Check that the state is reset.
- EXPECT_TRUE(matcher->ApproximatelyMatches(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
-
- matcher = pattern->Matcher(UTF8ToUnicodeText("3a", /*do_copy=*/false));
- EXPECT_FALSE(matcher->Matches(&status));
- EXPECT_FALSE(matcher->ApproximatelyMatches(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
-}
-
-TEST_F(UniLibTest, RegexGroups) {
- // The smiley face is a 4-byte UTF8 codepoint 0x1F60B, and it's important to
- // test the regex functionality with it to verify we are handling the indices
- // correctly.
- const UnicodeText regex_pattern =
- UTF8ToUnicodeText("([0-9])([0-9]+)😋", /*do_copy=*/false);
- std::unique_ptr<UniLib::RegexPattern> pattern =
- unilib_.CreateRegexPattern(regex_pattern);
- int status;
- std::unique_ptr<UniLib::RegexMatcher> matcher;
-
- matcher = pattern->Matcher(
- UTF8ToUnicodeText("hello😋😋 0123😋 world", /*do_copy=*/false));
- EXPECT_TRUE(matcher->Find(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Start(0, &status), 8);
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Start(1, &status), 8);
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Start(2, &status), 9);
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->End(0, &status), 13);
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->End(1, &status), 9);
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->End(2, &status), 12);
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Group(0, &status).ToUTF8String(), "0123😋");
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Group(1, &status).ToUTF8String(), "0");
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Group(2, &status).ToUTF8String(), "123");
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
-}
-
-TEST_F(UniLibTest, RegexGroupsNotAllGroupsInvolved) {
- const UnicodeText regex_pattern =
- UTF8ToUnicodeText("([0-9])([a-z])?", /*do_copy=*/false);
- std::unique_ptr<UniLib::RegexPattern> pattern =
- unilib_.CreateRegexPattern(regex_pattern);
- int status;
- std::unique_ptr<UniLib::RegexMatcher> matcher;
-
- matcher = pattern->Matcher(UTF8ToUnicodeText("7", /*do_copy=*/false));
- EXPECT_TRUE(matcher->Find(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Group(0, &status).ToUTF8String(), "7");
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Group(1, &status).ToUTF8String(), "7");
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Group(2, &status).ToUTF8String(), "");
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
-}
-
-TEST_F(UniLibTest, RegexGroupsEmptyResult) {
- const UnicodeText regex_pattern =
- UTF8ToUnicodeText("(.*)", /*do_copy=*/false);
- std::unique_ptr<UniLib::RegexPattern> pattern =
- unilib_.CreateRegexPattern(regex_pattern);
- int status;
- std::unique_ptr<UniLib::RegexMatcher> matcher;
-
- matcher = pattern->Matcher(UTF8ToUnicodeText("", /*do_copy=*/false));
- EXPECT_TRUE(matcher->Find(&status));
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Group(0, &status).ToUTF8String(), "");
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
- EXPECT_EQ(matcher->Group(1, &status).ToUTF8String(), "");
- EXPECT_EQ(status, UniLib::RegexMatcher::kNoError);
-}
-
-TEST_F(UniLibTest, BreakIterator) {
- const UnicodeText text = UTF8ToUnicodeText("some text", /*do_copy=*/false);
- std::unique_ptr<UniLib::BreakIterator> iterator =
- unilib_.CreateBreakIterator(text);
- std::vector<int> break_indices;
- int break_index = 0;
- while ((break_index = iterator->Next()) != UniLib::BreakIterator::kDone) {
- break_indices.push_back(break_index);
- }
- EXPECT_THAT(break_indices, ElementsAre(4, 5, 9));
-}
-
-TEST_F(UniLibTest, BreakIterator4ByteUTF8) {
- const UnicodeText text = UTF8ToUnicodeText("😀😂😋", /*do_copy=*/false);
- std::unique_ptr<UniLib::BreakIterator> iterator =
- unilib_.CreateBreakIterator(text);
- std::vector<int> break_indices;
- int break_index = 0;
- while ((break_index = iterator->Next()) != UniLib::BreakIterator::kDone) {
- break_indices.push_back(break_index);
- }
- EXPECT_THAT(break_indices, ElementsAre(1, 2, 3));
-}
-
-TEST_F(UniLibTest, Integer32Parse) {
- int result;
- EXPECT_TRUE(
- unilib_.ParseInt32(UTF8ToUnicodeText("123", /*do_copy=*/false), &result));
- EXPECT_EQ(result, 123);
-}
-
-TEST_F(UniLibTest, Integer32ParseFloatNumber) {
- int result;
- EXPECT_FALSE(unilib_.ParseInt32(UTF8ToUnicodeText("12.3", /*do_copy=*/false),
- &result));
-}
-
-TEST_F(UniLibTest, Integer32ParseLongNumber) {
- int32 result;
- EXPECT_TRUE(unilib_.ParseInt32(
- UTF8ToUnicodeText("1000000000", /*do_copy=*/false), &result));
- EXPECT_EQ(result, 1000000000);
-}
-
-TEST_F(UniLibTest, Integer32ParseEmptyString) {
- int result;
- EXPECT_FALSE(
- unilib_.ParseInt32(UTF8ToUnicodeText("", /*do_copy=*/false), &result));
-}
-
-TEST_F(UniLibTest, Integer32ParseFullWidth) {
- int result;
- // The input string here is full width
- EXPECT_TRUE(unilib_.ParseInt32(UTF8ToUnicodeText("１２３", /*do_copy=*/false),
- &result));
- EXPECT_EQ(result, 123);
-}
-
-TEST_F(UniLibTest, Integer32ParseNotNumber) {
- int result;
- // The input string here is full width
- EXPECT_FALSE(unilib_.ParseInt32(UTF8ToUnicodeText("１a３", /*do_copy=*/false),
- &result));
- // Strings starting with "nan" are not numbers.
- EXPECT_FALSE(unilib_.ParseInt32(UTF8ToUnicodeText("Nancy",
- /*do_copy=*/false),
- &result));
- // Strings starting with "inf" are not numbers
- EXPECT_FALSE(unilib_.ParseInt32(
- UTF8ToUnicodeText("Information", /*do_copy=*/false), &result));
-}
-
-TEST_F(UniLibTest, Integer64Parse) {
- int64 result;
- EXPECT_TRUE(
- unilib_.ParseInt64(UTF8ToUnicodeText("123", /*do_copy=*/false), &result));
- EXPECT_EQ(result, 123);
-}
-
-TEST_F(UniLibTest, Integer64ParseFloatNumber) {
- int64 result;
- EXPECT_FALSE(unilib_.ParseInt64(UTF8ToUnicodeText("12.3", /*do_copy=*/false),
- &result));
-}
-
-TEST_F(UniLibTest, Integer64ParseLongNumber) {
- int64 result;
- // The limitation comes from the javaicu implementation: parseDouble does not
- // have ICU support and parseInt limit the size of the number.
- EXPECT_TRUE(unilib_.ParseInt64(
- UTF8ToUnicodeText("1000000000", /*do_copy=*/false), &result));
- EXPECT_EQ(result, 1000000000);
-}
-
-TEST_F(UniLibTest, Integer64ParseEmptyString) {
- int64 result;
- EXPECT_FALSE(
- unilib_.ParseInt64(UTF8ToUnicodeText("", /*do_copy=*/false), &result));
-}
-
-TEST_F(UniLibTest, Integer64ParseFullWidth) {
- int64 result;
- // The input string here is full width
- EXPECT_TRUE(unilib_.ParseInt64(UTF8ToUnicodeText("１２３", /*do_copy=*/false),
- &result));
- EXPECT_EQ(result, 123);
-}
-
-TEST_F(UniLibTest, Integer64ParseNotNumber) {
- int64 result;
- // The input string here is full width
- EXPECT_FALSE(unilib_.ParseInt64(UTF8ToUnicodeText("１a４", /*do_copy=*/false),
- &result));
- // Strings starting with "nan" are not numbers.
- EXPECT_FALSE(unilib_.ParseInt64(UTF8ToUnicodeText("Nancy",
- /*do_copy=*/false),
- &result));
- // Strings starting with "inf" are not numbers
- EXPECT_FALSE(unilib_.ParseInt64(
- UTF8ToUnicodeText("Information", /*do_copy=*/false), &result));
-}
-
-TEST_F(UniLibTest, DoubleParse) {
- double result;
- EXPECT_TRUE(unilib_.ParseDouble(UTF8ToUnicodeText("1.23", /*do_copy=*/false),
- &result));
- EXPECT_EQ(result, 1.23);
-}
-
-TEST_F(UniLibTest, DoubleParseLongNumber) {
- double result;
- // The limitation comes from the javaicu implementation: parseDouble does not
- // have ICU support and parseInt limit the size of the number.
- EXPECT_TRUE(unilib_.ParseDouble(
- UTF8ToUnicodeText("999999999.999999999", /*do_copy=*/false), &result));
- EXPECT_EQ(result, 999999999.999999999);
-}
-
-TEST_F(UniLibTest, DoubleParseWithoutFractionalPart) {
- double result;
- EXPECT_TRUE(unilib_.ParseDouble(UTF8ToUnicodeText("123", /*do_copy=*/false),
- &result));
- EXPECT_EQ(result, 123);
-}
-
-TEST_F(UniLibTest, DoubleParseEmptyString) {
- double result;
- EXPECT_FALSE(
- unilib_.ParseDouble(UTF8ToUnicodeText("", /*do_copy=*/false), &result));
-}
-
-TEST_F(UniLibTest, DoubleParsePrecedingDot) {
- double result;
- EXPECT_FALSE(unilib_.ParseDouble(UTF8ToUnicodeText(".123", /*do_copy=*/false),
- &result));
-}
-
-TEST_F(UniLibTest, DoubleParseLeadingDot) {
- double result;
- EXPECT_FALSE(unilib_.ParseDouble(UTF8ToUnicodeText("123.", /*do_copy=*/false),
- &result));
-}
-
-TEST_F(UniLibTest, DoubleParseMultipleDots) {
- double result;
- EXPECT_FALSE(unilib_.ParseDouble(
- UTF8ToUnicodeText("1.2.3", /*do_copy=*/false), &result));
-}
-
-TEST_F(UniLibTest, DoubleParseFullWidth) {
- double result;
- // The input string here is full width
- EXPECT_TRUE(unilib_.ParseDouble(
- UTF8ToUnicodeText("１.２３", /*do_copy=*/false), &result));
- EXPECT_EQ(result, 1.23);
-}
-
-TEST_F(UniLibTest, DoubleParseNotNumber) {
- double result;
- // The input string here is full width
- EXPECT_FALSE(unilib_.ParseDouble(
- UTF8ToUnicodeText("１a５", /*do_copy=*/false), &result));
- // Strings starting with "nan" are not numbers.
- EXPECT_FALSE(unilib_.ParseDouble(
- UTF8ToUnicodeText("Nancy", /*do_copy=*/false), &result));
- // Strings starting with "inf" are not numbers
- EXPECT_FALSE(unilib_.ParseDouble(
- UTF8ToUnicodeText("Information", /*do_copy=*/false), &result));
-}
-
-} // namespace test_internal
-} // namespace libtextclassifier3
diff --git a/native/utils/utf8/unilib_test-include.h b/native/utils/utf8/unilib_test-include.h
deleted file mode 100644
index 342a00c..0000000
--- a/native/utils/utf8/unilib_test-include.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LIBTEXTCLASSIFIER_UTILS_UTF8_UNILIB_TEST_INCLUDE_H_
-#define LIBTEXTCLASSIFIER_UTILS_UTF8_UNILIB_TEST_INCLUDE_H_
-
-#include "utils/utf8/unilib.h"
-#include "gtest/gtest.h"
-
-#if defined TC3_UNILIB_ICU
-#define TC3_TESTING_CREATE_UNILIB_INSTANCE(VAR) VAR()
-#elif defined TC3_UNILIB_JAVAICU
-#include <jni.h>
-extern JNIEnv* g_jenv;
-#define TC3_TESTING_CREATE_UNILIB_INSTANCE(VAR) VAR(JniCache::Create(g_jenv))
-#elif defined TC3_UNILIB_APPLE
-#define TC3_TESTING_CREATE_UNILIB_INSTANCE(VAR) VAR()
-#elif defined TC3_UNILIB_DUMMY
-#define TC3_TESTING_CREATE_UNILIB_INSTANCE(VAR) VAR()
-#endif
-
-namespace libtextclassifier3 {
-namespace test_internal {
-
-class UniLibTest : public ::testing::Test {
- protected:
- UniLibTest() : TC3_TESTING_CREATE_UNILIB_INSTANCE(unilib_) {}
- UniLib unilib_;
-};
-
-} // namespace test_internal
-} // namespace libtextclassifier3
-
-#endif // LIBTEXTCLASSIFIER_UTILS_UTF8_UNILIB_TEST_INCLUDE_H_
diff --git a/native/utils/utf8/unilib_test.cc b/native/utils/utf8/unilib_test.cc
deleted file mode 100644
index 01b5164..0000000
--- a/native/utils/utf8/unilib_test.cc
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// The actual code of the test is in the following include:
-#include "utils/utf8/unilib_test-include.h"
diff --git a/native/utils/variant.h b/native/utils/variant.h
index 153a4e8..cb206ee 100644
--- a/native/utils/variant.h
+++ b/native/utils/variant.h
@@ -19,6 +19,7 @@
#include <map>
#include <string>
+#include <vector>
#include "utils/base/integral_types.h"
#include "utils/base/logging.h"
@@ -41,6 +42,9 @@
TYPE_DOUBLE_VALUE = 8,
TYPE_BOOL_VALUE = 9,
TYPE_STRING_VALUE = 10,
+ TYPE_STRING_VECTOR_VALUE = 11,
+ TYPE_FLOAT_VECTOR_VALUE = 12,
+ TYPE_INT_VECTOR_VALUE = 13,
};
Variant() : type_(TYPE_EMPTY) {}
@@ -68,6 +72,12 @@
: type_(TYPE_STRING_VALUE), string_value_(value) {}
explicit Variant(const bool value)
: type_(TYPE_BOOL_VALUE), bool_value_(value) {}
+ explicit Variant(const std::vector<std::string>& value)
+ : type_(TYPE_STRING_VECTOR_VALUE), string_vector_value_(value) {}
+ explicit Variant(const std::vector<float>& value)
+ : type_(TYPE_FLOAT_VECTOR_VALUE), float_vector_value_(value) {}
+ explicit Variant(const std::vector<int>& value)
+ : type_(TYPE_INT_VECTOR_VALUE), int_vector_value_(value) {}
Variant& operator=(const Variant&) = default;
@@ -121,6 +131,21 @@
return string_value_;
}
+ const std::vector<std::string>& StringVectorValue() const {
+ TC3_CHECK(HasStringVector());
+ return string_vector_value_;
+ }
+
+ const std::vector<float>& FloatVectorValue() const {
+ TC3_CHECK(HasFloatVector());
+ return float_vector_value_;
+ }
+
+ const std::vector<int>& IntVectorValue() const {
+ TC3_CHECK(HasIntVector());
+ return int_vector_value_;
+ }
+
// Converts the value of this variant to its string representation, regardless
// of the type of the actual value.
std::string ToString() const;
@@ -145,6 +170,12 @@
bool HasString() const { return type_ == TYPE_STRING_VALUE; }
+ bool HasStringVector() const { return type_ == TYPE_STRING_VECTOR_VALUE; }
+
+ bool HasFloatVector() const { return type_ == TYPE_FLOAT_VECTOR_VALUE; }
+
+ bool HasIntVector() const { return type_ == TYPE_INT_VECTOR_VALUE; }
+
Type GetType() const { return type_; }
bool HasValue() const { return type_ != TYPE_EMPTY; }
@@ -163,6 +194,9 @@
bool bool_value_;
};
std::string string_value_;
+ std::vector<std::string> string_vector_value_;
+ std::vector<float> float_vector_value_;
+ std::vector<int> int_vector_value_;
};
// Pretty-printing function for Variant.
diff --git a/native/utils/variant_test.cc b/native/utils/variant_test.cc
deleted file mode 100644
index 347d53f..0000000
--- a/native/utils/variant_test.cc
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/variant.h"
-
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-
-namespace libtextclassifier3 {
-namespace {
-
-TEST(VariantTest, GetType) {
- EXPECT_EQ(Variant().GetType(), Variant::TYPE_EMPTY);
- EXPECT_EQ(Variant(static_cast<int8_t>(9)).GetType(),
- Variant::TYPE_INT8_VALUE);
- EXPECT_EQ(Variant(static_cast<uint8_t>(9)).GetType(),
- Variant::TYPE_UINT8_VALUE);
- EXPECT_EQ(Variant(static_cast<int>(9)).GetType(), Variant::TYPE_INT_VALUE);
- EXPECT_EQ(Variant(static_cast<uint>(9)).GetType(), Variant::TYPE_UINT_VALUE);
- EXPECT_EQ(Variant(static_cast<int64>(9)).GetType(),
- Variant::TYPE_INT64_VALUE);
- EXPECT_EQ(Variant(static_cast<uint64>(9)).GetType(),
- Variant::TYPE_UINT64_VALUE);
- EXPECT_EQ(Variant(static_cast<float>(9)).GetType(),
- Variant::TYPE_FLOAT_VALUE);
- EXPECT_EQ(Variant(static_cast<double>(9)).GetType(),
- Variant::TYPE_DOUBLE_VALUE);
- EXPECT_EQ(Variant(true).GetType(), Variant::TYPE_BOOL_VALUE);
- EXPECT_EQ(Variant("hello").GetType(), Variant::TYPE_STRING_VALUE);
-}
-
-TEST(VariantTest, HasValue) {
- EXPECT_FALSE(Variant().HasValue());
- EXPECT_TRUE(Variant(static_cast<int8_t>(9)).HasValue());
- EXPECT_TRUE(Variant(static_cast<uint8_t>(9)).HasValue());
- EXPECT_TRUE(Variant(static_cast<int>(9)).HasValue());
- EXPECT_TRUE(Variant(static_cast<uint>(9)).HasValue());
- EXPECT_TRUE(Variant(static_cast<int64>(9)).HasValue());
- EXPECT_TRUE(Variant(static_cast<uint64>(9)).HasValue());
- EXPECT_TRUE(Variant(static_cast<float>(9)).HasValue());
- EXPECT_TRUE(Variant(static_cast<double>(9)).HasValue());
- EXPECT_TRUE(Variant(true).HasValue());
- EXPECT_TRUE(Variant("hello").HasValue());
-}
-
-TEST(VariantTest, Value) {
- EXPECT_EQ(Variant(static_cast<int8_t>(9)).Int8Value(), 9);
- EXPECT_EQ(Variant(static_cast<uint8_t>(9)).UInt8Value(), 9);
- EXPECT_EQ(Variant(static_cast<int>(9)).IntValue(), 9);
- EXPECT_EQ(Variant(static_cast<uint>(9)).UIntValue(), 9);
- EXPECT_EQ(Variant(static_cast<int64>(9)).Int64Value(), 9);
- EXPECT_EQ(Variant(static_cast<uint64>(9)).UInt64Value(), 9);
- EXPECT_EQ(Variant(static_cast<float>(9)).FloatValue(), 9);
- EXPECT_EQ(Variant(static_cast<double>(9)).DoubleValue(), 9);
- EXPECT_EQ(Variant(true).BoolValue(), true);
- EXPECT_EQ(Variant("hello").StringValue(), "hello");
-}
-
-} // namespace
-} // namespace libtextclassifier3
