| /* |
| * Copyright (C) 2018 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include "annotator/duration/duration.h" |
| |
| #include <string> |
| #include <vector> |
| |
| #include "annotator/collections.h" |
| #include "annotator/model_generated.h" |
| #include "annotator/types-test-util.h" |
| #include "annotator/types.h" |
| #include "utils/test-utils.h" |
| #include "utils/utf8/unicodetext.h" |
| #include "utils/utf8/unilib.h" |
| #include "gmock/gmock.h" |
| #include "gtest/gtest.h" |
| |
| namespace libtextclassifier3 { |
| namespace { |
| |
| using testing::AllOf; |
| using testing::ElementsAre; |
| using testing::Field; |
| |
| const DurationAnnotatorOptions* TestingDurationAnnotatorOptions() { |
| static const flatbuffers::DetachedBuffer* options_data = []() { |
| DurationAnnotatorOptionsT options; |
| options.enabled = true; |
| |
| options.week_expressions.push_back("week"); |
| options.week_expressions.push_back("weeks"); |
| |
| options.day_expressions.push_back("day"); |
| options.day_expressions.push_back("days"); |
| |
| options.hour_expressions.push_back("hour"); |
| options.hour_expressions.push_back("hours"); |
| |
| options.minute_expressions.push_back("minute"); |
| options.minute_expressions.push_back("minutes"); |
| |
| options.second_expressions.push_back("second"); |
| options.second_expressions.push_back("seconds"); |
| |
| options.filler_expressions.push_back("and"); |
| options.filler_expressions.push_back("a"); |
| options.filler_expressions.push_back("an"); |
| options.filler_expressions.push_back("one"); |
| |
| options.half_expressions.push_back("half"); |
| |
| flatbuffers::FlatBufferBuilder builder; |
| builder.Finish(DurationAnnotatorOptions::Pack(builder, &options)); |
| return new flatbuffers::DetachedBuffer(builder.Release()); |
| }(); |
| |
| return flatbuffers::GetRoot<DurationAnnotatorOptions>(options_data->data()); |
| } |
| |
| FeatureProcessor BuildFeatureProcessor(const UniLib* unilib) { |
| static const flatbuffers::DetachedBuffer* options_data = []() { |
| FeatureProcessorOptionsT options; |
| options.context_size = 1; |
| options.max_selection_span = 1; |
| options.snap_label_span_boundaries_to_containing_tokens = false; |
| options.ignored_span_boundary_codepoints.push_back(','); |
| |
| options.tokenization_codepoint_config.emplace_back( |
| new TokenizationCodepointRangeT()); |
| auto& config = options.tokenization_codepoint_config.back(); |
| config->start = 32; |
| config->end = 33; |
| config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR; |
| |
| flatbuffers::FlatBufferBuilder builder; |
| builder.Finish(FeatureProcessorOptions::Pack(builder, &options)); |
| return new flatbuffers::DetachedBuffer(builder.Release()); |
| }(); |
| |
| const FeatureProcessorOptions* feature_processor_options = |
| flatbuffers::GetRoot<FeatureProcessorOptions>(options_data->data()); |
| |
| return FeatureProcessor(feature_processor_options, unilib); |
| } |
| |
| class DurationAnnotatorTest : public ::testing::Test { |
| protected: |
| DurationAnnotatorTest() |
| : INIT_UNILIB_FOR_TESTING(unilib_), |
| feature_processor_(BuildFeatureProcessor(&unilib_)), |
| duration_annotator_(TestingDurationAnnotatorOptions(), |
| &feature_processor_) {} |
| |
| std::vector<Token> Tokenize(const UnicodeText& text) { |
| return feature_processor_.Tokenize(text); |
| } |
| |
| UniLib unilib_; |
| FeatureProcessor feature_processor_; |
| DurationAnnotator duration_annotator_; |
| }; |
| |
| TEST_F(DurationAnnotatorTest, ClassifiesSimpleDuration) { |
| ClassificationResult classification; |
| EXPECT_TRUE(duration_annotator_.ClassifyText( |
| UTF8ToUnicodeText("Wake me up in 15 minutes ok?"), {14, 24}, |
| AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification)); |
| |
| EXPECT_THAT(classification, |
| AllOf(Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, 15 * 60 * 1000))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, ClassifiesWhenTokensDontAlignWithSelection) { |
| ClassificationResult classification; |
| EXPECT_TRUE(duration_annotator_.ClassifyText( |
| UTF8ToUnicodeText("Wake me up in15 minutesok?"), {13, 23}, |
| AnnotationUsecase_ANNOTATION_USECASE_RAW, &classification)); |
| |
| EXPECT_THAT(classification, |
| AllOf(Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, 15 * 60 * 1000))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, FindsSimpleDuration) { |
| const UnicodeText text = UTF8ToUnicodeText("Wake me up in 15 minutes ok?"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| EXPECT_THAT( |
| result, |
| ElementsAre( |
| AllOf(Field(&AnnotatedSpan::span, CodepointSpan(14, 24)), |
| Field(&AnnotatedSpan::classification, |
| ElementsAre(AllOf( |
| Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, |
| 15 * 60 * 1000))))))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, FindsDurationWithHalfExpression) { |
| const UnicodeText text = |
| UTF8ToUnicodeText("Set a timer for 3 and half minutes ok?"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| EXPECT_THAT( |
| result, |
| ElementsAre( |
| AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 34)), |
| Field(&AnnotatedSpan::classification, |
| ElementsAre(AllOf( |
| Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, |
| 3.5 * 60 * 1000))))))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, FindsComposedDuration) { |
| const UnicodeText text = |
| UTF8ToUnicodeText("Wake me up in 3 hours and 5 seconds ok?"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| EXPECT_THAT( |
| result, |
| ElementsAre( |
| AllOf(Field(&AnnotatedSpan::span, CodepointSpan(14, 35)), |
| Field(&AnnotatedSpan::classification, |
| ElementsAre(AllOf( |
| Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, |
| 3 * 60 * 60 * 1000 + 5 * 1000))))))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, FindsHalfAnHour) { |
| const UnicodeText text = UTF8ToUnicodeText("Set a timer for half an hour"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| EXPECT_THAT( |
| result, |
| ElementsAre( |
| AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 28)), |
| Field(&AnnotatedSpan::classification, |
| ElementsAre(AllOf( |
| Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, |
| 0.5 * 60 * 60 * 1000))))))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, FindsWhenHalfIsAfterGranularitySpecification) { |
| const UnicodeText text = |
| UTF8ToUnicodeText("Set a timer for 1 hour and a half"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| EXPECT_THAT( |
| result, |
| ElementsAre( |
| AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 33)), |
| Field(&AnnotatedSpan::classification, |
| ElementsAre(AllOf( |
| Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, |
| 1.5 * 60 * 60 * 1000))))))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, FindsAnHourAndAHalf) { |
| const UnicodeText text = |
| UTF8ToUnicodeText("Set a timer for an hour and a half"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| EXPECT_THAT( |
| result, |
| ElementsAre( |
| AllOf(Field(&AnnotatedSpan::span, CodepointSpan(19, 34)), |
| Field(&AnnotatedSpan::classification, |
| ElementsAre(AllOf( |
| Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, |
| 1.5 * 60 * 60 * 1000))))))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, |
| FindsCorrectlyWhenSecondsComeSecondAndDontHaveNumber) { |
| const UnicodeText text = |
| UTF8ToUnicodeText("Set a timer for 10 minutes and a second ok?"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| EXPECT_THAT( |
| result, |
| ElementsAre( |
| AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 39)), |
| Field(&AnnotatedSpan::classification, |
| ElementsAre(AllOf( |
| Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, |
| 10 * 60 * 1000 + 1 * 1000))))))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, DoesNotGreedilyTakeFillerWords) { |
| const UnicodeText text = UTF8ToUnicodeText( |
| "Set a timer for a a a 10 minutes and 2 seconds an and an ok?"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| EXPECT_THAT( |
| result, |
| ElementsAre( |
| AllOf(Field(&AnnotatedSpan::span, CodepointSpan(22, 46)), |
| Field(&AnnotatedSpan::classification, |
| ElementsAre(AllOf( |
| Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, |
| 10 * 60 * 1000 + 2 * 1000))))))); |
| } |
| |
| TEST_F(DurationAnnotatorTest, DoesNotCrashWhenJustHalfIsSaid) { |
| const UnicodeText text = UTF8ToUnicodeText("Set a timer for half ok?"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| ASSERT_EQ(result.size(), 0); |
| } |
| |
| TEST_F(DurationAnnotatorTest, StripsPunctuationFromTokens) { |
| const UnicodeText text = |
| UTF8ToUnicodeText("Set a timer for 10 ,minutes, ,and, ,2, seconds, ok?"); |
| std::vector<Token> tokens = Tokenize(text); |
| std::vector<AnnotatedSpan> result; |
| EXPECT_TRUE(duration_annotator_.FindAll( |
| text, tokens, AnnotationUsecase_ANNOTATION_USECASE_RAW, &result)); |
| |
| EXPECT_THAT( |
| result, |
| ElementsAre( |
| AllOf(Field(&AnnotatedSpan::span, CodepointSpan(16, 46)), |
| Field(&AnnotatedSpan::classification, |
| ElementsAre(AllOf( |
| Field(&ClassificationResult::collection, "duration"), |
| Field(&ClassificationResult::duration_ms, |
| 10 * 60 * 1000 + 2 * 1000))))))); |
| } |
| |
| } // namespace |
| } // namespace libtextclassifier3 |
