| /* |
| * Copyright (C) 2017 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include "cached-features.h" |
| |
| #include "model-executor.h" |
| #include "tensor-view.h" |
| |
| #include "gmock/gmock.h" |
| #include "gtest/gtest.h" |
| |
| using testing::ElementsAreArray; |
| using testing::FloatEq; |
| using testing::Matcher; |
| |
| namespace libtextclassifier2 { |
| namespace { |
| |
| Matcher<std::vector<float>> ElementsAreFloat(const std::vector<float>& values) { |
| std::vector<Matcher<float>> matchers; |
| for (const float value : values) { |
| matchers.push_back(FloatEq(value)); |
| } |
| return ElementsAreArray(matchers); |
| } |
| |
| // EmbeddingExecutor that always returns features based on |
| class FakeEmbeddingExecutor : public EmbeddingExecutor { |
| public: |
| bool AddEmbedding(const TensorView<int>& sparse_features, float* dest, |
| int dest_size) override { |
| TC_CHECK_GE(dest_size, 2); |
| EXPECT_EQ(sparse_features.size(), 1); |
| |
| dest[0] = sparse_features.data()[0] * 11.0f; |
| dest[1] = -sparse_features.data()[0] * 11.0f; |
| return true; |
| } |
| |
| private: |
| std::vector<float> storage_; |
| }; |
| |
| std::vector<float> GetCachedClickContextFeatures( |
| const CachedFeatures& cached_features, int click_pos) { |
| std::vector<float> output_features; |
| cached_features.AppendClickContextFeaturesForClick(click_pos, |
| &output_features); |
| return output_features; |
| } |
| |
| std::vector<float> GetCachedBoundsSensitiveFeatures( |
| const CachedFeatures& cached_features, TokenSpan selected_span) { |
| std::vector<float> output_features; |
| cached_features.AppendBoundsSensitiveFeaturesForSpan(selected_span, |
| &output_features); |
| return output_features; |
| } |
| |
| TEST(CachedFeaturesTest, ClickContext) { |
| FeatureProcessorOptionsT options; |
| options.context_size = 2; |
| options.feature_version = 1; |
| flatbuffers::FlatBufferBuilder builder; |
| builder.Finish(CreateFeatureProcessorOptions(builder, &options)); |
| flatbuffers::DetachedBuffer options_fb = builder.Release(); |
| |
| std::vector<std::vector<int>> sparse_features(9); |
| for (int i = 0; i < sparse_features.size(); ++i) { |
| sparse_features[i].push_back(i + 1); |
| } |
| std::vector<std::vector<float>> dense_features(9); |
| for (int i = 0; i < dense_features.size(); ++i) { |
| dense_features[i].push_back((i + 1) * 0.1); |
| } |
| |
| std::vector<int> padding_sparse_features = {10203}; |
| std::vector<float> padding_dense_features = {321.0}; |
| |
| FakeEmbeddingExecutor executor; |
| const std::unique_ptr<CachedFeatures> cached_features = |
| CachedFeatures::Create( |
| {3, 10}, sparse_features, dense_features, padding_sparse_features, |
| padding_dense_features, |
| flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()), |
| &executor, /*feature_vector_size=*/3); |
| ASSERT_TRUE(cached_features); |
| |
| EXPECT_THAT(GetCachedClickContextFeatures(*cached_features, 5), |
| ElementsAreFloat({11.0, -11.0, 0.1, 22.0, -22.0, 0.2, 33.0, -33.0, |
| 0.3, 44.0, -44.0, 0.4, 55.0, -55.0, 0.5})); |
| |
| EXPECT_THAT(GetCachedClickContextFeatures(*cached_features, 6), |
| ElementsAreFloat({22.0, -22.0, 0.2, 33.0, -33.0, 0.3, 44.0, -44.0, |
| 0.4, 55.0, -55.0, 0.5, 66.0, -66.0, 0.6})); |
| |
| EXPECT_THAT(GetCachedClickContextFeatures(*cached_features, 7), |
| ElementsAreFloat({33.0, -33.0, 0.3, 44.0, -44.0, 0.4, 55.0, -55.0, |
| 0.5, 66.0, -66.0, 0.6, 77.0, -77.0, 0.7})); |
| } |
| |
| TEST(CachedFeaturesTest, BoundsSensitive) { |
| std::unique_ptr<FeatureProcessorOptions_::BoundsSensitiveFeaturesT> config( |
| new FeatureProcessorOptions_::BoundsSensitiveFeaturesT()); |
| config->enabled = true; |
| config->num_tokens_before = 2; |
| config->num_tokens_inside_left = 2; |
| config->num_tokens_inside_right = 2; |
| config->num_tokens_after = 2; |
| config->include_inside_bag = true; |
| config->include_inside_length = true; |
| FeatureProcessorOptionsT options; |
| options.bounds_sensitive_features = std::move(config); |
| options.feature_version = 2; |
| flatbuffers::FlatBufferBuilder builder; |
| builder.Finish(CreateFeatureProcessorOptions(builder, &options)); |
| flatbuffers::DetachedBuffer options_fb = builder.Release(); |
| |
| std::vector<std::vector<int>> sparse_features(6); |
| for (int i = 0; i < sparse_features.size(); ++i) { |
| sparse_features[i].push_back(i + 1); |
| } |
| std::vector<std::vector<float>> dense_features(6); |
| for (int i = 0; i < dense_features.size(); ++i) { |
| dense_features[i].push_back((i + 1) * 0.1); |
| } |
| |
| std::vector<int> padding_sparse_features = {10203}; |
| std::vector<float> padding_dense_features = {321.0}; |
| |
| FakeEmbeddingExecutor executor; |
| const std::unique_ptr<CachedFeatures> cached_features = |
| CachedFeatures::Create( |
| {3, 9}, sparse_features, dense_features, padding_sparse_features, |
| padding_dense_features, |
| flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()), |
| &executor, /*feature_vector_size=*/3); |
| ASSERT_TRUE(cached_features); |
| |
| EXPECT_THAT( |
| GetCachedBoundsSensitiveFeatures(*cached_features, {5, 8}), |
| ElementsAreFloat({11.0, -11.0, 0.1, 22.0, -22.0, 0.2, 33.0, |
| -33.0, 0.3, 44.0, -44.0, 0.4, 44.0, -44.0, |
| 0.4, 55.0, -55.0, 0.5, 66.0, -66.0, 0.6, |
| 112233.0, -112233.0, 321.0, 44.0, -44.0, 0.4, 3.0})); |
| |
| EXPECT_THAT( |
| GetCachedBoundsSensitiveFeatures(*cached_features, {5, 7}), |
| ElementsAreFloat({11.0, -11.0, 0.1, 22.0, -22.0, 0.2, 33.0, |
| -33.0, 0.3, 44.0, -44.0, 0.4, 33.0, -33.0, |
| 0.3, 44.0, -44.0, 0.4, 55.0, -55.0, 0.5, |
| 66.0, -66.0, 0.6, 38.5, -38.5, 0.35, 2.0})); |
| |
| EXPECT_THAT( |
| GetCachedBoundsSensitiveFeatures(*cached_features, {6, 8}), |
| ElementsAreFloat({22.0, -22.0, 0.2, 33.0, -33.0, 0.3, 44.0, |
| -44.0, 0.4, 55.0, -55.0, 0.5, 44.0, -44.0, |
| 0.4, 55.0, -55.0, 0.5, 66.0, -66.0, 0.6, |
| 112233.0, -112233.0, 321.0, 49.5, -49.5, 0.45, 2.0})); |
| |
| EXPECT_THAT( |
| GetCachedBoundsSensitiveFeatures(*cached_features, {6, 7}), |
| ElementsAreFloat({22.0, -22.0, 0.2, 33.0, -33.0, 0.3, |
| 44.0, -44.0, 0.4, 112233.0, -112233.0, 321.0, |
| 112233.0, -112233.0, 321.0, 44.0, -44.0, 0.4, |
| 55.0, -55.0, 0.5, 66.0, -66.0, 0.6, |
| 44.0, -44.0, 0.4, 1.0})); |
| } |
| |
| } // namespace |
| } // namespace libtextclassifier2 |