| // Copyright (C) 2019 Google LLC |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include <unistd.h> |
| |
| #include <cstdint> |
| #include <limits> |
| #include <memory> |
| #include <string> |
| #include <utility> |
| |
| #include "icing/text_classifier/lib3/utils/base/status.h" |
| #include "gmock/gmock.h" |
| #include "gtest/gtest.h" |
| #include "icing/document-builder.h" |
| #include "icing/file/filesystem.h" |
| #include "icing/file/mock-filesystem.h" |
| #include "icing/icing-search-engine.h" |
| #include "icing/jni/jni-cache.h" |
| #include "icing/join/join-processor.h" |
| #include "icing/portable/endian.h" |
| #include "icing/portable/equals-proto.h" |
| #include "icing/portable/platform.h" |
| #include "icing/proto/debug.pb.h" |
| #include "icing/proto/document.pb.h" |
| #include "icing/proto/document_wrapper.pb.h" |
| #include "icing/proto/initialize.pb.h" |
| #include "icing/proto/logging.pb.h" |
| #include "icing/proto/optimize.pb.h" |
| #include "icing/proto/persist.pb.h" |
| #include "icing/proto/reset.pb.h" |
| #include "icing/proto/schema.pb.h" |
| #include "icing/proto/scoring.pb.h" |
| #include "icing/proto/search.pb.h" |
| #include "icing/proto/status.pb.h" |
| #include "icing/proto/storage.pb.h" |
| #include "icing/proto/term.pb.h" |
| #include "icing/proto/usage.pb.h" |
| #include "icing/query/query-features.h" |
| #include "icing/schema-builder.h" |
| #include "icing/store/document-log-creator.h" |
| #include "icing/testing/common-matchers.h" |
| #include "icing/testing/fake-clock.h" |
| #include "icing/testing/icu-data-file-helper.h" |
| #include "icing/testing/jni-test-helpers.h" |
| #include "icing/testing/test-data.h" |
| #include "icing/testing/tmp-directory.h" |
| |
| namespace icing { |
| namespace lib { |
| |
| namespace { |
| |
| using ::icing::lib::portable_equals_proto::EqualsProto; |
| using ::testing::Eq; |
| using ::testing::Ge; |
| using ::testing::Gt; |
| using ::testing::HasSubstr; |
| using ::testing::Lt; |
| using ::testing::Return; |
| |
| // For mocking purpose, we allow tests to provide a custom Filesystem. |
| class TestIcingSearchEngine : public IcingSearchEngine { |
| public: |
| TestIcingSearchEngine(const IcingSearchEngineOptions& options, |
| std::unique_ptr<const Filesystem> filesystem, |
| std::unique_ptr<const IcingFilesystem> icing_filesystem, |
| std::unique_ptr<Clock> clock, |
| std::unique_ptr<JniCache> jni_cache) |
| : IcingSearchEngine(options, std::move(filesystem), |
| std::move(icing_filesystem), std::move(clock), |
| std::move(jni_cache)) {} |
| }; |
| |
| std::string GetTestBaseDir() { return GetTestTempDir() + "/icing"; } |
| |
| // This test is meant to cover all tests relating to |
| // IcingSearchEngine::Optimize. |
| class IcingSearchEngineOptimizeTest : public testing::Test { |
| protected: |
| void SetUp() override { |
| if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { |
| // If we've specified using the reverse-JNI method for segmentation (i.e. |
| // not ICU), then we won't have the ICU data file included to set up. |
| // Technically, we could choose to use reverse-JNI for segmentation AND |
| // include an ICU data file, but that seems unlikely and our current BUILD |
| // setup doesn't do this. |
| // File generated via icu_data_file rule in //icing/BUILD. |
| std::string icu_data_file_path = |
| GetTestFilePath("icing/icu.dat"); |
| ICING_ASSERT_OK( |
| icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); |
| } |
| filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); |
| } |
| |
| void TearDown() override { |
| filesystem_.DeleteDirectoryRecursively(GetTestBaseDir().c_str()); |
| } |
| |
| const Filesystem* filesystem() const { return &filesystem_; } |
| |
| private: |
| Filesystem filesystem_; |
| }; |
| |
| // Non-zero value so we don't override it to be the current time |
| constexpr int64_t kDefaultCreationTimestampMs = 1575492852000; |
| |
| IcingSearchEngineOptions GetDefaultIcingOptions() { |
| IcingSearchEngineOptions icing_options; |
| icing_options.set_base_dir(GetTestBaseDir()); |
| return icing_options; |
| } |
| |
| ScoringSpecProto GetDefaultScoringSpec() { |
| ScoringSpecProto scoring_spec; |
| scoring_spec.set_rank_by(ScoringSpecProto::RankingStrategy::DOCUMENT_SCORE); |
| return scoring_spec; |
| } |
| |
| // TODO(b/272145329): create SearchSpecBuilder, JoinSpecBuilder, |
| // SearchResultProtoBuilder and ResultProtoBuilder for unit tests and build all |
| // instances by them. |
| |
| TEST_F(IcingSearchEngineOptimizeTest, |
| AllPageTokensShouldBeInvalidatedAfterOptimization) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document1 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document2 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| |
| ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); |
| |
| SearchSpecProto search_spec; |
| search_spec.set_term_match_type(TermMatchType::PREFIX); |
| search_spec.set_query("message"); |
| |
| ResultSpecProto result_spec; |
| result_spec.set_num_per_page(1); |
| |
| // Searches and gets the first page, 1 result |
| SearchResultProto expected_search_result_proto; |
| expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); |
| *expected_search_result_proto.mutable_results()->Add()->mutable_document() = |
| document2; |
| SearchResultProto search_result_proto = |
| icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); |
| EXPECT_THAT(search_result_proto.next_page_token(), Gt(kInvalidNextPageToken)); |
| uint64_t next_page_token = search_result_proto.next_page_token(); |
| // Since the token is a random number, we don't need to verify |
| expected_search_result_proto.set_next_page_token(next_page_token); |
| EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| // Now document1 is still to be fetched. |
| |
| OptimizeResultProto optimize_result_proto; |
| optimize_result_proto.mutable_status()->set_code(StatusProto::OK); |
| optimize_result_proto.mutable_status()->set_message(""); |
| OptimizeResultProto actual_result = icing.Optimize(); |
| actual_result.clear_optimize_stats(); |
| ASSERT_THAT(actual_result, EqualsProto(optimize_result_proto)); |
| |
| // Tries to fetch the second page, no results since all tokens have been |
| // invalidated during Optimize() |
| expected_search_result_proto.clear_results(); |
| expected_search_result_proto.clear_next_page_token(); |
| search_result_proto = icing.GetNextPage(next_page_token); |
| EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, OptimizationShouldRemoveDeletedDocs) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document1 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| IcingSearchEngineOptions icing_options = GetDefaultIcingOptions(); |
| |
| GetResultProto expected_get_result_proto; |
| expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND); |
| expected_get_result_proto.mutable_status()->set_message( |
| "Document (namespace, uri1) not found."); |
| { |
| IcingSearchEngine icing(icing_options, GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); |
| |
| // Deletes document1 |
| ASSERT_THAT(icing.Delete("namespace", "uri1").status(), ProtoIsOk()); |
| const std::string document_log_path = |
| icing_options.base_dir() + "/document_dir/" + |
| DocumentLogCreator::GetDocumentLogFilename(); |
| int64_t document_log_size_before = |
| filesystem()->GetFileSize(document_log_path.c_str()); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| int64_t document_log_size_after = |
| filesystem()->GetFileSize(document_log_path.c_str()); |
| |
| // Validates that document can't be found right after Optimize() |
| EXPECT_THAT( |
| icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| // Validates that document is actually removed from document log |
| EXPECT_THAT(document_log_size_after, Lt(document_log_size_before)); |
| } // Destroys IcingSearchEngine to make sure nothing is cached. |
| |
| IcingSearchEngine icing(icing_options, GetTestJniCache()); |
| EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| EXPECT_THAT( |
| icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, |
| OptimizationShouldDeleteTemporaryDirectory) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| IcingSearchEngineOptions icing_options = GetDefaultIcingOptions(); |
| IcingSearchEngine icing(icing_options, GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| |
| // Create a tmp dir that will be used in Optimize() to swap files, |
| // this validates that any tmp dirs will be deleted before using. |
| const std::string tmp_dir = |
| icing_options.base_dir() + "/document_dir_optimize_tmp"; |
| |
| const std::string tmp_file = tmp_dir + "/file"; |
| ASSERT_TRUE(filesystem()->CreateDirectory(tmp_dir.c_str())); |
| ScopedFd fd(filesystem()->OpenForWrite(tmp_file.c_str())); |
| ASSERT_TRUE(fd.is_valid()); |
| ASSERT_TRUE(filesystem()->Write(fd.get(), "1234", 4)); |
| fd.reset(); |
| |
| EXPECT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| |
| EXPECT_FALSE(filesystem()->DirectoryExists(tmp_dir.c_str())); |
| EXPECT_FALSE(filesystem()->FileExists(tmp_file.c_str())); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, GetOptimizeInfoHasCorrectStats) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document1 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document2 = DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .SetCreationTimestampMs(100) |
| .SetTtlMs(500) |
| .Build(); |
| |
| { |
| auto fake_clock = std::make_unique<FakeClock>(); |
| fake_clock->SetSystemTimeMilliseconds(1000); |
| |
| TestIcingSearchEngine icing(GetDefaultIcingOptions(), |
| std::make_unique<Filesystem>(), |
| std::make_unique<IcingFilesystem>(), |
| std::move(fake_clock), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| |
| // Just initialized, nothing is optimizable yet. |
| GetOptimizeInfoResultProto optimize_info = icing.GetOptimizeInfo(); |
| EXPECT_THAT(optimize_info.status(), ProtoIsOk()); |
| EXPECT_THAT(optimize_info.optimizable_docs(), Eq(0)); |
| EXPECT_THAT(optimize_info.estimated_optimizable_bytes(), Eq(0)); |
| EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(0)); |
| |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); |
| |
| // Only have active documents, nothing is optimizable yet. |
| optimize_info = icing.GetOptimizeInfo(); |
| EXPECT_THAT(optimize_info.status(), ProtoIsOk()); |
| EXPECT_THAT(optimize_info.optimizable_docs(), Eq(0)); |
| EXPECT_THAT(optimize_info.estimated_optimizable_bytes(), Eq(0)); |
| EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(0)); |
| |
| // Deletes document1 |
| ASSERT_THAT(icing.Delete("namespace", "uri1").status(), ProtoIsOk()); |
| |
| optimize_info = icing.GetOptimizeInfo(); |
| EXPECT_THAT(optimize_info.status(), ProtoIsOk()); |
| EXPECT_THAT(optimize_info.optimizable_docs(), Eq(1)); |
| EXPECT_THAT(optimize_info.estimated_optimizable_bytes(), Gt(0)); |
| EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(0)); |
| int64_t first_estimated_optimizable_bytes = |
| optimize_info.estimated_optimizable_bytes(); |
| |
| // Add a second document, but it'll be expired since the time (1000) is |
| // greater than the document's creation timestamp (100) + the document's ttl |
| // (500) |
| ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); |
| |
| optimize_info = icing.GetOptimizeInfo(); |
| EXPECT_THAT(optimize_info.status(), ProtoIsOk()); |
| EXPECT_THAT(optimize_info.optimizable_docs(), Eq(2)); |
| EXPECT_THAT(optimize_info.estimated_optimizable_bytes(), |
| Gt(first_estimated_optimizable_bytes)); |
| EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(0)); |
| |
| // Optimize |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| } |
| |
| { |
| // Recreate with new time |
| auto fake_clock = std::make_unique<FakeClock>(); |
| fake_clock->SetSystemTimeMilliseconds(5000); |
| |
| TestIcingSearchEngine icing(GetDefaultIcingOptions(), |
| std::make_unique<Filesystem>(), |
| std::make_unique<IcingFilesystem>(), |
| std::move(fake_clock), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| |
| // Nothing is optimizable now that everything has been optimized away. |
| GetOptimizeInfoResultProto optimize_info = icing.GetOptimizeInfo(); |
| EXPECT_THAT(optimize_info.status(), ProtoIsOk()); |
| EXPECT_THAT(optimize_info.optimizable_docs(), Eq(0)); |
| EXPECT_THAT(optimize_info.estimated_optimizable_bytes(), Eq(0)); |
| EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(4000)); |
| } |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, GetAndPutShouldWorkAfterOptimization) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document1 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document2 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document3 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri3") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body three") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document4 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri4") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body four") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document5 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri5") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body five") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| GetResultProto expected_get_result_proto; |
| expected_get_result_proto.mutable_status()->set_code(StatusProto::OK); |
| |
| { |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| |
| ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document3).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Delete("namespace", "uri2").status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| |
| // Validates that Get() and Put() are good right after Optimize() |
| *expected_get_result_proto.mutable_document() = document1; |
| EXPECT_THAT( |
| icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| EXPECT_THAT( |
| icing.Get("namespace", "uri2", GetResultSpecProto::default_instance()) |
| .status() |
| .code(), |
| Eq(StatusProto::NOT_FOUND)); |
| *expected_get_result_proto.mutable_document() = document3; |
| EXPECT_THAT( |
| icing.Get("namespace", "uri3", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| EXPECT_THAT(icing.Put(document4).status(), ProtoIsOk()); |
| } // Destroys IcingSearchEngine to make sure nothing is cached. |
| |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| *expected_get_result_proto.mutable_document() = document1; |
| EXPECT_THAT( |
| icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| EXPECT_THAT( |
| icing.Get("namespace", "uri2", GetResultSpecProto::default_instance()) |
| .status() |
| .code(), |
| Eq(StatusProto::NOT_FOUND)); |
| *expected_get_result_proto.mutable_document() = document3; |
| EXPECT_THAT( |
| icing.Get("namespace", "uri3", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| *expected_get_result_proto.mutable_document() = document4; |
| EXPECT_THAT( |
| icing.Get("namespace", "uri4", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| |
| EXPECT_THAT(icing.Put(document5).status(), ProtoIsOk()); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, |
| GetAndPutShouldWorkAfterOptimizationWithEmptyDocuments) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto empty_document1 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto empty_document2 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto empty_document3 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri3") |
| .SetSchema("Message") |
| .AddStringProperty("body", "") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| GetResultProto expected_get_result_proto; |
| expected_get_result_proto.mutable_status()->set_code(StatusProto::OK); |
| |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(std::move(schema)).status(), ProtoIsOk()); |
| |
| ASSERT_THAT(icing.Put(empty_document1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(empty_document2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Delete("namespace", "uri2").status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| |
| // Validates that Get() and Put() are good right after Optimize() |
| *expected_get_result_proto.mutable_document() = empty_document1; |
| EXPECT_THAT( |
| icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| EXPECT_THAT( |
| icing.Get("namespace", "uri2", GetResultSpecProto::default_instance()) |
| .status() |
| .code(), |
| Eq(StatusProto::NOT_FOUND)); |
| EXPECT_THAT(icing.Put(empty_document3).status(), ProtoIsOk()); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, DeleteShouldWorkAfterOptimization) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document1 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document2 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| { |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| |
| // Validates that Delete() works right after Optimize() |
| EXPECT_THAT(icing.Delete("namespace", "uri1").status(), ProtoIsOk()); |
| |
| GetResultProto expected_get_result_proto; |
| expected_get_result_proto.mutable_status()->set_code( |
| StatusProto::NOT_FOUND); |
| expected_get_result_proto.mutable_status()->set_message( |
| "Document (namespace, uri1) not found."); |
| EXPECT_THAT( |
| icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| |
| expected_get_result_proto.mutable_status()->set_code(StatusProto::OK); |
| expected_get_result_proto.mutable_status()->clear_message(); |
| *expected_get_result_proto.mutable_document() = document2; |
| EXPECT_THAT( |
| icing.Get("namespace", "uri2", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| } // Destroys IcingSearchEngine to make sure nothing is cached. |
| |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| EXPECT_THAT(icing.Delete("namespace", "uri2").status(), ProtoIsOk()); |
| |
| GetResultProto expected_get_result_proto; |
| expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND); |
| expected_get_result_proto.mutable_status()->set_message( |
| "Document (namespace, uri1) not found."); |
| EXPECT_THAT( |
| icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| |
| expected_get_result_proto.mutable_status()->set_message( |
| "Document (namespace, uri2) not found."); |
| EXPECT_THAT( |
| icing.Get("namespace", "uri2", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, OptimizationFailureUninitializesIcing) { |
| // Setup filesystem to fail |
| auto mock_filesystem = std::make_unique<MockFilesystem>(); |
| bool just_swapped_files = false; |
| auto create_dir_lambda = [this, &just_swapped_files](const char* dir_name) { |
| if (just_swapped_files) { |
| // We should fail the first call immediately after swapping files. |
| just_swapped_files = false; |
| return false; |
| } |
| return filesystem()->CreateDirectoryRecursively(dir_name); |
| }; |
| ON_CALL(*mock_filesystem, CreateDirectoryRecursively) |
| .WillByDefault(create_dir_lambda); |
| |
| auto swap_lambda = [&just_swapped_files](const char* first_dir, |
| const char* second_dir) { |
| just_swapped_files = true; |
| return false; |
| }; |
| IcingSearchEngineOptions options = GetDefaultIcingOptions(); |
| ON_CALL(*mock_filesystem, SwapFiles(HasSubstr("document_dir_optimize_tmp"), |
| HasSubstr("document_dir"))) |
| .WillByDefault(swap_lambda); |
| TestIcingSearchEngine icing(options, std::move(mock_filesystem), |
| std::make_unique<IcingFilesystem>(), |
| std::make_unique<FakeClock>(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| |
| // The mocks should cause an unrecoverable error during Optimize - returning |
| // INTERNAL. |
| ASSERT_THAT(icing.Optimize().status(), ProtoStatusIs(StatusProto::INTERNAL)); |
| |
| // Ordinary operations should fail safely. |
| SchemaProto simple_schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("type0").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("prop0") |
| .SetDataType(TYPE_STRING) |
| .SetCardinality(CARDINALITY_OPTIONAL))) |
| .Build(); |
| |
| DocumentProto simple_doc = DocumentBuilder() |
| .SetKey("namespace0", "uri0") |
| .SetSchema("type0") |
| .AddStringProperty("prop0", "foo") |
| .Build(); |
| |
| SearchSpecProto search_spec; |
| search_spec.set_query("foo"); |
| search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); |
| ResultSpecProto result_spec; |
| ScoringSpecProto scoring_spec; |
| scoring_spec.set_rank_by( |
| ScoringSpecProto::RankingStrategy::CREATION_TIMESTAMP); |
| |
| EXPECT_THAT(icing.SetSchema(simple_schema).status(), |
| ProtoStatusIs(StatusProto::FAILED_PRECONDITION)); |
| EXPECT_THAT(icing.Put(simple_doc).status(), |
| ProtoStatusIs(StatusProto::FAILED_PRECONDITION)); |
| EXPECT_THAT(icing |
| .Get(simple_doc.namespace_(), simple_doc.uri(), |
| GetResultSpecProto::default_instance()) |
| .status(), |
| ProtoStatusIs(StatusProto::FAILED_PRECONDITION)); |
| EXPECT_THAT(icing.Search(search_spec, scoring_spec, result_spec).status(), |
| ProtoStatusIs(StatusProto::FAILED_PRECONDITION)); |
| |
| // Reset should get icing back to a safe (empty) and working state. |
| EXPECT_THAT(icing.Reset().status(), ProtoIsOk()); |
| EXPECT_THAT(icing.SetSchema(simple_schema).status(), ProtoIsOk()); |
| EXPECT_THAT(icing.Put(simple_doc).status(), ProtoIsOk()); |
| EXPECT_THAT(icing |
| .Get(simple_doc.namespace_(), simple_doc.uri(), |
| GetResultSpecProto::default_instance()) |
| .status(), |
| ProtoIsOk()); |
| EXPECT_THAT(icing.Search(search_spec, scoring_spec, result_spec).status(), |
| ProtoIsOk()); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, SetSchemaShouldWorkAfterOptimization) { |
| // Creates 3 test schemas |
| SchemaProto schema1 = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| SchemaProto schema2 = SchemaProto(schema1); |
| *schema2.mutable_types(0)->add_properties() = |
| PropertyConfigBuilder() |
| .SetName("property2") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_OPTIONAL) |
| .Build(); |
| |
| SchemaProto schema3 = SchemaProto(schema2); |
| *schema3.mutable_types(0)->add_properties() = |
| PropertyConfigBuilder() |
| .SetName("property3") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_OPTIONAL) |
| .Build(); |
| |
| { |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| |
| // Validates that SetSchema() works right after Optimize() |
| EXPECT_THAT(icing.SetSchema(schema2).status(), ProtoIsOk()); |
| } // Destroys IcingSearchEngine to make sure nothing is cached. |
| |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| EXPECT_THAT(icing.SetSchema(schema3).status(), ProtoIsOk()); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, SearchShouldWorkAfterOptimization) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder() |
| .SetType("Message") |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, |
| TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED)) |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("indexableInteger") |
| .SetDataTypeInt64(NUMERIC_MATCH_RANGE) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document = |
| DocumentBuilder() |
| .SetKey("namespace", "uri") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body") |
| .AddInt64Property("indexableInteger", 123) |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| SearchSpecProto search_spec1; |
| search_spec1.set_term_match_type(TermMatchType::PREFIX); |
| search_spec1.set_query("m"); |
| |
| SearchSpecProto search_spec2; |
| search_spec2.set_query("indexableInteger == 123"); |
| search_spec2.set_search_type( |
| SearchSpecProto::SearchType::EXPERIMENTAL_ICING_ADVANCED_QUERY); |
| search_spec2.add_enabled_features(std::string(kNumericSearchFeature)); |
| |
| SearchResultProto expected_search_result_proto; |
| expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); |
| *expected_search_result_proto.mutable_results()->Add()->mutable_document() = |
| document; |
| |
| { |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| |
| // Validates that Search() works right after Optimize() |
| // Term search |
| SearchResultProto search_result_proto1 = |
| icing.Search(search_spec1, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Numeric (integer) search |
| SearchResultProto search_result_google::protobuf = |
| icing.Search(search_spec2, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| } // Destroys IcingSearchEngine to make sure nothing is cached. |
| |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| |
| // Verify term search |
| SearchResultProto search_result_proto1 = |
| icing.Search(search_spec1, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Verify numeric (integer) search |
| SearchResultProto search_result_google::protobuf = |
| icing.Search(search_spec2, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, |
| JoinShouldWorkAfterOptimizationDeleteParent) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("name") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .AddType(SchemaTypeConfigBuilder() |
| .SetType("Message") |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, |
| TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED)) |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("senderQualifiedId") |
| .SetDataTypeJoinableString( |
| JOINABLE_VALUE_TYPE_QUALIFIED_ID) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto person1 = |
| DocumentBuilder() |
| .SetKey("namespace", "person1") |
| .SetSchema("Person") |
| .AddStringProperty("name", "person one") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto person2 = |
| DocumentBuilder() |
| .SetKey("namespace", "person2") |
| .SetSchema("Person") |
| .AddStringProperty("name", "person two") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| DocumentProto message1 = |
| DocumentBuilder() |
| .SetKey("namespace", "message1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .AddStringProperty("senderQualifiedId", "namespace#person1") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto message2 = |
| DocumentBuilder() |
| .SetKey("namespace", "message2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .AddStringProperty("senderQualifiedId", "namespace#person1") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto message3 = |
| DocumentBuilder() |
| .SetKey("namespace", "message3") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body three") |
| .AddStringProperty("senderQualifiedId", "namespace#person2") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| // Prepare join search spec to join a query for `name:person` with a child |
| // query for `body:message` based on the child's `senderQualifiedId` field. |
| SearchSpecProto search_spec; |
| search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); |
| search_spec.set_query("name:person"); |
| JoinSpecProto* join_spec = search_spec.mutable_join_spec(); |
| join_spec->set_parent_property_expression( |
| std::string(JoinProcessor::kQualifiedIdExpr)); |
| join_spec->set_child_property_expression("senderQualifiedId"); |
| join_spec->set_aggregation_scoring_strategy( |
| JoinSpecProto::AggregationScoringStrategy::COUNT); |
| JoinSpecProto::NestedSpecProto* nested_spec = |
| join_spec->mutable_nested_spec(); |
| SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec(); |
| nested_search_spec->set_term_match_type(TermMatchType::EXACT_ONLY); |
| nested_search_spec->set_query("body:message"); |
| *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec(); |
| *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance(); |
| |
| ResultSpecProto result_spec = ResultSpecProto::default_instance(); |
| result_spec.set_max_joined_children_per_parent_to_return( |
| std::numeric_limits<int32_t>::max()); |
| |
| // Person1 is going to be deleted below. Only person2 which is joined with |
| // message3 should match the query. |
| SearchResultProto expected_search_result_proto; |
| expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); |
| SearchResultProto::ResultProto* result_proto = |
| expected_search_result_proto.mutable_results()->Add(); |
| *result_proto->mutable_document() = person2; |
| *result_proto->mutable_joined_results()->Add()->mutable_document() = message3; |
| |
| { |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(person1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(person2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(message1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(message2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(message3).status(), ProtoIsOk()); |
| // Delete parent document: person1 |
| ASSERT_THAT(icing.Delete("namespace", "person1").status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| |
| // Validates that join search query works right after Optimize() |
| SearchResultProto search_result_proto = |
| icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); |
| EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| } // Destroys IcingSearchEngine to make sure nothing is cached. |
| |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| |
| SearchResultProto search_result_proto = |
| icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); |
| EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, |
| JoinShouldWorkAfterOptimizationDeleteChild) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("name") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .AddType(SchemaTypeConfigBuilder() |
| .SetType("Message") |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, |
| TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED)) |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("senderQualifiedId") |
| .SetDataTypeJoinableString( |
| JOINABLE_VALUE_TYPE_QUALIFIED_ID) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto person1 = |
| DocumentBuilder() |
| .SetKey("namespace", "person1") |
| .SetSchema("Person") |
| .AddStringProperty("name", "person one") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto person2 = |
| DocumentBuilder() |
| .SetKey("namespace", "person2") |
| .SetSchema("Person") |
| .AddStringProperty("name", "person two") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| DocumentProto message1 = |
| DocumentBuilder() |
| .SetKey("namespace", "message1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .AddStringProperty("senderQualifiedId", "namespace#person1") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto message2 = |
| DocumentBuilder() |
| .SetKey("namespace", "message2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .AddStringProperty("senderQualifiedId", "namespace#person1") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto message3 = |
| DocumentBuilder() |
| .SetKey("namespace", "message3") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body three") |
| .AddStringProperty("senderQualifiedId", "namespace#person2") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| // Prepare join search spec to join a query for `name:person` with a child |
| // query for `body:message` based on the child's `senderQualifiedId` field. |
| SearchSpecProto search_spec; |
| search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); |
| search_spec.set_query("name:person"); |
| JoinSpecProto* join_spec = search_spec.mutable_join_spec(); |
| join_spec->set_parent_property_expression( |
| std::string(JoinProcessor::kQualifiedIdExpr)); |
| join_spec->set_child_property_expression("senderQualifiedId"); |
| join_spec->set_aggregation_scoring_strategy( |
| JoinSpecProto::AggregationScoringStrategy::COUNT); |
| JoinSpecProto::NestedSpecProto* nested_spec = |
| join_spec->mutable_nested_spec(); |
| SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec(); |
| nested_search_spec->set_term_match_type(TermMatchType::EXACT_ONLY); |
| nested_search_spec->set_query("body:message"); |
| *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec(); |
| *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance(); |
| |
| ResultSpecProto result_spec = ResultSpecProto::default_instance(); |
| result_spec.set_max_joined_children_per_parent_to_return( |
| std::numeric_limits<int32_t>::max()); |
| |
| // Message1 and message3 are going to be deleted below. Both person1 and |
| // person2 should be included even though person2 has no child (since we're |
| // doing left join). |
| SearchResultProto expected_search_result_proto; |
| expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); |
| SearchResultProto::ResultProto* result_proto1 = |
| expected_search_result_proto.mutable_results()->Add(); |
| *result_proto1->mutable_document() = person1; |
| *result_proto1->mutable_joined_results()->Add()->mutable_document() = |
| message2; |
| SearchResultProto::ResultProto* result_google::protobuf = |
| expected_search_result_proto.mutable_results()->Add(); |
| *result_google::protobuf->mutable_document() = person2; |
| |
| { |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(person1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(person2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(message1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(message2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(message3).status(), ProtoIsOk()); |
| // Delete child documents: message1 and message3 |
| ASSERT_THAT(icing.Delete("namespace", "message1").status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Delete("namespace", "message3").status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| |
| // Validates that join search query works right after Optimize() |
| SearchResultProto search_result_proto = |
| icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); |
| EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| } // Destroys IcingSearchEngine to make sure nothing is cached. |
| |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| |
| SearchResultProto search_result_proto = |
| icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); |
| EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, |
| IcingShouldWorkFineIfOptimizationIsAborted) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("name") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .AddType(SchemaTypeConfigBuilder() |
| .SetType("Message") |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, |
| TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED)) |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("indexableInteger") |
| .SetDataTypeInt64(NUMERIC_MATCH_RANGE) |
| .SetCardinality(CARDINALITY_REQUIRED)) |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("senderQualifiedId") |
| .SetDataTypeJoinableString( |
| JOINABLE_VALUE_TYPE_QUALIFIED_ID) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto person = |
| DocumentBuilder() |
| .SetKey("namespace", "person") |
| .SetSchema("Person") |
| .AddStringProperty("name", "person") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| DocumentProto message1 = |
| DocumentBuilder() |
| .SetKey("namespace", "message1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .AddInt64Property("indexableInteger", 123) |
| .AddStringProperty("senderQualifiedId", "namespace#person") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| { |
| // Initializes a normal icing to create files needed |
| IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(person).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(message1).status(), ProtoIsOk()); |
| } |
| |
| // Creates a mock filesystem in which DeleteDirectoryRecursively() always |
| // fails. This will fail IcingSearchEngine::OptimizeDocumentStore() and makes |
| // it return ABORTED_ERROR. |
| auto mock_filesystem = std::make_unique<MockFilesystem>(); |
| ON_CALL(*mock_filesystem, |
| DeleteDirectoryRecursively(HasSubstr("_optimize_tmp"))) |
| .WillByDefault(Return(false)); |
| |
| TestIcingSearchEngine icing(GetDefaultIcingOptions(), |
| std::move(mock_filesystem), |
| std::make_unique<IcingFilesystem>(), |
| std::make_unique<FakeClock>(), GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| EXPECT_THAT(icing.Optimize().status(), ProtoStatusIs(StatusProto::ABORTED)); |
| |
| // Now optimization is aborted, we verify that document-related functions |
| // still work as expected. |
| |
| GetResultProto expected_get_result_proto; |
| expected_get_result_proto.mutable_status()->set_code(StatusProto::OK); |
| *expected_get_result_proto.mutable_document() = message1; |
| EXPECT_THAT(icing.Get("namespace", "message1", |
| GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| |
| DocumentProto message2 = |
| DocumentBuilder() |
| .SetKey("namespace", "message2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .AddInt64Property("indexableInteger", 123) |
| .AddStringProperty("senderQualifiedId", "namespace#person") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| EXPECT_THAT(icing.Put(message2).status(), ProtoIsOk()); |
| |
| SearchResultProto expected_search_result_proto; |
| expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); |
| *expected_search_result_proto.mutable_results()->Add()->mutable_document() = |
| message2; |
| *expected_search_result_proto.mutable_results()->Add()->mutable_document() = |
| message1; |
| |
| // Verify term search |
| SearchSpecProto search_spec1; |
| search_spec1.set_query("body:m"); |
| search_spec1.set_term_match_type(TermMatchType::PREFIX); |
| |
| SearchResultProto search_result_proto1 = |
| icing.Search(search_spec1, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Verify numeric (integer) search |
| SearchSpecProto search_spec2; |
| search_spec2.set_query("indexableInteger == 123"); |
| search_spec2.set_search_type( |
| SearchSpecProto::SearchType::EXPERIMENTAL_ICING_ADVANCED_QUERY); |
| search_spec2.add_enabled_features(std::string(kNumericSearchFeature)); |
| |
| SearchResultProto search_result_google::protobuf = |
| icing.Search(search_spec2, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Verify join search: join a query for `name:person` with a child query for |
| // `body:message` based on the child's `senderQualifiedId` field. |
| SearchSpecProto search_spec3; |
| search_spec3.set_term_match_type(TermMatchType::EXACT_ONLY); |
| search_spec3.set_query("name:person"); |
| JoinSpecProto* join_spec = search_spec3.mutable_join_spec(); |
| join_spec->set_parent_property_expression( |
| std::string(JoinProcessor::kQualifiedIdExpr)); |
| join_spec->set_child_property_expression("senderQualifiedId"); |
| join_spec->set_aggregation_scoring_strategy( |
| JoinSpecProto::AggregationScoringStrategy::COUNT); |
| JoinSpecProto::NestedSpecProto* nested_spec = |
| join_spec->mutable_nested_spec(); |
| SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec(); |
| nested_search_spec->set_term_match_type(TermMatchType::EXACT_ONLY); |
| nested_search_spec->set_query("body:message"); |
| *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec(); |
| *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance(); |
| |
| ResultSpecProto result_spec3 = ResultSpecProto::default_instance(); |
| result_spec3.set_max_joined_children_per_parent_to_return( |
| std::numeric_limits<int32_t>::max()); |
| |
| SearchResultProto expected_join_search_result_proto; |
| expected_join_search_result_proto.mutable_status()->set_code(StatusProto::OK); |
| SearchResultProto::ResultProto* result_proto = |
| expected_join_search_result_proto.mutable_results()->Add(); |
| *result_proto->mutable_document() = person; |
| *result_proto->mutable_joined_results()->Add()->mutable_document() = message2; |
| *result_proto->mutable_joined_results()->Add()->mutable_document() = message1; |
| |
| SearchResultProto search_result_proto3 = |
| icing.Search(search_spec3, GetDefaultScoringSpec(), result_spec3); |
| EXPECT_THAT(search_result_proto3, EqualsSearchResultIgnoreStatsAndScores( |
| expected_join_search_result_proto)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, |
| OptimizationShouldRecoverIfFileDirectoriesAreMissing) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder() |
| .SetType("Message") |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, |
| TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED)) |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("indexableInteger") |
| .SetDataTypeInt64(NUMERIC_MATCH_RANGE) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body") |
| .AddInt64Property("indexableInteger", 123) |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| // Creates a mock filesystem in which SwapFiles() always fails and deletes the |
| // directories. This will fail IcingSearchEngine::OptimizeDocumentStore(). |
| auto mock_filesystem = std::make_unique<MockFilesystem>(); |
| ON_CALL(*mock_filesystem, SwapFiles(HasSubstr("document_dir_optimize_tmp"), |
| HasSubstr("document_dir"))) |
| .WillByDefault([this](const char* one, const char* two) { |
| filesystem()->DeleteDirectoryRecursively(one); |
| filesystem()->DeleteDirectoryRecursively(two); |
| return false; |
| }); |
| |
| TestIcingSearchEngine icing(GetDefaultIcingOptions(), |
| std::move(mock_filesystem), |
| std::make_unique<IcingFilesystem>(), |
| std::make_unique<FakeClock>(), GetTestJniCache()); |
| |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document).status(), ProtoIsOk()); |
| |
| // Optimize() fails due to filesystem error |
| OptimizeResultProto result = icing.Optimize(); |
| EXPECT_THAT(result.status(), ProtoStatusIs(StatusProto::WARNING_DATA_LOSS)); |
| // Should rebuild the index for data loss. |
| EXPECT_THAT(result.optimize_stats().index_restoration_mode(), |
| Eq(OptimizeStatsProto::FULL_INDEX_REBUILD)); |
| |
| // Document is not found because original file directory is missing |
| GetResultProto expected_get_result_proto; |
| expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND); |
| expected_get_result_proto.mutable_status()->set_message( |
| "Document (namespace, uri) not found."); |
| EXPECT_THAT( |
| icing.Get("namespace", "uri", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| |
| DocumentProto new_document = |
| DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "new body") |
| .AddInt64Property("indexableInteger", 456) |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| EXPECT_THAT(icing.Put(new_document).status(), ProtoIsOk()); |
| |
| SearchSpecProto search_spec1; |
| search_spec1.set_query("m"); |
| search_spec1.set_term_match_type(TermMatchType::PREFIX); |
| |
| SearchSpecProto search_spec2; |
| search_spec2.set_query("indexableInteger == 123"); |
| search_spec2.set_search_type( |
| SearchSpecProto::SearchType::EXPERIMENTAL_ICING_ADVANCED_QUERY); |
| search_spec2.add_enabled_features(std::string(kNumericSearchFeature)); |
| |
| SearchResultProto expected_search_result_proto; |
| expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); |
| |
| // Searching old content returns nothing because original file directory is |
| // missing |
| // Term search |
| SearchResultProto search_result_proto1 = |
| icing.Search(search_spec1, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Numeric (integer) search |
| SearchResultProto search_result_google::protobuf = |
| icing.Search(search_spec2, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Searching new content returns the new document |
| *expected_search_result_proto.mutable_results()->Add()->mutable_document() = |
| new_document; |
| // Term search |
| search_spec1.set_query("n"); |
| search_result_proto1 = icing.Search(search_spec1, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Numeric (integer) search |
| search_spec2.set_query("indexableInteger == 456"); |
| search_result_google::protobuf = icing.Search(search_spec2, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, |
| OptimizationShouldRecoverIfDataFilesAreMissing) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder() |
| .SetType("Message") |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, |
| TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED)) |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("indexableInteger") |
| .SetDataTypeInt64(NUMERIC_MATCH_RANGE) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body") |
| .AddInt64Property("indexableInteger", 123) |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| // Creates a mock filesystem in which SwapFiles() always fails and empties the |
| // directories. This will fail IcingSearchEngine::OptimizeDocumentStore(). |
| auto mock_filesystem = std::make_unique<MockFilesystem>(); |
| ON_CALL(*mock_filesystem, SwapFiles(HasSubstr("document_dir_optimize_tmp"), |
| HasSubstr("document_dir"))) |
| .WillByDefault([this](const char* one, const char* two) { |
| filesystem()->DeleteDirectoryRecursively(one); |
| filesystem()->CreateDirectoryRecursively(one); |
| filesystem()->DeleteDirectoryRecursively(two); |
| filesystem()->CreateDirectoryRecursively(two); |
| return false; |
| }); |
| |
| TestIcingSearchEngine icing(GetDefaultIcingOptions(), |
| std::move(mock_filesystem), |
| std::make_unique<IcingFilesystem>(), |
| std::make_unique<FakeClock>(), GetTestJniCache()); |
| |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document).status(), ProtoIsOk()); |
| |
| // Optimize() fails due to filesystem error |
| OptimizeResultProto result = icing.Optimize(); |
| EXPECT_THAT(result.status(), ProtoStatusIs(StatusProto::WARNING_DATA_LOSS)); |
| // Should rebuild the index for data loss. |
| EXPECT_THAT(result.optimize_stats().index_restoration_mode(), |
| Eq(OptimizeStatsProto::FULL_INDEX_REBUILD)); |
| |
| // Document is not found because original files are missing |
| GetResultProto expected_get_result_proto; |
| expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND); |
| expected_get_result_proto.mutable_status()->set_message( |
| "Document (namespace, uri) not found."); |
| EXPECT_THAT( |
| icing.Get("namespace", "uri", GetResultSpecProto::default_instance()), |
| EqualsProto(expected_get_result_proto)); |
| |
| DocumentProto new_document = |
| DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "new body") |
| .AddInt64Property("indexableInteger", 456) |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| EXPECT_THAT(icing.Put(new_document).status(), ProtoIsOk()); |
| |
| SearchSpecProto search_spec1; |
| search_spec1.set_query("m"); |
| search_spec1.set_term_match_type(TermMatchType::PREFIX); |
| |
| SearchSpecProto search_spec2; |
| search_spec2.set_query("indexableInteger == 123"); |
| search_spec2.set_search_type( |
| SearchSpecProto::SearchType::EXPERIMENTAL_ICING_ADVANCED_QUERY); |
| search_spec2.add_enabled_features(std::string(kNumericSearchFeature)); |
| |
| SearchResultProto expected_search_result_proto; |
| expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); |
| |
| // Searching old content returns nothing because original files are missing |
| // Term search |
| SearchResultProto search_result_proto1 = |
| icing.Search(search_spec1, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Numeric (integer) search |
| SearchResultProto search_result_google::protobuf = |
| icing.Search(search_spec2, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Searching new content returns the new document |
| *expected_search_result_proto.mutable_results()->Add()->mutable_document() = |
| new_document; |
| // Term search |
| search_spec1.set_query("n"); |
| search_result_proto1 = icing.Search(search_spec1, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| |
| // Numeric (integer) search |
| search_spec2.set_query("indexableInteger == 456"); |
| search_result_google::protobuf = icing.Search(search_spec2, GetDefaultScoringSpec(), |
| ResultSpecProto::default_instance()); |
| EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores( |
| expected_search_result_proto)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, OptimizeThresholdTest) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder() |
| .SetType("Message") |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, |
| TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED)) |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("indexableInteger") |
| .SetDataTypeInt64(NUMERIC_MATCH_RANGE) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document1 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .AddInt64Property("indexableInteger", 1) |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document2 = DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .AddInt64Property("indexableInteger", 2) |
| .SetCreationTimestampMs(9000) |
| .SetTtlMs(500) |
| .Build(); |
| DocumentProto document3 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri3") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body three") |
| .AddInt64Property("indexableInteger", 3) |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| auto fake_clock = std::make_unique<FakeClock>(); |
| fake_clock->SetTimerElapsedMilliseconds(5); |
| fake_clock->SetSystemTimeMilliseconds(10000); |
| IcingSearchEngineOptions options = GetDefaultIcingOptions(); |
| // Set the threshold to 0.9 to test that the threshold works. |
| options.set_optimize_rebuild_index_threshold(0.9); |
| auto icing = std::make_unique<TestIcingSearchEngine>( |
| options, std::make_unique<Filesystem>(), |
| std::make_unique<IcingFilesystem>(), std::move(fake_clock), |
| GetTestJniCache()); |
| ASSERT_THAT(icing->Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing->SetSchema(schema).status(), ProtoIsOk()); |
| |
| // Add three documents. |
| ASSERT_THAT(icing->Put(document1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing->Put(document2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing->Put(document3).status(), ProtoIsOk()); |
| |
| // Delete the first document. |
| ASSERT_THAT(icing->Delete(document1.namespace_(), document1.uri()).status(), |
| ProtoIsOk()); |
| ASSERT_THAT(icing->PersistToDisk(PersistType::FULL).status(), ProtoIsOk()); |
| |
| OptimizeStatsProto expected; |
| expected.set_latency_ms(5); |
| expected.set_document_store_optimize_latency_ms(5); |
| expected.set_index_restoration_latency_ms(5); |
| expected.set_num_original_documents(3); |
| expected.set_num_deleted_documents(1); |
| expected.set_num_expired_documents(1); |
| expected.set_num_original_namespaces(1); |
| expected.set_num_deleted_namespaces(0); |
| expected.set_index_restoration_mode(OptimizeStatsProto::INDEX_TRANSLATION); |
| |
| // Run Optimize |
| OptimizeResultProto result = icing->Optimize(); |
| // Depending on how many blocks the documents end up spread across, it's |
| // possible that Optimize can remove documents without shrinking storage. The |
| // first Optimize call will also write the OptimizeStatusProto for the first |
| // time which will take up 1 block. So make sure that before_size is no less |
| // than after_size - 1 block. |
| uint32_t page_size = getpagesize(); |
| EXPECT_THAT(result.optimize_stats().storage_size_before(), |
| Ge(result.optimize_stats().storage_size_after() - page_size)); |
| result.mutable_optimize_stats()->clear_storage_size_before(); |
| result.mutable_optimize_stats()->clear_storage_size_after(); |
| EXPECT_THAT(result.optimize_stats(), EqualsProto(expected)); |
| |
| fake_clock = std::make_unique<FakeClock>(); |
| fake_clock->SetTimerElapsedMilliseconds(5); |
| fake_clock->SetSystemTimeMilliseconds(20000); |
| icing = std::make_unique<TestIcingSearchEngine>( |
| options, std::make_unique<Filesystem>(), |
| std::make_unique<IcingFilesystem>(), std::move(fake_clock), |
| GetTestJniCache()); |
| ASSERT_THAT(icing->Initialize().status(), ProtoIsOk()); |
| |
| expected = OptimizeStatsProto(); |
| expected.set_latency_ms(5); |
| expected.set_document_store_optimize_latency_ms(5); |
| expected.set_index_restoration_latency_ms(5); |
| expected.set_num_original_documents(1); |
| expected.set_num_deleted_documents(0); |
| expected.set_num_expired_documents(0); |
| expected.set_num_original_namespaces(1); |
| expected.set_num_deleted_namespaces(0); |
| expected.set_time_since_last_optimize_ms(10000); |
| expected.set_index_restoration_mode(OptimizeStatsProto::INDEX_TRANSLATION); |
| |
| // Run Optimize |
| result = icing->Optimize(); |
| EXPECT_THAT(result.optimize_stats().storage_size_before(), |
| Eq(result.optimize_stats().storage_size_after())); |
| result.mutable_optimize_stats()->clear_storage_size_before(); |
| result.mutable_optimize_stats()->clear_storage_size_after(); |
| EXPECT_THAT(result.optimize_stats(), EqualsProto(expected)); |
| |
| // Delete the last document. |
| ASSERT_THAT(icing->Delete(document3.namespace_(), document3.uri()).status(), |
| ProtoIsOk()); |
| |
| expected = OptimizeStatsProto(); |
| expected.set_latency_ms(5); |
| expected.set_document_store_optimize_latency_ms(5); |
| expected.set_index_restoration_latency_ms(5); |
| expected.set_num_original_documents(1); |
| expected.set_num_deleted_documents(1); |
| expected.set_num_expired_documents(0); |
| expected.set_num_original_namespaces(1); |
| expected.set_num_deleted_namespaces(1); |
| expected.set_time_since_last_optimize_ms(0); |
| // Should rebuild the index since all documents are removed. |
| expected.set_index_restoration_mode(OptimizeStatsProto::FULL_INDEX_REBUILD); |
| |
| // Run Optimize |
| result = icing->Optimize(); |
| EXPECT_THAT(result.optimize_stats().storage_size_before(), |
| Ge(result.optimize_stats().storage_size_after())); |
| result.mutable_optimize_stats()->clear_storage_size_before(); |
| result.mutable_optimize_stats()->clear_storage_size_after(); |
| EXPECT_THAT(result.optimize_stats(), EqualsProto(expected)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, OptimizeStatsProtoTest) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder() |
| .SetType("Message") |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, |
| TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED)) |
| .AddProperty(PropertyConfigBuilder() |
| .SetName("indexableInteger") |
| .SetDataTypeInt64(NUMERIC_MATCH_RANGE) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| |
| DocumentProto document1 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .AddInt64Property("indexableInteger", 1) |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document2 = DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .AddInt64Property("indexableInteger", 2) |
| .SetCreationTimestampMs(9000) |
| .SetTtlMs(500) |
| .Build(); |
| DocumentProto document3 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri3") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body three") |
| .AddInt64Property("indexableInteger", 3) |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| |
| auto fake_clock = std::make_unique<FakeClock>(); |
| fake_clock->SetTimerElapsedMilliseconds(5); |
| fake_clock->SetSystemTimeMilliseconds(10000); |
| // Use the default Icing options, so that a change to the default value will |
| // require updating this test. |
| auto icing = std::make_unique<TestIcingSearchEngine>( |
| GetDefaultIcingOptions(), std::make_unique<Filesystem>(), |
| std::make_unique<IcingFilesystem>(), std::move(fake_clock), |
| GetTestJniCache()); |
| ASSERT_THAT(icing->Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing->SetSchema(schema).status(), ProtoIsOk()); |
| |
| // Add three documents. |
| ASSERT_THAT(icing->Put(document1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing->Put(document2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing->Put(document3).status(), ProtoIsOk()); |
| |
| // Delete the first document. |
| ASSERT_THAT(icing->Delete(document1.namespace_(), document1.uri()).status(), |
| ProtoIsOk()); |
| ASSERT_THAT(icing->PersistToDisk(PersistType::FULL).status(), ProtoIsOk()); |
| |
| OptimizeStatsProto expected; |
| expected.set_latency_ms(5); |
| expected.set_document_store_optimize_latency_ms(5); |
| expected.set_index_restoration_latency_ms(5); |
| expected.set_num_original_documents(3); |
| expected.set_num_deleted_documents(1); |
| expected.set_num_expired_documents(1); |
| expected.set_num_original_namespaces(1); |
| expected.set_num_deleted_namespaces(0); |
| expected.set_index_restoration_mode(OptimizeStatsProto::FULL_INDEX_REBUILD); |
| |
| // Run Optimize |
| OptimizeResultProto result = icing->Optimize(); |
| // Depending on how many blocks the documents end up spread across, it's |
| // possible that Optimize can remove documents without shrinking storage. The |
| // first Optimize call will also write the OptimizeStatusProto for the first |
| // time which will take up 1 block. So make sure that before_size is no less |
| // than after_size - 1 block. |
| uint32_t page_size = getpagesize(); |
| EXPECT_THAT(result.optimize_stats().storage_size_before(), |
| Ge(result.optimize_stats().storage_size_after() - page_size)); |
| result.mutable_optimize_stats()->clear_storage_size_before(); |
| result.mutable_optimize_stats()->clear_storage_size_after(); |
| EXPECT_THAT(result.optimize_stats(), EqualsProto(expected)); |
| |
| fake_clock = std::make_unique<FakeClock>(); |
| fake_clock->SetTimerElapsedMilliseconds(5); |
| fake_clock->SetSystemTimeMilliseconds(20000); |
| // Use the default Icing options, so that a change to the default value will |
| // require updating this test. |
| icing = std::make_unique<TestIcingSearchEngine>( |
| GetDefaultIcingOptions(), std::make_unique<Filesystem>(), |
| std::make_unique<IcingFilesystem>(), std::move(fake_clock), |
| GetTestJniCache()); |
| ASSERT_THAT(icing->Initialize().status(), ProtoIsOk()); |
| |
| expected = OptimizeStatsProto(); |
| expected.set_latency_ms(5); |
| expected.set_document_store_optimize_latency_ms(5); |
| expected.set_index_restoration_latency_ms(5); |
| expected.set_num_original_documents(1); |
| expected.set_num_deleted_documents(0); |
| expected.set_num_expired_documents(0); |
| expected.set_num_original_namespaces(1); |
| expected.set_num_deleted_namespaces(0); |
| expected.set_time_since_last_optimize_ms(10000); |
| expected.set_index_restoration_mode(OptimizeStatsProto::FULL_INDEX_REBUILD); |
| |
| // Run Optimize |
| result = icing->Optimize(); |
| EXPECT_THAT(result.optimize_stats().storage_size_before(), |
| Eq(result.optimize_stats().storage_size_after())); |
| result.mutable_optimize_stats()->clear_storage_size_before(); |
| result.mutable_optimize_stats()->clear_storage_size_after(); |
| EXPECT_THAT(result.optimize_stats(), EqualsProto(expected)); |
| |
| // Delete the last document. |
| ASSERT_THAT(icing->Delete(document3.namespace_(), document3.uri()).status(), |
| ProtoIsOk()); |
| |
| expected = OptimizeStatsProto(); |
| expected.set_latency_ms(5); |
| expected.set_document_store_optimize_latency_ms(5); |
| expected.set_index_restoration_latency_ms(5); |
| expected.set_num_original_documents(1); |
| expected.set_num_deleted_documents(1); |
| expected.set_num_expired_documents(0); |
| expected.set_num_original_namespaces(1); |
| expected.set_num_deleted_namespaces(1); |
| expected.set_time_since_last_optimize_ms(0); |
| expected.set_index_restoration_mode(OptimizeStatsProto::FULL_INDEX_REBUILD); |
| |
| // Run Optimize |
| result = icing->Optimize(); |
| EXPECT_THAT(result.optimize_stats().storage_size_before(), |
| Ge(result.optimize_stats().storage_size_after())); |
| result.mutable_optimize_stats()->clear_storage_size_before(); |
| result.mutable_optimize_stats()->clear_storage_size_after(); |
| EXPECT_THAT(result.optimize_stats(), EqualsProto(expected)); |
| } |
| |
| TEST_F(IcingSearchEngineOptimizeTest, |
| OptimizationRewritesDocsWithNewCompressionLevel) { |
| SchemaProto schema = |
| SchemaBuilder() |
| .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( |
| PropertyConfigBuilder() |
| .SetName("body") |
| .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) |
| .SetCardinality(CARDINALITY_REQUIRED))) |
| .Build(); |
| DocumentProto document1 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri1") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body one") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| DocumentProto document2 = |
| DocumentBuilder() |
| .SetKey("namespace", "uri2") |
| .SetSchema("Message") |
| .AddStringProperty("body", "message body two") |
| .SetCreationTimestampMs(kDefaultCreationTimestampMs) |
| .Build(); |
| IcingSearchEngineOptions icing_options = GetDefaultIcingOptions(); |
| icing_options.set_compression_level(3); |
| int64_t document_log_size_compression_3; |
| int64_t document_log_size_after_opti_no_compression; |
| int64_t document_log_size_after_opti_compression_3; |
| const std::string document_log_path = |
| icing_options.base_dir() + "/document_dir/" + |
| DocumentLogCreator::GetDocumentLogFilename(); |
| { |
| IcingSearchEngine icing(icing_options, GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); |
| ASSERT_THAT(icing.PersistToDisk(PersistType::FULL).status(), ProtoIsOk()); |
| document_log_size_compression_3 = |
| filesystem()->GetFileSize(document_log_path.c_str()); |
| } // Destroys IcingSearchEngine to make sure nothing is cached. |
| |
| // Turn off compression |
| icing_options.set_compression_level(0); |
| |
| { |
| IcingSearchEngine icing(icing_options, GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| // Document log size is the same even after reopening with a different |
| // compression level |
| ASSERT_EQ(document_log_size_compression_3, |
| filesystem()->GetFileSize(document_log_path.c_str())); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| document_log_size_after_opti_no_compression = |
| filesystem()->GetFileSize(document_log_path.c_str()); |
| // Document log size is larger after optimizing since optimizing rewrites |
| // with the new compression level which is 0 or none |
| ASSERT_GT(document_log_size_after_opti_no_compression, |
| document_log_size_compression_3); |
| } |
| |
| // Restore the original compression level |
| icing_options.set_compression_level(3); |
| |
| { |
| IcingSearchEngine icing(icing_options, GetTestJniCache()); |
| ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); |
| // Document log size is the same even after reopening with a different |
| // compression level |
| ASSERT_EQ(document_log_size_after_opti_no_compression, |
| filesystem()->GetFileSize(document_log_path.c_str())); |
| ASSERT_THAT(icing.Optimize().status(), ProtoIsOk()); |
| document_log_size_after_opti_compression_3 = |
| filesystem()->GetFileSize(document_log_path.c_str()); |
| // Document log size should be the same as it was originally |
| ASSERT_EQ(document_log_size_after_opti_compression_3, |
| document_log_size_compression_3); |
| } |
| } |
| |
| } // namespace |
| } // namespace lib |
| } // namespace icing |
