Merge 25Q1 (ab/12770256) to aosp-main-future Bug: 385190204 Merged-In: I3d1adcdbd8c83f278969072cf1efcd68e1b5f27d Change-Id: Ib971cea2691b405d7f2cd472401dda952074371b
diff --git a/Android.bp b/Android.bp index 0b038f5..f0a7b19 100644 --- a/Android.bp +++ b/Android.bp
@@ -123,7 +123,8 @@ "icing/query/**/*_test.cc", "icing/scoring/**/*_test.cc", "icing/store/*_test.cc", - "icing/testing/*test*.cc", + "icing/testing/*test.cc", + "icing/testing/*test-data.cc", "icing/tokenization/*_test.cc", "icing/util/*_test.cc", ],
diff --git a/build.gradle b/build.gradle index 97cc5e1..38ae38d 100644 --- a/build.gradle +++ b/build.gradle
@@ -53,21 +53,6 @@ } } -protobuf { - protoc { - artifact = libs.protobufCompiler.get() - } - generateProtoTasks { - all().each { task -> - task.builtins { - java { - option 'lite' - } - } - } - } -} - androidx { mavenVersion = LibraryVersions.APPSEARCH }
diff --git a/icing/feature-flags.h b/icing/feature-flags.h new file mode 100644 index 0000000..63482f9 --- /dev/null +++ b/icing/feature-flags.h
@@ -0,0 +1,56 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_FEATURE_FLAGS_H_ +#define ICING_FEATURE_FLAGS_H_ + +namespace icing { +namespace lib { + +class FeatureFlags { + public: + explicit FeatureFlags(bool enable_scorable_properties, + bool enable_embedding_quantization, + bool enable_repeated_field_joins) + : enable_scorable_properties_(enable_scorable_properties), + enable_embedding_quantization_(enable_embedding_quantization), + enable_repeated_field_joins_(enable_repeated_field_joins) {} + + bool enable_scorable_properties() const { + return enable_scorable_properties_; + } + + bool enable_embedding_quantization() const { + return enable_embedding_quantization_; + } + + bool enable_repeated_field_joins() const { + return enable_repeated_field_joins_; + } + + private: + bool enable_scorable_properties_; + + // Whether to enable quantization for embedding vectors. If false, all + // embedding vectors will not be quantized. Otherwise, quantization will be + // controlled by the quantization type specified in the schema. + bool enable_embedding_quantization_; + + bool enable_repeated_field_joins_; +}; + +} // namespace lib +} // namespace icing + +#endif // ICING_FEATURE_FLAGS_H_
diff --git a/icing/file/file-backed-vector.h b/icing/file/file-backed-vector.h index 72b7fe3..8bd7161 100644 --- a/icing/file/file-backed-vector.h +++ b/icing/file/file-backed-vector.h
@@ -435,9 +435,9 @@ int32_t size() const { return len_; } - // Set the mutable array slice (starting at idx) by the given element array. - // It handles SetDirty properly for the file-backed-vector when modifying - // elements. + // Sets the mutable array slice (starting at idx) by the given element + // array. It handles SetDirty properly for the file-backed-vector when + // modifying elements. // // REQUIRES: arr is valid && arr_len >= 0 && idx >= 0 && idx + arr_len <= // size(), otherwise the behavior is undefined. @@ -448,6 +448,19 @@ } } + // Fills the mutable array slice, starting at idx with the given length, by + // the given value. It handles SetDirty properly for the file-backed-vector + // when modifying elements. + // + // REQUIRES: len >= 0 && idx >= 0 && idx + len <= size(), otherwise the + // behavior is undefined. + void Fill(int32_t idx, int32_t len, const T& value) { + for (int32_t i = 0; i < len; ++i) { + SetDirty(idx + i); + data_[idx + i] = value; + } + } + private: MutableArrayView(FileBackedVector<T>* vector, T* data, int32_t len) : vector_(vector),
diff --git a/icing/file/file-backed-vector_test.cc b/icing/file/file-backed-vector_test.cc index a4f5bf8..4561e39 100644 --- a/icing/file/file-backed-vector_test.cc +++ b/icing/file/file-backed-vector_test.cc
@@ -500,6 +500,96 @@ ElementsAre(1, 1, 1, 1, 1)); } +TEST_F(FileBackedVectorTest, MutableArrayViewFill) { + // Create a vector and add some data. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<FileBackedVector<int>> vector, + FileBackedVector<int>::Create( + filesystem_, file_path_, + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC)); + Insert(vector.get(), /*idx=*/0, std::vector<int>(/*count=*/100, /*value=*/1)); + EXPECT_THAT(vector->GetChecksum(), Eq(Crc32(2494890115U))); + EXPECT_THAT(vector->UpdateChecksum(), IsOkAndHolds(Crc32(2494890115U))); + EXPECT_THAT(vector->GetChecksum(), Eq(Crc32(2494890115U))); + + constexpr int kArrayViewOffset = 3; + constexpr int kArrayViewLen = 5; + ICING_ASSERT_OK_AND_ASSIGN( + FileBackedVector<int>::MutableArrayView mutable_arr, + vector->GetMutable(kArrayViewOffset, kArrayViewLen)); + + mutable_arr.Fill(/*idx=*/0, /*len=*/3, 1234); + EXPECT_THAT(Get(vector.get(), kArrayViewOffset, kArrayViewLen), + ElementsAre(1234, 1234, 1234, 1, 1)); + + mutable_arr.Fill(/*idx=*/2, /*len=*/3, 5678); + EXPECT_THAT(Get(vector.get(), kArrayViewOffset, kArrayViewLen), + ElementsAre(1234, 1234, 5678, 5678, 5678)); +} + +TEST_F(FileBackedVectorTest, MutableArrayViewFillWithZeroLength) { + // Create a vector and add some data. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<FileBackedVector<int>> vector, + FileBackedVector<int>::Create( + filesystem_, file_path_, + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC)); + Insert(vector.get(), /*idx=*/0, std::vector<int>(/*count=*/100, /*value=*/1)); + EXPECT_THAT(vector->GetChecksum(), Eq(Crc32(2494890115U))); + EXPECT_THAT(vector->UpdateChecksum(), IsOkAndHolds(Crc32(2494890115U))); + EXPECT_THAT(vector->GetChecksum(), Eq(Crc32(2494890115U))); + + constexpr int kArrayViewOffset = 3; + constexpr int kArrayViewLen = 5; + ICING_ASSERT_OK_AND_ASSIGN( + FileBackedVector<int>::MutableArrayView mutable_arr, + vector->GetMutable(kArrayViewOffset, kArrayViewLen)); + + // Zero len should work and change nothing + mutable_arr.Fill(/*idx=*/0, /*len=*/0, 1234); + EXPECT_THAT(Get(vector.get(), kArrayViewOffset, kArrayViewLen), + ElementsAre(1, 1, 1, 1, 1)); +} + +TEST_F(FileBackedVectorTest, MutableArrayViewFillShouldSetDirty) { + // Create a vector and add some data. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<FileBackedVector<int>> vector, + FileBackedVector<int>::Create( + filesystem_, file_path_, + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC)); + Insert(vector.get(), /*idx=*/0, std::vector<int>(/*count=*/100, /*value=*/1)); + EXPECT_THAT(vector->GetChecksum(), Eq(Crc32(2494890115U))); + EXPECT_THAT(vector->UpdateChecksum(), IsOkAndHolds(Crc32(2494890115U))); + EXPECT_THAT(vector->GetChecksum(), Eq(Crc32(2494890115U))); + + std::string_view reconstructed_view( + reinterpret_cast<const char*>(vector->array()), + vector->num_elements() * sizeof(int)); + + constexpr int kArrayViewOffset = 3; + constexpr int kArrayViewLen = 5; + ICING_ASSERT_OK_AND_ASSIGN( + FileBackedVector<int>::MutableArrayView mutable_arr, + vector->GetMutable(kArrayViewOffset, kArrayViewLen)); + + // Use Fill() to mutate elements + // MutableArrayView should set the affected element indices dirty so that + // UpdateChecksum() can pick up the change and compute the checksum correctly. + // Validate by mapping another array on top. + mutable_arr.Fill(/*idx=*/0, /*len=*/3, 1234); + ASSERT_THAT(Get(vector.get(), kArrayViewOffset, kArrayViewLen), + ElementsAre(1234, 1234, 1234, 1, 1)); + ICING_ASSERT_OK_AND_ASSIGN(Crc32 crc1, vector->UpdateChecksum()); + EXPECT_THAT(crc1, Eq(Crc32(reconstructed_view))); + + mutable_arr.Fill(/*idx=*/2, /*len=*/3, 5678); + ASSERT_THAT(Get(vector.get(), kArrayViewOffset, kArrayViewLen), + ElementsAre(1234, 1234, 5678, 5678, 5678)); + ICING_ASSERT_OK_AND_ASSIGN(Crc32 crc2, vector->UpdateChecksum()); + EXPECT_THAT(crc2, Eq(Crc32(reconstructed_view))); +} + TEST_F(FileBackedVectorTest, MutableArrayViewIndexOperatorShouldSetDirty) { // Create an array with some data. ICING_ASSERT_OK_AND_ASSIGN(
diff --git a/icing/file/memory-mapped-file-backed-proto-log.h b/icing/file/memory-mapped-file-backed-proto-log.h index a5a9d16..cc8f35f 100644 --- a/icing/file/memory-mapped-file-backed-proto-log.h +++ b/icing/file/memory-mapped-file-backed-proto-log.h
@@ -15,6 +15,7 @@ #ifndef ICING_FILE_MEMORY_MAPPED_FILE_BACKED_PROTO_LOG_H_ #define ICING_FILE_MEMORY_MAPPED_FILE_BACKED_PROTO_LOG_H_ +#include <cinttypes> #include <cstdint> #include <cstring> #include <memory> @@ -84,6 +85,22 @@ // INTERNAL_ERROR on IO error libtextclassifier3::StatusOr<Crc32> UpdateChecksum(); + // Calculates and returns the disk usage in bytes. Rounds up to the nearest + // block size. + // + // Returns: + // Disk usage on success + // INTERNAL_ERROR on IO error + libtextclassifier3::StatusOr<int64_t> GetDiskUsage() const; + + // Returns the file size of the all the elements held in the log. File size + // is in bytes. This excludes the size of the header of the log file. + // + // Returns: + // File size on success + // INTERNAL_ERROR on IO error + libtextclassifier3::StatusOr<int64_t> GetElementsFileSize() const; + // Reads the proto at the given index. // // Returns: @@ -160,6 +177,18 @@ } template <typename ProtoT> +libtextclassifier3::StatusOr<int64_t> +MemoryMappedFileBackedProtoLog<ProtoT>::GetDiskUsage() const { + return proto_fbv_->GetDiskUsage(); +} + +template <typename ProtoT> +libtextclassifier3::StatusOr<int64_t> +MemoryMappedFileBackedProtoLog<ProtoT>::GetElementsFileSize() const { + return proto_fbv_->GetElementsFileSize(); +} + +template <typename ProtoT> libtextclassifier3::StatusOr< std::unique_ptr<MemoryMappedFileBackedProtoLog<ProtoT>>> MemoryMappedFileBackedProtoLog<ProtoT>::Create(const Filesystem& filesystem, @@ -187,9 +216,10 @@ IcingStringUtil::StringPrintf("Index, %d, is less than 0", index)); } if (index + sizeof(ProtoMetadata) >= proto_fbv_->num_elements()) { + uint64_t upper_index = proto_fbv_->num_elements() - sizeof(ProtoMetadata); return absl_ports::OutOfRangeError(IcingStringUtil::StringPrintf( - "Index, %d, is greater/equal than the upper bound, %d", index, - proto_fbv_->num_elements() - sizeof(ProtoMetadata))); + "Index, %" PRId32 ", is greater/equal than the upper bound, %" PRIu64, + index, upper_index)); } ProtoMetadata proto_metadata;
diff --git a/icing/file/portable-file-backed-proto-log.h b/icing/file/portable-file-backed-proto-log.h index faccc6e..81010e8 100644 --- a/icing/file/portable-file-backed-proto-log.h +++ b/icing/file/portable-file-backed-proto-log.h
@@ -116,14 +116,14 @@ explicit Options( bool compress_in, const int32_t max_proto_size_in = constants::kMaxProtoSize, - const int32_t compression_level_in = kDeflateCompressionLevel) + const int32_t compression_level_in = kDefaultCompressionLevel) : compress(compress_in), max_proto_size(max_proto_size_in), compression_level(compression_level_in) {} }; // Level of compression, BEST_SPEED = 1, BEST_COMPRESSION = 9 - static constexpr int kDeflateCompressionLevel = 3; + static constexpr int kDefaultCompressionLevel = 3; // Number of bytes we reserve for the heading at the beginning of the proto // log. We reserve this so the header can grow without running into the
diff --git a/icing/file/portable-file-backed-proto-log_test.cc b/icing/file/portable-file-backed-proto-log_test.cc index b564a2c..587bfdc 100644 --- a/icing/file/portable-file-backed-proto-log_test.cc +++ b/icing/file/portable-file-backed-proto-log_test.cc
@@ -74,7 +74,7 @@ std::string file_path_; bool compress_ = true; int32_t compression_level_ = - PortableFileBackedProtoLog<DocumentProto>::kDeflateCompressionLevel; + PortableFileBackedProtoLog<DocumentProto>::kDefaultCompressionLevel; int64_t max_proto_size_ = 256 * 1024; // 256 KiB };
diff --git a/icing/file/version-util.cc b/icing/file/version-util.cc index e750b0c..af5ea50 100644 --- a/icing/file/version-util.cc +++ b/icing/file/version-util.cc
@@ -66,8 +66,7 @@ // Real error. return std::move(existing_flash_index_magic).status(); } - if (existing_flash_index_magic.ValueOrDie() == - kVersionZeroFlashIndexMagic) { + if (existing_flash_index_magic.ValueOrDie() == kVersionZeroFlashIndexMagic) { existing_version_info.version = 0; if (existing_version_info.max_version == -1) { existing_version_info.max_version = 0; @@ -233,7 +232,8 @@ /*needs_schema_store_derived_files_rebuild=*/true, /*needs_term_index_rebuild=*/true, /*needs_integer_index_rebuild=*/true, - /*needs_qualified_id_join_index_rebuild=*/true); + /*needs_qualified_id_join_index_rebuild=*/true, + /*needs_embedding_index_rebuild=*/true); } // 2. Compare the previous enabled features with the current enabled features @@ -263,7 +263,8 @@ /*needs_schema_store_derived_files_rebuild=*/true, /*needs_term_index_rebuild=*/true, /*needs_integer_index_rebuild=*/true, - /*needs_qualified_id_join_index_rebuild=*/true); + /*needs_qualified_id_join_index_rebuild=*/true, + /*needs_embedding_index_rebuild=*/true); } if (curr_features.find(prev_feature) == curr_features.end()) { DerivedFilesRebuildResult required_rebuilds = @@ -318,6 +319,10 @@ // version 3 -> version 4 upgrade, no need to rebuild break; } + case 4: { + // version 4 -> version 5 upgrade, no need to rebuild + break; + } default: // This should not happen. Rebuild anyway if unsure. should_rebuild |= true; @@ -330,13 +335,62 @@ DerivedFilesRebuildResult GetFeatureDerivedFilesRebuildResult( IcingSearchEngineFeatureInfoProto::FlaggedFeatureType feature) { switch (feature) { + case IcingSearchEngineFeatureInfoProto::FEATURE_SCORABLE_PROPERTIES: { + return DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild=*/true, + /*needs_schema_store_derived_files_rebuild=*/false, + /*needs_term_index_rebuild=*/false, + /*needs_integer_index_rebuild=*/false, + /*needs_qualified_id_join_index_rebuild=*/false, + /*needs_embedding_index_rebuild=*/false); + } case IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR: { return DerivedFilesRebuildResult( /*needs_document_store_derived_files_rebuild=*/false, /*needs_schema_store_derived_files_rebuild=*/false, /*needs_term_index_rebuild=*/true, /*needs_integer_index_rebuild=*/false, - /*needs_qualified_id_join_index_rebuild=*/false); + /*needs_qualified_id_join_index_rebuild=*/false, + /*needs_embedding_index_rebuild=*/false); + } + case IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX: { + return DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild=*/false, + /*needs_schema_store_derived_files_rebuild=*/false, + /*needs_term_index_rebuild=*/false, + /*needs_integer_index_rebuild=*/false, + /*needs_qualified_id_join_index_rebuild=*/false, + /*needs_embedding_index_rebuild=*/true); + } + case IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_QUANTIZATION: { + return DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild=*/false, + /*needs_schema_store_derived_files_rebuild=*/false, + /*needs_term_index_rebuild=*/false, + /*needs_integer_index_rebuild=*/false, + /*needs_qualified_id_join_index_rebuild=*/false, + /*needs_embedding_index_rebuild=*/true); + } + case IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE: { + // The schema database feature requires schema-store migration, which is + // done separately from derived files rebuild. + return DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild=*/false, + /*needs_schema_store_derived_files_rebuild=*/false, + /*needs_term_index_rebuild=*/false, + /*needs_integer_index_rebuild=*/false, + /*needs_qualified_id_join_index_rebuild=*/false, + /*needs_embedding_index_rebuild=*/false); + } + case IcingSearchEngineFeatureInfoProto:: + FEATURE_QUALIFIED_ID_JOIN_INDEX_V3_AND_DELETE_PROPAGATE_FROM: { + return DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild=*/false, + /*needs_schema_store_derived_files_rebuild=*/false, + /*needs_term_index_rebuild=*/false, + /*needs_integer_index_rebuild=*/false, + /*needs_qualified_id_join_index_rebuild=*/true, + /*needs_embedding_index_rebuild=*/false); } case IcingSearchEngineFeatureInfoProto::UNKNOWN: return DerivedFilesRebuildResult( @@ -344,10 +398,27 @@ /*needs_schema_store_derived_files_rebuild=*/true, /*needs_term_index_rebuild=*/true, /*needs_integer_index_rebuild=*/true, - /*needs_qualified_id_join_index_rebuild=*/true); + /*needs_qualified_id_join_index_rebuild=*/true, + /*needs_embedding_index_rebuild=*/true); } } +bool SchemaDatabaseMigrationRequired( + const IcingSearchEngineVersionProto& prev_version_proto) { + if (prev_version_proto.version() < kSchemaDatabaseVersion) { + return true; + } + for (const auto& feature : prev_version_proto.enabled_features()) { + // The schema database feature was enabled in the previous version, so no + // need to migrate. + if (feature.feature_type() == + IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE) { + return false; + } + } + return true; +} + IcingSearchEngineFeatureInfoProto GetFeatureInfoProto( IcingSearchEngineFeatureInfoProto::FlaggedFeatureType feature) { IcingSearchEngineFeatureInfoProto info; @@ -363,6 +434,7 @@ info.set_needs_integer_index_rebuild(result.needs_integer_index_rebuild); info.set_needs_qualified_id_join_index_rebuild( result.needs_qualified_id_join_index_rebuild); + info.set_needs_embedding_index_rebuild(result.needs_embedding_index_rebuild); return info; } @@ -375,6 +447,31 @@ enabled_features->Add(GetFeatureInfoProto( IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR)); } + // EmbeddingIndex feature + if (options.enable_embedding_index()) { + enabled_features->Add(GetFeatureInfoProto( + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX)); + } + if (options.enable_scorable_properties()) { + enabled_features->Add(GetFeatureInfoProto( + IcingSearchEngineFeatureInfoProto::FEATURE_SCORABLE_PROPERTIES)); + } + // EmbeddingQuantization feature + if (options.enable_embedding_quantization()) { + enabled_features->Add(GetFeatureInfoProto( + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_QUANTIZATION)); + } + // SchemaDatabase feature + if (options.enable_schema_database()) { + enabled_features->Add(GetFeatureInfoProto( + IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE)); + } + // QualifiedIdJoinIndex V3 and delete propagation type PROPAGATE_FROM feature + if (options.enable_qualified_id_join_index_v3_and_delete_propagate_from()) { + enabled_features->Add(GetFeatureInfoProto( + IcingSearchEngineFeatureInfoProto:: + FEATURE_QUALIFIED_ID_JOIN_INDEX_V3_AND_DELETE_PROPAGATE_FROM)); + } } } // namespace version_util
diff --git a/icing/file/version-util.h b/icing/file/version-util.h index 7e93b5c..0f5e8a4 100644 --- a/icing/file/version-util.h +++ b/icing/file/version-util.h
@@ -37,18 +37,20 @@ // (There were no M-2023-10, M-2023-12). // - Version 3: M-2024-02. Schema is compatible with v1 and v2. // - Version 4: Android V base. Schema is compatible with v1, v2 and v3. -// -// TODO(b/314816301): Bump kVersion to 4 for Android V rollout with v2 version -// detection -inline static constexpr int32_t kVersion = 4; +// - Version 5: M-2025-02. Schema is compatible with v1, v2, v3 and v4. +inline static constexpr int32_t kVersion = 5; inline static constexpr int32_t kVersionOne = 1; inline static constexpr int32_t kVersionTwo = 2; inline static constexpr int32_t kVersionThree = 3; inline static constexpr int32_t kVersionFour = 4; +inline static constexpr int32_t kVersionFive = 5; // Version at which v2 version file is introduced. inline static constexpr int32_t kFirstV2Version = kVersionFour; +// Version at which the database field is introduced to the schema proto. +inline static constexpr int32_t kSchemaDatabaseVersion = kVersionFive; + inline static constexpr int kVersionZeroFlashIndexMagic = 0x6dfba6ae; inline static constexpr std::string_view kVersionFilenameV1 = "version"; @@ -95,6 +97,7 @@ bool needs_term_index_rebuild = false; bool needs_integer_index_rebuild = false; bool needs_qualified_id_join_index_rebuild = false; + bool needs_embedding_index_rebuild = false; DerivedFilesRebuildResult() = default; @@ -102,7 +105,8 @@ bool needs_document_store_derived_files_rebuild_in, bool needs_schema_store_derived_files_rebuild_in, bool needs_term_index_rebuild_in, bool needs_integer_index_rebuild_in, - bool needs_qualified_id_join_index_rebuild_in) + bool needs_qualified_id_join_index_rebuild_in, + bool needs_embedding_index_rebuild_in) : needs_document_store_derived_files_rebuild( needs_document_store_derived_files_rebuild_in), needs_schema_store_derived_files_rebuild( @@ -110,13 +114,15 @@ needs_term_index_rebuild(needs_term_index_rebuild_in), needs_integer_index_rebuild(needs_integer_index_rebuild_in), needs_qualified_id_join_index_rebuild( - needs_qualified_id_join_index_rebuild_in) {} + needs_qualified_id_join_index_rebuild_in), + needs_embedding_index_rebuild(needs_embedding_index_rebuild_in) {} bool IsRebuildNeeded() const { return needs_document_store_derived_files_rebuild || needs_schema_store_derived_files_rebuild || needs_term_index_rebuild || needs_integer_index_rebuild || - needs_qualified_id_join_index_rebuild; + needs_qualified_id_join_index_rebuild || + needs_embedding_index_rebuild; } bool operator==(const DerivedFilesRebuildResult& other) const { @@ -127,23 +133,20 @@ needs_term_index_rebuild == other.needs_term_index_rebuild && needs_integer_index_rebuild == other.needs_integer_index_rebuild && needs_qualified_id_join_index_rebuild == - other.needs_qualified_id_join_index_rebuild; + other.needs_qualified_id_join_index_rebuild && + needs_embedding_index_rebuild == other.needs_embedding_index_rebuild; } void CombineWithOtherRebuildResultOr(const DerivedFilesRebuildResult& other) { - needs_document_store_derived_files_rebuild = - needs_document_store_derived_files_rebuild || + needs_document_store_derived_files_rebuild |= other.needs_document_store_derived_files_rebuild; - needs_schema_store_derived_files_rebuild = - needs_schema_store_derived_files_rebuild || + needs_schema_store_derived_files_rebuild |= other.needs_schema_store_derived_files_rebuild; - needs_term_index_rebuild = - needs_term_index_rebuild || other.needs_term_index_rebuild; - needs_integer_index_rebuild = - needs_integer_index_rebuild || other.needs_integer_index_rebuild; - needs_qualified_id_join_index_rebuild = - needs_qualified_id_join_index_rebuild || + needs_term_index_rebuild |= other.needs_term_index_rebuild; + needs_integer_index_rebuild |= other.needs_integer_index_rebuild; + needs_qualified_id_join_index_rebuild |= other.needs_qualified_id_join_index_rebuild; + needs_embedding_index_rebuild |= other.needs_embedding_index_rebuild; } }; @@ -167,7 +170,6 @@ return VersionInfo(version_proto.version(), version_proto.max_version()); } - // Reads the IcingSearchEngineVersionProto from the version files of the // existing data. // @@ -247,6 +249,14 @@ bool ShouldRebuildDerivedFiles(const VersionInfo& existing_version_info, int32_t curr_version = kVersion); +// Returns whether the schema database migration is required. +// +// This is true if the previous version is less than the version at which the +// database field is introduced, or if the schema database feature was +// not enabled in the previous version. +bool SchemaDatabaseMigrationRequired( + const IcingSearchEngineVersionProto& prev_version_proto); + // Returns the derived files rebuilds required for a given feature. DerivedFilesRebuildResult GetFeatureDerivedFilesRebuildResult( IcingSearchEngineFeatureInfoProto::FlaggedFeatureType feature);
diff --git a/icing/file/version-util_test.cc b/icing/file/version-util_test.cc index f2922e2..d881adc 100644 --- a/icing/file/version-util_test.cc +++ b/icing/file/version-util_test.cc
@@ -525,7 +525,55 @@ VersionUtilStateChangeTestParam( /*existing_version_info_in=*/VersionInfo(3, 4), /*curr_version_in=*/2, - /*expected_state_change_in=*/StateChange::kRollBack))); + /*expected_state_change_in=*/StateChange::kRollBack), + + // - version 3, max_version 3 + // - Current version = 4 + // - Result: upgrade + VersionUtilStateChangeTestParam( + /*existing_version_info_in=*/VersionInfo(3, 3), + /*curr_version_in=*/4, + /*expected_state_change_in=*/StateChange::kUpgrade), + + // - version 4, max_version 4 + // - Current version = 3 + // - Result: rollback + VersionUtilStateChangeTestParam( + /*existing_version_info_in=*/VersionInfo(4, 4), + /*curr_version_in=*/3, + /*expected_state_change_in=*/StateChange::kRollBack), + + // - version 4, max_version 5 + // - Current version = 4 + // - Result: compatible + VersionUtilStateChangeTestParam( + /*existing_version_info_in=*/VersionInfo(4, 5), + /*curr_version_in=*/4, + /*expected_state_change_in=*/StateChange::kCompatible), + + // - version 3, max_version 4 + // - Current version = 5 + // - Result: rollforward + VersionUtilStateChangeTestParam( + /*existing_version_info_in=*/VersionInfo(3, 4), + /*curr_version_in=*/5, + /*expected_state_change_in=*/StateChange::kRollForward), + + // - version 3, max_version 3 + // - Current version = 5 + // - Result: upgrade + VersionUtilStateChangeTestParam( + /*existing_version_info_in=*/VersionInfo(3, 3), + /*curr_version_in=*/5, + /*expected_state_change_in=*/StateChange::kUpgrade), + + // - version 4, max_version 5 + // - Current version = 5 + // - Result: rollforward + VersionUtilStateChangeTestParam( + /*existing_version_info_in=*/VersionInfo(4, 5), + /*curr_version_in=*/5, + /*expected_state_change_in=*/StateChange::kRollForward))); struct VersionUtilDerivedFilesRebuildTestParam { int32_t existing_version; @@ -588,11 +636,12 @@ /*curr_enabled_features_in=*/{}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/true, - /*needs_schema_store_derived_files_rebuild=*/true, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/true, - /*needs_qualified_id_join_index_rebuild=*/true)), + /*needs_document_store_derived_files_rebuild_in=*/true, + /*needs_schema_store_derived_files_rebuild_in=*/true, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/true, + /*needs_qualified_id_join_index_rebuild_in=*/true, + /*needs_embedding_index_rebuild_in=*/true)), // - Existing version -1, max_version 2 (invalid) // - Existing enabled features = {} @@ -606,11 +655,12 @@ /*curr_enabled_features_in=*/{}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/true, - /*needs_schema_store_derived_files_rebuild=*/true, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/true, - /*needs_qualified_id_join_index_rebuild=*/true)), + /*needs_document_store_derived_files_rebuild_in=*/true, + /*needs_schema_store_derived_files_rebuild_in=*/true, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/true, + /*needs_qualified_id_join_index_rebuild_in=*/true, + /*needs_embedding_index_rebuild_in=*/true)), // - Existing version 3, max_version 3 (pre v2 version check) // - Existing enabled features = {} @@ -624,11 +674,12 @@ /*curr_enabled_features_in=*/{}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/false, - /*needs_schema_store_derived_files_rebuild=*/false, - /*needs_term_index_rebuild=*/false, - /*needs_integer_index_rebuild=*/false, - /*needs_qualified_id_join_index_rebuild=*/false)), + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false)), // - Existing version 3, max_version 3 (pre v2 version check) // - Existing enabled features = {} @@ -643,11 +694,32 @@ {IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/false, - /*needs_schema_store_derived_files_rebuild=*/false, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/false, - /*needs_qualified_id_join_index_rebuild=*/false)), + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false)), + + // - Existing version 3, max_version 3 (pre v2 version check) + // - Existing enabled features = {} + // - Current version = 4 + // - Current enabled features = {FEATURE_EMBEDDING_INDEX} + // + // - Result: rebuild embedding index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/3, /*max_version_in=*/3, + /*existing_enabled_features_in=*/{}, /*curr_version_in=*/4, + /*curr_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/true)), // - Existing version 4, max_version 4 // - Existing enabled features = {} @@ -661,11 +733,31 @@ /*curr_enabled_features_in=*/{}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/false, - /*needs_schema_store_derived_files_rebuild=*/false, - /*needs_term_index_rebuild=*/false, - /*needs_integer_index_rebuild=*/false, - /*needs_qualified_id_join_index_rebuild=*/false)), + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false)), + + // - Existing version 4, max_version 4 + // - Existing enabled features = {} + // - Current version = 5 + // - Current enabled features = {} + // + // - Result: 4 -> 5 upgrade -- don't rebuild anything + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/4, /*max_version_in=*/4, + /*existing_enabled_features_in=*/{}, /*curr_version_in=*/5, + /*curr_enabled_features_in=*/{}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false)), // - Existing version 4, max_version 5 // - Existing enabled features = {} @@ -679,11 +771,12 @@ /*curr_enabled_features_in=*/{}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/true, - /*needs_schema_store_derived_files_rebuild=*/true, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/true, - /*needs_qualified_id_join_index_rebuild=*/true)), + /*needs_document_store_derived_files_rebuild_in=*/true, + /*needs_schema_store_derived_files_rebuild_in=*/true, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/true, + /*needs_qualified_id_join_index_rebuild_in=*/true, + /*needs_embedding_index_rebuild_in=*/true)), // - Existing version 5, max_version 5 // - Existing enabled features = {} @@ -697,11 +790,12 @@ /*curr_enabled_features_in=*/{}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/true, - /*needs_schema_store_derived_files_rebuild=*/true, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/true, - /*needs_qualified_id_join_index_rebuild=*/true)), + /*needs_document_store_derived_files_rebuild_in=*/true, + /*needs_schema_store_derived_files_rebuild_in=*/true, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/true, + /*needs_qualified_id_join_index_rebuild_in=*/true, + /*needs_embedding_index_rebuild_in=*/true)), // - Existing version 4, max_version 4 // - Existing enabled features = {} @@ -716,11 +810,78 @@ {IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/false, - /*needs_schema_store_derived_files_rebuild=*/false, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/false, - /*needs_qualified_id_join_index_rebuild=*/false)), + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false)), + + // - Existing version 4, max_version 4 + // - Existing enabled features = {} + // - Current version = 4 + // - Current enabled features = {FEATURE_EMBEDDING_INDEX} + // + // - Result: rebuild embedding index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/4, /*max_version_in=*/4, + /*existing_enabled_features_in=*/{}, /*curr_version_in=*/4, + /*curr_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/true)), + + // - Existing version 4, max_version 4 + // - Existing enabled features = {} + // - Current version = 4 + // - Current enabled features = {FEATURE_EMBEDDING_INDEX, + // FEATURE_EMBEDDING_QUANTIZATION} + // + // - Result: rebuild embedding index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/4, /*max_version_in=*/4, + /*existing_enabled_features_in=*/{}, /*curr_version_in=*/4, + /*curr_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX, + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_QUANTIZATION}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/true)), + + // - Existing version 4, max_version 4 + // - Existing enabled features = {FEATURE_EMBEDDING_INDEX} + // - Current version = 4 + // - Current enabled features = {FEATURE_EMBEDDING_INDEX, + // FEATURE_EMBEDDING_QUANTIZATION} + // + // - Result: rebuild embedding index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/4, /*max_version_in=*/4, + /*existing_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX}, + /*curr_version_in=*/4, + /*curr_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX, + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_QUANTIZATION}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/true)), // - Existing version 4, max_version 4 // - Existing enabled features = {FEATURE_HAS_PROPERTY_OPERATOR} @@ -735,11 +896,77 @@ /*curr_version_in=*/4, /*curr_enabled_features_in=*/{}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/false, - /*needs_schema_store_derived_files_rebuild=*/false, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/false, - /*needs_qualified_id_join_index_rebuild=*/false)), + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false)), + + // - Existing version 4, max_version 4 + // - Existing enabled features = {FEATURE_EMBEDDING_INDEX} + // - Current version = 4 + // - Current enabled features = {} + // + // - Result: rebuild embedding index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/4, /*max_version_in=*/4, + /*existing_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX}, + /*curr_version_in=*/4, /*curr_enabled_features_in=*/{}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/true)), + + // - Existing version 4, max_version 4 + // - Existing enabled features = {FEATURE_EMBEDDING_INDEX, + // FEATURE_EMBEDDING_QUANTIZATION} + // - Current version = 4 + // - Current enabled features = {} + // + // - Result: rebuild embedding index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/4, /*max_version_in=*/4, + /*existing_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX, + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_QUANTIZATION}, + /*curr_version_in=*/4, /*curr_enabled_features_in=*/{}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/true)), + + // - Existing version 4, max_version 4 + // - Existing enabled features = {FEATURE_EMBEDDING_INDEX, + // FEATURE_EMBEDDING_QUANTIZATION} + // - Current version = 4 + // - Current enabled features = {FEATURE_EMBEDDING_INDEX} + // + // - Result: rebuild embedding index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/4, /*max_version_in=*/4, + /*existing_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX, + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_QUANTIZATION}, + /*curr_version_in=*/4, /*curr_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/true)), // - Existing version 4, max_version 4 // - Existing enabled features = {UNKNOWN} @@ -755,11 +982,96 @@ {IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR}, /*expected_derived_files_rebuild_result_in=*/ DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/true, - /*needs_schema_store_derived_files_rebuild=*/true, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/true, - /*needs_qualified_id_join_index_rebuild=*/true)))); + /*needs_document_store_derived_files_rebuild_in=*/true, + /*needs_schema_store_derived_files_rebuild_in=*/true, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/true, + /*needs_qualified_id_join_index_rebuild_in=*/true, + /*needs_embedding_index_rebuild_in=*/true)), + + // - Existing version 5, max_version 5 + // - Existing enabled features = {} + // - Current version = 5 + // - Current enabled features = {FEATURE_HAS_PROPERTY_OPERATOR} + // + // - Result: rebuild term index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/5, /*max_version_in=*/5, + /*existing_enabled_features_in=*/{}, /*curr_version_in=*/5, + /*curr_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false)), + + // - Existing version 5, max_version 5 + // - Existing enabled features = {} + // - Current version = 5 + // - Current enabled features = {FEATURE_SCHEMA_DATABASE} + // + // - Result: no rebuild + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/5, /*max_version_in=*/5, + /*existing_enabled_features_in=*/{}, + /*curr_version_in=*/5, /*curr_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false)), + + // - Existing version 5, max_version 5 + // - Existing enabled features = {} + // - Current version = 5 + // - Current enabled features = + // {FEATURE_QUALIFIED_ID_JOIN_INDEX_V3_AND_DELETE_PROPAGATE_FROM} + // + // - Result: rebuild qualified id join index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/5, /*max_version_in=*/5, + /*existing_enabled_features_in=*/{}, + /*curr_version_in=*/5, /*curr_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto:: + FEATURE_QUALIFIED_ID_JOIN_INDEX_V3_AND_DELETE_PROPAGATE_FROM}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/true, + /*needs_embedding_index_rebuild_in=*/false)), + + // - Existing version 5, max_version 5 + // - Existing enabled features = + // {FEATURE_QUALIFIED_ID_JOIN_INDEX_V3_AND_DELETE_PROPAGATE_FROM} + // - Current version = 5 + // - Current enabled features = {} + // + // - Result: rebuild qualified id join index + VersionUtilDerivedFilesRebuildTestParam( + /*existing_version_in=*/5, /*max_version_in=*/5, + /*existing_enabled_features_in=*/ + {IcingSearchEngineFeatureInfoProto:: + FEATURE_QUALIFIED_ID_JOIN_INDEX_V3_AND_DELETE_PROPAGATE_FROM}, + /*curr_version_in=*/5, /*curr_enabled_features_in=*/{}, + /*expected_derived_files_rebuild_result_in=*/ + DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/true, + /*needs_embedding_index_rebuild_in=*/false)))); TEST(VersionUtilTest, ShouldRebuildDerivedFilesUndeterminedVersion) { EXPECT_THAT( @@ -792,6 +1104,10 @@ // (2 -> 0), 0 -> 1 EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(0, 2), /*curr_version=*/1), IsTrue()); + + // (5 -> 0), 0 -> 5 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(0, 5), /*curr_version=*/5), + IsTrue()); } TEST(VersionUtilTest, ShouldRebuildDerivedFilesRollBack) { @@ -806,6 +1122,10 @@ // (3 -> 2), 2 -> 1 EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(2, 3), /*curr_version=*/1), IsTrue()); + + // (5 -> 4), 4 -> 3 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(4, 5), /*curr_version=*/3), + IsTrue()); } TEST(VersionUtilTest, ShouldRebuildDerivedFilesRollForward) { @@ -820,6 +1140,10 @@ // (3 -> 1), 1 -> 2 EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(1, 3), /*curr_version=*/2), IsTrue()); + + // (5 -> 4), 4 -> 5 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(4, 5), /*curr_version=*/5), + IsTrue()); } TEST(VersionUtilTest, ShouldRebuildDerivedFilesCompatible) { @@ -863,17 +1187,39 @@ EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(kVersionOne, kVersionOne), /*curr_version=*/kVersionFour), IsFalse()); + + // kVersionFour -> kVersionFive. + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(kVersionFour, kVersionFour), + /*curr_version=*/kVersionFive), + IsFalse()); + + // kVersionThree -> kVersionFive. + EXPECT_THAT( + ShouldRebuildDerivedFiles(VersionInfo(kVersionThree, kVersionThree), + /*curr_version=*/kVersionFive), + IsFalse()); + + // kVersionTwo -> kVersionFour + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(kVersionTwo, kVersionTwo), + /*curr_version=*/kVersionFive), + IsFalse()); + + // kVersionOne -> kVersionFour. + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(kVersionOne, kVersionOne), + /*curr_version=*/kVersionFive), + IsFalse()); } TEST(VersionUtilTest, GetFeatureDerivedFilesRebuildResult_unknown) { EXPECT_THAT(GetFeatureDerivedFilesRebuildResult( IcingSearchEngineFeatureInfoProto::UNKNOWN), Eq(DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/true, - /*needs_schema_store_derived_files_rebuild=*/true, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/true, - /*needs_qualified_id_join_index_rebuild=*/true))); + /*needs_document_store_derived_files_rebuild_in=*/true, + /*needs_schema_store_derived_files_rebuild_in=*/true, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/true, + /*needs_qualified_id_join_index_rebuild_in=*/true, + /*needs_embedding_index_rebuild_in=*/true))); } TEST(VersionUtilTest, @@ -882,11 +1228,112 @@ GetFeatureDerivedFilesRebuildResult( IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR), Eq(DerivedFilesRebuildResult( - /*needs_document_store_derived_files_rebuild=*/false, - /*needs_schema_store_derived_files_rebuild=*/false, - /*needs_term_index_rebuild=*/true, - /*needs_integer_index_rebuild=*/false, - /*needs_qualified_id_join_index_rebuild=*/false))); + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/true, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false))); +} + +TEST(VersionUtilTest, + GetFeatureDerivedFilesRebuildResult_featureEmbeddingIndex) { + EXPECT_THAT(GetFeatureDerivedFilesRebuildResult( + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX), + Eq(DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/true))); +} + +TEST(VersionUtilTest, + GetFeatureDerivedFilesRebuildResult_featureEmbeddingQuantization) { + EXPECT_THAT( + GetFeatureDerivedFilesRebuildResult( + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_QUANTIZATION), + Eq(DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/true))); +} + +TEST(VersionUtilTest, + GetFeatureDerivedFilesRebuildResult_featureSchemaDatabase) { + EXPECT_THAT(GetFeatureDerivedFilesRebuildResult( + IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE), + Eq(DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/false, + /*needs_embedding_index_rebuild_in=*/false))); +} + +TEST( + VersionUtilTest, + GetFeatureDerivedFilesRebuildResult_featureQualifiedIdJoinIndexV3AndDeletePropagateFrom) { + EXPECT_THAT( + GetFeatureDerivedFilesRebuildResult( + IcingSearchEngineFeatureInfoProto:: + FEATURE_QUALIFIED_ID_JOIN_INDEX_V3_AND_DELETE_PROPAGATE_FROM), + Eq(DerivedFilesRebuildResult( + /*needs_document_store_derived_files_rebuild_in=*/false, + /*needs_schema_store_derived_files_rebuild_in=*/false, + /*needs_term_index_rebuild_in=*/false, + /*needs_integer_index_rebuild_in=*/false, + /*needs_qualified_id_join_index_rebuild_in=*/true, + /*needs_embedding_index_rebuild_in=*/false))); +} + +TEST(VersionUtilTest, SchemaDatabaseMigrationRequired) { + // Migration is required if the previous version is less than the version at + // which the database field is introduced. + IcingSearchEngineVersionProto previous_version_proto; + previous_version_proto.set_version(kSchemaDatabaseVersion - 1); + previous_version_proto.set_max_version(kSchemaDatabaseVersion - 1); + previous_version_proto.add_enabled_features()->set_feature_type( + IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE); + EXPECT_TRUE(SchemaDatabaseMigrationRequired(previous_version_proto)); + + // Migration is required if the schema database feature was not enabled in the + // previous version. + previous_version_proto.set_version(kSchemaDatabaseVersion); + previous_version_proto.set_max_version(kSchemaDatabaseVersion); + previous_version_proto.mutable_enabled_features()->Clear(); + // Add a feature that is not the schema database feature. + previous_version_proto.add_enabled_features()->set_feature_type( + IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR); + EXPECT_TRUE(SchemaDatabaseMigrationRequired(previous_version_proto)); + + previous_version_proto.set_version(kSchemaDatabaseVersion + 1); + previous_version_proto.set_max_version(kSchemaDatabaseVersion + 1); + previous_version_proto.mutable_enabled_features()->Clear(); + EXPECT_TRUE(SchemaDatabaseMigrationRequired(previous_version_proto)); +} + +TEST(VersionUtilTest, SchemaDatabaseMigrationNotRequired) { + // Migration is not required if previous version is >= the version at which + // the schema database is introduced and the schema database feature was + // enabled in the previous version. + IcingSearchEngineVersionProto previous_version_proto; + previous_version_proto.set_version(kSchemaDatabaseVersion); + previous_version_proto.set_max_version(kSchemaDatabaseVersion); + previous_version_proto.add_enabled_features()->set_feature_type( + IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE); + EXPECT_FALSE(SchemaDatabaseMigrationRequired(previous_version_proto)); + + previous_version_proto.set_version(kSchemaDatabaseVersion + 1); + previous_version_proto.set_max_version(kSchemaDatabaseVersion + 1); + previous_version_proto.add_enabled_features()->set_feature_type( + IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE); + EXPECT_FALSE(SchemaDatabaseMigrationRequired(previous_version_proto)); } class VersionUtilFeatureProtoTest @@ -913,13 +1360,21 @@ Eq(rebuild_result.needs_integer_index_rebuild)); EXPECT_THAT(feature_info.needs_qualified_id_join_index_rebuild(), Eq(rebuild_result.needs_qualified_id_join_index_rebuild)); + EXPECT_THAT(feature_info.needs_embedding_index_rebuild(), + Eq(rebuild_result.needs_embedding_index_rebuild)); } INSTANTIATE_TEST_SUITE_P( VersionUtilFeatureProtoTest, VersionUtilFeatureProtoTest, testing::Values( IcingSearchEngineFeatureInfoProto::UNKNOWN, - IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR)); + IcingSearchEngineFeatureInfoProto::FEATURE_HAS_PROPERTY_OPERATOR, + IcingSearchEngineFeatureInfoProto::FEATURE_SCORABLE_PROPERTIES, + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_INDEX, + IcingSearchEngineFeatureInfoProto::FEATURE_EMBEDDING_QUANTIZATION, + IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE, + IcingSearchEngineFeatureInfoProto:: + FEATURE_QUALIFIED_ID_JOIN_INDEX_V3_AND_DELETE_PROPAGATE_FROM)); } // namespace
diff --git a/icing/icing-search-engine.cc b/icing/icing-search-engine.cc index 769e5de..8cfcef9 100644 --- a/icing/icing-search-engine.cc +++ b/icing/icing-search-engine.cc
@@ -32,6 +32,7 @@ #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/mutex.h" #include "icing/absl_ports/str_cat.h" +#include "icing/feature-flags.h" #include "icing/file/destructible-file.h" #include "icing/file/file-backed-proto.h" #include "icing/file/filesystem.h" @@ -50,8 +51,8 @@ #include "icing/jni/jni-cache.h" #include "icing/join/join-children-fetcher.h" #include "icing/join/join-processor.h" -#include "icing/join/qualified-id-join-index-impl-v1.h" #include "icing/join/qualified-id-join-index-impl-v2.h" +#include "icing/join/qualified-id-join-index-impl-v3.h" #include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id-join-indexing-handler.h" #include "icing/legacy/index/icing-filesystem.h" @@ -101,6 +102,7 @@ #include "icing/util/status-macros.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" +#include <google/protobuf/repeated_field.h> namespace icing { namespace lib { @@ -238,25 +240,59 @@ return libtextclassifier3::Status::OK; } -bool IsV2QualifiedIdJoinIndexEnabled(const IcingSearchEngineOptions& options) { - return true; +libtextclassifier3::Status ValidateScoringSpec( + const ScoringSpecProto& scoring_spec) { + std::unordered_set<std::string> alias_schema_types; + for (const SchemaTypeAliasMapProto& alias_map_proto : + scoring_spec.schema_type_alias_map_protos()) { + if (alias_map_proto.alias_schema_type().empty()) { + return absl_ports::InvalidArgumentError( + "SchemaTypeAliasMapProto contains alias_schema_type with empty " + "string"); + } + if (alias_map_proto.schema_types().empty()) { + return absl_ports::InvalidArgumentError( + absl_ports::StrCat("SchemaTypeAliasMapProto contains empty " + "schema_types for alias_schema_type: ", + alias_map_proto.alias_schema_type())); + } + if (alias_schema_types.find(alias_map_proto.alias_schema_type()) != + alias_schema_types.end()) { + return absl_ports::InvalidArgumentError( + absl_ports::StrCat("SchemaTypeAliasMapProto contains multiple " + "entries with the same alias_schema_type: ", + alias_map_proto.alias_schema_type())); + } + alias_schema_types.insert(alias_map_proto.alias_schema_type()); + } + return libtextclassifier3::Status::OK; +} + +ScoringSpecProto CopyParentSchemaTypeAliasMapToChild( + const ScoringSpecProto& parent_scoring_spec, + const ScoringSpecProto& child_scoring_spec) { + ScoringSpecProto new_child_scoring_spec = std::move(child_scoring_spec); + for (const SchemaTypeAliasMapProto& alias_map_proto : + parent_scoring_spec.schema_type_alias_map_protos()) { + *new_child_scoring_spec.add_schema_type_alias_map_protos() = + alias_map_proto; + } + return new_child_scoring_spec; } libtextclassifier3::StatusOr<std::unique_ptr<QualifiedIdJoinIndex>> CreateQualifiedIdJoinIndex(const Filesystem& filesystem, std::string qualified_id_join_index_dir, - const IcingSearchEngineOptions& options) { - if (IsV2QualifiedIdJoinIndexEnabled(options)) { + const IcingSearchEngineOptions& options, + const FeatureFlags& feature_flags) { + if (options.enable_qualified_id_join_index_v3_and_delete_propagate_from()) { + return QualifiedIdJoinIndexImplV3::Create( + filesystem, std::move(qualified_id_join_index_dir), feature_flags); + } else { // V2 return QualifiedIdJoinIndexImplV2::Create( filesystem, std::move(qualified_id_join_index_dir), options.pre_mapping_fbv()); - } else { - // V1 - // TODO(b/275121148): deprecate this part after rollout v2. - return QualifiedIdJoinIndexImplV1::Create( - filesystem, std::move(qualified_id_join_index_dir), - options.pre_mapping_fbv(), options.use_persistent_hash_map()); } } @@ -481,6 +517,9 @@ std::unique_ptr<const IcingFilesystem> icing_filesystem, std::unique_ptr<Clock> clock, std::unique_ptr<const JniCache> jni_cache) : options_(std::move(options)), + feature_flags_(options_.enable_scorable_properties(), + options_.enable_embedding_quantization(), + options_.enable_repeated_field_joins()), filesystem_(std::move(filesystem)), icing_filesystem_(std::move(icing_filesystem)), clock_(std::move(clock)), @@ -659,13 +698,15 @@ std::max(stored_version_info.max_version, version_util::kVersion)); version_util::AddEnabledFeatures(options_, ¤t_version_proto); - // Step 1: If versions are incompatible, migrate schema according to the - // version state change. - if (version_state_change != version_util::StateChange::kCompatible) { - ICING_RETURN_IF_ERROR(SchemaStore::MigrateSchema( - filesystem_.get(), MakeSchemaDirectoryPath(options_.base_dir()), - version_state_change, version_util::kVersion)); - } + // Step 1: Perform schema migration if needed. This is a no-op if the schema + // is fully compatible with the current version. + bool perform_schema_database_migration = + version_util::SchemaDatabaseMigrationRequired(stored_version_proto) && + options_.enable_schema_database(); + ICING_RETURN_IF_ERROR(SchemaStore::MigrateSchema( + filesystem_.get(), MakeSchemaDirectoryPath(options_.base_dir()), + version_state_change, version_util::kVersion, + perform_schema_database_migration)); // Step 2: Discard derived files that need to be rebuilt version_util::DerivedFilesRebuildResult required_derived_files_rebuild = @@ -727,7 +768,8 @@ } if (options_.enable_blob_store()) { ICING_RETURN_IF_ERROR( - InitializeBlobStore(options_.orphan_blob_time_to_live_ms())); + InitializeBlobStore(options_.orphan_blob_time_to_live_ms(), + options_.blob_store_compression_level())); } ICING_ASSIGN_OR_RETURN( bool document_store_derived_files_regenerated, @@ -748,7 +790,8 @@ // unused. if (options_.enable_blob_store()) { ICING_RETURN_IF_ERROR( - InitializeBlobStore(options_.orphan_blob_time_to_live_ms())); + InitializeBlobStore(options_.orphan_blob_time_to_live_ms(), + options_.blob_store_compression_level())); } ICING_RETURN_IF_ERROR(InitializeDocumentStore( /*force_recovery_and_revalidate_documents=*/true, initialize_stats)); @@ -786,8 +829,9 @@ *filesystem_, qualified_id_join_index_dir)); ICING_ASSIGN_OR_RETURN( qualified_id_join_index_, - CreateQualifiedIdJoinIndex( - *filesystem_, std::move(qualified_id_join_index_dir), options_)); + CreateQualifiedIdJoinIndex(*filesystem_, + std::move(qualified_id_join_index_dir), + options_, feature_flags_)); // Discard embedding index directory and instantiate a new one. std::string embedding_index_dir = @@ -796,7 +840,8 @@ EmbeddingIndex::Discard(*filesystem_, embedding_index_dir)); ICING_ASSIGN_OR_RETURN( embedding_index_, - EmbeddingIndex::Create(filesystem_.get(), embedding_index_dir)); + EmbeddingIndex::Create(filesystem_.get(), embedding_index_dir, + clock_.get(), &feature_flags_)); std::unique_ptr<Timer> restore_timer = clock_->GetNewTimer(); IndexRestorationResult restore_result = RestoreIndexIfNeeded(); @@ -825,7 +870,8 @@ } else if (version_state_change != version_util::StateChange::kCompatible) { if (options_.enable_blob_store()) { ICING_RETURN_IF_ERROR( - InitializeBlobStore(options_.orphan_blob_time_to_live_ms())); + InitializeBlobStore(options_.orphan_blob_time_to_live_ms(), + options_.blob_store_compression_level())); } ICING_ASSIGN_OR_RETURN(bool document_store_derived_files_regenerated, InitializeDocumentStore( @@ -852,7 +898,8 @@ } else { if (options_.enable_blob_store()) { ICING_RETURN_IF_ERROR( - InitializeBlobStore(options_.orphan_blob_time_to_live_ms())); + InitializeBlobStore(options_.orphan_blob_time_to_live_ms(), + options_.blob_store_compression_level())); } ICING_ASSIGN_OR_RETURN( bool document_store_derived_files_regenerated, @@ -889,7 +936,10 @@ initialize_stats->set_qualified_id_join_index_restoration_cause( InitializeStatsProto::FEATURE_FLAG_CHANGED); } - // TODO(b/326656531): Update version-util to consider embedding index. + if (required_derived_files_rebuild.needs_embedding_index_rebuild) { + initialize_stats->set_embedding_index_restoration_cause( + InitializeStatsProto::FEATURE_FLAG_CHANGED); + } } if (status.ok()) { @@ -914,8 +964,10 @@ absl_ports::StrCat("Could not create directory: ", schema_store_dir)); } ICING_ASSIGN_OR_RETURN( - schema_store_, SchemaStore::Create(filesystem_.get(), schema_store_dir, - clock_.get(), initialize_stats)); + schema_store_, + SchemaStore::Create(filesystem_.get(), schema_store_dir, clock_.get(), + &feature_flags_, options_.enable_schema_database(), + initialize_stats)); return libtextclassifier3::Status::OK; } @@ -936,8 +988,7 @@ DocumentStore::CreateResult create_result, DocumentStore::Create( filesystem_.get(), document_dir, clock_.get(), schema_store_.get(), - force_recovery_and_revalidate_documents, - /*document_store_namespace_id_fingerprint=*/true, + &feature_flags_, force_recovery_and_revalidate_documents, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, options_.compression_level(), initialize_stats)); document_store_ = std::move(create_result.document_store); @@ -945,7 +996,7 @@ } libtextclassifier3::Status IcingSearchEngine::InitializeBlobStore( - int32_t orphan_blob_time_to_live_ms) { + int32_t orphan_blob_time_to_live_ms, int32_t blob_store_compression_level) { std::string blob_dir = MakeBlobDirectoryPath(options_.base_dir()); // Make sure the sub-directory exists if (!filesystem_->CreateDirectoryRecursively(blob_dir.c_str())) { @@ -956,7 +1007,8 @@ ICING_ASSIGN_OR_RETURN( auto blob_store_or, BlobStore::Create(filesystem_.get(), blob_dir, clock_.get(), - orphan_blob_time_to_live_ms)); + orphan_blob_time_to_live_ms, + blob_store_compression_level)); blob_store_ = std::make_unique<BlobStore>(std::move(blob_store_or)); return libtextclassifier3::Status::OK; } @@ -1035,7 +1087,7 @@ MakeQualifiedIdJoinIndexWorkingPath(options_.base_dir()); InitializeStatsProto::RecoveryCause qualified_id_join_index_recovery_cause; if (document_store_derived_files_regenerated && - IsV2QualifiedIdJoinIndexEnabled(options_)) { + !options_.enable_qualified_id_join_index_v3_and_delete_propagate_from()) { // V2 qualified id join index depends on document store derived files, so we // have to rebuild it from scratch if // document_store_derived_files_regenerated is true. @@ -1044,14 +1096,15 @@ ICING_ASSIGN_OR_RETURN( qualified_id_join_index_, - CreateQualifiedIdJoinIndex( - *filesystem_, std::move(qualified_id_join_index_dir), options_)); + CreateQualifiedIdJoinIndex(*filesystem_, + std::move(qualified_id_join_index_dir), + options_, feature_flags_)); qualified_id_join_index_recovery_cause = InitializeStatsProto::DEPENDENCIES_CHANGED; } else { auto qualified_id_join_index_or = CreateQualifiedIdJoinIndex( - *filesystem_, qualified_id_join_index_dir, options_); + *filesystem_, qualified_id_join_index_dir, options_, feature_flags_); if (!qualified_id_join_index_or.ok()) { ICING_RETURN_IF_ERROR(QualifiedIdJoinIndex::Discard( *filesystem_, qualified_id_join_index_dir)); @@ -1061,8 +1114,9 @@ // Try recreating it from scratch and rebuild everything. ICING_ASSIGN_OR_RETURN( qualified_id_join_index_, - CreateQualifiedIdJoinIndex( - *filesystem_, std::move(qualified_id_join_index_dir), options_)); + CreateQualifiedIdJoinIndex(*filesystem_, + std::move(qualified_id_join_index_dir), + options_, feature_flags_)); } else { // Qualified id join index was created fine. qualified_id_join_index_ = @@ -1078,8 +1132,8 @@ const std::string embedding_dir = MakeEmbeddingIndexWorkingPath(options_.base_dir()); InitializeStatsProto::RecoveryCause embedding_index_recovery_cause; - auto embedding_index_or = - EmbeddingIndex::Create(filesystem_.get(), embedding_dir); + auto embedding_index_or = EmbeddingIndex::Create( + filesystem_.get(), embedding_dir, clock_.get(), &feature_flags_); if (!embedding_index_or.ok()) { ICING_RETURN_IF_ERROR(EmbeddingIndex::Discard(*filesystem_, embedding_dir)); @@ -1088,7 +1142,8 @@ // Try recreating it from scratch and re-indexing everything. ICING_ASSIGN_OR_RETURN( embedding_index_, - EmbeddingIndex::Create(filesystem_.get(), embedding_dir)); + EmbeddingIndex::Create(filesystem_.get(), embedding_dir, clock_.get(), + &feature_flags_)); } else { // Embedding index was created fine. embedding_index_ = std::move(embedding_index_or).ValueOrDie(); @@ -1267,6 +1322,19 @@ } } + if (feature_flags_.enable_scorable_properties()) { + if (!set_schema_result.schema_types_scorable_property_inconsistent_by_id + .empty()) { + status = document_store_->RegenerateScorablePropertyCache( + set_schema_result + .schema_types_scorable_property_inconsistent_by_id); + if (!status.ok()) { + TransformStatus(status, result_status); + return result_proto; + } + } + } + result_status->set_code(StatusProto::OK); } else { result_status->set_code(StatusProto::FAILED_PRECONDITION); @@ -1298,6 +1366,29 @@ return result_proto; } +GetSchemaResultProto IcingSearchEngine::GetSchema(std::string_view database) { + GetSchemaResultProto result_proto; + StatusProto* result_status = result_proto.mutable_status(); + + absl_ports::shared_lock l(&mutex_); + if (!initialized_) { + result_status->set_code(StatusProto::FAILED_PRECONDITION); + result_status->set_message("IcingSearchEngine has not been initialized!"); + return result_proto; + } + + libtextclassifier3::StatusOr<SchemaProto> schema = + schema_store_->GetSchema(std::string(database)); + if (!schema.ok()) { + TransformStatus(schema.status(), result_status); + return result_proto; + } + + result_status->set_code(StatusProto::OK); + *result_proto.mutable_schema() = std::move(schema).ValueOrDie(); + return result_proto; +} + GetSchemaTypeResultProto IcingSearchEngine::GetSchemaType( std::string_view schema_type) { GetSchemaTypeResultProto result_proto; @@ -1362,8 +1453,10 @@ TransformStatus(put_result_or.status(), result_status); return result_proto; } + DocumentId old_document_id = put_result_or.ValueOrDie().old_document_id; DocumentId document_id = put_result_or.ValueOrDie().new_document_id; - result_proto.set_was_replacement(put_result_or.ValueOrDie().was_replacement); + result_proto.set_was_replacement( + put_result_or.ValueOrDie().was_replacement()); auto data_indexing_handlers_or = CreateDataIndexingHandlers(); if (!data_indexing_handlers_or.ok()) { @@ -1374,7 +1467,7 @@ std::move(data_indexing_handlers_or).ValueOrDie(), clock_.get()); auto index_status = index_processor.IndexDocument( - tokenized_document, document_id, put_document_stats); + tokenized_document, document_id, old_document_id, put_document_stats); // Getting an internal error from the index could possibly mean that the index // is broken. Try to rebuild them to recover. if (absl_ports::IsInternal(index_status)) { @@ -1515,27 +1608,63 @@ DeleteStatsProto* delete_stats = result_proto.mutable_delete_stats(); delete_stats->set_delete_type(DeleteStatsProto::DeleteType::SINGLE); + libtextclassifier3::Status status; + libtextclassifier3::Status propagate_delete_status; + int num_documents_deleted = 0; std::unique_ptr<Timer> delete_timer = clock_->GetNewTimer(); - // TODO(b/216487496): Implement a more robust version of TC_RETURN_IF_ERROR - // that can support error logging. - int64_t current_time_ms = clock_->GetSystemTimeMilliseconds(); - libtextclassifier3::Status status = - document_store_->Delete(name_space, uri, current_time_ms); + + libtextclassifier3::StatusOr<DocumentId> document_id_or = + document_store_->GetDocumentId(name_space, uri); + if (!document_id_or.ok()) { + status = std::move(document_id_or).status(); + } else { + DocumentId document_id = document_id_or.ValueOrDie(); + + // TODO(b/216487496): Implement a more robust version of TC_RETURN_IF_ERROR + // that can support error logging. + int64_t current_time_ms = clock_->GetSystemTimeMilliseconds(); + status = document_store_->Delete(document_id, current_time_ms); + if (status.ok()) { + ++num_documents_deleted; + } + + // It is possible that the document has expired and the delete operation + // fails with NOT_FOUND_ERROR. In this case, we should still propagate the + // delete operation, regardless of the outcome of the delete operation. + libtextclassifier3::StatusOr<int> propagated_child_docs_deleted_or = + PropagateDelete(/*deleted_document_ids=*/{document_id}, + current_time_ms); + if (propagated_child_docs_deleted_or.ok()) { + num_documents_deleted += propagated_child_docs_deleted_or.ValueOrDie(); + } else { + propagate_delete_status = + std::move(propagated_child_docs_deleted_or).status(); + } + } + delete_stats->set_num_documents_deleted(num_documents_deleted); + delete_stats->set_latency_ms(delete_timer->GetElapsedMilliseconds()); + if (!status.ok()) { LogSeverity::Code severity = ERROR; if (absl_ports::IsNotFound(status)) { severity = DBG; } ICING_LOG(severity) << status.error_message() - << "Failed to delete Document. namespace: " + << ". Failed to delete Document. namespace: " << name_space << ", uri: " << uri; TransformStatus(status, result_status); return result_proto; } + if (!propagate_delete_status.ok()) { + ICING_LOG(ERROR) << propagate_delete_status.error_message() + << ". Failed to propagate delete for document. namespace: " + << name_space << ", uri: " << uri; + TransformStatus(propagate_delete_status, result_status); + return result_proto; + } + result_status->set_code(StatusProto::OK); - delete_stats->set_latency_ms(delete_timer->GetElapsedMilliseconds()); - delete_stats->set_num_documents_deleted(1); return result_proto; } @@ -1645,7 +1774,8 @@ auto query_processor_or = QueryProcessor::Create( index_.get(), integer_index_.get(), embedding_index_.get(), language_segmenter_.get(), normalizer_.get(), document_store_.get(), - schema_store_.get(), clock_.get()); + schema_store_.get(), /*join_children_fetcher=*/nullptr, clock_.get(), + &feature_flags_); if (!query_processor_or.ok()) { TransformStatus(query_processor_or.status(), result_status); delete_stats->set_parse_query_latency_ms( @@ -1722,6 +1852,44 @@ return result_proto; } +libtextclassifier3::StatusOr<int> IcingSearchEngine::PropagateDelete( + const std::unordered_set<DocumentId>& deleted_document_ids, + int64_t current_time_ms) { + int propagated_child_docs_deleted = 0; + + if (!options_.enable_qualified_id_join_index_v3_and_delete_propagate_from() || + qualified_id_join_index_->version() != + QualifiedIdJoinIndex::Version::kV3) { + // No-op if delete propagation is disabled or the join index is not v3. + return propagated_child_docs_deleted; + } + + // Create join processor to get propagated child documents to delete. + JoinProcessor join_processor(document_store_.get(), schema_store_.get(), + qualified_id_join_index_.get(), current_time_ms); + ICING_ASSIGN_OR_RETURN( + std::unordered_set<DocumentId> child_docs_to_delete, + join_processor.GetPropagatedChildDocumentsToDelete(deleted_document_ids)); + + // Delete all propagated child documents. + for (DocumentId child_doc_id : child_docs_to_delete) { + auto status = document_store_->Delete(child_doc_id, current_time_ms); + if (!status.ok()) { + if (absl_ports::IsNotFound(status)) { + // The child document has already been deleted or expired, so skip the + // error. + continue; + } + + // Real error. + return status; + } + ++propagated_child_docs_deleted; + } + + return propagated_child_docs_deleted; +} + PersistToDiskResultProto IcingSearchEngine::PersistToDisk( PersistType::Code persist_type) { ICING_VLOG(1) << "Persisting data to disk"; @@ -1863,7 +2031,8 @@ } libtextclassifier3::Status embedding_index_optimize_status = - embedding_index_->Optimize(optimize_result.document_id_old_to_new, + embedding_index_->Optimize(document_store_.get(), schema_store_.get(), + optimize_result.document_id_old_to_new, document_store_->last_added_document_id()); if (!embedding_index_optimize_status.ok()) { ICING_LOG(WARNING) << "Failed to optimize embedding index. Error: " @@ -2039,6 +2208,24 @@ schema_store_->GetStorageInfo(); *result.mutable_storage_info()->mutable_index_storage_info() = index_->GetStorageInfo(); + if (blob_store_ != nullptr) { + auto namespace_blob_storage_infos_or = blob_store_->GetStorageInfo(); + if (!namespace_blob_storage_infos_or.ok()) { + result.mutable_status()->set_code(StatusProto::INTERNAL); + result.mutable_status()->set_message( + namespace_blob_storage_infos_or.status().error_message()); + return result; + } + std::vector<NamespaceBlobStorageInfoProto> namespace_blob_storage_infos = + std::move(namespace_blob_storage_infos_or).ValueOrDie(); + + for (NamespaceBlobStorageInfoProto& namespace_blob_storage_info : + namespace_blob_storage_infos) { + *result.mutable_storage_info() + ->mutable_namespace_blob_storage_info() + ->Add() = std::move(namespace_blob_storage_info); + } + } // TODO(b/259744228): add stats for integer index result.mutable_status()->set_code(StatusProto::OK); return result; @@ -2090,8 +2277,8 @@ PersistType::Code persist_type) { if (blob_store_ != nullptr) { // For all valid PersistTypes, we persist the ground truth. The ground truth - // in the schema_store is always persisted immediately after changes are - // applied. So there is nothing to do if persist_type is LITE. + // in the blob_store is a proto log file, which is need to be called when + // persist_type is LITE. ICING_RETURN_IF_ERROR(blob_store_->PersistToDisk()); } ICING_RETURN_IF_ERROR(document_store_->PersistToDisk(persist_type)); @@ -2201,6 +2388,11 @@ TransformStatus(status, result_status); return result_proto; } + status = ValidateScoringSpec(scoring_spec); + if (!status.ok()) { + TransformStatus(status, result_status); + return result_proto; + } const JoinSpecProto& join_spec = search_spec.join_spec(); std::unique_ptr<JoinChildrenFetcher> join_children_fetcher; @@ -2212,10 +2404,23 @@ QueryStatsProto::SearchStats* child_search_stats = query_stats->mutable_child_search_stats(); + // Build a child scoring spec by copying the parent schema type alias map. + // Note that this function will transfer the ownership of data from + // search_spec.join_spec().nested_spec().scoring_spec() to the + // child_scoring_spec. + // Hence, that following functions should NOT access + // search_spec.join_spec().nested_spec().scoring_spec() after this function + // call, but using child_scoring_spec instead. + // + // TODO(b/379288742): Avoid making the copy of the parent schema type alias + // map. + ScoringSpecProto child_scoring_spec = CopyParentSchemaTypeAliasMapToChild( + scoring_spec, search_spec.join_spec().nested_spec().scoring_spec()); + // Process child query + // TODO(b/372541905): Validate the child search spec. QueryScoringResults nested_query_scoring_results = ProcessQueryAndScore( - join_spec.nested_spec().search_spec(), - join_spec.nested_spec().scoring_spec(), + join_spec.nested_spec().search_spec(), child_scoring_spec, join_spec.nested_spec().result_spec(), /*join_children_fetcher=*/nullptr, current_time_ms, child_search_stats); if (!nested_query_scoring_results.status.ok()) { @@ -2226,23 +2431,21 @@ JoinProcessor join_processor(document_store_.get(), schema_store_.get(), qualified_id_join_index_.get(), current_time_ms); - // Building a JoinChildrenFetcher where child documents are grouped by - // their joinable values. - libtextclassifier3::StatusOr<JoinChildrenFetcher> join_children_fetcher_or = - join_processor.GetChildrenFetcher( + // Building a JoinChildrenFetcher for looking up child documents by parent + // document id. + libtextclassifier3::StatusOr<std::unique_ptr<JoinChildrenFetcher>> + join_children_fetcher_or = join_processor.GetChildrenFetcher( search_spec.join_spec(), std::move(nested_query_scoring_results.scored_document_hits)); if (!join_children_fetcher_or.ok()) { TransformStatus(join_children_fetcher_or.status(), result_status); return result_proto; } - join_children_fetcher = std::make_unique<JoinChildrenFetcher>( - std::move(join_children_fetcher_or).ValueOrDie()); + join_children_fetcher = std::move(join_children_fetcher_or).ValueOrDie(); // Assign child's ResultAdjustmentInfo. child_result_adjustment_info = std::make_unique<ResultAdjustmentInfo>( - join_spec.nested_spec().search_spec(), - join_spec.nested_spec().scoring_spec(), + join_spec.nested_spec().search_spec(), child_scoring_spec, join_spec.nested_spec().result_spec(), schema_store_.get(), std::move(nested_query_scoring_results.query_terms)); } @@ -2394,7 +2597,8 @@ auto query_processor_or = QueryProcessor::Create( index_.get(), integer_index_.get(), embedding_index_.get(), language_segmenter_.get(), normalizer_.get(), document_store_.get(), - schema_store_.get(), clock_.get()); + schema_store_.get(), join_children_fetcher, clock_.get(), + &feature_flags_); if (!query_processor_or.ok()) { search_stats->set_parse_query_latency_ms( component_timer->GetElapsedMilliseconds()); @@ -2441,7 +2645,7 @@ scoring_spec, /*default_semantic_metric_type=*/ search_spec.embedding_query_metric_type(), document_store_.get(), schema_store_.get(), current_time_ms, join_children_fetcher, - &query_results.embedding_query_results); + &query_results.embedding_query_results, &feature_flags_); if (!scoring_processor_or.ok()) { return QueryScoringResults(std::move(scoring_processor_or).status(), std::move(query_results.query_terms), @@ -2550,7 +2754,7 @@ } BlobProto IcingSearchEngine::OpenWriteBlob( - PropertyProto::BlobHandleProto blob_handle) { + const PropertyProto::BlobHandleProto& blob_handle) { BlobProto blob_proto; StatusProto* status = blob_proto.mutable_status(); @@ -2568,7 +2772,8 @@ return blob_proto; } - auto write_fd_or = blob_store_->OpenWrite(blob_handle); + libtextclassifier3::StatusOr<int> write_fd_or = + blob_store_->OpenWrite(blob_handle); if (!write_fd_or.ok()) { TransformStatus(write_fd_or.status(), status); return blob_proto; @@ -2578,8 +2783,35 @@ return blob_proto; } +BlobProto IcingSearchEngine::RemoveBlob( + const PropertyProto::BlobHandleProto& blob_handle) { + BlobProto blob_proto; + StatusProto* status = blob_proto.mutable_status(); + + absl_ports::unique_lock l(&mutex_); + if (blob_store_ == nullptr) { + status->set_code(StatusProto::FAILED_PRECONDITION); + status->set_message("Remove blob is not supported in this Icing instance!"); + return blob_proto; + } + + if (!initialized_) { + status->set_code(StatusProto::FAILED_PRECONDITION); + status->set_message("IcingSearchEngine has not been initialized!"); + return blob_proto; + } + + auto remove_result = blob_store_->RemoveBlob(blob_handle); + if (!remove_result.ok()) { + TransformStatus(remove_result, status); + return blob_proto; + } + status->set_code(StatusProto::OK); + return blob_proto; +} + BlobProto IcingSearchEngine::OpenReadBlob( - PropertyProto::BlobHandleProto blob_handle) { + const PropertyProto::BlobHandleProto& blob_handle) { BlobProto blob_proto; StatusProto* status = blob_proto.mutable_status(); absl_ports::shared_lock l(&mutex_); @@ -2608,7 +2840,7 @@ } BlobProto IcingSearchEngine::CommitBlob( - PropertyProto::BlobHandleProto blob_handle) { + const PropertyProto::BlobHandleProto& blob_handle) { BlobProto blob_proto; StatusProto* status = blob_proto.mutable_status(); absl_ports::unique_lock l(&mutex_); @@ -2692,8 +2924,8 @@ // system in the broken state for future operations. auto create_result_or = DocumentStore::Create( filesystem_.get(), current_document_dir, clock_.get(), - schema_store_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*document_store_namespace_id_fingerprint=*/true, + schema_store_.get(), &feature_flags_, + /*force_recovery_and_revalidate_documents=*/false, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, options_.compression_level(), /*initialize_stats=*/nullptr); // TODO(b/144458732): Implement a more robust version of @@ -2720,8 +2952,8 @@ // Recreates the doc store instance auto create_result_or = DocumentStore::Create( filesystem_.get(), current_document_dir, clock_.get(), - schema_store_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*document_store_namespace_id_fingerprint=*/true, + schema_store_.get(), &feature_flags_, + /*force_recovery_and_revalidate_documents=*/false, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, options_.compression_level(), /*initialize_stats=*/nullptr); if (!create_result_or.ok()) { @@ -2843,8 +3075,11 @@ TokenizedDocument tokenized_document( std::move(tokenized_document_or).ValueOrDie()); + // No valid old_document_id should be used here since we're in recovery mode + // and there is no "existing document replacement/update". libtextclassifier3::Status status = - index_processor.IndexDocument(tokenized_document, document_id); + index_processor.IndexDocument(tokenized_document, document_id, + /*old_document_id=*/kInvalidDocumentId); if (!status.ok()) { if (!absl_ports::IsDataLoss(status)) { // Real error. Stop recovering and pass it up. @@ -2919,7 +3154,8 @@ // Embedding index handler ICING_ASSIGN_OR_RETURN( std::unique_ptr<EmbeddingIndexingHandler> embedding_indexing_handler, - EmbeddingIndexingHandler::Create(clock_.get(), embedding_index_.get())); + EmbeddingIndexingHandler::Create(clock_.get(), embedding_index_.get(), + options_.enable_embedding_index())); handlers.push_back(std::move(embedding_indexing_handler)); return handlers; } @@ -3064,13 +3300,13 @@ // Schema store if (rebuild_result.needs_schema_store_derived_files_rebuild) { ICING_RETURN_IF_ERROR(SchemaStore::DiscardDerivedFiles( - filesystem_.get(), options_.base_dir())); + filesystem_.get(), MakeSchemaDirectoryPath(options_.base_dir()))); } // Document store if (rebuild_result.needs_document_store_derived_files_rebuild) { ICING_RETURN_IF_ERROR(DocumentStore::DiscardDerivedFiles( - filesystem_.get(), options_.base_dir())); + filesystem_.get(), MakeDocumentDirectoryPath(options_.base_dir()))); } // Term index @@ -3098,7 +3334,11 @@ } } - // TODO(b/326656531): Update version-util to consider embedding index. + // Embedding index. + if (rebuild_result.needs_embedding_index_rebuild) { + ICING_RETURN_IF_ERROR(EmbeddingIndex::Discard( + *filesystem_, MakeEmbeddingIndexWorkingPath(options_.base_dir()))); + } return libtextclassifier3::Status::OK; } @@ -3181,7 +3421,7 @@ auto suggestion_processor_or = SuggestionProcessor::Create( index_.get(), integer_index_.get(), embedding_index_.get(), language_segmenter_.get(), normalizer_.get(), document_store_.get(), - schema_store_.get(), clock_.get()); + schema_store_.get(), clock_.get(), &feature_flags_); if (!suggestion_processor_or.ok()) { TransformStatus(suggestion_processor_or.status(), response_status); return response;
diff --git a/icing/icing-search-engine.h b/icing/icing-search-engine.h index 299dc6d..d23767d 100644 --- a/icing/icing-search-engine.h +++ b/icing/icing-search-engine.h
@@ -27,6 +27,7 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/mutex.h" #include "icing/absl_ports/thread_annotations.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/version-util.h" #include "icing/index/data-indexing-handler.h" @@ -175,6 +176,21 @@ // INTERNAL_ERROR on IO error GetSchemaResultProto GetSchema() ICING_LOCKS_EXCLUDED(mutex_); + // Get Icing's current copy of the schema for the given database. + // + // NOTE: This is an expensive operation. It is recommended to call GetSchema() + // instead if you do not need to filter the schema by database, or if you're + // retrieving the only database in the schema. + // + // Returns: + // SchemaProto on success + // NOT_FOUND if a schema has not been set yet, or if the database is not + // present in the schema + // FAILED_PRECONDITION IcingSearchEngine has not been initialized yet. + // INTERNAL_ERROR on IO error + GetSchemaResultProto GetSchema(std::string_view database) + ICING_LOCKS_EXCLUDED(mutex_); + // Get Icing's copy of the SchemaTypeConfigProto of name schema_type // // Returns: @@ -357,9 +373,21 @@ // Returns: // File descriptor on success // InvalidArgumentError on invalid blob handle - // PermissionDeniedError on blob is committed + // FailedPreconditionError on blob is already opened for write + // AlreadyExistsError on blob is committed // INTERNAL_ERROR on IO error - BlobProto OpenWriteBlob(PropertyProto::BlobHandleProto blob_handle); + BlobProto OpenWriteBlob(const PropertyProto::BlobHandleProto& blob_handle); + + // Removes a blob file and blob handle from the blob store. + // + // This will remove the blob on any state. No matter it's committed or not or + // it has reference document links or not. + // + // Returns: + // InvalidArgumentError on invalid blob handle + // NotFoundError on blob is not found + // InternalError on IO error + BlobProto RemoveBlob(const PropertyProto::BlobHandleProto& blob_handle); // Gets or creates a file for read only purpose for the given blob handle. // The blob must be committed by calling commitBlob otherwise it is not @@ -369,7 +397,7 @@ // File descriptor on success // InvalidArgumentError on invalid blob handle // NotFoundError on blob is not found or is not committed - BlobProto OpenReadBlob(PropertyProto::BlobHandleProto blob_handle); + BlobProto OpenReadBlob(const PropertyProto::BlobHandleProto& blob_handle); // Commits the given blob, the blob is open to write via openWrite. // Before the blob is committed, it is not visible to any reader via openRead. @@ -381,7 +409,7 @@ // False on the blob is already committed. // InvalidArgumentError on invalid blob handle or digest is mismatch with // file content NotFoundError on blob is not found. - BlobProto CommitBlob(PropertyProto::BlobHandleProto blob_handle); + BlobProto CommitBlob(const PropertyProto::BlobHandleProto& blob_handle); // Makes sure that every update/delete received till this point is flushed // to disk. If the app crashes after a call to PersistToDisk(), Icing @@ -477,6 +505,7 @@ private: const IcingSearchEngineOptions options_; + const FeatureFlags feature_flags_; const std::unique_ptr<const Filesystem> filesystem_; const std::unique_ptr<const IcingFilesystem> icing_filesystem_; bool initialized_ ICING_GUARDED_BY(mutex_) = false; @@ -611,7 +640,7 @@ // OK on success // FAILED_PRECONDITION if initialize_stats is null libtextclassifier3::Status InitializeBlobStore( - int32_t orphan_blob_time_to_live_ms) + int32_t orphan_blob_time_to_live_ms, int32_t compression_level) ICING_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Do any initialization/recovery necessary to create term index, integer @@ -683,20 +712,15 @@ QueryStatsProto::SearchStats* search_stats) ICING_SHARED_LOCKS_REQUIRED(mutex_); - // Many of the internal components rely on other components' derived data. - // Check that everything is consistent with each other so that we're not - // using outdated derived data in some parts of our system. - // - // NOTE: this method can be called only at startup time or after - // PersistToDisk(), otherwise the check could fail due to any changes that are - // not persisted. + // Deletes documents propagated from the given deleted document ids via + // joinable properties with delete propagation enabled. // // Returns: - // OK on success - // NOT_FOUND if missing header file - // INTERNAL_ERROR on any IO errors or if header is inconsistent - libtextclassifier3::Status CheckConsistency() - ICING_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Number of propagated documents deleted on success + // INTERNAL_ERROR on any I/O errors + libtextclassifier3::StatusOr<int> PropagateDelete( + const std::unordered_set<DocumentId>& deleted_document_ids, + int64_t current_time_ms) ICING_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Discards derived data that requires rebuild based on rebuild_result. //
diff --git a/icing/icing-search-engine_benchmark.cc b/icing/icing-search-engine_benchmark.cc index 29dd92a..91924dc 100644 --- a/icing/icing-search-engine_benchmark.cc +++ b/icing/icing-search-engine_benchmark.cc
@@ -1114,7 +1114,7 @@ options.set_base_dir(test_dir); options.set_index_merge_size(kIcingFullIndexSize); options.set_document_store_namespace_id_fingerprint(true); - options.set_use_new_qualified_id_join_index(true); + options.set_enable_qualified_id_join_index_v3_and_delete_propagate_from(true); std::unique_ptr<IcingSearchEngine> icing = std::make_unique<IcingSearchEngine>(options);
diff --git a/icing/icing-search-engine_blob_test.cc b/icing/icing-search-engine_blob_test.cc index f32d6d1..72655ae 100644 --- a/icing/icing-search-engine_blob_test.cc +++ b/icing/icing-search-engine_blob_test.cc
@@ -22,6 +22,7 @@ #include <utility> #include <vector> +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" #include "icing/file/filesystem.h" @@ -29,6 +30,7 @@ #include "icing/jni/jni-cache.h" #include "icing/legacy/index/icing-filesystem.h" #include "icing/portable/equals-proto.h" +#include "icing/proto/storage.pb.h" #include "icing/schema-builder.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" @@ -44,7 +46,10 @@ namespace { -using ::testing::Eq; +using ::icing::lib::portable_equals_proto::EqualsProto; +using ::testing::IsEmpty; +using ::testing::SizeIs; +using ::testing::UnorderedElementsAre; // For mocking purpose, we allow tests to provide a custom Filesystem. class TestIcingSearchEngine : public IcingSearchEngine { @@ -60,14 +65,16 @@ }; std::string GetTestBaseDir() { return GetTestTempDir() + "/icing"; } -std::string GetTestBaseBlobStoreDir() { - return GetTestTempDir() + "/icing/blob_dir"; -} + +std::string GetTestBlobDir() { return GetTestTempDir() + "/icing/blob_dir"; } + +std::string GetTestBlobFileDir() { return GetTestBlobDir() + "/blob_files"; } // This test is meant to cover all tests relating to IcingSearchEngine::Delete*. class IcingSearchEngineBlobTest : public testing::Test { protected: void SetUp() override { + filesystem_.DeleteDirectoryRecursively(GetTestBaseDir().c_str()); filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); } @@ -134,8 +141,8 @@ TEST_F(IcingSearchEngineBlobTest, InvalidBlobHandle) { PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("blob"); blob_handle.set_digest("invalid"); + blob_handle.set_namespace_("namespaceA"); IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); @@ -160,10 +167,10 @@ ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("blob"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); blob_handle.set_digest((void*)digest.data(), digest.size()); + blob_handle.set_namespace_("namespaceA"); BlobProto write_blob_proto = icing.OpenWriteBlob(blob_handle); EXPECT_THAT(write_blob_proto.status(), @@ -181,10 +188,10 @@ ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("label"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); blob_handle.set_digest((void*)digest.data(), digest.size()); + blob_handle.set_namespace_("namespaceA"); BlobProto write_blob_proto = icing.OpenWriteBlob(blob_handle); ASSERT_THAT(write_blob_proto.status(), ProtoIsOk()); @@ -211,15 +218,90 @@ } } +TEST_F(IcingSearchEngineBlobTest, RemovePendingBlob) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + PropertyProto::BlobHandleProto blob_handle; + blob_handle.set_namespace_("namespace1"); + std::vector<unsigned char> data = GenerateRandomBytes(24); + std::array<uint8_t, 32> digest = CalculateDigest(data); + blob_handle.set_digest((void*)digest.data(), digest.size()); + + BlobProto write_blob_proto = icing.OpenWriteBlob(blob_handle); + ASSERT_THAT(write_blob_proto.status(), ProtoIsOk()); + { + ScopedFd write_fd(write_blob_proto.file_descriptor()); + ASSERT_TRUE(filesystem()->Write(write_fd.get(), data.data(), data.size())); + } + + std::vector<std::string> file_names; + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobFileDir().c_str(), &file_names)); + ASSERT_THAT(file_names, SizeIs(1)); + + EXPECT_THAT(icing.RemoveBlob(blob_handle).status(), ProtoIsOk()); + + // Commit will return NOT_FOUND since the blob is removed. + BlobProto commit_blob_proto = icing.CommitBlob(blob_handle); + EXPECT_THAT(commit_blob_proto.status(), + ProtoStatusIs(StatusProto::NOT_FOUND)); + + file_names = std::vector<std::string>(); + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobFileDir().c_str(), &file_names)); + // The pending file is deleted. + EXPECT_THAT(file_names, IsEmpty()); +} + +TEST_F(IcingSearchEngineBlobTest, RemoveCommittedBlob) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + PropertyProto::BlobHandleProto blob_handle; + blob_handle.set_namespace_("namespace1"); + std::vector<unsigned char> data = GenerateRandomBytes(24); + std::array<uint8_t, 32> digest = CalculateDigest(data); + blob_handle.set_digest((void*)digest.data(), digest.size()); + + BlobProto write_blob_proto = icing.OpenWriteBlob(blob_handle); + ASSERT_THAT(write_blob_proto.status(), ProtoIsOk()); + { + ScopedFd write_fd(write_blob_proto.file_descriptor()); + ASSERT_TRUE(filesystem()->Write(write_fd.get(), data.data(), data.size())); + } + + // commit the blob + ASSERT_THAT(icing.CommitBlob(blob_handle).status(), ProtoIsOk()); + + std::vector<std::string> file_names; + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobFileDir().c_str(), &file_names)); + ASSERT_THAT(file_names, SizeIs(1)); + + EXPECT_THAT(icing.RemoveBlob(blob_handle).status(), ProtoIsOk()); + + // Commit will return NOT_FOUND since the blob is removed. + BlobProto commit_blob_proto = icing.CommitBlob(blob_handle); + EXPECT_THAT(commit_blob_proto.status(), + ProtoStatusIs(StatusProto::NOT_FOUND)); + + file_names = std::vector<std::string>(); + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobFileDir().c_str(), &file_names)); + // The pending file is deleted. + EXPECT_THAT(file_names, IsEmpty()); +} + TEST_F(IcingSearchEngineBlobTest, WriteAndReadBlobByDocument) { IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("label"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); blob_handle.set_digest((void*)digest.data(), digest.size()); + blob_handle.set_namespace_("namespaceA"); BlobProto write_blob_proto = icing.OpenWriteBlob(blob_handle); ASSERT_THAT(write_blob_proto.status(), ProtoIsOk()); @@ -266,11 +348,11 @@ ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("blob1"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); blob_handle.set_digest(std::string(digest.begin(), digest.end())); + blob_handle.set_namespace_("namespaceA"); BlobProto write_blob_proto = icing.OpenWriteBlob(blob_handle); ASSERT_THAT(write_blob_proto.status(), ProtoIsOk()); @@ -292,11 +374,11 @@ EXPECT_THAT(icing1.Initialize().status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("blob1"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); blob_handle.set_digest((void*)digest.data(), digest.size()); + blob_handle.set_namespace_("namespaceA"); BlobProto write_blob_proto = icing1.OpenWriteBlob(blob_handle); ASSERT_THAT(write_blob_proto.status(), ProtoIsOk()); @@ -325,11 +407,11 @@ ASSERT_THAT(icing1.SetSchema(CreateBlobSchema()).status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("blob1"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); blob_handle.set_digest((void*)digest.data(), digest.size()); + blob_handle.set_namespace_("namespaceA"); BlobProto write_blob_proto = icing1.OpenWriteBlob(blob_handle); ASSERT_THAT(write_blob_proto.status(), ProtoIsOk()); @@ -367,11 +449,11 @@ ASSERT_THAT(icing1.SetSchema(CreateBlobSchema()).status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("blob1"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); blob_handle.set_digest((void*)digest.data(), digest.size()); + blob_handle.set_namespace_("namespaceA"); BlobProto write_blob_proto = icing1.OpenWriteBlob(blob_handle); ASSERT_THAT(write_blob_proto.status(), ProtoIsOk()); @@ -416,19 +498,11 @@ // set a schema to icing to avoid wipe out all directories. ASSERT_THAT(icing.SetSchema(CreateBlobSchema()).status(), ProtoIsOk()); - std::vector<std::string> file_names; - std::unordered_set<std::string> excludes; - ASSERT_TRUE(filesystem()->ListDirectory(GetTestBaseBlobStoreDir().c_str(), - excludes, /*recursive=*/false, - &file_names)); - int32_t file_count = file_names.size(); - PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("label"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); - std::string digestString = std::string(digest.begin(), digest.end()); - blob_handle.set_digest(std::move(digestString)); + blob_handle.set_digest(std::string(digest.begin(), digest.end())); + blob_handle.set_namespace_("namespaceA"); BlobProto writeBlobProto = icing.OpenWriteBlob(blob_handle); ASSERT_THAT(writeBlobProto.status(), ProtoIsOk()); @@ -437,11 +511,11 @@ ASSERT_TRUE(filesystem()->Write(write_fd.get(), data.data(), data.size())); } - file_names = std::vector<std::string>(); - ASSERT_TRUE(filesystem()->ListDirectory(GetTestBaseBlobStoreDir().c_str(), - excludes, /*recursive=*/false, - &file_names)); - ASSERT_THAT(file_names.size(), file_count + 1); + std::vector<std::string> file_names; + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobFileDir().c_str(), &file_names)); + // The blob file is created. + EXPECT_THAT(file_names, SizeIs(1)); BlobProto commitBlobProto = icing.CommitBlob(blob_handle); ASSERT_THAT(commitBlobProto.status(), ProtoIsOk()); @@ -473,15 +547,18 @@ std::string actual_data = std::string(buf.get(), buf.get() + size); EXPECT_EQ(expected_data, actual_data); + file_names = std::vector<std::string>(); + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobDir().c_str(), &file_names)); + int32_t cur_file_count = file_names.size(); // Optimize remove the expired orphan blob. - ASSERT_THAT(icing2.Optimize().status(), ProtoIsOk()); + EXPECT_THAT(icing2.Optimize().status(), ProtoIsOk()); EXPECT_THAT(icing2.OpenReadBlob(blob_handle).status(), ProtoStatusIs(StatusProto::NOT_FOUND)); file_names = std::vector<std::string>(); - ASSERT_TRUE(filesystem()->ListDirectory(GetTestBaseBlobStoreDir().c_str(), - excludes, /*recursive=*/false, - &file_names)); - ASSERT_THAT(file_names.size(), file_count); + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobDir().c_str(), &file_names)); + EXPECT_THAT(file_names, SizeIs(cur_file_count)); } TEST_F(IcingSearchEngineBlobTest, BlobOptimizeWithoutCommit) { @@ -498,11 +575,12 @@ // write two blobs but not commit PropertyProto::BlobHandleProto blob_handle1; - blob_handle1.set_label("label1"); std::vector<unsigned char> data1 = GenerateRandomBytes(24); std::array<uint8_t, 32> digest1 = CalculateDigest(data1); - std::string digestString1 = std::string(digest1.begin(), digest1.end()); - blob_handle1.set_digest(std::move(digestString1)); + std::string digest_string1 = std::string(digest1.begin(), digest1.end()); + blob_handle1.set_digest(std::move(digest_string1)); + blob_handle1.set_namespace_("namespaceA"); + BlobProto writeBlobProto = icing.OpenWriteBlob(blob_handle1); ASSERT_THAT(writeBlobProto.status(), ProtoIsOk()); { @@ -512,11 +590,10 @@ } PropertyProto::BlobHandleProto blob_handle2; - blob_handle2.set_label("label2"); std::vector<unsigned char> data2 = GenerateRandomBytes(24); std::array<uint8_t, 32> digest2 = CalculateDigest(data2); - std::string digestString2 = std::string(digest2.begin(), digest2.end()); - blob_handle2.set_digest(std::move(digestString2)); + blob_handle2.set_digest(std::string(digest2.begin(), digest2.end())); + blob_handle2.set_namespace_("namespaceA"); writeBlobProto = icing.OpenWriteBlob(blob_handle2); ASSERT_THAT(writeBlobProto.status(), ProtoIsOk()); { @@ -555,11 +632,10 @@ ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("label"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); - std::string digestString = std::string(digest.begin(), digest.end()); - blob_handle.set_digest(std::move(digestString)); + blob_handle.set_digest(std::string(digest.begin(), digest.end())); + blob_handle.set_namespace_("namespaceA"); BlobProto writeBlobProto = icing.OpenWriteBlob(blob_handle); ASSERT_THAT(writeBlobProto.status(), ProtoIsOk()); @@ -638,11 +714,10 @@ ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("label"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); - std::string digestString = std::string(digest.begin(), digest.end()); - blob_handle.set_digest(std::move(digestString)); + blob_handle.set_digest(std::string(digest.begin(), digest.end())); + blob_handle.set_namespace_("namespaceA"); BlobProto writeBlobProto = icing.OpenWriteBlob(blob_handle); ASSERT_THAT(writeBlobProto.status(), ProtoIsOk()); @@ -750,11 +825,10 @@ ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("label"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); - std::string digestString = std::string(digest.begin(), digest.end()); - blob_handle.set_digest(std::move(digestString)); + blob_handle.set_digest(std::string(digest.begin(), digest.end())); + blob_handle.set_namespace_("namespaceA"); BlobProto writeBlobProto = icing.OpenWriteBlob(blob_handle); ASSERT_THAT(writeBlobProto.status(), ProtoIsOk()); @@ -845,19 +919,12 @@ std::move(fake_clock), GetTestJniCache()); ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); - std::vector<std::string> file_names; - std::unordered_set<std::string> excludes; - ASSERT_TRUE(filesystem()->ListDirectory(GetTestBaseBlobStoreDir().c_str(), - excludes, /*recursive=*/false, - &file_names)); - int32_t file_count = file_names.size(); - PropertyProto::BlobHandleProto blob_handle1; - blob_handle1.set_label("label1"); std::vector<unsigned char> data1 = GenerateRandomBytes(24); std::array<uint8_t, 32> digest1 = CalculateDigest(data1); - std::string digestString1 = std::string(digest1.begin(), digest1.end()); - blob_handle1.set_digest(std::move(digestString1)); + std::string digest_string1 = std::string(digest1.begin(), digest1.end()); + blob_handle1.set_digest(std::move(digest_string1)); + blob_handle1.set_namespace_("namespaceA"); BlobProto writeBlobProto1 = icing.OpenWriteBlob(blob_handle1); ASSERT_THAT(writeBlobProto1.status(), ProtoIsOk()); @@ -871,11 +938,10 @@ ASSERT_THAT(commitBlobProto.status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle2; - blob_handle2.set_label("label2"); std::vector<unsigned char> data2 = GenerateRandomBytes(24); std::array<uint8_t, 32> digest2 = CalculateDigest(data2); - std::string digestString2 = std::string(digest2.begin(), digest2.end()); - blob_handle2.set_digest(std::move(digestString2)); + blob_handle2.set_digest(std::string(digest2.begin(), digest2.end())); + blob_handle2.set_namespace_("namespaceA"); BlobProto writeBlobProto2 = icing.OpenWriteBlob(blob_handle2); ASSERT_THAT(writeBlobProto2.status(), ProtoIsOk()); @@ -889,11 +955,10 @@ ASSERT_THAT(commitBlobProto2.status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle3; - blob_handle3.set_label("label3"); std::vector<unsigned char> data3 = GenerateRandomBytes(24); std::array<uint8_t, 32> digest3 = CalculateDigest(data3); - std::string digestString3 = std::string(digest3.begin(), digest3.end()); - blob_handle3.set_digest(std::move(digestString3)); + blob_handle3.set_digest(std::string(digest3.begin(), digest3.end())); + blob_handle3.set_namespace_("namespaceA"); BlobProto writeBlobProto3 = icing.OpenWriteBlob(blob_handle3); ASSERT_THAT(writeBlobProto3.status(), ProtoIsOk()); @@ -906,12 +971,11 @@ BlobProto commitBlobProto3 = icing.CommitBlob(blob_handle3); ASSERT_THAT(commitBlobProto3.status(), ProtoIsOk()); - file_names = std::vector<std::string>(); - ASSERT_TRUE(filesystem()->ListDirectory(GetTestBaseBlobStoreDir().c_str(), - excludes, /*recursive=*/false, - &file_names)); - ASSERT_THAT(file_names.size(), file_count + 3); - + std::vector<std::string> file_names; + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobFileDir().c_str(), &file_names)); + // 3 more blob files are created. + ASSERT_THAT(file_names, SizeIs(3)); // Set schema and put 3 documents that contains the blob handle ASSERT_THAT(icing.SetSchema(CreateBlobSchema()).status(), ProtoIsOk()); ASSERT_THAT( @@ -955,10 +1019,11 @@ ProtoStatusIs(StatusProto::NOT_FOUND)); ASSERT_THAT(icing2.OpenReadBlob(blob_handle3).status(), ProtoIsOk()); file_names = std::vector<std::string>(); - ASSERT_TRUE(filesystem()->ListDirectory(GetTestBaseBlobStoreDir().c_str(), - excludes, /*recursive=*/false, - &file_names)); - ASSERT_THAT(file_names.size(), file_count + 1); + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobFileDir().c_str(), &file_names)); + + // 2 blob files are removed + ASSERT_THAT(file_names, SizeIs(1)); // remove the last reference document, now the all blobs become orphan. ASSERT_THAT(icing2.Delete("namespace", "doc3").status(), ProtoIsOk()); @@ -967,10 +1032,10 @@ EXPECT_THAT(icing2.OpenReadBlob(blob_handle3).status(), ProtoStatusIs(StatusProto::NOT_FOUND)); file_names = std::vector<std::string>(); - ASSERT_TRUE(filesystem()->ListDirectory(GetTestBaseBlobStoreDir().c_str(), - excludes, /*recursive=*/false, - &file_names)); - ASSERT_THAT(file_names.size(), file_count); + ASSERT_TRUE( + filesystem()->ListDirectory(GetTestBlobFileDir().c_str(), &file_names)); + // the last blob file is removed. + ASSERT_THAT(file_names, SizeIs(0)); } TEST_F(IcingSearchEngineBlobTest, OptimizeBlobHandlesNoTTL) { @@ -990,11 +1055,10 @@ ASSERT_THAT(icing.SetSchema(CreateBlobSchema()).status(), ProtoIsOk()); PropertyProto::BlobHandleProto blob_handle; - blob_handle.set_label("label"); std::vector<unsigned char> data = GenerateRandomBytes(24); std::array<uint8_t, 32> digest = CalculateDigest(data); - std::string digestString = std::string(digest.begin(), digest.end()); - blob_handle.set_digest(std::move(digestString)); + blob_handle.set_digest(std::string(digest.begin(), digest.end())); + blob_handle.set_namespace_("namespaceA"); BlobProto writeBlobProto = icing.OpenWriteBlob(blob_handle); ASSERT_THAT(writeBlobProto.status(), ProtoIsOk()); @@ -1033,6 +1097,128 @@ EXPECT_EQ(expected_data, actual_data); } +TEST_F(IcingSearchEngineBlobTest, EmptyNamespace) { + auto fake_clock = std::make_unique<FakeClock>(); + fake_clock->SetSystemTimeMilliseconds(1000); + TestIcingSearchEngine icing(GetDefaultIcingOptions(), + std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::move(fake_clock), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + PropertyProto::BlobHandleProto blob_handle; + std::vector<unsigned char> data = GenerateRandomBytes(12); + std::array<uint8_t, 32> digest = CalculateDigest(data); + blob_handle.set_digest(std::string(digest.begin(), digest.end())); + BlobProto writeBlobProto = icing.OpenWriteBlob(blob_handle); + + EXPECT_THAT(writeBlobProto.status(), + ProtoStatusIs(StatusProto::INVALID_ARGUMENT)); +} + +TEST_F(IcingSearchEngineBlobTest, OptimizeNamespaceUsage) { + auto fake_clock = std::make_unique<FakeClock>(); + fake_clock->SetSystemTimeMilliseconds(1000); + TestIcingSearchEngine icing(GetDefaultIcingOptions(), + std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::move(fake_clock), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + // insert 3 blobs from 3 different namespaces + PropertyProto::BlobHandleProto blob_handle1; + std::vector<unsigned char> data1 = GenerateRandomBytes(12); + std::array<uint8_t, 32> digest1 = CalculateDigest(data1); + blob_handle1.set_digest(std::string(digest1.begin(), digest1.end())); + blob_handle1.set_namespace_("namespaceA"); + BlobProto writeBlobProto1 = icing.OpenWriteBlob(blob_handle1); + ASSERT_THAT(writeBlobProto1.status(), ProtoIsOk()); + { + ScopedFd write_fd(writeBlobProto1.file_descriptor()); + ASSERT_TRUE( + filesystem()->Write(write_fd.get(), data1.data(), data1.size())); + } + BlobProto commitBlobProto = icing.CommitBlob(blob_handle1); + ASSERT_THAT(commitBlobProto.status(), ProtoIsOk()); + + PropertyProto::BlobHandleProto blob_handle2; + std::vector<unsigned char> data2 = GenerateRandomBytes(24); + std::array<uint8_t, 32> digest2 = CalculateDigest(data2); + blob_handle2.set_digest(std::string(digest2.begin(), digest2.end())); + blob_handle2.set_namespace_("namespaceB"); + BlobProto writeBlobProto2 = icing.OpenWriteBlob(blob_handle2); + ASSERT_THAT(writeBlobProto2.status(), ProtoIsOk()); + { + ScopedFd write_fd(writeBlobProto2.file_descriptor()); + ASSERT_TRUE( + filesystem()->Write(write_fd.get(), data2.data(), data2.size())); + } + BlobProto commitBlobProto2 = icing.CommitBlob(blob_handle2); + ASSERT_THAT(commitBlobProto2.status(), ProtoIsOk()); + + PropertyProto::BlobHandleProto blob_handle3; + std::vector<unsigned char> data3 = GenerateRandomBytes(36); + std::array<uint8_t, 32> digest3 = CalculateDigest(data3); + blob_handle3.set_digest(std::string(digest3.begin(), digest3.end())); + blob_handle3.set_namespace_("namespaceC"); + BlobProto writeBlobProto3 = icing.OpenWriteBlob(blob_handle3); + ASSERT_THAT(writeBlobProto3.status(), ProtoIsOk()); + { + ScopedFd write_fd(writeBlobProto3.file_descriptor()); + ASSERT_TRUE( + filesystem()->Write(write_fd.get(), data3.data(), data3.size())); + } + BlobProto commitBlobProto3 = icing.CommitBlob(blob_handle3); + ASSERT_THAT(commitBlobProto3.status(), ProtoIsOk()); + + // Set schema and put a documents that contains the blob handle2 only + ASSERT_THAT(icing.SetSchema(CreateBlobSchema()).status(), ProtoIsOk()); + ASSERT_THAT( + icing.Put(CreateBlobDocument("namespace", "doc", blob_handle2)).status(), + ProtoIsOk()); + + // persist blob to disk + EXPECT_THAT(icing.PersistToDisk(PersistType::FULL).status(), ProtoIsOk()); + + // Verify namespace usage + StorageInfoResultProto storage_info_result = icing.GetStorageInfo(); + EXPECT_THAT(storage_info_result.status(), ProtoIsOk()); + NamespaceBlobStorageInfoProto namespace_info_a; + namespace_info_a.set_namespace_("namespaceA"); + namespace_info_a.set_blob_size(12); + namespace_info_a.set_num_blobs(1); + NamespaceBlobStorageInfoProto namespace_info_b; + namespace_info_b.set_namespace_("namespaceB"); + namespace_info_b.set_blob_size(24); + namespace_info_b.set_num_blobs(1); + NamespaceBlobStorageInfoProto namespace_info_c; + namespace_info_c.set_namespace_("namespaceC"); + namespace_info_c.set_blob_size(36); + namespace_info_c.set_num_blobs(1); + EXPECT_THAT(storage_info_result.storage_info().namespace_blob_storage_info(), + UnorderedElementsAre(EqualsProto(namespace_info_a), + EqualsProto(namespace_info_b), + EqualsProto(namespace_info_c))); + + // create second icing in 8 days later + auto fake_clock2 = std::make_unique<FakeClock>(); + fake_clock2->SetSystemTimeMilliseconds(1000 + 8 * 24 * 60 * 60 * + 1000); // pass 8 days + TestIcingSearchEngine icing2(GetDefaultIcingOptions(), + std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::move(fake_clock2), GetTestJniCache()); + ASSERT_THAT(icing2.Initialize().status(), ProtoIsOk()); + + // After optimize, blobs of namespaceA and namespaceC are removed. + ASSERT_THAT(icing2.Optimize().status(), ProtoIsOk()); + + storage_info_result = icing2.GetStorageInfo(); + EXPECT_THAT(storage_info_result.status(), ProtoIsOk()); + EXPECT_THAT(storage_info_result.storage_info().namespace_blob_storage_info(), + UnorderedElementsAre(EqualsProto(namespace_info_b))); +} + } // namespace } // namespace lib } // namespace icing
diff --git a/icing/icing-search-engine_delete_test.cc b/icing/icing-search-engine_delete_test.cc index 6546d2d..b85654f 100644 --- a/icing/icing-search-engine_delete_test.cc +++ b/icing/icing-search-engine_delete_test.cc
@@ -47,10 +47,10 @@ #include "icing/schema-builder.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/testing/tmp-directory.h" +#include "icing/util/icu-data-file-helper.h" namespace icing { namespace lib { @@ -97,7 +97,7 @@ std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); ICING_ASSERT_OK( - icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); + icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path)); } filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); } @@ -118,33 +118,32 @@ IcingSearchEngineOptions GetDefaultIcingOptions() { IcingSearchEngineOptions icing_options; icing_options.set_base_dir(GetTestBaseDir()); + icing_options.set_enable_qualified_id_join_index_v3_and_delete_propagate_from( + true); return icing_options; } -SchemaProto CreateMessageSchema() { - return SchemaBuilder() - .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( - PropertyConfigBuilder() - .SetName("body") - .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) - .SetCardinality(CARDINALITY_REQUIRED))) +SchemaTypeConfigProto CreateMessageSchemaTypeConfig() { + return SchemaTypeConfigBuilder() + .SetType("Message") + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) .Build(); } -SchemaProto CreateEmailSchema() { - return SchemaBuilder() - .AddType(SchemaTypeConfigBuilder() - .SetType("Email") - .AddProperty(PropertyConfigBuilder() - .SetName("body") - .SetDataTypeString(TERM_MATCH_PREFIX, - TOKENIZER_PLAIN) - .SetCardinality(CARDINALITY_REQUIRED)) - .AddProperty(PropertyConfigBuilder() - .SetName("subject") - .SetDataTypeString(TERM_MATCH_PREFIX, - TOKENIZER_PLAIN) - .SetCardinality(CARDINALITY_REQUIRED))) +SchemaTypeConfigProto CreateEmailSchemaTypeConfig() { + return SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) .Build(); } @@ -154,6 +153,186 @@ return scoring_spec; } +TEST_F(IcingSearchEngineDeleteTest, Delete) { + SchemaProto schema = + SchemaBuilder().AddType(CreateMessageSchemaTypeConfig()).Build(); + + DocumentProto document = + DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("Message") + .AddStringProperty("body", "message body1") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document).status(), ProtoIsOk()); + + // Sanity check that the document is present. + GetResultProto expected_get_result_proto; + expected_get_result_proto.mutable_status()->set_code(StatusProto::OK); + *expected_get_result_proto.mutable_document() = document; + ASSERT_THAT( + icing.Get("namespace", "uri", GetResultSpecProto::default_instance()), + EqualsProto(expected_get_result_proto)); + + // Delete "namespace", "uri". + EXPECT_THAT(icing.Delete("namespace", "uri").status(), ProtoIsOk()); + + // Get again. + expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND); + expected_get_result_proto.mutable_status()->set_message( + "Document (namespace, uri) not found."); + expected_get_result_proto.clear_document(); + EXPECT_THAT( + icing.Get("namespace", "uri", GetResultSpecProto::default_instance()), + EqualsProto(expected_get_result_proto)); +} + +TEST_F(IcingSearchEngineDeleteTest, DeleteWithJoinDeletePropagation) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Message") + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_NONE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + DocumentProto person1 = + DocumentBuilder() + .SetKey("namespace", "person1") + .SetSchema("Person") + .AddStringProperty("name", "Alice") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto person2 = + DocumentBuilder() + .SetKey("namespace", "person2") + .SetSchema("Person") + .AddStringProperty("name", "Bob") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto email1 = + DocumentBuilder() + .SetKey("namespace", "email1") + .SetSchema("Email") + .AddStringProperty("subject", "test") + .AddStringProperty("sender", "namespace#person1") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto email2 = + DocumentBuilder() + .SetKey("namespace", "email2") + .SetSchema("Email") + .AddStringProperty("subject", "test") + .AddStringProperty("sender", "namespace#person2") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto message1 = + DocumentBuilder() + .SetKey("namespace", "message1") + .SetSchema("Message") + .AddStringProperty("body", "test") + .AddStringProperty("sender", "namespace#person1") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto message2 = + DocumentBuilder() + .SetKey("namespace", "message2") + .SetSchema("Message") + .AddStringProperty("body", "test") + .AddStringProperty("sender", "namespace#person2") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(person1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(person2).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email2).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(message1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(message2).status(), ProtoIsOk()); + + // Delete person1. + DeleteResultProto delete_result = icing.Delete("namespace", "person1"); + EXPECT_THAT(delete_result.status(), ProtoIsOk()); + // Person1 and email1 should be deleted. + EXPECT_THAT(delete_result.delete_stats().num_documents_deleted(), Eq(2)); + + // Verify Get API for email and message documents. + // Email1 should be deleted. The joinable property "sender" in schema type + // "Email" has delete propagation type PROPAGATE_FROM and the referenced + // document "person1" is deleted. + GetResultProto expected_get_result_proto1; + expected_get_result_proto1.mutable_status()->set_code(StatusProto::NOT_FOUND); + expected_get_result_proto1.mutable_status()->set_message( + "Document (namespace, email1) not found."); + EXPECT_THAT( + icing.Get("namespace", "email1", GetResultSpecProto::default_instance()), + EqualsProto(expected_get_result_proto1)); + + // Email2 should still exist. The joinable property "sender" in schema type + // "Email" has delete propagation type PROPAGATE_FROM but the referenced + // document "person2" is not deleted. + GetResultProto expected_get_result_google::protobuf; + expected_get_result_google::protobuf.mutable_status()->set_code(StatusProto::OK); + *expected_get_result_google::protobuf.mutable_document() = email2; + EXPECT_THAT( + icing.Get("namespace", "email2", GetResultSpecProto::default_instance()), + EqualsProto(expected_get_result_google::protobuf)); + + // Message1 should still exist. The joinable property "sender" in schema type + // "Message" has delete propagation type NONE. + GetResultProto expected_get_result_proto3; + expected_get_result_proto3.mutable_status()->set_code(StatusProto::OK); + *expected_get_result_proto3.mutable_document() = message1; + EXPECT_THAT(icing.Get("namespace", "message1", + GetResultSpecProto::default_instance()), + EqualsProto(expected_get_result_proto3)); + + // Message2 should still exist. The joinable property "sender" in schema type + // "Message" has delete propagation type NONE, and the referenced document + // "person2" is not deleted. + GetResultProto expected_get_result_proto4; + expected_get_result_proto4.mutable_status()->set_code(StatusProto::OK); + *expected_get_result_proto4.mutable_document() = message2; + EXPECT_THAT(icing.Get("namespace", "message2", + GetResultSpecProto::default_instance()), + EqualsProto(expected_get_result_proto4)); +} + TEST_F(IcingSearchEngineDeleteTest, DeleteBySchemaType) { SchemaProto schema; // Add an email type @@ -258,22 +437,22 @@ } TEST_F(IcingSearchEngineDeleteTest, DeleteSchemaTypeByQuery) { - SchemaProto schema = CreateMessageSchema(); - // Add an email type - SchemaProto tmp = CreateEmailSchema(); - *schema.add_types() = tmp.types(0); + SchemaProto schema = SchemaBuilder() + .AddType(CreateMessageSchemaTypeConfig()) + .AddType(CreateEmailSchemaTypeConfig()) + .Build(); DocumentProto document1 = DocumentBuilder() .SetKey("namespace1", "uri1") - .SetSchema(schema.types(0).schema_type()) + .SetSchema("Message") .AddStringProperty("body", "message body1") .SetCreationTimestampMs(kDefaultCreationTimestampMs) .Build(); DocumentProto document2 = DocumentBuilder() .SetKey("namespace2", "uri2") - .SetSchema(schema.types(1).schema_type()) + .SetSchema("Email") .AddStringProperty("subject", "subject subject2") .AddStringProperty("body", "message body2") .SetCreationTimestampMs(kDefaultCreationTimestampMs) @@ -299,7 +478,7 @@ // Delete the first type. The first doc should be irretrievable. The // second should still be present. SearchSpecProto search_spec; - search_spec.add_schema_type_filters(schema.types(0).schema_type()); + search_spec.add_schema_type_filters("Message"); EXPECT_THAT(icing.DeleteByQuery(search_spec).status(), ProtoIsOk()); expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND); @@ -333,6 +512,9 @@ } TEST_F(IcingSearchEngineDeleteTest, DeleteByNamespace) { + SchemaProto schema = + SchemaBuilder().AddType(CreateMessageSchemaTypeConfig()).Build(); + DocumentProto document1 = DocumentBuilder() .SetKey("namespace1", "uri1") @@ -362,7 +544,7 @@ std::make_unique<IcingFilesystem>(), std::move(fake_clock), GetTestJniCache()); ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); - ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); ASSERT_THAT(icing.Put(document3).status(), ProtoIsOk()); @@ -434,6 +616,9 @@ } TEST_F(IcingSearchEngineDeleteTest, DeleteNamespaceByQuery) { + SchemaProto schema = + SchemaBuilder().AddType(CreateMessageSchemaTypeConfig()).Build(); + DocumentProto document1 = DocumentBuilder() .SetKey("namespace1", "uri1") @@ -451,7 +636,7 @@ IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); - EXPECT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); EXPECT_THAT(icing.Put(document1).status(), ProtoIsOk()); EXPECT_THAT(icing.Put(document2).status(), ProtoIsOk()); @@ -504,6 +689,9 @@ } TEST_F(IcingSearchEngineDeleteTest, DeleteByQuery) { + SchemaProto schema = + SchemaBuilder().AddType(CreateMessageSchemaTypeConfig()).Build(); + DocumentProto document1 = DocumentBuilder() .SetKey("namespace1", "uri1") @@ -526,7 +714,7 @@ std::make_unique<IcingFilesystem>(), std::move(fake_clock), GetTestJniCache()); EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); - EXPECT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); EXPECT_THAT(icing.Put(document1).status(), ProtoIsOk()); EXPECT_THAT(icing.Put(document2).status(), ProtoIsOk()); @@ -591,6 +779,9 @@ } TEST_F(IcingSearchEngineDeleteTest, DeleteByQueryReturnInfo) { + SchemaProto schema = + SchemaBuilder().AddType(CreateMessageSchemaTypeConfig()).Build(); + DocumentProto document1 = DocumentBuilder() .SetKey("namespace1", "uri1") @@ -620,7 +811,7 @@ std::make_unique<IcingFilesystem>(), std::move(fake_clock), GetTestJniCache()); ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); - ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); ASSERT_THAT(icing.Put(document3).status(), ProtoIsOk()); @@ -690,6 +881,9 @@ } TEST_F(IcingSearchEngineDeleteTest, DeleteByQueryNotFound) { + SchemaProto schema = + SchemaBuilder().AddType(CreateMessageSchemaTypeConfig()).Build(); + DocumentProto document1 = DocumentBuilder() .SetKey("namespace1", "uri1") @@ -707,7 +901,7 @@ IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); - EXPECT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); EXPECT_THAT(icing.Put(document1).status(), ProtoIsOk()); EXPECT_THAT(icing.Put(document2).status(), ProtoIsOk());
diff --git a/icing/icing-search-engine_fuzz_test.cc b/icing/icing-search-engine_fuzz_test.cc index 2cf19ad..6f1cf69 100644 --- a/icing/icing-search-engine_fuzz_test.cc +++ b/icing/icing-search-engine_fuzz_test.cc
@@ -26,9 +26,9 @@ #include "icing/proto/search.pb.h" #include "icing/proto/term.pb.h" #include "icing/schema-builder.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" #include "icing/testing/tmp-directory.h" +#include "icing/util/icu-data-file-helper.h" namespace icing { namespace lib { @@ -64,7 +64,7 @@ // Initialize IcingSearchEngineOptions icing_options = Setup(); std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); - if (!icu_data_file_helper::SetUpICUDataFile(icu_data_file_path).ok()) { + if (!icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path).ok()) { return 1; } IcingSearchEngine icing(icing_options);
diff --git a/icing/icing-search-engine_initialization_test.cc b/icing/icing-search-engine_initialization_test.cc index 3ffc2cf..7643e4c 100644 --- a/icing/icing-search-engine_initialization_test.cc +++ b/icing/icing-search-engine_initialization_test.cc
@@ -28,6 +28,7 @@ #include "gtest/gtest.h" #include "icing/absl_ports/str_cat.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" #include "icing/file/memory-mapped-file.h" @@ -36,6 +37,8 @@ #include "icing/file/version-util.h" #include "icing/icing-search-engine.h" #include "icing/index/data-indexing-handler.h" +#include "icing/index/embed/embedding-index.h" +#include "icing/index/embedding-indexing-handler.h" #include "icing/index/index-processor.h" #include "icing/index/index.h" #include "icing/index/integer-section-indexing-handler.h" @@ -44,8 +47,9 @@ #include "icing/index/numeric/numeric-index.h" #include "icing/index/term-indexing-handler.h" #include "icing/jni/jni-cache.h" +#include "icing/join/document-join-id-pair.h" #include "icing/join/join-processor.h" -#include "icing/join/qualified-id-join-index-impl-v2.h" +#include "icing/join/qualified-id-join-index-impl-v3.h" #include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id-join-indexing-handler.h" #include "icing/legacy/index/icing-filesystem.h" @@ -77,18 +81,20 @@ #include "icing/store/document-id.h" #include "icing/store/document-log-creator.h" #include "icing/store/document-store.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/testing/common-matchers.h" +#include "icing/testing/embedding-test-utils.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" @@ -107,6 +113,7 @@ using ::testing::IsEmpty; using ::testing::Matcher; using ::testing::Ne; +using ::testing::Pointee; using ::testing::Return; using ::testing::SizeIs; @@ -162,6 +169,8 @@ class IcingSearchEngineInitializationTest : public testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { // If we've specified using the reverse-JNI method for segmentation (i.e. // not ICU), then we won't have the ICU data file included to set up. @@ -172,7 +181,7 @@ std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); ICING_ASSERT_OK( - icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); + icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path)); } filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); @@ -197,6 +206,7 @@ const IcingFilesystem* icing_filesystem() const { return &icing_filesystem_; } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; IcingFilesystem icing_filesystem_; std::unique_ptr<LanguageSegmenter> lang_segmenter_; @@ -220,6 +230,10 @@ return GetTestBaseDir() + "/qualified_id_join_index_dir"; } +std::string GetEmbeddingIndexDir() { + return GetTestBaseDir() + "/embedding_index_dir"; +} + std::string GetSchemaDir() { return GetTestBaseDir() + "/schema_dir"; } std::string GetBlobDir() { return GetTestBaseDir() + "/blob_dir"; } @@ -232,7 +246,11 @@ IcingSearchEngineOptions icing_options; icing_options.set_base_dir(GetTestBaseDir()); icing_options.set_document_store_namespace_id_fingerprint(true); - icing_options.set_use_new_qualified_id_join_index(true); + icing_options.set_enable_embedding_index(true); + icing_options.set_enable_embedding_quantization(true); + icing_options.set_enable_blob_store(true); + icing_options.set_enable_qualified_id_join_index_v3_and_delete_propagate_from( + true); return icing_options; } @@ -919,7 +937,8 @@ FakeClock fake_clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock)); + SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock, + feature_flags_.get())); ICING_EXPECT_OK(schema_store->SetSchema( new_schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -1076,24 +1095,26 @@ FakeClock fake_clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock)); + SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( BlobStore blob_store, BlobStore::Create(filesystem(), GetBlobDir(), &fake_clock, - /*orphan_blob_time_to_live_ms=*/0)); + /*orphan_blob_time_to_live_ms=*/0, + PortableFileBackedProtoLog< + BlobInfoProto>::kDefaultCompressionLevel)); // Puts message2 into DocumentStore but doesn't index it. ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, DocumentStore::Create(filesystem(), GetDocumentDir(), &fake_clock, - schema_store.get(), + schema_store.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -1124,8 +1145,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( - HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing(icing_options, std::move(mock_filesystem), @@ -1226,14 +1248,12 @@ TEST_F(IcingSearchEngineInitializationTest, RecoverFromCorruptedDocumentStore) { // Test the following scenario: some document store derived files are - // corrupted. IcingSearchEngine should be able to recover the document store, - // and since NamespaceIds were reassigned, we should rebuild qualified id join - // index as well. Several additional behaviors are also tested: + // corrupted. IcingSearchEngine should be able to recover the document store. + // Several additional behaviors are also tested: // - Index directory handling: // - Term index directory should be unaffected. // - Integer index directory should be unaffected. - // - Should discard the entire qualified id join index directory and start - // it from scratch. + // - Qualified id join index directory should be unaffected. // - Truncate indices: // - "TruncateTo()" for term index shouldn't take effect. // - "Clear()" shouldn't be called for integer index, i.e. no integer index @@ -1328,7 +1348,8 @@ FakeClock fake_clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock)); + SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock, + feature_flags_.get())); // Manually corrupt one of the derived files of DocumentStore without // updating checksum in DocumentStore header. @@ -1367,14 +1388,15 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively(HasSubstr("/integer_index_dir/"))) .Times(0); - // Ensure qualified id join index directory should be discarded once, and + // Ensure qualified id join index directory should never be discarded, and // Clear() should never be called (i.e. storage sub directory // "*/qualified_id_join_index_dir/*" should never be discarded). EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) - .Times(1); - EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( - HasSubstr("/qualified_id_join_index_dir/"))) + .Times(0); + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing(icing_options, std::move(mock_filesystem), @@ -1388,16 +1410,15 @@ EXPECT_THAT( initialize_result.initialize_stats().document_store_recovery_cause(), Eq(InitializeStatsProto::IO_ERROR)); - // Term, integer index should be unaffected. + // Term, integer index and qualified id join index should be unaffected. EXPECT_THAT(initialize_result.initialize_stats().index_restoration_cause(), Eq(InitializeStatsProto::NONE)); EXPECT_THAT( initialize_result.initialize_stats().integer_index_restoration_cause(), Eq(InitializeStatsProto::NONE)); - // Qualified id join index should be rebuilt. EXPECT_THAT(initialize_result.initialize_stats() .qualified_id_join_index_restoration_cause(), - Eq(InitializeStatsProto::DEPENDENCIES_CHANGED)); + Eq(InitializeStatsProto::NONE)); // Verify join search: join a query for `name:person` with a child query for // `body:message` based on the child's `senderQualifiedId` field. message2 @@ -1558,8 +1579,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( - HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing(GetDefaultIcingOptions(), @@ -1704,8 +1726,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( - HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing(GetDefaultIcingOptions(), @@ -1791,9 +1814,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); static constexpr int32_t kNewIntegerIndexBucketSplitThreshold = 1000; @@ -1972,8 +1995,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(1); - EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( - HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing(GetDefaultIcingOptions(), @@ -2121,9 +2145,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing( @@ -2315,9 +2339,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing( @@ -2511,9 +2535,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing( @@ -2702,8 +2726,8 @@ index->set_last_added_document_id(original_last_added_doc_id + 1); Index::Editor editor = index->Edit(original_last_added_doc_id + 1, /*section_id=*/0, - TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TermMatchType::EXACT_ONLY)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); } @@ -2733,9 +2757,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); IcingSearchEngineOptions options = GetDefaultIcingOptions(); @@ -2948,8 +2972,8 @@ index->set_last_added_document_id(original_last_added_doc_id + 1); Index::Editor editor = index->Edit(original_last_added_doc_id + 1, /*section_id=*/0, - TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TermMatchType::EXACT_ONLY)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); } @@ -2979,9 +3003,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); IcingSearchEngineOptions options = GetDefaultIcingOptions(); @@ -3146,18 +3170,17 @@ // Add hits for document 0 and merge. ASSERT_THAT(index->last_added_document_id(), kInvalidDocumentId); index->set_last_added_document_id(0); - Index::Editor editor = - index->Edit(/*document_id=*/0, /*section_id=*/0, - TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + Index::Editor editor = index->Edit(/*document_id=*/0, /*section_id=*/0, + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TermMatchType::EXACT_ONLY)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(index->Merge()); // Add hits for document 1 and don't merge. index->set_last_added_document_id(1); editor = index->Edit(/*document_id=*/1, /*section_id=*/0, - TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("bar", TermMatchType::EXACT_ONLY)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); } @@ -3187,9 +3210,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(AtLeast(1)); TestIcingSearchEngine icing( @@ -3344,16 +3367,16 @@ index->set_last_added_document_id(original_last_added_doc_id + 1); Index::Editor editor = index->Edit(original_last_added_doc_id + 1, /*section_id=*/0, - TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TermMatchType::EXACT_ONLY)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(index->Merge()); // Add hits for document 5 and don't merge. index->set_last_added_document_id(original_last_added_doc_id + 2); editor = index->Edit(original_last_added_doc_id + 2, /*section_id=*/0, - TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("bar", TermMatchType::EXACT_ONLY)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); } @@ -3384,9 +3407,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing( @@ -3579,9 +3602,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(AtLeast(1)); TestIcingSearchEngine icing( @@ -3758,9 +3781,9 @@ EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( EndsWith("/qualified_id_join_index_dir"))) .Times(0); - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(0); TestIcingSearchEngine icing( @@ -3913,18 +3936,15 @@ Filesystem filesystem; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index, - QualifiedIdJoinIndexImplV2::Create(filesystem, - GetQualifiedIdJoinIndexDir(), - /*pre_mapping_fbv=*/false)); + QualifiedIdJoinIndexImplV3::Create( + filesystem, GetQualifiedIdJoinIndexDir(), *feature_flags_)); // Add data for document 0. ASSERT_THAT(qualified_id_join_index->last_added_document_id(), kInvalidDocumentId); qualified_id_join_index->set_last_added_document_id(0); ICING_ASSERT_OK(qualified_id_join_index->Put( - /*schema_type_id=*/0, /*joinable_property_id=*/0, /*document_id=*/0, - /*ref_namespace_fingerprint_ids=*/ - {NamespaceFingerprintIdentifier(/*namespace_id=*/0, - /*target_str=*/"uri")})); + DocumentJoinIdPair(/*document_id=*/0, /*joinable_property_id=*/0), + /*parent_document_ids=*/std::vector<DocumentId>{0})); } // 3. Create the index again. This should trigger index restoration. @@ -3953,9 +3973,9 @@ // Clear() should be called to truncate qualified id join index and thus // underlying storage sub directory (path_expr = // "*/qualified_id_join_index_dir/*") should be discarded. - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(AtLeast(1)); TestIcingSearchEngine icing( @@ -3985,14 +4005,11 @@ Filesystem filesystem; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index, - QualifiedIdJoinIndexImplV2::Create(filesystem, - GetQualifiedIdJoinIndexDir(), - /*pre_mapping_fbv=*/false)); - ICING_ASSERT_OK_AND_ASSIGN( - auto iterator, qualified_id_join_index->GetIterator( - /*schema_type_id=*/0, /*joinable_property_id=*/0)); - EXPECT_THAT(iterator->Advance(), - StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); + QualifiedIdJoinIndexImplV3::Create( + filesystem, GetQualifiedIdJoinIndexDir(), *feature_flags_)); + EXPECT_THAT(qualified_id_join_index, Pointee(IsEmpty())); + EXPECT_THAT(qualified_id_join_index->Get(/*parent_document_id=*/0), + IsOkAndHolds(IsEmpty())); } } @@ -4087,20 +4104,16 @@ Filesystem filesystem; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index, - QualifiedIdJoinIndexImplV2::Create(filesystem, - GetQualifiedIdJoinIndexDir(), - /*pre_mapping_fbv=*/false)); + QualifiedIdJoinIndexImplV3::Create( + filesystem, GetQualifiedIdJoinIndexDir(), *feature_flags_)); // Add data for document 4. DocumentId original_last_added_doc_id = qualified_id_join_index->last_added_document_id(); qualified_id_join_index->set_last_added_document_id( original_last_added_doc_id + 1); ICING_ASSERT_OK(qualified_id_join_index->Put( - /*schema_type_id=*/1, /*joinable_property_id=*/0, - /*document_id=*/original_last_added_doc_id + 1, - /*ref_namespace_fingerprint_ids=*/ - {NamespaceFingerprintIdentifier(/*namespace_id=*/0, - /*target_str=*/"person")})); + DocumentJoinIdPair(/*document_id=*/4, /*joinable_property_id=*/0), + /*parent_document_ids=*/std::vector<DocumentId>{0})); } // 3. Create the index again. This should trigger index restoration. @@ -4129,9 +4142,9 @@ // Clear() should be called to truncate qualified id join index and thus // underlying storage sub directory (path_expr = // "*/qualified_id_join_index_dir/*") should be discarded. - EXPECT_CALL( - *mock_filesystem, - DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + EXPECT_CALL(*mock_filesystem, DeleteFile(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteFile(HasSubstr("/qualified_id_join_index_dir/"))) .Times(AtLeast(1)); TestIcingSearchEngine icing( @@ -4509,9 +4522,7 @@ // Document store rewinds to previous checkpoint and all derived files were // regenerated. // - Last stored doc id will be consistent with last added document ids in - // term/integer indices, so there will be no index restoration. - // - Qualified id join index depends on document store derived files and - // since they were regenerated, we should rebuild qualified id join index. + // term/integer/join indices, so there will be no index restoration. EXPECT_THAT( initialize_result_proto.initialize_stats().index_restoration_cause(), Eq(InitializeStatsProto::NONE)); @@ -4520,10 +4531,10 @@ Eq(InitializeStatsProto::NONE)); EXPECT_THAT(initialize_result_proto.initialize_stats() .qualified_id_join_index_restoration_cause(), - Eq(InitializeStatsProto::DEPENDENCIES_CHANGED)); + Eq(InitializeStatsProto::NONE)); EXPECT_THAT(initialize_result_proto.initialize_stats() .index_restoration_latency_ms(), - Eq(10)); + Eq(0)); EXPECT_THAT(initialize_result_proto.initialize_stats() .schema_store_recovery_cause(), Eq(InitializeStatsProto::NONE)); @@ -4904,7 +4915,8 @@ FakeClock fake_clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock)); + SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock, + feature_flags_.get())); ICING_EXPECT_OK(schema_store->SetSchema( new_schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -5175,12 +5187,11 @@ auto mock_filesystem = std::make_unique<MockFilesystem>(); EXPECT_CALL(*mock_filesystem, PRead(A<const char*>(), _, _, _)) .WillRepeatedly(DoDefault()); - // This fails QualifiedIdJoinIndexImplV2::Create() once. - EXPECT_CALL( - *mock_filesystem, - PRead(Matcher<const char*>(Eq(qualified_id_join_index_metadata_file)), _, - _, _)) - .WillOnce(Return(false)) + // This fails QualifiedIdJoinIndexImplV3::Create() once. + EXPECT_CALL(*mock_filesystem, OpenForWrite(_)).WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, OpenForWrite(Matcher<const char*>( + Eq(qualified_id_join_index_metadata_file)))) + .WillOnce(Return(-1)) .WillRepeatedly(DoDefault()); auto fake_clock = std::make_unique<FakeClock>(); @@ -5275,10 +5286,10 @@ Eq(InitializeStatsProto::NONE)); EXPECT_THAT(initialize_result_proto.initialize_stats() .qualified_id_join_index_restoration_cause(), - Eq(InitializeStatsProto::DEPENDENCIES_CHANGED)); + Eq(InitializeStatsProto::NONE)); EXPECT_THAT( initialize_result_proto.initialize_stats().index_restoration_latency_ms(), - Eq(10)); + Eq(0)); EXPECT_THAT( initialize_result_proto.initialize_stats().schema_store_recovery_cause(), Eq(InitializeStatsProto::NONE)); @@ -5427,6 +5438,11 @@ .SetName("senderQualifiedId") .SetDataTypeJoinableString( JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("embedding") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); @@ -5451,6 +5467,8 @@ .AddStringProperty("body", "correct message") .AddInt64Property("indexableInteger", 123) .AddStringProperty("senderQualifiedId", "namespace#person/1") + .AddVectorProperty( + "embedding", CreateVector("my_model", {0.1, 0.2, 0.3, 0.4, 0.5})) .SetCreationTimestampMs(kDefaultCreationTimestampMs) .Build(); @@ -5480,24 +5498,26 @@ FakeClock fake_clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock)); + SchemaStore::Create(filesystem(), GetSchemaDir(), &fake_clock, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( BlobStore blob_store, BlobStore::Create(filesystem(), GetBlobDir(), &fake_clock, - /*orphan_blob_time_to_live_ms=*/0)); + /*orphan_blob_time_to_live_ms=*/0, + PortableFileBackedProtoLog< + BlobInfoProto>::kDefaultCompressionLevel)); // Put message into DocumentStore ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, DocumentStore::Create(filesystem(), GetDocumentDir(), &fake_clock, - schema_store.get(), + schema_store.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -5521,9 +5541,13 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index, - QualifiedIdJoinIndexImplV2::Create(*filesystem(), - GetQualifiedIdJoinIndexDir(), - /*pre_mapping_fbv=*/false)); + QualifiedIdJoinIndexImplV3::Create( + *filesystem(), GetQualifiedIdJoinIndexDir(), *feature_flags_)); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<EmbeddingIndex> embedding_index, + EmbeddingIndex::Create(filesystem(), GetEmbeddingIndexDir(), + &fake_clock, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<TermIndexingHandler> term_indexing_handler, @@ -5539,10 +5563,15 @@ qualified_id_join_indexing_handler, QualifiedIdJoinIndexingHandler::Create( &fake_clock, document_store.get(), qualified_id_join_index.get())); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<EmbeddingIndexingHandler> embedding_indexing_handler, + EmbeddingIndexingHandler::Create(&fake_clock, embedding_index.get(), + /*enable_embedding_index=*/true)); std::vector<std::unique_ptr<DataIndexingHandler>> handlers; handlers.push_back(std::move(term_indexing_handler)); handlers.push_back(std::move(integer_section_indexing_handler)); handlers.push_back(std::move(qualified_id_join_indexing_handler)); + handlers.push_back(std::move(embedding_indexing_handler)); IndexProcessor index_processor(std::move(handlers), &fake_clock); DocumentProto incorrect_message = @@ -5552,13 +5581,17 @@ .AddStringProperty("body", "wrong message") .AddInt64Property("indexableInteger", 456) .AddStringProperty("senderQualifiedId", "namespace#person/2") + .AddVectorProperty( + "embedding", + CreateVector("my_model", {-0.1, -0.2, -0.3, -0.4, -0.5})) .SetCreationTimestampMs(kDefaultCreationTimestampMs) .Build(); ICING_ASSERT_OK_AND_ASSIGN( TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store.get(), lang_segmenter_.get(), std::move(incorrect_message))); - ICING_ASSERT_OK(index_processor.IndexDocument(tokenized_document, doc_id)); + ICING_ASSERT_OK(index_processor.IndexDocument(tokenized_document, doc_id, + put_result.old_document_id)); // Rewrite existing data's version files ICING_ASSERT_OK( @@ -5604,6 +5637,7 @@ // - term index // - numeric index // - qualified id join index + // - embedding index InitializeStatsProto::RecoveryCause expected_recovery_cause = GetParam().existing_version_info.version != version_util::kVersion ? InitializeStatsProto::VERSION_CHANGED @@ -5622,6 +5656,9 @@ EXPECT_THAT(initialize_result.initialize_stats() .qualified_id_join_index_restoration_cause(), Eq(expected_recovery_cause)); + EXPECT_THAT( + initialize_result.initialize_stats().embedding_index_restoration_cause(), + Eq(expected_recovery_cause)); // Manually check version file ICING_ASSERT_OK_AND_ASSIGN( @@ -5698,6 +5735,22 @@ search_spec3, ScoringSpecProto::default_instance(), result_spec3); EXPECT_THAT(search_result_proto3, EqualsSearchResultIgnoreStatsAndScores( expected_join_search_result_proto)); + + // Verify embedding search + SearchSpecProto search_spec4; + search_spec4.set_query("semanticSearch(getEmbeddingParameter(0), 0)"); + *search_spec4.add_embedding_query_vectors() = + CreateVector("my_model", {1, 1, 1, 1, 1}); + search_spec4.set_embedding_query_metric_type( + SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT); + search_spec4.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + + SearchResultProto search_result_proto4 = + icing.Search(search_spec4, ScoringSpecProto::default_instance(), + ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto4, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); } INSTANTIATE_TEST_SUITE_P( @@ -5913,6 +5966,831 @@ testing::Values(std::make_tuple(false, false), std::make_tuple(false, true), std::make_tuple(true, false), std::make_tuple(true, true))); +class IcingSearchEngineInitializationChangeEmbeddingFlagTest + : public IcingSearchEngineInitializationTest, + public ::testing::WithParamInterface<std::vector<bool>> {}; +TEST_P(IcingSearchEngineInitializationChangeEmbeddingFlagTest, + ChangeEnableEmbeddingIndexFlagTest) { + std::vector<bool> enable_embedding_index_flags = GetParam(); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( + PropertyConfigBuilder() + .SetName("embedding") + .SetDataTypeVector(EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + + // Create a document with an embedding. + DocumentProto document = + DocumentBuilder() + .SetKey("icing", "uri") + .SetSchema("Message") + .SetCreationTimestampMs(1) + .AddVectorProperty( + "embedding", CreateVector("my_model", {0.1, 0.2, 0.3, 0.4, 0.5})) + .Build(); + + // Create icing with a document that has an embedding. + { + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_embedding_index(enable_embedding_index_flags[0]); + TestIcingSearchEngine icing(options, std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document).status(), ProtoIsOk()); + } + + // Create icing multiple times with different enable_embedding_index flags. + for (int i = 1; i < enable_embedding_index_flags.size(); ++i) { + bool flag_changed = + enable_embedding_index_flags[i] != enable_embedding_index_flags[i - 1]; + + // Ensure that the embedding index is rebuilt if the flag is changed. + auto mock_filesystem = std::make_unique<MockFilesystem>(); + EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively(_)) + .WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteDirectoryRecursively(EndsWith("/embedding_index_dir"))) + .Times(flag_changed ? 1 : 0); + + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_embedding_index(enable_embedding_index_flags[i]); + TestIcingSearchEngine icing(options, std::move(mock_filesystem), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + InitializeResultProto initialize_result = icing.Initialize(); + ASSERT_THAT(initialize_result.status(), ProtoIsOk()); + // Ensure that the embedding index is rebuilt if the flag is changed. + EXPECT_THAT(initialize_result.initialize_stats() + .embedding_index_restoration_cause(), + Eq(flag_changed ? InitializeStatsProto::FEATURE_FLAG_CHANGED + : InitializeStatsProto::NONE)); + EXPECT_THAT(initialize_result.initialize_stats().index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT( + initialize_result.initialize_stats().integer_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT(initialize_result.initialize_stats() + .qualified_id_join_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + + // Write a embedding query that should match the document if the embedding + // index is enabled. + SearchSpecProto search_spec; + search_spec.set_query("semanticSearch(getEmbeddingParameter(0), 0)"); + *search_spec.add_embedding_query_vectors() = + CreateVector("my_model", {1, 1, 1, 1, 1}); + search_spec.set_embedding_query_metric_type( + SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT); + search_spec.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + + SearchResultProto results = + icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + // The document should be returned if the embedding index is enabled. + if (enable_embedding_index_flags[i]) { + EXPECT_THAT(results.results(), SizeIs(1)); + EXPECT_THAT(results.results(0).document(), EqualsProto(document)); + } else { + EXPECT_THAT(results.results(), IsEmpty()); + } + } +} + +TEST_P(IcingSearchEngineInitializationChangeEmbeddingFlagTest, + ChangeEnableEmbeddingQuantizationFlagTest) { + std::vector<bool> enable_embedding_quantization_flags = GetParam(); + + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Message") + .AddProperty( + PropertyConfigBuilder() + .SetName("embeddingUnquantized") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH, + EmbeddingIndexingConfig::QuantizationType::NONE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("embeddingQuantized") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH, + EmbeddingIndexingConfig:: + QuantizationType::QUANTIZE_8_BIT) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // If quantization is enabled, this vector will be quantized to {0, 1, 255}. + PropertyProto::VectorProto vector = CreateVector("my_model", {0, 1.45, 255}); + // Create two documents with different quantization types. If quantization is + // enabled, then only document2's embedding will be quantized. Otherwise, both + // will be unquantized. + DocumentProto document1 = + DocumentBuilder() + .SetKey("icing", "uri1") + .SetSchema("Message") + .SetCreationTimestampMs(1) + .AddVectorProperty("embeddingUnquantized", vector) + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("icing", "uri2") + .SetSchema("Message") + .SetCreationTimestampMs(1) + .AddVectorProperty("embeddingQuantized", vector) + .Build(); + + // Create icing with a document that has an embedding. + { + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_embedding_index(true); + options.set_enable_embedding_quantization( + enable_embedding_quantization_flags[0]); + TestIcingSearchEngine icing(options, std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); + } + + // Create icing multiple times with different enable_embedding_index flags. + for (int i = 1; i < enable_embedding_quantization_flags.size(); ++i) { + bool flag_changed = enable_embedding_quantization_flags[i] != + enable_embedding_quantization_flags[i - 1]; + + // Ensure that the embedding index is rebuilt if the flag is changed. + auto mock_filesystem = std::make_unique<MockFilesystem>(); + EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively(_)) + .WillRepeatedly(DoDefault()); + EXPECT_CALL(*mock_filesystem, + DeleteDirectoryRecursively(EndsWith("/embedding_index_dir"))) + .Times(flag_changed ? 1 : 0); + + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_embedding_index(true); + options.set_enable_embedding_quantization( + enable_embedding_quantization_flags[i]); + TestIcingSearchEngine icing(options, std::move(mock_filesystem), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + InitializeResultProto initialize_result = icing.Initialize(); + ASSERT_THAT(initialize_result.status(), ProtoIsOk()); + // Ensure that the embedding index is rebuilt if the flag is changed. + EXPECT_THAT(initialize_result.initialize_stats() + .embedding_index_restoration_cause(), + Eq(flag_changed ? InitializeStatsProto::FEATURE_FLAG_CHANGED + : InitializeStatsProto::NONE)); + EXPECT_THAT(initialize_result.initialize_stats().index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT( + initialize_result.initialize_stats().integer_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT(initialize_result.initialize_stats() + .qualified_id_join_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + + // Write an embedding query that always matches all the documents. We will + // check quantization by checking the scores. + // + // This query will assign a dot product score that is equal to the sum of + // all dimensions of the matched embedding. As a result, if quantization is + // enabled, the score will be 0 + 1 + 255 = 256. Otherwise, the score will + // be 0 + 1.45 + 255 = 256.45. + SearchSpecProto search_spec; + search_spec.set_query("semanticSearch(getEmbeddingParameter(0))"); + *search_spec.add_embedding_query_vectors() = + CreateVector("my_model", {1, 1, 1}); + search_spec.set_embedding_query_metric_type( + SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT); + search_spec.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + // Set the advanced scoring expression to a constant so that the documents + // will be returned according to their order they were added. + scoring_spec.set_advanced_scoring_expression("0"); + // Set the additional advanced scoring expression to check the embedding + // scores. + scoring_spec.add_additional_advanced_scoring_expressions( + "sum(this.matchedSemanticScores(getEmbeddingParameter(0)))"); + + SearchResultProto results = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(2)); + + constexpr float eps = 0.0001f; + constexpr float unquantized_score = 256.45f; + constexpr float quantized_score = 256.0f; + EXPECT_THAT(results.results(0).document(), EqualsProto(document2)); + if (enable_embedding_quantization_flags[i]) { + // When quantization is enabled, only the embedding that is configured + // with quantization is quantized. So only document2's embedding is + // quantized. + EXPECT_NEAR(results.results(0).additional_scores(0), quantized_score, + eps); + } else { + // When quantization is disabled, all embeddings are unquantized. + EXPECT_NEAR(results.results(0).additional_scores(0), unquantized_score, + eps); + } + // document1's embedding is always unquantized, since it is not configured + // to be quantized. + EXPECT_THAT(results.results(1).document(), EqualsProto(document1)); + EXPECT_NEAR(results.results(1).additional_scores(0), unquantized_score, + eps); + } +} + +INSTANTIATE_TEST_SUITE_P( + IcingSearchEngineInitializationChangeEmbeddingFlagTest, + IcingSearchEngineInitializationChangeEmbeddingFlagTest, + testing::Values(std::vector<bool>{false, true, false, true, false, true}, + std::vector<bool>{true, false, true, false, true, false}, + std::vector<bool>{false, true, true, true, false, true}, + std::vector<bool>{true, false, false, false, true, false}, + std::vector<bool>{true, true, true, true}, + std::vector<bool>{false, false, false, false})); + +class IcingSearchEngineInitializationChangeEnableScorablePropertiesFlagTest + : public IcingSearchEngineInitializationTest, + public ::testing::WithParamInterface<std::vector<bool>> {}; +TEST_P(IcingSearchEngineInitializationChangeEnableScorablePropertiesFlagTest, + ChangeEnableScorablePropertiesFlagTest) { + std::vector<bool> enable_scorable_properties_flags = GetParam(); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("Message") + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("score") + .SetDataTypeInt64(NUMERIC_MATCH_UNKNOWN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + + // Create a document with an embedding. + int scorable_prop_value = 10; + DocumentProto document = DocumentBuilder() + .SetKey("icing", "uri") + .SetSchema("Message") + .SetCreationTimestampMs(1) + .AddStringProperty("body", "foo bar") + .AddInt64Property("score", scorable_prop_value) + .Build(); + + // Create icing with a document that has a scorable property. + { + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_scorable_properties( + enable_scorable_properties_flags.at(0)); + TestIcingSearchEngine icing(options, std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document).status(), ProtoIsOk()); + } + + // Create icing multiple times with different enable_scorable_properties + // flags. + for (int i = 1; i < enable_scorable_properties_flags.size(); ++i) { + bool flag_changed = enable_scorable_properties_flags[i] != + enable_scorable_properties_flags[i - 1]; + + // Ensure that the document store derived files are rebuilt if the flag is + // changed. + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_scorable_properties(enable_scorable_properties_flags[i]); + TestIcingSearchEngine icing(options, std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + InitializeResultProto initialize_result = icing.Initialize(); + ASSERT_THAT(initialize_result.status(), ProtoIsOk()); + + // Document store recovery cause should be FEATURE_FLAG_CHANGED if the flag + // is changed. + EXPECT_THAT( + initialize_result.initialize_stats().document_store_recovery_cause(), + Eq(flag_changed ? InitializeStatsProto::FEATURE_FLAG_CHANGED + : InitializeStatsProto::NONE)); + + // Schema store and all indices should be unaffected. + EXPECT_THAT( + initialize_result.initialize_stats().schema_store_recovery_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT(initialize_result.initialize_stats().index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT( + initialize_result.initialize_stats().integer_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT(initialize_result.initialize_stats() + .qualified_id_join_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT(initialize_result.initialize_stats() + .embedding_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + + // Write normal query that should retrieve the document. + SearchSpecProto search_spec; + search_spec.set_query("bar"); + search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); + search_spec.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + + ScoringSpecProto scoring_spec; + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression("this.creationTimestamp()"); + SearchResultProto results = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + EXPECT_THAT(results.results(0).document(), EqualsProto(document)); + EXPECT_THAT(results.results(0).score(), Eq(1)); + + // Now write a query that tries to access the scorable property. + scoring_spec.set_advanced_scoring_expression( + "this.creationTimestamp() + sum(getScorableProperty(\"Message\", " + "\"score\"))"); + SchemaTypeAliasMapProto* alias_map_proto = + scoring_spec.add_schema_type_alias_map_protos(); + alias_map_proto->set_alias_schema_type("Message"); + alias_map_proto->mutable_schema_types()->Add("Message"); + + results = icing.Search(search_spec, scoring_spec, + ResultSpecProto::default_instance()); + if (options.enable_scorable_properties()) { + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + EXPECT_THAT(results.results(0).document(), EqualsProto(document)); + EXPECT_THAT(results.results(0).score(), Eq(1 + scorable_prop_value)); + } else { + EXPECT_THAT(results.status(), + ProtoStatusIs(StatusProto::INVALID_ARGUMENT)); + } + + // Reindex the doc with a new scorable property value, regardless of whether + // scorable properties are enabled. This way we can confirm that Initialize + // is actually rebuilding the scorable property cache with new values. + scorable_prop_value += 10; + document = DocumentBuilder(document) + .ClearProperties() + .AddStringProperty("body", "foo bar") + .AddInt64Property("score", scorable_prop_value) + .Build(); + ASSERT_THAT(icing.Put(document).status(), ProtoIsOk()); + } +} + +INSTANTIATE_TEST_SUITE_P( + IcingSearchEngineInitializationChangeEnableScorablePropertiesFlagTest, + IcingSearchEngineInitializationChangeEnableScorablePropertiesFlagTest, + testing::Values(std::vector<bool>{false, true, false, true, false, true}, + std::vector<bool>{true, false, true, false, true, false}, + std::vector<bool>{false, true, true, true, false, true}, + std::vector<bool>{true, false, false, false, true, false}, + std::vector<bool>{true, true, true, true}, + std::vector<bool>{false, false, false, false})); + +class IcingSearchEngineInitializationSchemaDatabaseMigrationTest + : public IcingSearchEngineInitializationTest, + public ::testing::WithParamInterface<std::tuple<int32_t, bool, bool>> {}; + +TEST_P(IcingSearchEngineInitializationSchemaDatabaseMigrationTest, + InitializeWithSchemaDatabaseMigration) { + int32_t existing_version = std::get<0>(GetParam()); + bool previous_version_has_schema_database_enabled = std::get<1>(GetParam()); + bool enable_schema_database = std::get<2>(GetParam()); + + IcingSearchEngineVersionProto previous_version_proto; + previous_version_proto.set_version(existing_version); + previous_version_proto.set_max_version(existing_version); + if (previous_version_has_schema_database_enabled) { + previous_version_proto.add_enabled_features()->set_feature_type( + IcingSearchEngineFeatureInfoProto::FEATURE_SCHEMA_DATABASE); + } + + SchemaTypeConfigProto db1_email_type = + SchemaTypeConfigBuilder() + .SetType("db1/email") + .AddProperty(PropertyConfigBuilder() + .SetName("db1Subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + SchemaTypeConfigProto db2_email_type = + SchemaTypeConfigBuilder() + .SetType("db2/email") + .AddProperty(PropertyConfigBuilder() + .SetName("db2Subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("db2Id") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + + if (previous_version_has_schema_database_enabled) { + // Populate the database field for the db1/email type. + db1_email_type = + SchemaTypeConfigBuilder(db1_email_type).SetDatabase("db1").Build(); + if (existing_version >= version_util::kSchemaDatabaseVersion) { + // Populate the database field for the db2/email type only if previous + // version is a post schema-database version. + db2_email_type = + SchemaTypeConfigBuilder(db2_email_type).SetDatabase("db2").Build(); + } + // Otherwise, the database field is not populated for db2/email type. This + // is to simulate the following situation: + // 1. Icing is initialized on a version>kSchemaDatabaseVersion with schema + // database enabled, and db1/email is set with the database field + // populated. + // 2. Icing gets rolled back to pre-schema database version, db2/email is + // set during this time so the database field is not populated. + } + SchemaProto previous_version_schema = + SchemaBuilder().AddType(db1_email_type).AddType(db2_email_type).Build(); + + DocumentProto db1_email_doc = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("db1/email") + .AddStringProperty("db1Subject", "subject") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto db2_email_doc = + DocumentBuilder() + .SetKey("namespace", "uri3") + .SetSchema("db2/email") + .AddStringProperty("db2Subject", "subject") + .AddInt64Property("db2Id", 123) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + { // Initialize IcingSearchEngine + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_schema_database( + previous_version_has_schema_database_enabled); + TestIcingSearchEngine icing(options, std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + // 1. Set schema. + if (options.enable_schema_database()) { + // Need to set schemas with a single database field at a time. + ASSERT_THAT( + icing.SetSchema(SchemaBuilder().AddType(db1_email_type).Build()) + .status(), + ProtoIsOk()); + ASSERT_THAT( + icing.SetSchema(SchemaBuilder().AddType(db2_email_type).Build()) + .status(), + ProtoIsOk()); + } else { + ASSERT_THAT(icing.SetSchema(previous_version_schema).status(), + ProtoIsOk()); + } + // 2. Put two documents + ASSERT_THAT(icing.Put(db1_email_doc).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(db2_email_doc).status(), ProtoIsOk()); + // 3. Rewrite version files + // - Only need to rewrite v1 version file to write an older version + // number. + // - FeatureInfo rewritting (v2 version file) is not needed as it should + // be handled by IcingSearchEngine. + ICING_ASSERT_OK(version_util::WriteV1Version( + *filesystem(), GetVersionFileDir(), + version_util::VersionInfo(existing_version, existing_version))); + } // This should shut down IcingSearchEngine and persist anything it needs to + + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_schema_database(enable_schema_database); + + TestIcingSearchEngine icing(options, std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), GetTestJniCache()); + InitializeResultProto initialize_result = icing.Initialize(); + ASSERT_THAT(initialize_result.status(), ProtoIsOk()); + + SearchResultProto db1_email_search_result_proto; + db1_email_search_result_proto.mutable_status()->set_code(StatusProto::OK); + *db1_email_search_result_proto.mutable_results()->Add()->mutable_document() = + db1_email_doc; + + SearchResultProto db2_email_search_result_proto; + db2_email_search_result_proto.mutable_status()->set_code(StatusProto::OK); + *db2_email_search_result_proto.mutable_results()->Add()->mutable_document() = + db2_email_doc; + + SearchResultProto all_email_search_result_proto; + all_email_search_result_proto.mutable_status()->set_code(StatusProto::OK); + *all_email_search_result_proto.mutable_results()->Add()->mutable_document() = + db2_email_doc; + *all_email_search_result_proto.mutable_results()->Add()->mutable_document() = + db1_email_doc; + + // Verify term search + SearchSpecProto search_spec1; + search_spec1.set_query("db1Subject:subject"); + search_spec1.set_term_match_type(TermMatchType::EXACT_ONLY); + SearchResultProto search_result_proto1 = + icing.Search(search_spec1, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores( + db1_email_search_result_proto)); + + SearchSpecProto search_spec2; + search_spec2.set_query("subject"); + search_spec2.set_term_match_type(TermMatchType::EXACT_ONLY); + SearchResultProto search_result_google::protobuf = + icing.Search(search_spec2, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores( + all_email_search_result_proto)); + + // Verify numeric (integer) search + SearchSpecProto search_spec3; + search_spec3.set_query("db2Id == 123"); + search_spec3.add_enabled_features(std::string(kNumericSearchFeature)); + + SearchResultProto search_result_proto3 = + icing.Search(search_spec3, ScoringSpecProto::default_instance(), + ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto3, EqualsSearchResultIgnoreStatsAndScores( + db2_email_search_result_proto)); + + // Verify GetSchema + if (enable_schema_database) { + SchemaTypeConfigProto db1_email_type_with_db = + SchemaTypeConfigBuilder(db1_email_type).SetDatabase("db1").Build(); + SchemaTypeConfigProto db2_email_type_with_db = + SchemaTypeConfigBuilder(db2_email_type).SetDatabase("db2").Build(); + SchemaProto full_schema_with_database = SchemaBuilder() + .AddType(db1_email_type_with_db) + .AddType(db2_email_type_with_db) + .Build(); + SchemaProto db1_schema = + SchemaBuilder().AddType(db1_email_type_with_db).Build(); + SchemaProto db2_schema = + SchemaBuilder().AddType(db2_email_type_with_db).Build(); + + GetSchemaResultProto expected_get_schema_result_proto_full; + expected_get_schema_result_proto_full.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_full.mutable_schema() = + full_schema_with_database; + EXPECT_THAT(icing.GetSchema(), + EqualsProto(expected_get_schema_result_proto_full)); + + GetSchemaResultProto expected_get_schema_result_proto_db1; + expected_get_schema_result_proto_db1.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_db1.mutable_schema() = db1_schema; + EXPECT_THAT(icing.GetSchema("db1"), + EqualsProto(expected_get_schema_result_proto_db1)); + + GetSchemaResultProto expected_get_schema_result_proto_db2; + expected_get_schema_result_proto_db2.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_db2.mutable_schema() = db2_schema; + EXPECT_THAT(icing.GetSchema("db2"), + EqualsProto(expected_get_schema_result_proto_db2)); + } else { + GetSchemaResultProto expected_get_schema_result_proto; + expected_get_schema_result_proto.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto.mutable_schema() = + previous_version_schema; + EXPECT_THAT(icing.GetSchema(), + EqualsProto(expected_get_schema_result_proto)); + } +} + +INSTANTIATE_TEST_SUITE_P( + IcingSearchEngineInitializationSchemaDatabaseMigrationTest, + IcingSearchEngineInitializationSchemaDatabaseMigrationTest, + testing::Values( + std::make_tuple( + /*previous_version=*/version_util::kSchemaDatabaseVersion - 1, + /*prev_version_schema_database_enabled=*/false, + /*enable_schema_database=*/false), + std::make_tuple( + /*previous_version=*/version_util::kSchemaDatabaseVersion - 1, + /*prev_version_schema_database_enabled=*/false, + /*enable_schema_database=*/true), + // The next two cases simulate the following scenario: + // 1. Icing is initialized on a version>kSchemaDatabaseVersion for + // sometime, and schemas are set with the database field populated. + // 2. Icing gets rolled back to pre-schema database version, so new + // schema types no longer populate the database field. + // 3. Icing gets rolled forward to post-schema database version again, + // and we should verify that database migration happens correctly. + std::make_tuple( + /*previous_version=*/version_util::kSchemaDatabaseVersion - 1, + /*prev_version_schema_database_enabled=*/true, + /*enable_schema_database=*/false), + std::make_tuple( + /*previous_version=*/version_util::kSchemaDatabaseVersion - 1, + /*prev_version_schema_database_enabled=*/true, + /*enable_schema_database=*/true), + std::make_tuple( + /*previous_version=*/version_util::kSchemaDatabaseVersion, + /*prev_version_schema_database_enabled=*/false, + /*enable_schema_database=*/false), + std::make_tuple( + /*previous_version=*/version_util::kSchemaDatabaseVersion, + /*prev_version_schema_database_enabled=*/false, + /*enable_schema_database=*/true), + std::make_tuple( + /*previous_version=*/version_util::kSchemaDatabaseVersion, + /*prev_version_schema_database_enabled=*/true, + /*enable_schema_database=*/false), + std::make_tuple( + /*previous_version=*/version_util::kSchemaDatabaseVersion, + /*prev_version_schema_database_enabled=*/true, + /*enable_schema_database=*/true))); + +class IcingSearchEngineInitializationChangeEnableJoinIndexV3FlagTest + : public IcingSearchEngineInitializationTest, + public ::testing::WithParamInterface<std::vector<bool>> {}; +TEST_P(IcingSearchEngineInitializationChangeEnableJoinIndexV3FlagTest, + ChangeEnableJoinIndexV3FlagTest) { + std::vector<bool> enable_join_index_v3_flags = GetParam(); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Message") + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("senderQualifiedId") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + + DocumentProto person = + DocumentBuilder() + .SetKey("namespace", "person") + .SetSchema("Person") + .AddStringProperty("name", "person") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto message = + DocumentBuilder() + .SetKey("namespace", "message/1") + .SetSchema("Message") + .AddStringProperty("body", "message body") + .AddStringProperty("senderQualifiedId", "namespace#person") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + { + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_qualified_id_join_index_v3_and_delete_propagate_from( + enable_join_index_v3_flags.at(0)); + TestIcingSearchEngine icing(options, std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(person).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(message).status(), ProtoIsOk()); + } + + // Create icing multiple times with different + // enable_qualified_id_join_index_v3_and_propagate_delete flags. + for (int i = 1; i < enable_join_index_v3_flags.size(); ++i) { + bool flag_changed = + enable_join_index_v3_flags[i] != enable_join_index_v3_flags[i - 1]; + + // Ensure that the qualified id join index is rebuilt if the flag is + // changed. + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_qualified_id_join_index_v3_and_delete_propagate_from( + enable_join_index_v3_flags[i]); + TestIcingSearchEngine icing(options, std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + InitializeResultProto initialize_result = icing.Initialize(); + ASSERT_THAT(initialize_result.status(), ProtoIsOk()); + + // Qualified id join index recovery cause should be FEATURE_FLAG_CHANGED if + // flag is changed. + EXPECT_THAT(initialize_result.initialize_stats() + .qualified_id_join_index_restoration_cause(), + Eq(flag_changed ? InitializeStatsProto::FEATURE_FLAG_CHANGED + : InitializeStatsProto::NONE)); + + // Schema store, document store and all other indices should be unaffected. + EXPECT_THAT( + initialize_result.initialize_stats().schema_store_recovery_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT( + initialize_result.initialize_stats().document_store_recovery_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT(initialize_result.initialize_stats().index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT( + initialize_result.initialize_stats().integer_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT(initialize_result.initialize_stats() + .embedding_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + + // Prepare join search spec to join a query for `name:person` with a child + // query for `body:message` based on the child's `senderQualifiedId` field. + // + // No matter what the flag value is, the join API should always return the + // expected result. + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); + search_spec.set_query("name:person"); + JoinSpecProto* join_spec = search_spec.mutable_join_spec(); + join_spec->set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec->set_child_property_expression("senderQualifiedId"); + join_spec->set_aggregation_scoring_strategy( + JoinSpecProto::AggregationScoringStrategy::COUNT); + JoinSpecProto::NestedSpecProto* nested_spec = + join_spec->mutable_nested_spec(); + SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec(); + nested_search_spec->set_term_match_type(TermMatchType::EXACT_ONLY); + nested_search_spec->set_query("body:message"); + *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec(); + *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance(); + + ResultSpecProto result_spec = ResultSpecProto::default_instance(); + result_spec.set_max_joined_children_per_parent_to_return( + std::numeric_limits<int32_t>::max()); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); + SearchResultProto::ResultProto* result_proto = + expected_search_result_proto.mutable_results()->Add(); + *result_proto->mutable_document() = person; + *result_proto->mutable_joined_results()->Add()->mutable_document() = + message; + + SearchResultProto search_result_proto = + icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); + } +} + +INSTANTIATE_TEST_SUITE_P( + IcingSearchEngineInitializationChangeEnableJoinIndexV3FlagTest, + IcingSearchEngineInitializationChangeEnableJoinIndexV3FlagTest, + testing::Values(std::vector<bool>{false, true, false, true, false, true}, + std::vector<bool>{true, false, true, false, true, false}, + std::vector<bool>{false, true, true, true, false, true}, + std::vector<bool>{true, false, false, false, true, false}, + std::vector<bool>{true, true, true, true}, + std::vector<bool>{false, false, false, false})); + } // namespace } // namespace lib } // namespace icing
diff --git a/icing/icing-search-engine_optimize_test.cc b/icing/icing-search-engine_optimize_test.cc index 4a9dd71..f5c1ced 100644 --- a/icing/icing-search-engine_optimize_test.cc +++ b/icing/icing-search-engine_optimize_test.cc
@@ -19,6 +19,7 @@ #include <memory> #include <string> #include <utility> +#include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "gmock/gmock.h" @@ -52,10 +53,10 @@ #include "icing/store/document-log-creator.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/testing/tmp-directory.h" +#include "icing/util/icu-data-file-helper.h" namespace icing { namespace lib { @@ -63,6 +64,7 @@ namespace { using ::icing::lib::portable_equals_proto::EqualsProto; +using ::testing::ElementsAre; using ::testing::Eq; using ::testing::Ge; using ::testing::Gt; @@ -100,7 +102,7 @@ std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); ICING_ASSERT_OK( - icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); + icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path)); } filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); } @@ -120,7 +122,10 @@ IcingSearchEngineOptions GetDefaultIcingOptions() { IcingSearchEngineOptions icing_options; + icing_options.set_enable_scorable_properties(true); icing_options.set_base_dir(GetTestBaseDir()); + icing_options.set_enable_qualified_id_join_index_v3_and_delete_propagate_from( + true); return icing_options; } @@ -130,6 +135,25 @@ return scoring_spec; } +void AddSchemaTypeAliasMap(ScoringSpecProto* scoring_spec, + const std::string& alias_schema_type, + const std::vector<std::string>& schema_types) { + SchemaTypeAliasMapProto* alias_map_proto = + scoring_spec->add_schema_type_alias_map_protos(); + alias_map_proto->set_alias_schema_type(alias_schema_type); + alias_map_proto->mutable_schema_types()->Add(schema_types.begin(), + schema_types.end()); +} + +std::vector<double> GetScoresFromSearchResults( + const SearchResultProto& search_result_proto) { + std::vector<double> result_scores; + for (int i = 0; i < search_result_proto.results_size(); i++) { + result_scores.push_back(search_result_proto.results(i).score()); + } + return result_scores; +} + // TODO(b/272145329): create SearchSpecBuilder, JoinSpecBuilder, // SearchResultProtoBuilder and ResultProtoBuilder for unit tests and build all // instances by them. @@ -1842,6 +1866,268 @@ } } +TEST_F(IcingSearchEngineOptimizeTest, + GetScorablePropertyShouldWorkAfterOptimization) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("scoreInt64") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + DocumentProto document1 = DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("Email") + .AddStringProperty("subject", "subject foo1") + .AddInt64Property("scoreInt64", 1) + .SetCreationTimestampMs(0) + .Build(); + double ranking_score_doc1 = 1; + DocumentProto document2 = DocumentBuilder() + .SetKey("foo", "2") + .SetSchema("Email") + .AddStringProperty("subject", "subject foo2") + .AddInt64Property("scoreInt64", 2) + .SetCreationTimestampMs(0) + .Build(); + double ranking_score_doc2 = 2; + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + + ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); + + { + // getScorableProperty works as expected before optimization. + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec; + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Email\", \"scoreInt64\"))"); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + AddSchemaTypeAliasMap(&scoring_spec, "Email", {"Email"}); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(ranking_score_doc2, ranking_score_doc1)); + } + + { + // getScorableProperty works as expected after optimization. + icing.Optimize(); + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec; + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Email\", \"scoreInt64\"))"); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + AddSchemaTypeAliasMap(&scoring_spec, "Email", {"Email"}); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(ranking_score_doc2, ranking_score_doc1)); + } +} + +TEST_F(IcingSearchEngineOptimizeTest, + GetScorablePropertyShouldWorkAfterOverwriteAndOptimization) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("scoreInt64") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + DocumentProto document1 = DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("Email") + .AddStringProperty("subject", "subject foo1") + .AddInt64Property("scoreInt64", 1) + .SetCreationTimestampMs(0) + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("foo", "2") + .SetSchema("Email") + .AddStringProperty("subject", "subject foo2") + .AddInt64Property("scoreInt64", 2) + .SetCreationTimestampMs(0) + .Build(); + double ranking_score_doc2 = 2; + DocumentProto document_overwrite_doc1 = + DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("Email") + .AddStringProperty("subject", "subject foo1") + .AddInt64Property("scoreInt64", 3) + .SetCreationTimestampMs(0) + .Build(); + double ranking_score_doc1_after_overwrite = 3; + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + + ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document_overwrite_doc1).status(), ProtoIsOk()); + + { + // getScorableProperty works as expected before optimization. + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec; + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Email\", \"scoreInt64\"))"); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + AddSchemaTypeAliasMap(&scoring_spec, "Email", {"Email"}); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + EXPECT_THAT( + GetScoresFromSearchResults(search_result_proto), + ElementsAre(ranking_score_doc1_after_overwrite, ranking_score_doc2)); + } + + { + // getScorableProperty works as expected after optimization. + icing.Optimize(); + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec; + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Email\", \"scoreInt64\"))"); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + AddSchemaTypeAliasMap(&scoring_spec, "Email", {"Email"}); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + EXPECT_THAT( + GetScoresFromSearchResults(search_result_proto), + ElementsAre(ranking_score_doc1_after_overwrite, ranking_score_doc2)); + } +} + +TEST_F(IcingSearchEngineOptimizeTest, + GetScorablePropertyShouldWorkAfterDeletionAndOptimization) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("scoreInt64") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + DocumentProto document1 = DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("Email") + .AddStringProperty("subject", "subject foo1") + .AddInt64Property("scoreInt64", 1) + .SetCreationTimestampMs(0) + .Build(); + double ranking_score_doc1 = 1; + DocumentProto document2 = DocumentBuilder() + .SetKey("foo", "2") + .SetSchema("Email") + .AddStringProperty("subject", "subject foo2") + .AddInt64Property("scoreInt64", 2) + .SetCreationTimestampMs(0) + .Build(); + double ranking_score_doc2 = 2; + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + + ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk()); + + { + // getScorableProperty works as expected before deletion and optimization. + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec; + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Email\", \"scoreInt64\"))"); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + AddSchemaTypeAliasMap(&scoring_spec, "Email", {"Email"}); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(ranking_score_doc2, ranking_score_doc1)); + } + + { + ASSERT_THAT(icing.Delete(document1.namespace_(), document1.uri()).status(), + ProtoIsOk()); + icing.Optimize(); + + // After document1 is deleted, the document2 will get a new id internally. + // After the Optimize() is called, we verify that the scorable property is + // still working as expected. + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec; + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Email\", \"scoreInt64\"))"); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + AddSchemaTypeAliasMap(&scoring_spec, "Email", {"Email"}); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(ranking_score_doc2)); + } +} + } // namespace } // namespace lib } // namespace icing
diff --git a/icing/icing-search-engine_put_test.cc b/icing/icing-search-engine_put_test.cc index c5f8e2b..a6cc463 100644 --- a/icing/icing-search-engine_put_test.cc +++ b/icing/icing-search-engine_put_test.cc
@@ -47,11 +47,11 @@ #include "icing/schema-builder.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/random-string.h" #include "icing/testing/test-data.h" #include "icing/testing/tmp-directory.h" +#include "icing/util/icu-data-file-helper.h" namespace icing { namespace lib { @@ -111,7 +111,7 @@ std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); ICING_ASSERT_OK( - icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); + icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path)); } filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); } @@ -528,7 +528,7 @@ PropertyProto::BlobHandleProto blob_handle; blob_handle.set_digest(std::string(32, ' ')); - blob_handle.set_label("label"); + blob_handle.set_namespace_("namespace"); DocumentProto document = DocumentBuilder() @@ -570,7 +570,7 @@ PropertyProto::BlobHandleProto blob_handle; blob_handle.set_digest("invalid digest"); - blob_handle.set_label("label"); + blob_handle.set_namespace_("namespace"); DocumentProto document = DocumentBuilder()
diff --git a/icing/icing-search-engine_schema_test.cc b/icing/icing-search-engine_schema_test.cc index 115e4cf..89f9546 100644 --- a/icing/icing-search-engine_schema_test.cc +++ b/icing/icing-search-engine_schema_test.cc
@@ -17,6 +17,7 @@ #include <memory> #include <string> #include <utility> +#include <vector> #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -26,6 +27,7 @@ #include "icing/icing-search-engine.h" #include "icing/jni/jni-cache.h" #include "icing/join/join-processor.h" +#include "icing/legacy/index/icing-filesystem.h" #include "icing/portable/equals-proto.h" #include "icing/portable/platform.h" #include "icing/proto/debug.pb.h" @@ -46,10 +48,11 @@ #include "icing/schema/section.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/testing/tmp-directory.h" +#include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" namespace icing { namespace lib { @@ -92,7 +95,7 @@ std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); ICING_ASSERT_OK( - icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); + icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path)); } filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); } @@ -116,7 +119,9 @@ IcingSearchEngineOptions icing_options; icing_options.set_base_dir(GetTestBaseDir()); icing_options.set_document_store_namespace_id_fingerprint(true); - icing_options.set_use_new_qualified_id_join_index(true); + icing_options.set_enable_schema_database(true); + icing_options.set_enable_qualified_id_join_index_v3_and_delete_propagate_from( + true); return icing_options; } @@ -829,6 +834,368 @@ EqualsSearchResultIgnoreStatsAndScores(expected_search_result_proto3)); } +TEST_F(IcingSearchEngineSchemaTest, SetSchemaMultipleDatabases) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + // Create and set schema in db1 with 2 properties: + // - 'a': string type, indexed. + // - 'b': int64 type, indexed. + SchemaTypeConfigProto db1_type = + SchemaTypeConfigBuilder() + .SetType("db1_type") + .SetDatabase("db1") + .AddProperty(PropertyConfigBuilder() + .SetName("a") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("b") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REQUIRED)) + .Build(); + SchemaProto db1_schema = SchemaBuilder().AddType(db1_type).Build(); + SetSchemaResultProto set_schema_result = icing.SetSchema(db1_schema); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + SetSchemaResultProto expected_set_schema_result; + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_new_schema_types()->Add("db1_type"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + DocumentProto db1_document = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("db1_type") + .AddStringProperty("a", "message body") + .AddInt64Property("b", 123) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(db1_document).status(), ProtoIsOk()); + + SearchResultProto expected_search_result_proto_db1; + expected_search_result_proto_db1.mutable_status()->set_code(StatusProto::OK); + *expected_search_result_proto_db1.mutable_results() + ->Add() + ->mutable_document() = db1_document; + + // Verify term search + SearchSpecProto search_spec1; + search_spec1.set_query("message"); + search_spec1.set_term_match_type(TermMatchType::EXACT_ONLY); + + SearchResultProto actual_results = + icing.Search(search_spec1, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto_db1)); + + // Verify numeric (integer) search + SearchSpecProto search_spec2; + search_spec2.set_query("b == 123"); + search_spec2.add_enabled_features(std::string(kNumericSearchFeature)); + + actual_results = icing.Search(search_spec2, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto_db1)); + + // Add a schema for db2: + // - 'a': string type, indexed. + // - 'c': int64 type, indexed. + SchemaTypeConfigProto db2_type = + SchemaTypeConfigBuilder() + .SetType("db2_type") + .SetDatabase("db2") + .AddProperty( + PropertyConfigBuilder() + .SetName("a") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("c") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REQUIRED)) + .Build(); + SchemaProto db2_schema = SchemaBuilder().AddType(db2_type).Build(); + set_schema_result = icing.SetSchema(db2_schema); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + expected_set_schema_result = SetSchemaResultProto(); + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_new_schema_types()->Add("db2_type"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + DocumentProto db2_document = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("db2_type") + .AddStringProperty("a", "message body") + .AddInt64Property("c", 123) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + EXPECT_THAT(icing.Put(db2_document).status(), ProtoIsOk()); + + SearchResultProto expected_search_result_proto_both_docs; + expected_search_result_proto_both_docs.mutable_status()->set_code( + StatusProto::OK); + *expected_search_result_proto_both_docs.mutable_results() + ->Add() + ->mutable_document() = db2_document; + *expected_search_result_proto_both_docs.mutable_results() + ->Add() + ->mutable_document() = db1_document; + + // Verify term search: will get both documents for query "a:message". + actual_results = icing.Search(search_spec1, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto_both_docs)); + + // Verify numeric (integer) search: will only get first document for query + // "b == 123". + actual_results = icing.Search(search_spec2, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto_db1)); + + // Get full schema + GetSchemaResultProto expected_get_schema_result_proto_full; + expected_get_schema_result_proto_full.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_full.mutable_schema() = + SchemaBuilder().AddType(db1_type).AddType(db2_type).Build(); + EXPECT_THAT(icing.GetSchema(), + EqualsProto(expected_get_schema_result_proto_full)); + + // Get db1 schema + GetSchemaResultProto expected_get_schema_result_proto_db1_full; + expected_get_schema_result_proto_db1_full.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_db1_full.mutable_schema() = db1_schema; + EXPECT_THAT(icing.GetSchema("db1"), + EqualsProto(expected_get_schema_result_proto_db1_full)); + + // Get db2 schema + GetSchemaResultProto expected_get_schema_result_proto_db2_full; + expected_get_schema_result_proto_db2_full.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_db2_full.mutable_schema() = db2_schema; + EXPECT_THAT(icing.GetSchema("db2"), + EqualsProto(expected_get_schema_result_proto_db2_full)); +} + +TEST_F(IcingSearchEngineSchemaTest, SetSchemaUpdateExistingDatabaseOk) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + // Create and set schema in db1 with 2 properties: + // - 'b': string type, indexed. + // - 'c': int64 type, indexed. + SchemaTypeConfigProto db1_type = + SchemaTypeConfigBuilder() + .SetType("db1_type") + .SetDatabase("db1") + .AddProperty(PropertyConfigBuilder() + .SetName("b") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("c") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REQUIRED)) + .Build(); + SchemaProto db1_schema = SchemaBuilder().AddType(db1_type).Build(); + SetSchemaResultProto set_schema_result = icing.SetSchema(db1_schema); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + SetSchemaResultProto expected_set_schema_result; + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_new_schema_types()->Add("db1_type"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + // Add a schema for db2: + // - 'b': string type, indexed. + // - 'd': int64 type, indexed. + SchemaTypeConfigProto db2_type = + SchemaTypeConfigBuilder() + .SetType("db2_type") + .SetDatabase("db2") + .AddProperty( + PropertyConfigBuilder() + .SetName("b") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("d") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REQUIRED)) + .Build(); + SchemaProto db2_schema = SchemaBuilder().AddType(db2_type).Build(); + set_schema_result = icing.SetSchema(db2_schema); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + expected_set_schema_result = SetSchemaResultProto(); + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_new_schema_types()->Add("db2_type"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + // Add documents + DocumentProto db1_document1 = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("db1_type") + .AddStringProperty("b", "message body") + .AddInt64Property("c", 123) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto db2_document = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("db2_type") + .AddStringProperty("b", "message body") + .AddInt64Property("d", 123) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + EXPECT_THAT(icing.Put(db1_document1).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(db2_document).status(), ProtoIsOk()); + + // Verify term search. Should match both docs. + SearchSpecProto search_spec1; + search_spec1.set_query("b:message"); + search_spec1.set_term_match_type(TermMatchType::EXACT_ONLY); + + SearchResultProto expected_result_db1doc1; + expected_result_db1doc1.mutable_status()->set_code(StatusProto::OK); + *expected_result_db1doc1.mutable_results()->Add()->mutable_document() = + db1_document1; + + SearchResultProto expected_result_db1doc1_db2; + expected_result_db1doc1_db2.mutable_status()->set_code(StatusProto::OK); + *expected_result_db1doc1_db2.mutable_results()->Add()->mutable_document() = + db2_document; + *expected_result_db1doc1_db2.mutable_results()->Add()->mutable_document() = + db1_document1; + + SearchResultProto actual_results = + icing.Search(search_spec1, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_result_db1doc1_db2)); + + // Verify numeric (integer) search. Should only match db1_document1. + SearchSpecProto search_spec2; + search_spec2.set_query("c == 123"); + search_spec2.add_enabled_features(std::string(kNumericSearchFeature)); + + actual_results = icing.Search(search_spec2, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(expected_result_db1doc1)); + + // Update the db1 schema: + // - db1_schema: + // - 'a': string type, indexed. (new) + // - 'b': string type, indexed. + // - 'c': int64 type, indexed. + // - db2_schema: + // - 'b': string type, indexed. + // - 'd': int64 type, indexed. + db1_type.mutable_properties()->Add( + PropertyConfigBuilder() + .SetName("a") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL) + .Build()); + db1_schema = SchemaBuilder().AddType(db1_type).Build(); + set_schema_result = icing.SetSchema(db1_schema); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + expected_set_schema_result = SetSchemaResultProto(); + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_index_incompatible_changed_schema_types() + ->Add("db1_type"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + // Add new document + DocumentProto db1_document2 = + DocumentBuilder() + .SetKey("namespace", "uri3") + .SetSchema("db1_type") + .AddStringProperty("a", "message body") + .AddStringProperty("b", "string value") + .AddInt64Property("c", 123) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + EXPECT_THAT(icing.Put(db1_document2).status(), ProtoIsOk()); + + // Check that the original docs are still searchable. + // Verify term search. "b:message" should still only match db1_document1 and + // db2_document. + actual_results = icing.Search(search_spec1, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_result_db1doc1_db2)); + + // Verify numeric (integer) search. "c == 123" should now match match + // db1_document1 and db1_document2. + SearchResultProto expected_result_all_db1; + expected_result_all_db1.mutable_status()->set_code(StatusProto::OK); + *expected_result_all_db1.mutable_results()->Add()->mutable_document() = + db1_document2; + *expected_result_all_db1.mutable_results()->Add()->mutable_document() = + db1_document1; + + actual_results = icing.Search(search_spec2, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(expected_result_all_db1)); + + // "message" should match all docs + SearchSpecProto search_spec3; + search_spec3.set_query("message"); + search_spec3.set_term_match_type(TermMatchType::PREFIX); + + SearchResultProto expected_result_all_docs; + expected_result_all_docs.mutable_status()->set_code(StatusProto::OK); + *expected_result_all_docs.mutable_results()->Add()->mutable_document() = + db1_document2; + *expected_result_all_docs.mutable_results()->Add()->mutable_document() = + db2_document; + *expected_result_all_docs.mutable_results()->Add()->mutable_document() = + db1_document1; + + actual_results = icing.Search(search_spec3, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(expected_result_all_docs)); + + // Get full schema + GetSchemaResultProto expected_get_schema_result_proto_full; + expected_get_schema_result_proto_full.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_full.mutable_schema() = + SchemaBuilder().AddType(db1_type).AddType(db2_type).Build(); + EXPECT_THAT(icing.GetSchema(), + EqualsProto(expected_get_schema_result_proto_full)); + + // Get db1 schema + GetSchemaResultProto expected_get_schema_result_proto_db1_full; + expected_get_schema_result_proto_db1_full.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_db1_full.mutable_schema() = db1_schema; + EXPECT_THAT(icing.GetSchema("db1"), + EqualsProto(expected_get_schema_result_proto_db1_full)); + + // Get db2 schema + GetSchemaResultProto expected_get_schema_result_proto_db2_full; + expected_get_schema_result_proto_db2_full.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_db2_full.mutable_schema() = db2_schema; + EXPECT_THAT(icing.GetSchema("db2"), + EqualsProto(expected_get_schema_result_proto_db2_full)); +} + TEST_F(IcingSearchEngineSchemaTest, SetSchemaNewIndexedStringPropertyTriggersIndexRestorationAndReturnsOk) { IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); @@ -2928,6 +3295,125 @@ EXPECT_THAT(icing.GetSchema(), EqualsProto(expected_get_schema_result_proto)); } +TEST_F(IcingSearchEngineSchemaTest, GetSchemaDatabaseOk) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + // Add a schema for db1: + SchemaTypeConfigProto db1_type = + SchemaTypeConfigBuilder() + .SetType("db1_type") + .SetDatabase("db1") + .AddProperty(PropertyConfigBuilder() + .SetName("a") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("b") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REQUIRED)) + .Build(); + + SchemaProto db1_schema = SchemaBuilder().AddType(db1_type).Build(); + SetSchemaResultProto set_schema_result = icing.SetSchema(db1_schema); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + SetSchemaResultProto expected_set_schema_result; + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_new_schema_types()->Add("db1_type"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + // Add a schema for db2: + SchemaTypeConfigProto db2_type = + SchemaTypeConfigBuilder() + .SetType("db2_type") + .SetDatabase("db2") + .AddProperty( + PropertyConfigBuilder() + .SetName("a") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("c") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REQUIRED)) + .Build(); + + SchemaProto db2_schema = SchemaBuilder().AddType(db2_type).Build(); + set_schema_result = icing.SetSchema(db2_schema); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + expected_set_schema_result = SetSchemaResultProto(); + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_new_schema_types()->Add("db2_type"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + // GetSchema per database + GetSchemaResultProto expected_get_schema_result_proto_db1; + expected_get_schema_result_proto_db1.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_db1.mutable_schema() = db1_schema; + EXPECT_THAT(icing.GetSchema("db1"), + EqualsProto(expected_get_schema_result_proto_db1)); + GetSchemaResultProto expected_get_schema_result_proto_db2; + expected_get_schema_result_proto_db2.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_db2.mutable_schema() = db2_schema; + EXPECT_THAT(icing.GetSchema("db2"), + EqualsProto(expected_get_schema_result_proto_db2)); + + // Get full schema + GetSchemaResultProto expected_get_schema_result_proto_full; + expected_get_schema_result_proto_full.mutable_status()->set_code( + StatusProto::OK); + *expected_get_schema_result_proto_full.mutable_schema() = + SchemaBuilder().AddType(db1_type).AddType(db2_type).Build(); + EXPECT_THAT(icing.GetSchema(), + EqualsProto(expected_get_schema_result_proto_full)); +} + +TEST_F(IcingSearchEngineSchemaTest, GetSchemaDatabaseNotFound) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + GetSchemaResultProto get_schema_result_proto = + icing.GetSchema("nonexistent_db"); + EXPECT_THAT(get_schema_result_proto.status(), + ProtoStatusIs(StatusProto::NOT_FOUND)); + EXPECT_THAT(get_schema_result_proto.status().message(), + HasSubstr("No schema found")); + + // Add a schema for db1: + SchemaProto db1_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_type") + .SetDatabase("db1") + .AddProperty(PropertyConfigBuilder() + .SetName("a") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("b") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + SetSchemaResultProto set_schema_result = icing.SetSchema(db1_schema); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + SetSchemaResultProto expected_set_schema_result; + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_new_schema_types()->Add("db1_type"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + get_schema_result_proto = icing.GetSchema("nonexistent_db"); + EXPECT_THAT(get_schema_result_proto.status(), + ProtoStatusIs(StatusProto::NOT_FOUND)); + EXPECT_THAT(get_schema_result_proto.status().message(), + HasSubstr("No schema found")); +} + TEST_F(IcingSearchEngineSchemaTest, GetSchemaTypeFailedPrecondition) { IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
diff --git a/icing/icing-search-engine_search_test.cc b/icing/icing-search-engine_search_test.cc index 72121ac..346bb19 100644 --- a/icing/icing-search-engine_search_test.cc +++ b/icing/icing-search-engine_search_test.cc
@@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <algorithm> #include <cstdint> #include <initializer_list> #include <limits> @@ -53,11 +54,11 @@ #include "icing/testing/common-matchers.h" #include "icing/testing/embedding-test-utils.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/testing/tmp-directory.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/snippet-helpers.h" namespace icing { @@ -107,7 +108,7 @@ std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); ICING_ASSERT_OK( - icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); + icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path)); } filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); } @@ -131,7 +132,12 @@ IcingSearchEngineOptions icing_options; icing_options.set_base_dir(GetTestBaseDir()); icing_options.set_document_store_namespace_id_fingerprint(true); - icing_options.set_use_new_qualified_id_join_index(true); + icing_options.set_enable_schema_database(true); + icing_options.set_enable_embedding_index(true); + icing_options.set_enable_scorable_properties(true); + icing_options.set_enable_embedding_quantization(true); + icing_options.set_enable_qualified_id_join_index_v3_and_delete_propagate_from( + true); return icing_options; } @@ -247,6 +253,25 @@ return result_uris; } +std::vector<int32_t> GetScoresFromSearchResults( + SearchResultProto& search_result_proto) { + std::vector<int32_t> result_scores; + for (int i = 0; i < search_result_proto.results_size(); i++) { + result_scores.push_back(search_result_proto.results(i).score()); + } + return result_scores; +} + +void AddSchemaTypeAliasMap(ScoringSpecProto* scoring_spec, + const std::string& alias_schema_type, + const std::vector<std::string>& schema_types) { + SchemaTypeAliasMapProto* alias_map_proto = + scoring_spec->add_schema_type_alias_map_protos(); + alias_map_proto->set_alias_schema_type(alias_schema_type); + alias_map_proto->mutable_schema_types()->Add(schema_types.begin(), + schema_types.end()); +} + TEST_F(IcingSearchEngineSearchTest, SearchReturnsValidResults) { IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); @@ -6014,6 +6039,245 @@ ElementsAre(EqualsProto(expected_result_google::protobuf))); } +TEST_F(IcingSearchEngineSearchTest, JoinAfterUpdatingParent) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty(PropertyConfigBuilder() + .SetName("firstName") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("lastName") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("emailAddress") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("personQualifiedId") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + DocumentProto person = + DocumentBuilder() + .SetKey("pkg$db/namespace", "person") + .SetSchema("Person") + .AddStringProperty("firstName", "first") + .AddStringProperty("lastName", "last") + .AddStringProperty("emailAddress", "email@gmail.com") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetScore(1) + .Build(); + DocumentProto email = + DocumentBuilder() + .SetKey("namespace", "email") + .SetSchema("Email") + .AddStringProperty("subject", "test subject") + .AddStringProperty("personQualifiedId", "pkg$db/namespace#person") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetScore(3) + .Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(person).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email).status(), ProtoIsOk()); + + // Parent SearchSpec + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query("firstName:first"); + + // JoinSpec + JoinSpecProto* join_spec = search_spec.mutable_join_spec(); + join_spec->set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec->set_child_property_expression("personQualifiedId"); + join_spec->set_aggregation_scoring_strategy( + JoinSpecProto::AggregationScoringStrategy::MAX); + JoinSpecProto::NestedSpecProto* nested_spec = + join_spec->mutable_nested_spec(); + SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec(); + nested_search_spec->set_term_match_type(TermMatchType::PREFIX); + nested_search_spec->set_query("subject:test"); + + *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec(); + *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance(); + + // Parent ScoringSpec + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + + // Parent ResultSpec + ResultSpecProto result_spec; + result_spec.set_num_per_page(1); + result_spec.set_max_joined_children_per_parent_to_return( + std::numeric_limits<int32_t>::max()); + + SearchResultProto expected_result; + expected_result.mutable_status()->set_code(StatusProto::OK); + SearchResultProto::ResultProto* result_proto = + expected_result.mutable_results()->Add(); + *result_proto->mutable_document() = person; + *result_proto->mutable_joined_results()->Add()->mutable_document() = email; + + EXPECT_THAT(icing.Search(search_spec, scoring_spec, result_spec), + EqualsSearchResultIgnoreStatsAndScores(expected_result)); + + // Put person document again to update the parent document. + DocumentProto updated_person = + DocumentBuilder() + .SetKey("pkg$db/namespace", "person") + .SetSchema("Person") + .AddStringProperty("firstName", "first updated") + .AddStringProperty("lastName", "last") + .AddStringProperty("emailAddress", "email@gmail.com") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetScore(1) + .Build(); + ASSERT_THAT(icing.Put(updated_person).status(), ProtoIsOk()); + + // Search again to verify join API still works. + *expected_result.mutable_results(0)->mutable_document() = updated_person; + EXPECT_THAT(icing.Search(search_spec, scoring_spec, result_spec), + EqualsSearchResultIgnoreStatsAndScores(expected_result)); +} + +TEST_F(IcingSearchEngineSearchTest, JoinAfterUpdatingChild) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty(PropertyConfigBuilder() + .SetName("firstName") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("lastName") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("emailAddress") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("personQualifiedId") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + DocumentProto person = + DocumentBuilder() + .SetKey("pkg$db/namespace", "person") + .SetSchema("Person") + .AddStringProperty("firstName", "first") + .AddStringProperty("lastName", "last") + .AddStringProperty("emailAddress", "email@gmail.com") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetScore(1) + .Build(); + DocumentProto email = + DocumentBuilder() + .SetKey("namespace", "email") + .SetSchema("Email") + .AddStringProperty("subject", "test subject") + .AddStringProperty("personQualifiedId", "pkg$db/namespace#person") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetScore(3) + .Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(person).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email).status(), ProtoIsOk()); + + // Parent SearchSpec + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query("firstName:first"); + + // JoinSpec + JoinSpecProto* join_spec = search_spec.mutable_join_spec(); + join_spec->set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec->set_child_property_expression("personQualifiedId"); + join_spec->set_aggregation_scoring_strategy( + JoinSpecProto::AggregationScoringStrategy::MAX); + JoinSpecProto::NestedSpecProto* nested_spec = + join_spec->mutable_nested_spec(); + SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec(); + nested_search_spec->set_term_match_type(TermMatchType::PREFIX); + nested_search_spec->set_query("subject:test"); + + *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec(); + *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance(); + + // Parent ScoringSpec + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + + // Parent ResultSpec + ResultSpecProto result_spec; + result_spec.set_num_per_page(1); + result_spec.set_max_joined_children_per_parent_to_return( + std::numeric_limits<int32_t>::max()); + + SearchResultProto expected_result; + expected_result.mutable_status()->set_code(StatusProto::OK); + SearchResultProto::ResultProto* result_proto = + expected_result.mutable_results()->Add(); + *result_proto->mutable_document() = person; + *result_proto->mutable_joined_results()->Add()->mutable_document() = email; + + EXPECT_THAT(icing.Search(search_spec, scoring_spec, result_spec), + EqualsSearchResultIgnoreStatsAndScores(expected_result)); + + // Put person document again to update the parent document. + DocumentProto updated_email = + DocumentBuilder() + .SetKey("namespace", "email") + .SetSchema("Email") + .AddStringProperty("subject", "test subject updated") + .AddStringProperty("personQualifiedId", "pkg$db/namespace#person") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetScore(3) + .Build(); + ASSERT_THAT(icing.Put(updated_email).status(), ProtoIsOk()); + + // Search again to verify join API still works. + *expected_result.mutable_results(0) + ->mutable_joined_results(0) + ->mutable_document() = updated_email; + EXPECT_THAT(icing.Search(search_spec, scoring_spec, result_spec), + EqualsSearchResultIgnoreStatsAndScores(expected_result)); +} + TEST_F(IcingSearchEngineSearchTest, JoinSnippet) { SchemaProto schema = SchemaBuilder() @@ -7621,6 +7885,124 @@ DoubleNear(0 + 0.5, kEps)); } +TEST_F(IcingSearchEngineSearchTest, EmbeddingSearchWithQuantizedProperty) { + constexpr float eps = 0.0001f; + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("embedding") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("embeddingQuantized") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH, + QUANTIZATION_TYPE_QUANTIZE_8_BIT) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + // If quantization is enabled, this vector will be quantized to {0, 1, 255}. + PropertyProto::VectorProto vector1 = CreateVector("my_model", {0, 1.45, 255}); + // If quantization is enabled, this vector will be quantized to {0, -2, -255}. + PropertyProto::VectorProto vector2 = + CreateVector("my_model", {0, -2.15, -255}); + + DocumentProto document_with_original_embedding = + DocumentBuilder() + .SetKey("icing", "uri0") + .SetSchema("Email") + .SetCreationTimestampMs(1) + .AddStringProperty("body", "foo") + .AddVectorProperty("embedding", vector1, vector2) + .Build(); + DocumentProto document_with_quantized_embedding = + DocumentBuilder() + .SetKey("icing", "uri1") + .SetSchema("Email") + .SetCreationTimestampMs(1) + .AddVectorProperty("embeddingQuantized", vector1, vector2) + .Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document_with_original_embedding).status(), + ProtoIsOk()); + ASSERT_THAT(icing.Put(document_with_quantized_embedding).status(), + ProtoIsOk()); + + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); + search_spec.set_embedding_query_metric_type( + SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT); + search_spec.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + // Add an embedding query with semantic scores: + // - [0 + 1.45 + 255 = 256.45, 0 - 2.15 - 255 = -257.15] for + // document_with_original_embedding + // - [0 + 1 + 255 = 256, 0 - 2 - 255 = -257] for + // document_with_quantized_embedding + *search_spec.add_embedding_query_vectors() = + CreateVector("my_model", {1, 1, 1}); + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + + // All embeddings should be matched for the range of [-1000, 1000]. + search_spec.set_query( + "semanticSearch(getEmbeddingParameter(0), -1000, 1000)"); + scoring_spec.set_advanced_scoring_expression( + "len(this.matchedSemanticScores(getEmbeddingParameter(0)))"); + scoring_spec.add_additional_advanced_scoring_expressions( + "sum(this.matchedSemanticScores(getEmbeddingParameter(0)))"); + SearchResultProto results = icing.Search(search_spec, scoring_spec, + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(2)); + // Check results for document_with_quantized_embedding. + EXPECT_THAT(results.results(0).document(), + EqualsProto(document_with_quantized_embedding)); + EXPECT_THAT(results.results(0).score(), 2); + EXPECT_THAT(results.results(0).additional_scores(0), + DoubleNear(256 + (-257), eps)); + // Check results for document_with_original_embedding. + EXPECT_THAT(results.results(1).document(), + EqualsProto(document_with_original_embedding)); + EXPECT_THAT(results.results(1).score(), 2); + EXPECT_THAT(results.results(1).additional_scores(0), + DoubleNear(256.45 + (-257.15), eps)); + + // Only one embedding (with score of 256.45, or 256 if quantized) should be + // matched for the range of [0, 1000]. + search_spec.set_query("semanticSearch(getEmbeddingParameter(0), 0, 1000)"); + scoring_spec.set_advanced_scoring_expression( + "len(this.matchedSemanticScores(getEmbeddingParameter(0)))"); + scoring_spec.add_additional_advanced_scoring_expressions( + "sum(this.matchedSemanticScores(getEmbeddingParameter(0)))"); + results = icing.Search(search_spec, scoring_spec, + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(2)); + // Check results for document_with_quantized_embedding. + EXPECT_THAT(results.results(0).document(), + EqualsProto(document_with_quantized_embedding)); + EXPECT_THAT(results.results(0).score(), 1); + EXPECT_THAT(results.results(0).additional_scores(0), DoubleNear(256, eps)); + // Check results for document_with_original_embedding. + EXPECT_THAT(results.results(1).document(), + EqualsProto(document_with_original_embedding)); + EXPECT_THAT(results.results(1).score(), 1); + EXPECT_THAT(results.results(1).additional_scores(0), DoubleNear(256.45, eps)); +} + TEST_F(IcingSearchEngineSearchTest, AdditionalScoresOnlyAllowedInAdvancedScoring) { SchemaProto schema = @@ -7823,6 +8205,1930 @@ EXPECT_THAT(results.results(), IsEmpty()); } +TEST_F(IcingSearchEngineSearchTest, SearchWithUriFilters) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + + // Add three documents with different URIs, namespaces, and "body" values. + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace1", "uri1") + .SetSchema("Message") + .AddStringProperty("body", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + DocumentProto document_two = + DocumentBuilder() + .SetKey("namespace2", "uri2") + .SetSchema("Message") + .AddStringProperty("body", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + DocumentProto document_three = + DocumentBuilder() + .SetKey("namespace2", "uri3") + .SetSchema("Message") + .AddStringProperty("body", "bar") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(document_three).status(), ProtoIsOk()); + + // Create a search spec with an empty query and uri filters that should match + // document1 and document3. + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query(""); + NamespaceDocumentUriGroup* namespace1_uris = + search_spec.add_document_uri_filters(); + namespace1_uris->set_namespace_("namespace1"); + namespace1_uris->add_document_uris("uri1"); + NamespaceDocumentUriGroup* namespace2_uris = + search_spec.add_document_uri_filters(); + namespace2_uris->set_namespace_("namespace2"); + namespace2_uris->add_document_uris("uri3"); + + // Check results + SearchResultProto results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(2)); + EXPECT_THAT(results.results(0).document(), EqualsProto(document_three)); + EXPECT_THAT(results.results(1).document(), EqualsProto(document_one)); + + // Now set the query to "foo" and check that only document_one is returned. + search_spec.set_query("foo"); + results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + EXPECT_THAT(results.results(0).document(), EqualsProto(document_one)); +} + +TEST_F(IcingSearchEngineSearchTest, + NamespacesInUriFiltersMustAppearInNamespaceFilters) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + + // Add three documents with different URIs and namespaces. + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace1", "uri1") + .SetSchema("Message") + .AddStringProperty("body", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + DocumentProto document_two = + DocumentBuilder() + .SetKey("namespace2", "uri2") + .SetSchema("Message") + .AddStringProperty("body", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + DocumentProto document_three = + DocumentBuilder() + .SetKey("namespace2", "uri3") + .SetSchema("Message") + .AddStringProperty("body", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(document_three).status(), ProtoIsOk()); + + // Create a search spec with uri filters that include a namespace that is not + // in the namespace filters. + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query("foo"); + NamespaceDocumentUriGroup* namespace1_uris = + search_spec.add_document_uri_filters(); + namespace1_uris->set_namespace_("namespace1"); + namespace1_uris->add_document_uris("uri1"); + NamespaceDocumentUriGroup* namespace2_uris = + search_spec.add_document_uri_filters(); + namespace2_uris->set_namespace_("namespace2"); + namespace2_uris->add_document_uris("uri3"); + search_spec.add_namespace_filters("namespace1"); + + // Check results + SearchResultProto results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status().message(), + HasSubstr("does not appear in the namespace filter")); +} + +TEST_F(IcingSearchEngineSearchTest, UriFiltersWorkWithPropertyFilters) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreateEmailSchema()).status(), ProtoIsOk()); + + // Add three documents with different URIs. + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("Email") + .AddStringProperty("subject", "foo") + .AddStringProperty("body", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + DocumentProto document_two = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("Email") + .AddStringProperty("subject", "foo") + .AddStringProperty("body", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + DocumentProto document_three = + DocumentBuilder() + .SetKey("namespace", "uri3") + .SetSchema("Email") + .AddStringProperty("subject", "baz") + .AddStringProperty("body", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + ASSERT_THAT(icing.Put(document_three).status(), ProtoIsOk()); + + // Create a search spec with uri filters that should only match document1 and + // document3, and a property filter that only searches for "subject". As a + // result, only document1 should be returned. + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query("foo"); + NamespaceDocumentUriGroup* uris = search_spec.add_document_uri_filters(); + uris->set_namespace_("namespace"); + uris->add_document_uris("uri1"); + uris->add_document_uris("uri3"); + TypePropertyMask* property_filters = search_spec.add_type_property_filters(); + property_filters->set_schema_type("Email"); + property_filters->add_paths("subject"); + + // Check results + SearchResultProto results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + EXPECT_THAT(results.results(0).document(), EqualsProto(document_one)); +} + +TEST_F(IcingSearchEngineSearchTest, SearchWithRankingByScorableProperty) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "John") + .AddDoubleProperty("income", 10000, 20000) + .Build(); + DocumentProto document1 = DocumentBuilder() + .SetKey("icing", "person1") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "Johnson") + .AddDoubleProperty("income", 5000, 30000) + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("icing", "person2") + .SetSchema("Person") + .SetScore(20) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "Jack") + .Build(); + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document0).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document1).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document2).status(), ProtoIsOk()); + + // "J" will match all 3 documents + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query("J"); + + // Ranking by documentScore and income. + ScoringSpecProto scoring_spec_by_income = GetDefaultScoringSpec(); + scoring_spec_by_income.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec_by_income.set_advanced_scoring_expression( + "this.documentScore() + sum(getScorableProperty(\"Person\", " + "\"income\"))"); + scoring_spec_by_income.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + AddSchemaTypeAliasMap(&scoring_spec_by_income, "Person", {"Person"}); + + int expected_person0_score = 10 + 10000 + 20000; + int expected_person1_score = 10 + 5000 + 30000; + int expected_person2_score = 20 + 0; + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec_by_income, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + + // Verify that the search results are ranked as expected. + EXPECT_THAT(GetUrisFromSearchResults(search_result_proto), + ElementsAre("person1", "person0", "person2")); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(expected_person1_score, expected_person0_score, + expected_person2_score)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithRankingByScorablePropertyWithFeatureDisabled) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + + // Ranking by documentScore and income. + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "this.documentScore() + sum(getScorableProperty(\"Person\", " + "\"income\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "Person", {"Person"}); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code( + StatusProto::INVALID_ARGUMENT); + expected_search_result_proto.mutable_status()->set_message( + "SCORABLE_PROPERTY_RANKING feature is not enabled."); + + SearchResultProto actual_results = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithRankingByScorableProperty_WithNestedSchemas) { + SchemaProto schema_proto = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty(PropertyConfigBuilder() + .SetName("id") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("age") + .SetDataType(PropertyConfigProto::DataType::INT64) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("isStarred") + .SetDataType(PropertyConfigProto::DataType::BOOLEAN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeDocument( + "Person", /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("receiver") + .SetDataTypeDocument( + "Person", /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("scoreInt64") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + DocumentProto document = + DocumentBuilder() + .SetKey("namespace", "email_id0") + .SetSchema("Email") + .SetScore(5) + .AddInt64Property("scoreInt64", 1, 2, 3) + .AddDocumentProperty("receiver", + DocumentBuilder() + .SetKey("namespace", "receiver0") + .SetSchema("Person") + .AddInt64Property("age", 30) + .AddDoubleProperty("income", 10, 20) + .AddBooleanProperty("isStarred", true) + .Build(), + DocumentBuilder() + .SetKey("namespace", "receiver1") + .SetSchema("Person") + .AddInt64Property("age", 35) + .AddDoubleProperty("income", 11, 12) + .AddBooleanProperty("isStarred", false) + .Build()) + .AddDocumentProperty("sender", + DocumentBuilder() + .SetKey("namespace", "sender0") + .SetSchema("Person") + .AddInt64Property("age", 50) + .AddDoubleProperty("income", 21, 22) + .AddBooleanProperty("isStarred", false) + .Build()) + .Build(); + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema_proto).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "this.documentScore() + " + "sum(getScorableProperty(\"Email\", \"scoreInt64\")) + " + "sum(getScorableProperty(\"Email\", \"sender.income\")) + " + "5 * sum(getScorableProperty(\"Email\", \"receiver.income\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "Person", {"Person"}); + AddSchemaTypeAliasMap(&scoring_spec, "Email", {"Email"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + int expected_score = 5 + (1 + 2 + 3) + (21 + 22) + 5 * (10 + 20 + 11 + 12); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + + EXPECT_THAT(GetUrisFromSearchResults(search_result_proto), + ElementsAre("email_id0")); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(expected_score)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithRankingByScorableProperty_WithInvalidPropertyName) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "John") + .AddDoubleProperty("income", 10000, 20000) + .Build(); + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document0).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "this.documentScore() + sum(getScorableProperty(\"Person\", " + "\"not_exist\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "Person", {"Person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code( + StatusProto::INVALID_ARGUMENT); + expected_search_result_proto.mutable_status()->set_message( + "'not_exist' is not defined as a scorable property under schema type 0"); + + SearchResultProto actual_search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT( + actual_search_result_proto, + EqualsSearchResultIgnoreStatsAndScores(expected_search_result_proto)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithRankingByScorableProperty_ScorablePropertyDisabled) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "John") + .AddDoubleProperty("income", 10000, 20000) + .Build(); + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_scorable_properties(false); + IcingSearchEngine icing(options, GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document0).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "this.documentScore() + sum(getScorableProperty(\"Person\", " + "\"not_exist\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "Person", {"Person"}); + + SearchResultProto results = icing.Search(search_spec, scoring_spec, + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoStatusIs(StatusProto::INVALID_ARGUMENT)); +} + +TEST_F(IcingSearchEngineSearchTest, JoinSearchWithRankingByScorableProperty) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("customizedScore") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("isStarred") + .SetDataType(PropertyConfigProto::DataType::BOOLEAN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("CallLogSignalDoc") + .AddProperty(PropertyConfigBuilder() + .SetName("personQualifiedId") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("rfsScore") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("SmsLogSignalDoc") + .AddProperty(PropertyConfigBuilder() + .SetName("personQualifiedId") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("rfsScore") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + // John has 2 call logs and 1 sms log to join. + DocumentProto person_john_doc = + DocumentBuilder() + .SetKey("namespace", "person_john_id") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("Person") + .SetScore(10) + .AddBooleanProperty("isStarred", true) + .AddDoubleProperty("customizedScore", 30) + .AddStringProperty("name", "John") + .Build(); + // Kevin has 2 call logs and 1 sms log to join. + DocumentProto person_kevin_doc = + DocumentBuilder() + .SetKey("namespace", "person_kevin_id") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("Person") + .SetScore(20) + .AddBooleanProperty("isStarred", false) + .AddDoubleProperty("customizedScore", 40) + .AddStringProperty("name", "Kevin") + .Build(); + // Tim has no signal docs to join. + DocumentProto person_tim_doc = + DocumentBuilder() + .SetKey("namespace", "person_tim_id") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("Person") + .SetScore(50) + .AddDoubleProperty("customizedScore", 60) + .AddStringProperty("name", "Tim") + .Build(); + DocumentProto john_call_log_doc1 = + DocumentBuilder() + .SetKey("namespace", "call_log_id1") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("CallLogSignalDoc") + .SetScore(5) + .AddDoubleProperty("rfsScore", 100, 200) + .AddStringProperty("personQualifiedId", "namespace#person_john_id") + .Build(); + DocumentProto john_call_log_doc2 = + DocumentBuilder() + .SetKey("namespace", "call_log_id2") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("CallLogSignalDoc") + .SetScore(5) + .AddDoubleProperty("rfsScore", 200, 300) + .AddStringProperty("personQualifiedId", "namespace#person_john_id") + .Build(); + DocumentProto kevin_call_log_doc1 = + DocumentBuilder() + .SetKey("namespace", "call_log_id3") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("CallLogSignalDoc") + .SetScore(5) + .AddDoubleProperty("rfsScore", 300, 400) + .AddStringProperty("personQualifiedId", "namespace#person_kevin_id") + .Build(); + DocumentProto kevin_call_log_doc2 = + DocumentBuilder() + .SetKey("namespace", "call_log_id4") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("CallLogSignalDoc") + .SetScore(5) + .AddDoubleProperty("rfsScore", 500, 800) + .AddStringProperty("personQualifiedId", "namespace#person_kevin_id") + .Build(); + DocumentProto john_sms_log_doc1 = + DocumentBuilder() + .SetKey("namespace", "sms_log_id1") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("SmsLogSignalDoc") + .SetScore(5) + .AddDoubleProperty("rfsScore", 1000, 2000) + .AddStringProperty("personQualifiedId", "namespace#person_john_id") + .Build(); + DocumentProto kevin_sms_log_doc1 = + DocumentBuilder() + .SetKey("namespace", "sms_log_id2") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("SmsLogSignalDoc") + .SetScore(5) + .AddDoubleProperty("rfsScore", 2000, 3000) + .AddStringProperty("personQualifiedId", "namespace#person_kevin_id") + .Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(person_john_doc).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(person_kevin_doc).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(person_tim_doc).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(john_call_log_doc1).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(john_call_log_doc2).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(kevin_call_log_doc1).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(kevin_call_log_doc2).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(john_sms_log_doc1).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(kevin_sms_log_doc1).status(), ProtoIsOk()); + + // all people will be matched. + SearchSpecProto parent_search_spec; + parent_search_spec.add_schema_type_filters("Person"); + + // Child scoring spec: ranking by call logs and sms logs' rfsScore. + ScoringSpecProto child_scoring_spec = GetDefaultScoringSpec(); + child_scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + child_scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"CallLogSignalDoc\", \"rfsScore\")) + " + "sum(getScorableProperty(\"SmsLogSignalDoc\", \"rfsScore\"))"); + child_scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + JoinSpecProto* join_spec = parent_search_spec.mutable_join_spec(); + join_spec->set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec->set_child_property_expression("personQualifiedId"); + JoinSpecProto::NestedSpecProto* nested_spec = + join_spec->mutable_nested_spec(); + *nested_spec->mutable_scoring_spec() = child_scoring_spec; + nested_spec->mutable_search_spec()->add_schema_type_filters( + "CallLogSignalDoc"); + nested_spec->mutable_search_spec()->add_schema_type_filters( + "SmsLogSignalDoc"); + + // Parent ScoringSpec + ScoringSpecProto parent_scoring_spec = GetDefaultScoringSpec(); + parent_scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + parent_scoring_spec.set_advanced_scoring_expression( + "this.documentScore() + " + "sum(getScorableProperty(\"Person\", \"customizedScore\")) + " + "20 * sum(getScorableProperty(\"Person\", \"isStarred\")) + " + "maxOrDefault(this.childrenRankingSignals(), 0)"); + AddSchemaTypeAliasMap(&parent_scoring_spec, "Person", {"Person"}); + AddSchemaTypeAliasMap(&parent_scoring_spec, "CallLogSignalDoc", + {"CallLogSignalDoc"}); + AddSchemaTypeAliasMap(&parent_scoring_spec, "SmsLogSignalDoc", + {"SmsLogSignalDoc"}); + parent_scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + int person_john_expected_score = + 10 + 30 + 20 * /*isStarred=*/1 + + std::max({100 + 200, 300 + 400, 1000 + 2000}); + int person_kevin_expected_score = + 20 + 40 + 20 * /*isStarred=*/0 + + std::max({300 + 400, 500 + 800, 2000 + 3000}); + int person_tim_expected_score = + 50 + 60 + 20 * /*isStarred=*/0 + /*child_ranking_signal=*/0; + + SearchResultProto search_result_proto = + icing.Search(parent_search_spec, parent_scoring_spec, + ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + + // Verify that the search results are ranked as expected. + EXPECT_THAT( + GetUrisFromSearchResults(search_result_proto), + ElementsAre("person_kevin_id", "person_john_id", "person_tim_id")); + EXPECT_THAT( + GetScoresFromSearchResults(search_result_proto), + ElementsAre(person_kevin_expected_score, person_john_expected_score, + person_tim_expected_score)); +} + +TEST_F(IcingSearchEngineSearchTest, + JoinSearchWithRankingByScorableProperty_ScorablePropertyDisabled) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("CallLogSignalDoc") + .AddProperty(PropertyConfigBuilder() + .SetName("personQualifiedId") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("rfsScore") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + // John has 2 call logs and 1 sms log to join. + DocumentProto person_john_doc = + DocumentBuilder() + .SetKey("namespace", "person_john_id") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("Person") + .SetScore(10) + .AddStringProperty("name", "John") + .Build(); + DocumentProto john_call_log_doc1 = + DocumentBuilder() + .SetKey("namespace", "call_log_id1") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .SetSchema("CallLogSignalDoc") + .SetScore(5) + .AddDoubleProperty("rfsScore", 100, 200) + .AddStringProperty("personQualifiedId", "namespace#person_john_id") + .Build(); + + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + options.set_enable_scorable_properties(false); + IcingSearchEngine icing(options, GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(person_john_doc).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(john_call_log_doc1).status(), ProtoIsOk()); + + // all people will be matched. + SearchSpecProto parent_search_spec; + parent_search_spec.add_schema_type_filters("Person"); + + // Child scoring spec: ranking by call logs and sms logs' rfsScore. + ScoringSpecProto child_scoring_spec = GetDefaultScoringSpec(); + child_scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + child_scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"CallLogSignalDoc\", \"rfsScore\"))"); + AddSchemaTypeAliasMap(&child_scoring_spec, "Person", {"Person"}); + AddSchemaTypeAliasMap(&child_scoring_spec, "CallLogSignalDoc", + {"CallLogSignalDoc"}); + + JoinSpecProto* join_spec = parent_search_spec.mutable_join_spec(); + join_spec->set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec->set_child_property_expression("personQualifiedId"); + JoinSpecProto::NestedSpecProto* nested_spec = + join_spec->mutable_nested_spec(); + *nested_spec->mutable_scoring_spec() = child_scoring_spec; + nested_spec->mutable_search_spec()->add_schema_type_filters( + "CallLogSignalDoc"); + + // Parent ScoringSpec + ScoringSpecProto parent_scoring_spec = GetDefaultScoringSpec(); + parent_scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + parent_scoring_spec.set_advanced_scoring_expression( + "this.documentScore() + " + "maxOrDefault(this.childrenRankingSignals(), 0)"); + AddSchemaTypeAliasMap(&parent_scoring_spec, "Person", {"Person"}); + AddSchemaTypeAliasMap(&parent_scoring_spec, "CallLogSignalDoc", + {"CallLogSignalDoc"}); + + SearchResultProto results = + icing.Search(parent_search_spec, parent_scoring_spec, + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoStatusIs(StatusProto::INVALID_ARGUMENT)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithScorableProperty_DocumentsFromMultipleMatchedSchemaTypes) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("pkg1/db1/person") + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("pkg2/db2/person") + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("pkg1/db1/person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddDoubleProperty("income", 10000, 20000) + .Build(); + DocumentProto document1 = DocumentBuilder() + .SetKey("icing", "person1") + .SetSchema("pkg2/db2/person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddDoubleProperty("income", 30000, 40000) + .Build(); + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document0).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document1).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + + // Ranking by income. + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Person\", \"income\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "Person", + {"pkg1/db1/person", "pkg2/db2/person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + int expected_person0_score = 10000 + 20000; + int expected_person1_score = 30000 + 40000; + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + + // Verify that the search results are ranked as expected. + EXPECT_THAT(GetUrisFromSearchResults(search_result_proto), + ElementsAre("person1", "person0")); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(expected_person1_score, expected_person0_score)); +} + +TEST_F(IcingSearchEngineSearchTest, + InvalidScoringSpecAliasMapWithEmptyAliasSchemaType) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + AddSchemaTypeAliasMap(&scoring_spec, /*alias_schema_type=*/"", {"Person"}); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code( + StatusProto::INVALID_ARGUMENT); + expected_search_result_proto.mutable_status()->set_message( + "SchemaTypeAliasMapProto contains alias_schema_type with empty string"); + EXPECT_THAT( + icing.Search(search_spec, scoring_spec, + ResultSpecProto::default_instance()), + EqualsSearchResultIgnoreStatsAndScores(expected_search_result_proto)); +} + +TEST_F(IcingSearchEngineSearchTest, + InvalidScoringSpecAliasMapWithEmptySchemaTypes) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + AddSchemaTypeAliasMap(&scoring_spec, /*alias_schema_type=*/"Person", {}); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code( + StatusProto::INVALID_ARGUMENT); + expected_search_result_proto.mutable_status()->set_message( + "SchemaTypeAliasMapProto contains empty schema_types for " + "alias_schema_type: Person"); + EXPECT_THAT( + icing.Search(search_spec, scoring_spec, + ResultSpecProto::default_instance()), + EqualsSearchResultIgnoreStatsAndScores(expected_search_result_proto)); +} + +TEST_F(IcingSearchEngineSearchTest, + InvalidScoringSpecAliasMapWithDuplicatedAliasSchemaTypes) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + AddSchemaTypeAliasMap(&scoring_spec, /*alias_schema_type=*/"aliasPerson", + {"PersonA"}); + AddSchemaTypeAliasMap(&scoring_spec, /*alias_schema_type=*/"aliasPerson", + {"PersonB"}); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code( + StatusProto::INVALID_ARGUMENT); + expected_search_result_proto.mutable_status()->set_message( + "SchemaTypeAliasMapProto contains multiple entries with the same " + "alias_schema_type: aliasPerson"); + EXPECT_THAT( + icing.Search(search_spec, scoring_spec, + ResultSpecProto::default_instance()), + EqualsSearchResultIgnoreStatsAndScores(expected_search_result_proto)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithScorablePropertyWithUpdatePropertyAsScorable) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "John") + .AddDoubleProperty("income", 20000) + .Build(); + DocumentProto document1 = DocumentBuilder() + .SetKey("icing", "person1") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "John") + .AddDoubleProperty("income", 10000) + .Build(); + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document0).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document1).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + AddSchemaTypeAliasMap(&scoring_spec, "Person", {"Person"}); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Person\", \"income\"))"); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code( + StatusProto::INVALID_ARGUMENT); + expected_search_result_proto.mutable_status()->set_message( + "'income' is not defined as a scorable property under schema type 0"); + + SearchResultProto actual_search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT( + actual_search_result_proto, + EqualsSearchResultIgnoreStatsAndScores(expected_search_result_proto)); + + // Update the schema to set Person.income as a scorable property. + SchemaProto new_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + EXPECT_THAT(icing.SetSchema(new_schema).status(), ProtoIsOk()); + + actual_search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(actual_search_result_proto.status(), ProtoIsOk()); + + // Check the search results. + EXPECT_THAT(GetUrisFromSearchResults(actual_search_result_proto), + ElementsAre("person0", "person1")); + EXPECT_THAT(GetScoresFromSearchResults(actual_search_result_proto), + ElementsAre(20000, 10000)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithScorablePropertyWithFlipPropertyScorability) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "John") + .AddDoubleProperty("income", 20000) + .Build(); + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document0).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + AddSchemaTypeAliasMap(&scoring_spec, "Person", {"Person"}); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Person\", \"income\"))"); + + // Update the schema to set Person.income as not scorable. It would erase the + // data the scorable property cache. + schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_DISABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code( + StatusProto::INVALID_ARGUMENT); + expected_search_result_proto.mutable_status()->set_message( + "'income' is not defined as a scorable property under schema type 0"); + SearchResultProto actual_search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT( + actual_search_result_proto, + EqualsSearchResultIgnoreStatsAndScores(expected_search_result_proto)); + + // Update the schema to set Person.income as scorable again. It would + // re-populate the scorable property cache. + schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + + // Check the search results. + EXPECT_THAT(GetUrisFromSearchResults(search_result_proto), + ElementsAre("person0")); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(20000)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithScorablePropertyWithAddNewPropertyAsScorable) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(5) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "John") + .Build(); + DocumentProto document1 = DocumentBuilder() + .SetKey("icing", "person1") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "John") + .Build(); + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document0).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document1).status(), ProtoIsOk()); + + // Update the schema to add Person.income as a scorable property. + // It would populate the scorable property cache with the default values. + SchemaProto new_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + EXPECT_THAT(icing.SetSchema(new_schema).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + AddSchemaTypeAliasMap(&scoring_spec, "Person", {"Person"}); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "this.documentScore() +sum(getScorableProperty(\"Person\", \"income\"))"); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + + // Check the search results. + EXPECT_THAT(GetUrisFromSearchResults(search_result_proto), + ElementsAre("person1", "person0")); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(10, 5)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithScorablePropertyWithReorderScorableProperties) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("age") + .SetDataType(PropertyConfigProto::DataType::INT64) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(5) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "John") + .AddInt64Property("age", 20) + .AddDoubleProperty("income", 10000, 20000) + .Build(); + DocumentProto document1 = DocumentBuilder() + .SetKey("icing", "person1") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddStringProperty("name", "Kevin") + .AddInt64Property("age", 25) + .AddDoubleProperty("income", 30000, 40000) + .Build(); + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document0).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document1).status(), ProtoIsOk()); + + // Update the schema change the order of the properties "age" and "income". + SchemaProto new_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("age") + .SetDataType(PropertyConfigProto::DataType::INT64) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + EXPECT_THAT(icing.SetSchema(new_schema).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + AddSchemaTypeAliasMap(&scoring_spec, "Person", {"Person"}); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "this.documentScore() + sum(getScorableProperty(\"Person\", \"income\")) " + "+ sum(getScorableProperty(\"Person\", \"age\"))"); + int expected_score0 = 5 + (10000 + 20000) + 20; + int expected_score1 = 10 + (30000 + 40000) + 25; + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + + // Check the search results. + EXPECT_THAT(GetUrisFromSearchResults(search_result_proto), + ElementsAre("person1", "person0")); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(expected_score1, expected_score0)); +} + +TEST_F(IcingSearchEngineSearchTest, + SearchWithScorablePropertyWithUpdatePropertyAsScorableInNestedSchema) { + SchemaProto schema_proto = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty(PropertyConfigBuilder() + .SetName("id") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_DISABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeDocument( + "Person", /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("receiver") + .SetDataTypeDocument( + "Person", /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + DocumentProto document = + DocumentBuilder() + .SetKey("namespace", "email_id0") + .SetSchema("Email") + .AddDocumentProperty("receiver", + DocumentBuilder() + .SetKey("namespace", "receiver0") + .SetSchema("Person") + .AddDoubleProperty("income", 10, 20) + .Build(), + DocumentBuilder() + .SetKey("namespace", "receiver1") + .SetSchema("Person") + .AddDoubleProperty("income", 11, 12) + .Build()) + .AddDocumentProperty("sender", + DocumentBuilder() + .SetKey("namespace", "sender0") + .SetSchema("Person") + .AddDoubleProperty("income", 21, 22) + .Build()) + .Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + EXPECT_THAT(icing.Initialize().status(), ProtoIsOk()); + EXPECT_THAT(icing.SetSchema(schema_proto).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(document).status(), ProtoIsOk()); + + // Update the 'Person' schema by setting Person.income as scorable. + // It would trigger the re-generation of the scorable property cache for the + // the schema 'Email', as it is a parent schema of 'Person'. + schema_proto = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty(PropertyConfigBuilder() + .SetName("id") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeDocument( + "Person", /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("receiver") + .SetDataTypeDocument( + "Person", /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + EXPECT_THAT(icing.SetSchema(schema_proto).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + AddSchemaTypeAliasMap(&scoring_spec, "Email", {"Email"}); + scoring_spec.set_rank_by( + ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); + scoring_spec.set_advanced_scoring_expression( + "sum(getScorableProperty(\"Email\", \"sender.income\")) + " + "sum(getScorableProperty(\"Email\", \"receiver.income\"))"); + int expected_score = (21 + 22) + (10 + 20 + 11 + 12); + + SearchResultProto search_result_proto = icing.Search( + search_spec, scoring_spec, ResultSpecProto::default_instance()); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + + EXPECT_THAT(GetUrisFromSearchResults(search_result_proto), + ElementsAre("email_id0")); + EXPECT_THAT(GetScoresFromSearchResults(search_result_proto), + ElementsAre(expected_score)); +} + +TEST_F(IcingSearchEngineSearchTest, MatchScoreExpression) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + + // Add three documents with different document scores and "body" values. + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("Message") + .AddStringProperty("body", "foo") + .SetScore(2) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto document_two = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("Message") + .AddStringProperty("body", "foo") + .SetScore(3) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto document_three = + DocumentBuilder() + .SetKey("namespace", "uri3") + .SetSchema("Message") + .AddStringProperty("body", "bar") + .SetScore(4) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(document_three).status(), ProtoIsOk()); + + // Get documents with a document score in [3, 4], which matches document 2 + // and 3. + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); + search_spec.add_enabled_features( + std::string(kMatchScoreExpressionFunctionFeature)); + search_spec.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + search_spec.set_query("matchScoreExpression(\"this.documentScore()\", 3, 4)"); + SearchResultProto results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(2)); + EXPECT_THAT(results.results(0).document(), EqualsProto(document_three)); + EXPECT_THAT(results.results(1).document(), EqualsProto(document_two)); + + // Get documents with a document score in [3, 4] AND a "foo" body, which + // matches document 2. + search_spec.set_query( + "foo AND matchScoreExpression(\"this.documentScore()\", 3, 4)"); + results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + EXPECT_THAT(results.results(0).document(), EqualsProto(document_two)); + + // Get documents with a document score in [3, 4] OR a "foo" body, which + // matches all documents. + search_spec.set_query( + "foo OR matchScoreExpression(\"this.documentScore()\", 3, 4)"); + results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(3)); + EXPECT_THAT(results.results(0).document(), EqualsProto(document_three)); + EXPECT_THAT(results.results(1).document(), EqualsProto(document_two)); + EXPECT_THAT(results.results(2).document(), EqualsProto(document_one)); +} + +TEST_F(IcingSearchEngineSearchTest, MatchScoreExpressionNotSupportedFunctions) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); + search_spec.add_enabled_features( + std::string(kMatchScoreExpressionFunctionFeature)); + search_spec.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + + search_spec.set_query( + "matchScoreExpression(\"len(this.childrenRankingSignals())\", 0, 1)"); + SearchResultProto results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status().message(), + HasSubstr("childrenRankingSignals must only be used with join")); + EXPECT_THAT(results.status().message(), + HasSubstr("matchScoreExpression: Failed to handle")); + + search_spec.set_query( + "matchScoreExpression(\"sum(this.matchedSemanticScores(" + "getEmbeddingParameter(0)))\", 0, 1)"); + results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status().message(), + HasSubstr("matchedSemanticScores function is not available")); + EXPECT_THAT(results.status().message(), + HasSubstr("matchScoreExpression: Failed to handle")); + + search_spec.set_query( + "matchScoreExpression(\"sum(this.propertyWeights())\", 0, 1)"); + results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status().message(), + HasSubstr("propertyWeights function is not available")); + EXPECT_THAT(results.status().message(), + HasSubstr("matchScoreExpression: Failed to handle")); + + search_spec.set_query( + "matchScoreExpression(\"this.relevanceScore()\", 0, 1)"); + results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status().message(), + HasSubstr("relevanceScore function is not available")); + EXPECT_THAT(results.status().message(), + HasSubstr("matchScoreExpression: Failed to handle")); +} + +TEST_F(IcingSearchEngineSearchTest, MatchScoreExpressionNotEnabled) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); + search_spec.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + search_spec.set_query("matchScoreExpression(\"this.documentScore()\", 3, 4)"); + SearchResultProto results = icing.Search(search_spec, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(results.status().message(), + HasSubstr("Attempted use of unenabled feature")); +} + +TEST_F(IcingSearchEngineSearchTest, JoinWithMatchScoreExpression) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("personQualifiedId") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // person1 has children email1 and email2. + DocumentProto person1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "person1") + .SetSchema("Person") + .AddStringProperty("name", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + // person2 has a single child email3 + DocumentProto person2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "person2") + .SetSchema("Person") + .AddStringProperty("name", "bar") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + // person3 has no child. + DocumentProto person3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "person3") + .SetSchema("Person") + .AddStringProperty("name", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + DocumentProto email1 = + DocumentBuilder() + .SetKey("namespace", "email1") + .SetSchema("Email") + .AddStringProperty("subject", "test subject 1") + .AddStringProperty("personQualifiedId", "pkg$db/namespace#person1") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto email2 = + DocumentBuilder() + .SetKey("namespace", "email2") + .SetSchema("Email") + .AddStringProperty("subject", "test subject 2") + .AddStringProperty("personQualifiedId", "pkg$db/namespace#person1") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto email3 = + DocumentBuilder() + .SetKey("namespace", "email3") + .SetSchema("Email") + .AddStringProperty("subject", "test subject 3") + .AddStringProperty("personQualifiedId", "pkg$db/namespace#person2") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(person1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(person2).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(person3).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email2).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email3).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); + search_spec.add_enabled_features( + std::string(kMatchScoreExpressionFunctionFeature)); + search_spec.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + + // JoinSpec + JoinSpecProto* join_spec = search_spec.mutable_join_spec(); + join_spec->set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec->set_child_property_expression("personQualifiedId"); + JoinSpecProto::NestedSpecProto* nested_spec = + join_spec->mutable_nested_spec(); + SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec(); + nested_search_spec->set_term_match_type(TermMatchType::PREFIX); + nested_search_spec->set_query("subject:test"); + *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec(); + *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance(); + + // Get all parent documents that have at least 1 join children, which should + // match person1 and person2, since person3 has no children. + search_spec.set_query( + "matchScoreExpression(\"len(this.childrenRankingSignals())\", 1)"); + ResultSpecProto result_spec; + result_spec.set_max_joined_children_per_parent_to_return( + std::numeric_limits<int32_t>::max()); + SearchResultProto results = + icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + ASSERT_THAT(results.status(), ProtoIsOk()); + ASSERT_THAT(results.results(), SizeIs(2)); + EXPECT_THAT(results.results(0).document().uri(), Eq("person2")); + EXPECT_THAT(results.results(0).joined_results_size(), Eq(1)); + EXPECT_THAT(results.results(0).joined_results(0).document().uri(), + Eq("email3")); + EXPECT_THAT(results.results(1).document().uri(), Eq("person1")); + EXPECT_THAT(results.results(1).joined_results_size(), Eq(2)); + EXPECT_THAT(results.results(1).joined_results(0).document().uri(), + Eq("email2")); + EXPECT_THAT(results.results(1).joined_results(1).document().uri(), + Eq("email1")); + + // Get all parent documents with "foo" name and join with email documents, + // which should match person1 and person3. + search_spec.set_query("foo"); + results = icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + ASSERT_THAT(results.status(), ProtoIsOk()); + ASSERT_THAT(results.results(), SizeIs(2)); + EXPECT_THAT(results.results(0).document().uri(), Eq("person3")); + EXPECT_THAT(results.results(0).joined_results_size(), Eq(0)); + EXPECT_THAT(results.results(1).document().uri(), Eq("person1")); + EXPECT_THAT(results.results(1).joined_results_size(), Eq(2)); + EXPECT_THAT(results.results(1).joined_results(0).document().uri(), + Eq("email2")); + EXPECT_THAT(results.results(1).joined_results(1).document().uri(), + Eq("email1")); + + // Get all parent documents with "foo" name OR with at least 1 join children, + // which should match all 3 documents. + // This is analogous to a SQL "full outer join." + search_spec.set_query( + "foo OR matchScoreExpression(\"len(this.childrenRankingSignals())\", 1)"); + results = icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + ASSERT_THAT(results.status(), ProtoIsOk()); + ASSERT_THAT(results.results(), SizeIs(3)); + // Person3 is returned because it has "foo" matched, even though it has no + // join children. + EXPECT_THAT(results.results(0).document().uri(), Eq("person3")); + EXPECT_THAT(results.results(0).joined_results_size(), Eq(0)); + EXPECT_THAT(results.results(1).document().uri(), Eq("person2")); + EXPECT_THAT(results.results(1).joined_results_size(), Eq(1)); + EXPECT_THAT(results.results(1).joined_results(0).document().uri(), + Eq("email3")); + EXPECT_THAT(results.results(2).document().uri(), Eq("person1")); + EXPECT_THAT(results.results(2).joined_results_size(), Eq(2)); + EXPECT_THAT(results.results(2).joined_results(0).document().uri(), + Eq("email2")); + EXPECT_THAT(results.results(2).joined_results(1).document().uri(), + Eq("email1")); + + // Get all parent documents with "foo" name AND with at least 1 join children, + // which should only match person1. + // This is analogous to a SQL "inner join." + search_spec.set_query( + "foo AND " + "matchScoreExpression(\"len(this.childrenRankingSignals())\", 1)"); + results = icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + ASSERT_THAT(results.status(), ProtoIsOk()); + ASSERT_THAT(results.results(), SizeIs(1)); + EXPECT_THAT(results.results(0).document().uri(), Eq("person1")); + EXPECT_THAT(results.results(0).joined_results_size(), Eq(2)); + EXPECT_THAT(results.results(0).joined_results(0).document().uri(), + Eq("email2")); + EXPECT_THAT(results.results(0).joined_results(1).document().uri(), + Eq("email1")); +} + +TEST_F(IcingSearchEngineSearchTest, + HybridEmbeddingSearchUsingJoinWithMatchScoreExpression) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Email").AddProperty( + PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REPEATED))) + .AddType(SchemaTypeConfigBuilder() + .SetType("EmbeddingAttachment") + .AddProperty(PropertyConfigBuilder() + .SetName("embedding") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("emailQualifiedId") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + PropertyProto::VectorProto query_embedding = + CreateVector("my_model", {1, 1, 1}); + // Create email1 with body "foo", and with an embedding attachment of score + // 0.1 + 0.2 + 0.3 = 0.6 by join. + DocumentProto email1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email1") + .SetSchema("Email") + .AddStringProperty("body", "foo") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto embedding1 = + DocumentBuilder() + .SetKey("namespace", "embedding1") + .SetSchema("EmbeddingAttachment") + .AddVectorProperty("embedding", + CreateVector("my_model", {0.1, 0.2, 0.3})) + .AddStringProperty("emailQualifiedId", "pkg$db/namespace#email1") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + // Create email2 with body "bar", and with an embedding attachment of score + // 0.2 + 0.3 + 0.4 = 0.9 by join. + DocumentProto email2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email2") + .SetSchema("Email") + .AddStringProperty("body", "bar") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto embedding2 = + DocumentBuilder() + .SetKey("namespace", "embedding2") + .SetSchema("EmbeddingAttachment") + .AddVectorProperty("embedding", + CreateVector("my_model", {0.2, 0.3, 0.4})) + .AddStringProperty("emailQualifiedId", "pkg$db/namespace#email2") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + // Create email3 with body "baz", and with an embedding attachment of score + // 0.1 + 0.2 + 0.3 = 0.6 by join. + DocumentProto email3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email3") + .SetSchema("Email") + .AddStringProperty("body", "baz") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + DocumentProto embedding3 = + DocumentBuilder() + .SetKey("namespace", "embedding3") + .SetSchema("EmbeddingAttachment") + .AddVectorProperty("embedding", + CreateVector("my_model", {0.1, 0.2, 0.3})) + .AddStringProperty("emailQualifiedId", "pkg$db/namespace#email3") + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(embedding1).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email2).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(embedding2).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(email3).status(), ProtoIsOk()); + ASSERT_THAT(icing.Put(embedding3).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + search_spec.add_enabled_features( + std::string(kMatchScoreExpressionFunctionFeature)); + search_spec.add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + + // JoinSpec + JoinSpecProto* join_spec = search_spec.mutable_join_spec(); + join_spec->set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec->set_child_property_expression("emailQualifiedId"); + JoinSpecProto::NestedSpecProto* nested_spec = + join_spec->mutable_nested_spec(); + *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec(); + *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance(); + SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec(); + nested_search_spec->add_enabled_features( + std::string(kListFilterQueryLanguageFeature)); + + // Get all parent documents with "foo" OR an embedding attachment of + // score >= 0.8, which should match email1 (because it has "foo") and email2 + // (because it has an embedding attachment of score 0.9). This is analogous to + // a SQL "full outer join." + search_spec.set_term_match_type(TermMatchType::EXACT_ONLY); + search_spec.set_query( + "foo OR matchScoreExpression(\"len(this.childrenRankingSignals())\", 1)"); + *nested_search_spec->add_embedding_query_vectors() = query_embedding; + nested_search_spec->set_embedding_query_metric_type( + SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT); + nested_search_spec->set_query( + "semanticSearch(getEmbeddingParameter(0), 0.8)"); + + ResultSpecProto result_spec; + result_spec.set_max_joined_children_per_parent_to_return( + std::numeric_limits<int32_t>::max()); + SearchResultProto results = + icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + ASSERT_THAT(results.status(), ProtoIsOk()); + ASSERT_THAT(results.results(), SizeIs(2)); + EXPECT_THAT(results.results(0).document().uri(), Eq("email2")); + EXPECT_THAT(results.results(0).joined_results_size(), Eq(1)); + EXPECT_THAT(results.results(0).joined_results(0).document().uri(), + Eq("embedding2")); + EXPECT_THAT(results.results(1).document().uri(), Eq("email1")); + // email1's embedding attachment is not matched, so it is not returned. + EXPECT_THAT(results.results(1).joined_results(), IsEmpty()); +} + } // namespace } // namespace lib } // namespace icing
diff --git a/icing/icing-search-engine_suggest_test.cc b/icing/icing-search-engine_suggest_test.cc index 7344241..1f814aa 100644 --- a/icing/icing-search-engine_suggest_test.cc +++ b/icing/icing-search-engine_suggest_test.cc
@@ -46,10 +46,10 @@ #include "icing/schema-builder.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/testing/tmp-directory.h" +#include "icing/util/icu-data-file-helper.h" namespace icing { namespace lib { @@ -92,7 +92,7 @@ std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); ICING_ASSERT_OK( - icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); + icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path)); } filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); }
diff --git a/icing/icing-search-engine_test.cc b/icing/icing-search-engine_test.cc index 9ef1dfc..14b9b7d 100644 --- a/icing/icing-search-engine_test.cc +++ b/icing/icing-search-engine_test.cc
@@ -48,10 +48,10 @@ #include "icing/schema-builder.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/testing/tmp-directory.h" +#include "icing/util/icu-data-file-helper.h" namespace icing { namespace lib { @@ -99,7 +99,7 @@ std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); ICING_ASSERT_OK( - icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); + icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path)); } filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str()); }
diff --git a/icing/index/data-indexing-handler.h b/icing/index/data-indexing-handler.h index 16a1796..317eca6 100644 --- a/icing/index/data-indexing-handler.h +++ b/icing/index/data-indexing-handler.h
@@ -32,16 +32,25 @@ virtual ~DataIndexingHandler() = default; // Handles the indexing process: add data into the specific type index (e.g. - // term index, integer index, qualified id type joinable index) for all - // contents in the corresponding type of data in tokenized_document. - // For example, IntegerSectionIndexingHandler::Handle should add data into - // integer index for all contents in tokenized_document.integer_sections. + // term index, integer index, qualified id type joinable index, embedding + // index) for all contents in the corresponding type of data in + // tokenized_document. For example, IntegerSectionIndexingHandler::Handle + // should add data into integer index for all contents in + // tokenized_document.integer_sections. + // + // old_document_id is provided. If valid, then it means the document with + // the same (namespace, uri) exists previously, and it is updated with new + // contents at this round. Each indexing handler should decide whether + // migrating existing data from old_document_id to (new) document_id according + // to each index's data logic. // // Also it should handle last added DocumentId properly (based on // recovery_mode_) to avoid adding previously indexed documents. // // tokenized_document: document object with different types of tokenized data. // document_id: id of the document. + // old_document_id: id of the document before the update. If it is a new + // document, then it will be kInvalidDocumentId. // recovery_mode: decides how to handle document_id <= // last_added_document_id. If in recovery_mode, then // Handle() will simply return OK immediately. Otherwise, @@ -57,7 +66,8 @@ // - Any other errors. It depends on each implementation. virtual libtextclassifier3::Status Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, - bool recovery_mode, PutDocumentStatsProto* put_document_stats) = 0; + DocumentId old_document_id, bool recovery_mode, + PutDocumentStatsProto* put_document_stats) = 0; protected: const Clock& clock_; // Does not own.
diff --git a/icing/index/embed/doc-hit-info-iterator-embedding.cc b/icing/index/embed/doc-hit-info-iterator-embedding.cc index 5a6e547..1fa5fac 100644 --- a/icing/index/embed/doc-hit-info-iterator-embedding.cc +++ b/icing/index/embed/doc-hit-info-iterator-embedding.cc
@@ -14,6 +14,7 @@ #include "icing/index/embed/doc-hit-info-iterator-embedding.h" +#include <cstdint> #include <memory> #include <string_view> #include <utility> @@ -29,10 +30,12 @@ #include "icing/index/embed/posting-list-embedding-hit-accessor.h" #include "icing/index/hit/doc-hit-info.h" #include "icing/index/hit/hit.h" -#include "icing/index/iterator/section-restrict-data.h" #include "icing/proto/search.pb.h" +#include "icing/schema/schema-store.h" #include "icing/schema/section.h" +#include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" +#include "icing/store/document-store.h" #include "icing/util/status-macros.h" namespace icing { @@ -44,10 +47,13 @@ SearchSpecProto::EmbeddingQueryMetricType::Code metric_type, double score_low, double score_high, EmbeddingQueryResults::EmbeddingQueryScoreMap* score_map, - const EmbeddingIndex* embedding_index) { + const EmbeddingIndex* embedding_index, const DocumentStore* document_store, + const SchemaStore* schema_store, int64_t current_time_ms) { ICING_RETURN_ERROR_IF_NULL(query); ICING_RETURN_ERROR_IF_NULL(embedding_index); ICING_RETURN_ERROR_IF_NULL(score_map); + ICING_RETURN_ERROR_IF_NULL(document_store); + ICING_RETURN_ERROR_IF_NULL(schema_store); libtextclassifier3::StatusOr<std::unique_ptr<PostingListEmbeddingHitAccessor>> pl_accessor_or = embedding_index->GetAccessorForVector(*query); @@ -69,7 +75,8 @@ return std::unique_ptr<DocHitInfoIteratorEmbedding>( new DocHitInfoIteratorEmbedding( query, metric_type, std::move(embedding_scorer), score_low, - score_high, score_map, embedding_index, std::move(pl_accessor))); + score_high, score_map, embedding_index, std::move(pl_accessor), + document_store, schema_store, current_time_ms)); } libtextclassifier3::StatusOr<const EmbeddingHit*> @@ -89,6 +96,14 @@ doc_hit_info_.set_document_id(embedding_hit.basic_hit().document_id()); current_allowed_sections_mask_ = ComputeAllowedSectionsMask(doc_hit_info_.document_id()); + + schema_type_id_ = document_store_.GetSchemaTypeId( + doc_hit_info_.document_id(), current_time_ms_); + if (schema_type_id_ == kInvalidSchemaTypeId) { + // This means that the document is deleted or expired, so update + // current_allowed_sections_mask_ to skip the document. + current_allowed_sections_mask_ = kSectionIdMaskNone; + } } else if (doc_hit_info_.document_id() != embedding_hit.basic_hit().document_id()) { return nullptr; @@ -105,6 +120,7 @@ } doc_hit_info_ = DocHitInfo(kInvalidDocumentId, kSectionIdMaskNone); + schema_type_id_ = kInvalidSchemaTypeId; std::vector<double>* matched_scores = nullptr; current_allowed_sections_mask_ = kSectionIdMaskAll; while (true) { @@ -121,15 +137,19 @@ continue; } - // Calculate the semantic score. - int dimension = query_.values_size(); + // The schema type id is guaranteed to be valid here. Otherwise, + // current_allowed_sections_mask_ should be assigned to kSectionIdMaskNone + // by AdvanceToNextEmbeddingHit, and the embedding hit should have been + // skipped above. ICING_ASSIGN_OR_RETURN( - const float* vector, - embedding_index_.GetEmbeddingVector(*embedding_hit, dimension)); - double semantic_score = - embedding_scorer_->Score(dimension, - /*v1=*/query_.values().data(), - /*v2=*/vector); + EmbeddingIndexingConfig::QuantizationType::Code quantization_type, + schema_store_.GetQuantizationType( + schema_type_id_, embedding_hit->basic_hit().section_id())); + // Calculate the semantic score. + ICING_ASSIGN_OR_RETURN( + float semantic_score, + embedding_index_.ScoreEmbeddingHit(*embedding_scorer_, query_, + *embedding_hit, quantization_type)); // If the semantic score is within the desired score range, update // doc_hit_info_ and score_map_.
diff --git a/icing/index/embed/doc-hit-info-iterator-embedding.h b/icing/index/embed/doc-hit-info-iterator-embedding.h index ae126c8..3076cd7 100644 --- a/icing/index/embed/doc-hit-info-iterator-embedding.h +++ b/icing/index/embed/doc-hit-info-iterator-embedding.h
@@ -15,6 +15,7 @@ #ifndef ICING_INDEX_EMBED_DOC_HIT_INFO_ITERATOR_EMBEDDING_H_ #define ICING_INDEX_EMBED_DOC_HIT_INFO_ITERATOR_EMBEDDING_H_ +#include <cstdint> #include <memory> #include <string> #include <string_view> @@ -32,7 +33,10 @@ #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/index/iterator/section-restrict-data.h" #include "icing/proto/search.pb.h" +#include "icing/schema/schema-store.h" #include "icing/schema/section.h" +#include "icing/store/document-filter-data.h" +#include "icing/store/document-store.h" namespace icing { namespace lib { @@ -59,7 +63,9 @@ SearchSpecProto::EmbeddingQueryMetricType::Code metric_type, double score_low, double score_high, EmbeddingQueryResults::EmbeddingQueryScoreMap* score_map, - const EmbeddingIndex* embedding_index); + const EmbeddingIndex* embedding_index, + const DocumentStore* document_store, const SchemaStore* schema_store, + int64_t current_time_ms); libtextclassifier3::Status Advance() override; @@ -92,7 +98,9 @@ double score_high, EmbeddingQueryResults::EmbeddingQueryScoreMap* score_map, const EmbeddingIndex* embedding_index, - std::unique_ptr<PostingListEmbeddingHitAccessor> posting_list_accessor) + std::unique_ptr<PostingListEmbeddingHitAccessor> posting_list_accessor, + const DocumentStore* document_store, const SchemaStore* schema_store, + int64_t current_time_ms) : query_(*query), metric_type_(metric_type), embedding_scorer_(std::move(embedding_scorer)), @@ -104,6 +112,10 @@ cached_embedding_hits_idx_(0), current_allowed_sections_mask_(kSectionIdMaskAll), no_more_hit_(false), + schema_type_id_(kInvalidSchemaTypeId), + document_store_(*document_store), + schema_store_(*schema_store), + current_time_ms_(current_time_ms), num_advance_calls_(0) {} // Advance to the next embedding hit of the current document. If the current @@ -152,7 +164,11 @@ int cached_embedding_hits_idx_; SectionIdMask current_allowed_sections_mask_; bool no_more_hit_; + SchemaTypeId schema_type_id_; // The schema type id for the current document. + const DocumentStore& document_store_; + const SchemaStore& schema_store_; + int64_t current_time_ms_; int num_advance_calls_; };
diff --git a/icing/index/embed/embedding-index.cc b/icing/index/embed/embedding-index.cc index 0f8ad53..663f53b 100644 --- a/icing/index/embed/embedding-index.cc +++ b/icing/index/embed/embedding-index.cc
@@ -27,6 +27,7 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" +#include "icing/feature-flags.h" #include "icing/file/destructible-directory.h" #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" @@ -34,11 +35,17 @@ #include "icing/file/posting_list/flash-index-storage.h" #include "icing/file/posting_list/posting-list-identifier.h" #include "icing/index/embed/embedding-hit.h" +#include "icing/index/embed/embedding-scorer.h" #include "icing/index/embed/posting-list-embedding-hit-accessor.h" +#include "icing/index/embed/quantizer.h" #include "icing/index/hit/hit.h" +#include "icing/schema/schema-store.h" +#include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" +#include "icing/store/document-store.h" #include "icing/store/dynamic-trie-key-mapper.h" #include "icing/store/key-mapper.h" +#include "icing/util/clock.h" #include "icing/util/crc32.h" #include "icing/util/encode-util.h" #include "icing/util/logging.h" @@ -72,6 +79,11 @@ return absl_ports::StrCat(working_path, "/embedding_vectors"); } +std::string GetQuantizedEmbeddingVectorsFilePath( + std::string_view working_path) { + return absl_ports::StrCat(working_path, "/quantized_embedding_vectors"); +} + // An injective function that maps the ordered pair (dimension, model_signature) // to a string, which is used to form a key for embedding_posting_list_mapper_. std::string GetPostingListKey(uint32_t dimension, @@ -94,17 +106,30 @@ } std::string GetPostingListKey(const PropertyProto::VectorProto& vector) { - return GetPostingListKey(vector.values_size(), vector.model_signature()); + return GetPostingListKey(vector.values().size(), vector.model_signature()); +} + +libtextclassifier3::StatusOr<Quantizer> CreateQuantizer( + const PropertyProto::VectorProto& vector) { + if (vector.values().empty()) { + return absl_ports::InvalidArgumentError("Vector dimension is 0"); + } + auto minmax_pair = + std::minmax_element(vector.values().begin(), vector.values().end()); + return Quantizer::Create(*minmax_pair.first, *minmax_pair.second); } } // namespace libtextclassifier3::StatusOr<std::unique_ptr<EmbeddingIndex>> -EmbeddingIndex::Create(const Filesystem* filesystem, std::string working_path) { +EmbeddingIndex::Create(const Filesystem* filesystem, std::string working_path, + const Clock* clock, const FeatureFlags* feature_flags) { ICING_RETURN_ERROR_IF_NULL(filesystem); + ICING_RETURN_ERROR_IF_NULL(clock); - std::unique_ptr<EmbeddingIndex> index = std::unique_ptr<EmbeddingIndex>( - new EmbeddingIndex(*filesystem, std::move(working_path))); + std::unique_ptr<EmbeddingIndex> index = + std::unique_ptr<EmbeddingIndex>(new EmbeddingIndex( + *filesystem, std::move(working_path), clock, feature_flags)); ICING_RETURN_IF_ERROR(index->Initialize()); return index; } @@ -133,6 +158,12 @@ filesystem_, GetEmbeddingVectorsFilePath(working_path_), MemoryMappedFile::READ_WRITE_AUTO_SYNC)); + ICING_ASSIGN_OR_RETURN( + quantized_embedding_vectors_, + FileBackedVector<char>::Create( + filesystem_, GetQuantizedEmbeddingVectorsFilePath(working_path_), + MemoryMappedFile::READ_WRITE_AUTO_SYNC)); + return libtextclassifier3::Status::OK; } @@ -193,6 +224,7 @@ flash_index_storage_.reset(); embedding_posting_list_mapper_.reset(); embedding_vectors_.reset(); + quantized_embedding_vectors_.reset(); if (filesystem_.DirectoryExists(working_path_.c_str())) { ICING_RETURN_IF_ERROR(Discard(filesystem_, working_path_)); } @@ -218,26 +250,56 @@ posting_list_id); } +libtextclassifier3::StatusOr<uint32_t> EmbeddingIndex::AppendEmbeddingVector( + const PropertyProto::VectorProto& vector, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type) { + uint32_t dimension = vector.values().size(); + uint32_t location; + if (!feature_flags_->enable_embedding_quantization() || + quantization_type == EmbeddingIndexingConfig::QuantizationType::NONE) { + location = embedding_vectors_->num_elements(); + ICING_ASSIGN_OR_RETURN( + FileBackedVector<float>::MutableArrayView mutable_arr, + embedding_vectors_->Allocate(dimension)); + mutable_arr.SetArray(/*idx=*/0, vector.values().data(), dimension); + } else { + ICING_ASSIGN_OR_RETURN(Quantizer quantizer, CreateQuantizer(vector)); + // Quantize the vector + std::vector<uint8_t> quantized_values; + quantized_values.reserve(vector.values().size()); + for (float value : vector.values()) { + quantized_values.push_back(quantizer.Quantize(value)); + } + + // Store the quantizer and the quantized vector + location = quantized_embedding_vectors_->num_elements(); + ICING_ASSIGN_OR_RETURN( + FileBackedVector<char>::MutableArrayView mutable_arr, + quantized_embedding_vectors_->Allocate(sizeof(Quantizer) + dimension)); + mutable_arr.SetArray(/*idx=*/0, reinterpret_cast<char*>(&quantizer), + sizeof(Quantizer)); + mutable_arr.SetArray(/*idx=*/sizeof(Quantizer), + reinterpret_cast<char*>(quantized_values.data()), + dimension); + } + return location; +} + libtextclassifier3::Status EmbeddingIndex::BufferEmbedding( - const BasicHit& basic_hit, const PropertyProto::VectorProto& vector) { - if (vector.values_size() == 0) { + const BasicHit& basic_hit, const PropertyProto::VectorProto& vector, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type) { + if (vector.values().empty()) { return absl_ports::InvalidArgumentError("Vector dimension is 0"); } ICING_RETURN_IF_ERROR(MarkIndexNonEmpty()); - uint32_t location = embedding_vectors_->num_elements(); - uint32_t dimension = vector.values_size(); std::string key = GetPostingListKey(vector); + ICING_ASSIGN_OR_RETURN(uint32_t location, + AppendEmbeddingVector(vector, quantization_type)); // Buffer the embedding hit. pending_embedding_hits_.push_back( {std::move(key), EmbeddingHit(basic_hit, location)}); - - // Put vector - ICING_ASSIGN_OR_RETURN(FileBackedVector<float>::MutableArrayView mutable_arr, - embedding_vectors_->Allocate(dimension)); - mutable_arr.SetArray(/*idx=*/0, vector.values().data(), dimension); - return libtextclassifier3::Status::OK; } @@ -303,13 +365,45 @@ return libtextclassifier3::Status::OK; } +libtextclassifier3::StatusOr<uint32_t> EmbeddingIndex::TransferEmbeddingVector( + const EmbeddingHit& old_hit, uint32_t dimension, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type, + EmbeddingIndex* new_index) const { + uint32_t new_location; + if (!feature_flags_->enable_embedding_quantization() || + quantization_type == EmbeddingIndexingConfig::QuantizationType::NONE) { + ICING_ASSIGN_OR_RETURN(const float* old_vector, + GetEmbeddingVector(old_hit, dimension)); + new_location = new_index->embedding_vectors_->num_elements(); + + // Copy the embedding vector of the hit to the new index. + ICING_ASSIGN_OR_RETURN( + FileBackedVector<float>::MutableArrayView mutable_arr, + new_index->embedding_vectors_->Allocate(dimension)); + mutable_arr.SetArray(/*idx=*/0, old_vector, dimension); + } else { + ICING_ASSIGN_OR_RETURN(const char* old_data, + GetQuantizedEmbeddingVector(old_hit, dimension)); + new_location = new_index->quantized_embedding_vectors_->num_elements(); + + // Copy the embedding vector of the hit to the new index. + ICING_ASSIGN_OR_RETURN(FileBackedVector<char>::MutableArrayView mutable_arr, + new_index->quantized_embedding_vectors_->Allocate( + sizeof(Quantizer) + dimension)); + mutable_arr.SetArray(/*idx=*/0, old_data, sizeof(Quantizer) + dimension); + } + return new_location; +} + libtextclassifier3::Status EmbeddingIndex::TransferIndex( + const DocumentStore& document_store, const SchemaStore& schema_store, const std::vector<DocumentId>& document_id_old_to_new, EmbeddingIndex* new_index) const { if (is_empty()) { return absl_ports::FailedPreconditionError("EmbeddingIndex is empty"); } + const int64_t current_time_ms = clock_.GetSystemTimeMilliseconds(); std::unique_ptr<KeyMapper<PostingListIdentifier>::Iterator> itr = embedding_posting_list_mapper_->GetIterator(); while (itr->Advance()) { @@ -330,6 +424,8 @@ PostingListEmbeddingHitAccessor::CreateFromExisting( flash_index_storage_.get(), posting_list_hit_serializer_.get(), /*existing_posting_list_id=*/itr->GetValue())); + DocumentId last_new_document_id = kInvalidDocumentId; + SchemaTypeId schema_type_id = kInvalidSchemaTypeId; while (true) { ICING_ASSIGN_OR_RETURN(std::vector<EmbeddingHit> batch, old_pl_accessor->GetNextHitsBatch()); @@ -340,8 +436,6 @@ // Safety checks to add robustness to the codebase, so to make sure // that we never access invalid memory, in case that hit from the // posting list is corrupted. - ICING_ASSIGN_OR_RETURN(const float* old_vector, - GetEmbeddingVector(old_hit, dimension)); if (old_hit.basic_hit().document_id() < 0 || old_hit.basic_hit().document_id() >= document_id_old_to_new.size()) { @@ -350,23 +444,36 @@ "too small, or the index may have been corrupted."); } - // Construct transferred hit + // Construct transferred hit and add the embedding vector to the new + // index. DocumentId new_document_id = document_id_old_to_new[old_hit.basic_hit().document_id()]; if (new_document_id == kInvalidDocumentId) { continue; } + if (new_document_id != last_new_document_id) { + schema_type_id = + document_store.GetSchemaTypeId(new_document_id, current_time_ms); + } + last_new_document_id = new_document_id; + if (schema_type_id == kInvalidSchemaTypeId) { + // This should not happen, since document store is optimized first, + // so that new_document_id here should be alive. + continue; + } + ICING_ASSIGN_OR_RETURN( + EmbeddingIndexingConfig::QuantizationType::Code quantization_type, + schema_store.GetQuantizationType(schema_type_id, + old_hit.basic_hit().section_id())); ICING_RETURN_IF_ERROR(new_index->MarkIndexNonEmpty()); - uint32_t new_location = new_index->embedding_vectors_->num_elements(); + + ICING_ASSIGN_OR_RETURN( + uint32_t new_location, + TransferEmbeddingVector(old_hit, dimension, quantization_type, + new_index)); new_hits.push_back(EmbeddingHit( BasicHit(old_hit.basic_hit().section_id(), new_document_id), new_location)); - - // Copy the embedding vector of the hit to the new index. - ICING_ASSIGN_OR_RETURN( - FileBackedVector<float>::MutableArrayView mutable_arr, - new_index->embedding_vectors_->Allocate(dimension)); - mutable_arr.SetArray(/*idx=*/0, old_vector, dimension); } } // No hit needs to be added to the new index. @@ -395,8 +502,11 @@ } libtextclassifier3::Status EmbeddingIndex::Optimize( + const DocumentStore* document_store, const SchemaStore* schema_store, const std::vector<DocumentId>& document_id_old_to_new, DocumentId new_last_added_document_id) { + ICING_RETURN_ERROR_IF_NULL(document_store); + ICING_RETURN_ERROR_IF_NULL(schema_store); if (is_empty()) { info().last_added_document_id = new_last_added_document_id; return libtextclassifier3::Status::OK; @@ -424,9 +534,11 @@ { ICING_ASSIGN_OR_RETURN( std::unique_ptr<EmbeddingIndex> new_index, - EmbeddingIndex::Create(&filesystem_, temporary_index_dir.dir())); - ICING_RETURN_IF_ERROR( - TransferIndex(document_id_old_to_new, new_index.get())); + EmbeddingIndex::Create(&filesystem_, temporary_index_dir.dir(), &clock_, + feature_flags_)); + ICING_RETURN_IF_ERROR(TransferIndex(*document_store, *schema_store, + document_id_old_to_new, + new_index.get())); new_index->set_last_added_document_id(new_last_added_document_id); ICING_RETURN_IF_ERROR(new_index->PersistToDisk()); } @@ -436,6 +548,7 @@ flash_index_storage_.reset(); embedding_posting_list_mapper_.reset(); embedding_vectors_.reset(); + quantized_embedding_vectors_.reset(); if (!filesystem_.SwapFiles(temporary_index_dir.dir().c_str(), working_path_.c_str())) { @@ -448,6 +561,32 @@ return Initialize(); } +libtextclassifier3::StatusOr<float> EmbeddingIndex::ScoreEmbeddingHit( + const EmbeddingScorer& scorer, const PropertyProto::VectorProto& query, + const EmbeddingHit& hit, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type) const { + int dimension = query.values().size(); + float semantic_score; + if (!feature_flags_->enable_embedding_quantization() || + quantization_type == EmbeddingIndexingConfig::QuantizationType::NONE) { + ICING_ASSIGN_OR_RETURN(const float* vector, + GetEmbeddingVector(hit, dimension)); + semantic_score = scorer.Score(dimension, + /*v1=*/query.values().data(), + /*v2=*/vector); + } else { + ICING_ASSIGN_OR_RETURN(const char* data, + GetQuantizedEmbeddingVector(hit, dimension)); + Quantizer quantizer(data); + const uint8_t* quantized_vector = + reinterpret_cast<const uint8_t*>(data + sizeof(Quantizer)); + semantic_score = scorer.Score(dimension, + /*v1=*/query.values().data(), + /*v2=*/quantized_vector, quantizer); + } + return semantic_score; +} + libtextclassifier3::Status EmbeddingIndex::PersistMetadataToDisk() { return metadata_mmapped_file_->PersistToDisk(); } @@ -461,6 +600,7 @@ } ICING_RETURN_IF_ERROR(embedding_posting_list_mapper_->PersistToDisk()); ICING_RETURN_IF_ERROR(embedding_vectors_->PersistToDisk()); + ICING_RETURN_IF_ERROR(quantized_embedding_vectors_->PersistToDisk()); return libtextclassifier3::Status::OK; } @@ -472,8 +612,11 @@ embedding_posting_list_mapper_->UpdateChecksum()); ICING_ASSIGN_OR_RETURN(Crc32 embedding_vectors_crc, embedding_vectors_->UpdateChecksum()); + ICING_ASSIGN_OR_RETURN(Crc32 quantized_embedding_vectors_crc, + quantized_embedding_vectors_->UpdateChecksum()); return Crc32(embedding_posting_list_mapper_crc.Get() ^ - embedding_vectors_crc.Get()); + embedding_vectors_crc.Get() ^ + quantized_embedding_vectors_crc.Get()); } libtextclassifier3::StatusOr<Crc32> EmbeddingIndex::GetStoragesChecksum() @@ -484,8 +627,11 @@ ICING_ASSIGN_OR_RETURN(Crc32 embedding_posting_list_mapper_crc, embedding_posting_list_mapper_->GetChecksum()); Crc32 embedding_vectors_crc = embedding_vectors_->GetChecksum(); + Crc32 quantized_embedding_vectors_crc = + quantized_embedding_vectors_->GetChecksum(); return Crc32(embedding_posting_list_mapper_crc.Get() ^ - embedding_vectors_crc.Get()); + embedding_vectors_crc.Get() ^ + quantized_embedding_vectors_crc.Get()); } } // namespace lib
diff --git a/icing/index/embed/embedding-index.h b/icing/index/embed/embedding-index.h index dd589c9..a2dfb7a 100644 --- a/icing/index/embed/embedding-index.h +++ b/icing/index/embed/embedding-index.h
@@ -25,6 +25,7 @@ #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" +#include "icing/feature-flags.h" #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" #include "icing/file/memory-mapped-file.h" @@ -32,12 +33,18 @@ #include "icing/file/posting_list/flash-index-storage.h" #include "icing/file/posting_list/posting-list-identifier.h" #include "icing/index/embed/embedding-hit.h" +#include "icing/index/embed/embedding-scorer.h" #include "icing/index/embed/posting-list-embedding-hit-accessor.h" #include "icing/index/embed/posting-list-embedding-hit-serializer.h" +#include "icing/index/embed/quantizer.h" #include "icing/index/hit/hit.h" +#include "icing/schema/schema-store.h" #include "icing/store/document-id.h" +#include "icing/store/document-store.h" #include "icing/store/key-mapper.h" +#include "icing/util/clock.h" #include "icing/util/crc32.h" +#include "icing/util/logging.h" namespace icing { namespace lib { @@ -84,7 +91,8 @@ // - Any error from MemoryMappedFile, FlashIndexStorage, // DynamicTrieKeyMapper, or FileBackedVector. static libtextclassifier3::StatusOr<std::unique_ptr<EmbeddingIndex>> Create( - const Filesystem* filesystem, std::string working_path); + const Filesystem* filesystem, std::string working_path, + const Clock* clock, const FeatureFlags* feature_flags); static libtextclassifier3::Status Discard(const Filesystem& filesystem, const std::string& working_path) { @@ -94,6 +102,14 @@ libtextclassifier3::Status Clear(); + ~EmbeddingIndex() override { + if (!PersistToDisk().ok()) { + ICING_LOG(WARNING) + << "Failed to persist embedding index to disk while destructing " + << working_path_; + } + } + // Buffer an embedding pending to be added to the index. This is required // since EmbeddingHits added in posting lists must be decreasing, which means // that section ids and location indexes for a single document must be added @@ -104,7 +120,8 @@ // - INVALID_ARGUMENT error if the dimension is 0. // - INTERNAL_ERROR on I/O error libtextclassifier3::Status BufferEmbedding( - const BasicHit& basic_hit, const PropertyProto::VectorProto& vector); + const BasicHit& basic_hit, const PropertyProto::VectorProto& vector, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type); // Commit the embedding hits in the buffer to the index. // @@ -147,6 +164,7 @@ // - INTERNAL_ERROR on IO error, this indicates that the index may be in an // invalid state and should be cleared. libtextclassifier3::Status Optimize( + const DocumentStore* document_store, const SchemaStore* schema_store, const std::vector<DocumentId>& document_id_old_to_new, DocumentId new_last_added_document_id); @@ -165,6 +183,31 @@ } return embedding_vectors_->array() + hit.location(); } + libtextclassifier3::StatusOr<const char*> GetQuantizedEmbeddingVector( + const EmbeddingHit& hit, uint32_t dimension) const { + // quantized_embedding_vectors_ stores data in char format. Every quantized + // embedding vector contains a Quantizer header followed by the actual + // vector, and every value in the vector is stored in uint8_t. + if (static_cast<int64_t>(hit.location()) + sizeof(Quantizer) + + sizeof(uint8_t) * dimension > + GetTotalQuantizedVectorSize()) { + return absl_ports::OutOfRangeError( + "Got an embedding hit that refers to a vector out of range."); + } + return quantized_embedding_vectors_->array() + hit.location(); + } + + // Calculates the score for the given embedding hit with the given query. + // + // Returns: + // - The score on success. + // - OUT_OF_RANGE_ERROR if the referred vector is out of range based on the + // location and dimension. + // - Any error from schema store when getting the quantization type. + libtextclassifier3::StatusOr<float> ScoreEmbeddingHit( + const EmbeddingScorer& scorer, const PropertyProto::VectorProto& query, + const EmbeddingHit& hit, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type) const; libtextclassifier3::StatusOr<const float*> GetRawEmbeddingData() const { if (is_empty()) { @@ -180,6 +223,13 @@ return embedding_vectors_->num_elements(); } + int32_t GetTotalQuantizedVectorSize() const { + if (is_empty()) { + return 0; + } + return quantized_embedding_vectors_->num_elements(); + } + DocumentId last_added_document_id() const { return info().last_added_document_id; } @@ -196,9 +246,12 @@ private: explicit EmbeddingIndex(const Filesystem& filesystem, - std::string working_path) + std::string working_path, const Clock* clock, + const FeatureFlags* feature_flags) : PersistentStorage(filesystem, std::move(working_path), - kWorkingPathType) {} + kWorkingPathType), + clock_(*clock), + feature_flags_(feature_flags) {} // Creates the storage data if the index is not empty. This will initialize // flash_index_storage_, embedding_posting_list_mapper_, embedding_vectors_. @@ -221,6 +274,17 @@ libtextclassifier3::Status Initialize(); + // Transfers the embedding vector of the given hit from the current index to + // the new index. + // + // Returns: + // - The location of the transferred vector in the new index on success. + // - Any error when allocating the vector storage in the new index. + libtextclassifier3::StatusOr<uint32_t> TransferEmbeddingVector( + const EmbeddingHit& old_hit, uint32_t dimension, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type, + EmbeddingIndex* new_index) const; + // Transfers embedding data and hits from the current index to new_index. // // Returns: @@ -230,6 +294,7 @@ // in an invalid state and the caller should handle it properly (e.g. // discard and rebuild) libtextclassifier3::Status TransferIndex( + const DocumentStore& document_store, const SchemaStore& schema_store, const std::vector<DocumentId>& document_id_old_to_new, EmbeddingIndex* new_index) const; @@ -251,6 +316,18 @@ libtextclassifier3::StatusOr<Crc32> GetStoragesChecksum() const override; + // Appends the given embedding vector to the appropriate vector storage + // (embedding_vectors_ or quantized_embedding_vectors_) based on the + // quantization type. + // + // Returns: + // - The location of the appended vector (i.e., the starting index within + // the vector storage). + // - Any error when allocating the vector storage. + libtextclassifier3::StatusOr<uint32_t> AppendEmbeddingVector( + const PropertyProto::VectorProto& vector, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type); + Crcs& crcs() override { return *reinterpret_cast<Crcs*>(metadata_mmapped_file_->mutable_region() + kCrcsMetadataBufferOffset); @@ -271,6 +348,9 @@ kInfoMetadataBufferOffset); } + const Clock& clock_; + const FeatureFlags* feature_flags_; // Does not own. + // In memory data: // Pending embedding hits with their embedding keys used for // embedding_posting_list_mapper_. @@ -301,6 +381,7 @@ // // null if the index is empty. std::unique_ptr<FileBackedVector<float>> embedding_vectors_; + std::unique_ptr<FileBackedVector<char>> quantized_embedding_vectors_; }; } // namespace lib
diff --git a/icing/index/embed/embedding-index_test.cc b/icing/index/embed/embedding-index_test.cc index 6171e92..ed44120 100644 --- a/icing/index/embed/embedding-index_test.cc +++ b/icing/index/embed/embedding-index_test.cc
@@ -28,17 +28,26 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/absl_ports/canonical_errors.h" +#include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-hit.h" -#include "icing/index/embed/posting-list-embedding-hit-accessor.h" +#include "icing/index/embed/quantizer.h" #include "icing/index/hit/hit.h" +#include "icing/legacy/index/icing-filesystem.h" #include "icing/proto/document.pb.h" +#include "icing/schema-builder.h" +#include "icing/schema/schema-store.h" +#include "icing/schema/section.h" #include "icing/store/document-id.h" +#include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/embedding-test-utils.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" +#include "icing/util/clock.h" #include "icing/util/crc32.h" -#include "icing/util/status-macros.h" namespace icing { namespace lib { @@ -47,67 +56,86 @@ using ::testing::ElementsAre; using ::testing::Eq; +using ::testing::FloatNear; +using ::testing::HasSubstr; using ::testing::IsEmpty; +using ::testing::Pointwise; using ::testing::Test; +static constexpr SectionId kSectionIdQuantizedEmbedding = 2; +static constexpr float kEpsQuantized = 0.01f; + class EmbeddingIndexTest : public Test { protected: void SetUp() override { - embedding_index_dir_ = GetTestTempDir() + "/embedding_index_test"; + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + test_dir_ = GetTestTempDir() + "/icing"; + embedding_index_dir_ = test_dir_ + "/embedding_index"; + document_store_dir_ = test_dir_ + "/document_store"; + schema_store_dir_ = test_dir_ + "/schema_store"; + filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); + filesystem_.CreateDirectoryRecursively(document_store_dir_.c_str()); + filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()); + + ICING_ASSERT_OK_AND_ASSIGN( + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &clock_, feature_flags_.get())); + + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + DocumentStore::Create(&filesystem_, document_store_dir_, &clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); + document_store_ = std::move(create_result.document_store); + ICING_ASSERT_OK_AND_ASSIGN( embedding_index_, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir_)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir_, &clock_, + feature_flags_.get())); + + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("type") + .AddProperty( + PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeVector(EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("prop2") + .SetDataTypeVector(EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL)) + // Quantized embedding + .AddProperty( + PropertyConfigBuilder() + .SetName("prop3") + .SetDataTypeVector(EMBEDDING_INDEXING_LINEAR_SEARCH, + QUANTIZATION_TYPE_QUANTIZE_8_BIT) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); } void TearDown() override { + document_store_.reset(); + schema_store_.reset(); embedding_index_.reset(); - filesystem_.DeleteDirectoryRecursively(embedding_index_dir_.c_str()); - } - - libtextclassifier3::StatusOr<std::vector<EmbeddingHit>> GetHits( - uint32_t dimension, std::string_view model_signature) { - return GetHits(embedding_index_.get(), dimension, model_signature); - } - - static libtextclassifier3::StatusOr<std::vector<EmbeddingHit>> GetHits( - const EmbeddingIndex* embedding_index, uint32_t dimension, - std::string_view model_signature) { - std::vector<EmbeddingHit> hits; - - libtextclassifier3::StatusOr< - std::unique_ptr<PostingListEmbeddingHitAccessor>> - pl_accessor_or = - embedding_index->GetAccessor(dimension, model_signature); - std::unique_ptr<PostingListEmbeddingHitAccessor> pl_accessor; - if (pl_accessor_or.ok()) { - pl_accessor = std::move(pl_accessor_or).ValueOrDie(); - } else if (absl_ports::IsNotFound(pl_accessor_or.status())) { - return hits; - } else { - return std::move(pl_accessor_or).status(); - } - - while (true) { - ICING_ASSIGN_OR_RETURN(std::vector<EmbeddingHit> batch, - pl_accessor->GetNextHitsBatch()); - if (batch.empty()) { - return hits; - } - hits.insert(hits.end(), batch.begin(), batch.end()); - } - } - - std::vector<float> GetRawEmbeddingData() { - return GetRawEmbeddingData(embedding_index_.get()); - } - - static std::vector<float> GetRawEmbeddingData( - const EmbeddingIndex* embedding_index) { - ICING_ASSIGN_OR_RETURN(const float* data, - embedding_index->GetRawEmbeddingData(), - std::vector<float>()); - return std::vector<float>(data, - data + embedding_index->GetTotalVectorSize()); + filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } libtextclassifier3::StatusOr<bool> IndexContainsMetadataOnly() { @@ -119,8 +147,16 @@ return sub_dirs.size() == 1 && sub_dirs[0] == "metadata"; } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; + IcingFilesystem icing_filesystem_; + std::string test_dir_; std::string embedding_index_dir_; + std::string schema_store_dir_; + std::string document_store_dir_; + Clock clock_; + std::unique_ptr<SchemaStore> schema_store_; + std::unique_ptr<DocumentStore> document_store_; std::unique_ptr<EmbeddingIndex> embedding_index_; }; @@ -128,6 +164,19 @@ EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(true)); } +TEST_F(EmbeddingIndexTest, InitializationShouldFailWithNullPointer) { + std::string embedding_index_dir = + GetTestTempDir() + "/embedding_index_test_local"; + + EXPECT_THAT(EmbeddingIndex::Create(nullptr, embedding_index_dir, &clock_, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + + EXPECT_THAT(EmbeddingIndex::Create(&filesystem_, embedding_index_dir, nullptr, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); +} + TEST_F(EmbeddingIndexTest, InitializationShouldFailWithoutPersistToDiskOrDestruction) { // 1. Create index and confirm that data was properly added. @@ -135,20 +184,22 @@ GetTestTempDir() + "/embedding_index_test_local"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndex> embedding_index, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir, &clock_, + feature_flags_.get())); PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); ICING_ASSERT_OK(embedding_index->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index->CommitBufferToIndex()); embedding_index->set_last_added_document_id(0); EXPECT_THAT( - GetHits(embedding_index.get(), /*dimension=*/3, - /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre(EmbeddingHit( BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(embedding_index.get()), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index.get()), ElementsAre(0.1, 0.2, 0.3)); EXPECT_EQ(embedding_index->last_added_document_id(), 0); // GetChecksum should succeed without updating the checksum. @@ -156,7 +207,8 @@ // 2. Try to create another index with the same directory. This should fail // due to checksum mismatch. - EXPECT_THAT(EmbeddingIndex::Create(&filesystem_, embedding_index_dir), + EXPECT_THAT(EmbeddingIndex::Create(&filesystem_, embedding_index_dir, &clock_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); embedding_index.reset(); @@ -169,20 +221,22 @@ GetTestTempDir() + "/embedding_index_test_local"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndex> embedding_index, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir, &clock_, + feature_flags_.get())); PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); ICING_ASSERT_OK(embedding_index->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index->CommitBufferToIndex()); embedding_index->set_last_added_document_id(0); EXPECT_THAT( - GetHits(embedding_index.get(), /*dimension=*/3, - /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre(EmbeddingHit( BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(embedding_index.get()), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index.get()), ElementsAre(0.1, 0.2, 0.3)); EXPECT_EQ(embedding_index->last_added_document_id(), 0); @@ -194,14 +248,15 @@ // 3. Create another index and confirm that the data is still there. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndex> embedding_index_two, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir, &clock_, + feature_flags_.get())); EXPECT_THAT( - GetHits(embedding_index_two.get(), /*dimension=*/3, - /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index_two.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre(EmbeddingHit( BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(embedding_index_two.get()), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_two.get()), ElementsAre(0.1, 0.2, 0.3)); EXPECT_EQ(embedding_index_two->last_added_document_id(), 0); @@ -216,20 +271,22 @@ GetTestTempDir() + "/embedding_index_test_local"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndex> embedding_index, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir, &clock_, + feature_flags_.get())); PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); ICING_ASSERT_OK(embedding_index->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index->CommitBufferToIndex()); embedding_index->set_last_added_document_id(0); EXPECT_THAT( - GetHits(embedding_index.get(), /*dimension=*/3, - /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre(EmbeddingHit( BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(embedding_index.get()), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index.get()), ElementsAre(0.1, 0.2, 0.3)); EXPECT_EQ(embedding_index->last_added_document_id(), 0); @@ -239,14 +296,15 @@ // 3. Create another index and confirm that the data is still there. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndex> embedding_index_two, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir, &clock_, + feature_flags_.get())); EXPECT_THAT( - GetHits(embedding_index_two.get(), /*dimension=*/3, - /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index_two.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre(EmbeddingHit( BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(embedding_index_two.get()), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_two.get()), ElementsAre(0.1, 0.2, 0.3)); EXPECT_EQ(embedding_index_two->last_added_document_id(), 0); @@ -255,18 +313,78 @@ filesystem_.DeleteDirectoryRecursively(embedding_index_dir.c_str()); } +TEST_F(EmbeddingIndexTest, GetEmbeddingVectorShouldFailWhenOutOfRange) { + BasicHit basic_hit(/*section_id=*/0, /*document_id=*/0); + PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding(basic_hit, vector, + QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); + + EmbeddingHit embedding_hit(basic_hit, /*location=*/0); + uint32_t dimension = 3; + ICING_ASSERT_OK( + embedding_index_->GetEmbeddingVector(embedding_hit, dimension)); + EXPECT_THAT( + embedding_index_->GetEmbeddingVector(embedding_hit, dimension + 1), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); +} + +TEST_F(EmbeddingIndexTest, + GetQuantizedEmbeddingVectorShouldFailWhenOutOfRange) { + BasicHit basic_hit(kSectionIdQuantizedEmbedding, /*document_id=*/0); + PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + basic_hit, vector, QUANTIZATION_TYPE_QUANTIZE_8_BIT)); + ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); + + EmbeddingHit embedding_hit(basic_hit, /*location=*/0); + uint32_t dimension = 3; + ICING_ASSERT_OK( + embedding_index_->GetQuantizedEmbeddingVector(embedding_hit, dimension)); + EXPECT_THAT(embedding_index_->GetQuantizedEmbeddingVector(embedding_hit, + dimension + 1), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); +} + TEST_F(EmbeddingIndexTest, AddSingleEmbedding) { PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(0); EXPECT_THAT( - GetHits(/*dimension=*/3, /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre(EmbeddingHit( BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(0.1, 0.2, 0.3)); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.1, 0.2, 0.3)); + EXPECT_EQ(embedding_index_->last_added_document_id(), 0); +} + +TEST_F(EmbeddingIndexTest, AddSingleQuantizedEmbedding) { + PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/0), vector, + QUANTIZATION_TYPE_QUANTIZE_8_BIT)); + ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); + embedding_index_->set_last_added_document_id(0); + + EmbeddingHit hit(BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/0), + /*location=*/0); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre(hit))); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + hit, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {0.1, 0.2, 0.3}))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); } @@ -275,41 +393,81 @@ PropertyProto::VectorProto vector2 = CreateVector("model", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector2)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector2, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(0); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0), EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/3)))); - EXPECT_THAT(GetRawEmbeddingData(), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, -0.1, -0.2, -0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); } +TEST_F(EmbeddingIndexTest, AddMultipleQuantizedEmbeddingsInTheSameSection) { + PropertyProto::VectorProto vector1 = CreateVector("model", {0.1, 0.2, 0.3}); + PropertyProto::VectorProto vector2 = + CreateVector("model", {-0.1, -0.2, -0.3}); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_QUANTIZE_8_BIT)); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/0), vector2, + QUANTIZATION_TYPE_QUANTIZE_8_BIT)); + ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); + embedding_index_->set_last_added_document_id(0); + + EmbeddingHit hit1(BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/0), + /*location=*/0); + EmbeddingHit hit2(BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/0), + /*location=*/3 + sizeof(Quantizer)); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre(hit1, hit2))); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(2 * (3 + sizeof(Quantizer)))); // Two quantized vectors + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + hit1, /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {0.1, 0.2, 0.3}))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + hit2, /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {-0.1, -0.2, -0.3}))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); + EXPECT_EQ(embedding_index_->last_added_document_id(), 0); +} + TEST_F(EmbeddingIndexTest, HitsWithLowerSectionIdReturnedFirst) { PropertyProto::VectorProto vector1 = CreateVector("model", {0.1, 0.2, 0.3}); PropertyProto::VectorProto vector2 = CreateVector("model", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/5, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/5, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/2, /*document_id=*/0), vector2)); + BasicHit(/*section_id=*/2, /*document_id=*/0), vector2, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(0); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/2, /*document_id=*/0), /*location=*/3), EmbeddingHit(BasicHit(/*section_id=*/5, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, -0.1, -0.2, -0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); } @@ -319,19 +477,22 @@ PropertyProto::VectorProto vector2 = CreateVector("model", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/1), vector2)); + BasicHit(/*section_id=*/0, /*document_id=*/1), vector2, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(1); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/1), /*location=*/3), EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, -0.1, -0.2, -0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 1); } @@ -341,24 +502,30 @@ PropertyProto::VectorProto vector2 = CreateVector("model2", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector2)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector2, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(0); - EXPECT_THAT(GetHits(/*dimension=*/2, /*model_signature=*/"model1"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/2, + /*model_signature=*/"model1"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model2"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model2"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/2)))); - EXPECT_THAT( - GetHits(/*dimension=*/5, /*model_signature=*/"non-existent-model"), - IsOkAndHolds(IsEmpty())); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(0.1, 0.2, -0.1, -0.2, -0.3)); + EXPECT_THAT(GetEmbeddingHitsFromIndex( + embedding_index_.get(), + /*dimension=*/5, /*model_signature=*/"non-existent-model"), + IsOkAndHolds(IsEmpty())); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.1, 0.2, -0.1, -0.2, -0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); } @@ -368,21 +535,26 @@ PropertyProto::VectorProto vector2 = CreateVector("model", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector2)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector2, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(0); - EXPECT_THAT(GetHits(/*dimension=*/2, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/2, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/2)))); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(0.1, 0.2, -0.1, -0.2, -0.3)); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.1, 0.2, -0.1, -0.2, -0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); } @@ -393,22 +565,41 @@ PropertyProto::VectorProto vector1 = CreateVector("model", {0.1, 0.2, 0.3}); PropertyProto::VectorProto vector2 = CreateVector("model", {-0.1, -0.2, -0.3}); - ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/1, /*document_id=*/0), vector1)); - ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/2, /*document_id=*/1), vector2)); - ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); - embedding_index_->set_last_added_document_id(1); + PropertyProto::VectorProto vector3 = CreateVector("model", {0.4, 0.5, 0.6}); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), - IsOkAndHolds(ElementsAre( - EmbeddingHit(BasicHit(/*section_id=*/2, /*document_id=*/1), - /*location=*/3), - EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/0), - /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(/*section_id=*/1, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(/*section_id=*/2, /*document_id=*/1), vector2, + QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/2), vector3, + QUANTIZATION_TYPE_QUANTIZE_8_BIT)); + ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); + embedding_index_->set_last_added_document_id(2); + + EmbeddingHit hit1(BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/2), + /*location=*/0); + EmbeddingHit hit2(BasicHit(/*section_id=*/2, /*document_id=*/1), + /*location=*/3); + EmbeddingHit hit3(BasicHit(/*section_id=*/1, /*document_id=*/0), + /*location=*/0); + + EXPECT_THAT( + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre(hit1, hit2, hit3))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, -0.1, -0.2, -0.3)); - EXPECT_EQ(embedding_index_->last_added_document_id(), 1); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + hit1, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), vector3.values()))); + EXPECT_EQ(embedding_index_->last_added_document_id(), 2); EXPECT_FALSE(embedding_index_->is_empty()); EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(false)); @@ -416,7 +607,9 @@ ICING_ASSERT_OK(embedding_index_->Clear()); EXPECT_TRUE(embedding_index_->is_empty()); EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(true)); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + IsEmpty()); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), Eq(0)); EXPECT_EQ(embedding_index_->last_added_document_id(), kInvalidDocumentId); } } @@ -428,22 +621,40 @@ PropertyProto::VectorProto vector1 = CreateVector("model", {0.1, 0.2, 0.3}); PropertyProto::VectorProto vector2 = CreateVector("model", {-0.1, -0.2, -0.3}); - ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/1, /*document_id=*/0), vector1)); - ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/2, /*document_id=*/1), vector2)); - ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); - embedding_index_->set_last_added_document_id(1); + PropertyProto::VectorProto vector3 = CreateVector("model", {0.4, 0.5, 0.6}); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), - IsOkAndHolds(ElementsAre( - EmbeddingHit(BasicHit(/*section_id=*/2, /*document_id=*/1), - /*location=*/3), - EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/0), - /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(/*section_id=*/1, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(/*section_id=*/2, /*document_id=*/1), vector2, + QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/2), vector3, + QUANTIZATION_TYPE_QUANTIZE_8_BIT)); + ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); + embedding_index_->set_last_added_document_id(2); + + EmbeddingHit hit1(BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/2), + /*location=*/0); + EmbeddingHit hit2(BasicHit(/*section_id=*/2, /*document_id=*/1), + /*location=*/3); + EmbeddingHit hit3(BasicHit(/*section_id=*/1, /*document_id=*/0), + /*location=*/0); + EXPECT_THAT( + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre(hit1, hit2, hit3))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, -0.1, -0.2, -0.3)); - EXPECT_EQ(embedding_index_->last_added_document_id(), 1); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + hit1, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), vector3.values()))); + EXPECT_EQ(embedding_index_->last_added_document_id(), 2); EXPECT_FALSE(embedding_index_->is_empty()); EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(false)); @@ -452,10 +663,13 @@ EmbeddingIndex::Discard(filesystem_, embedding_index_dir_); ICING_ASSERT_OK_AND_ASSIGN( embedding_index_, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir_)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir_, &clock_, + feature_flags_.get())); EXPECT_TRUE(embedding_index_->is_empty()); EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(true)); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + IsEmpty()); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), Eq(0)); EXPECT_EQ(embedding_index_->last_added_document_id(), kInvalidDocumentId); } } @@ -464,7 +678,8 @@ ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); EXPECT_TRUE(embedding_index_->is_empty()); EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(true)); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), Eq(0)); } TEST_F(EmbeddingIndexTest, MultipleCommits) { @@ -473,20 +688,23 @@ CreateVector("model", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/1, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/1, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector2)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector2, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/3), EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, -0.1, -0.2, -0.3)); } @@ -497,11 +715,13 @@ CreateVector("model", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/1, /*document_id=*/0), vector2)); + BasicHit(/*section_id=*/1, /*document_id=*/0), vector2, + QUANTIZATION_TYPE_NONE)); // Posting list with delta encoding can only allow decreasing values. EXPECT_THAT(embedding_index_->CommitBufferToIndex(), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); @@ -513,11 +733,13 @@ CreateVector("model", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/1), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/1), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector2)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector2, + QUANTIZATION_TYPE_NONE)); // Posting list with delta encoding can only allow decreasing values, which // means document ids must be committed increasingly, since document ids are // inverted in hit values. @@ -525,63 +747,184 @@ StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } +TEST_F(EmbeddingIndexTest, OptimizeShouldFailWithNullPointer) { + EXPECT_THAT(embedding_index_->Optimize( + /*document_store=*/nullptr, schema_store_.get(), + /*document_id_old_to_new=*/{}, + /*new_last_added_document_id=*/kInvalidDocumentId), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + + EXPECT_THAT(embedding_index_->Optimize( + document_store_.get(), /*schema_store=*/nullptr, + /*document_id_old_to_new=*/{}, + /*new_last_added_document_id=*/kInvalidDocumentId), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); +} + +TEST_F(EmbeddingIndexTest, OptimizeShouldFailWhenDocumentIdMapIsTooSmall) { + PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(/*section_id=*/0, /*document_id=*/2), vector, + QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); + embedding_index_->set_last_added_document_id(2); + + // Optimize should fail because the provided document_id_old_to_new map does + // not contain an entry for document id 2. + EXPECT_THAT(embedding_index_ + ->Optimize(document_store_.get(), schema_store_.get(), + /*document_id_old_to_new=*/{0, 1}, + /*new_last_added_document_id=*/2) + .error_message(), + HasSubstr("The provided map is too small")); +} + TEST_F(EmbeddingIndexTest, EmptyOptimizeIsOk) { ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), /*document_id_old_to_new=*/{}, /*new_last_added_document_id=*/kInvalidDocumentId)); EXPECT_TRUE(embedding_index_->is_empty()); EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(true)); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), Eq(0)); } TEST_F(EmbeddingIndexTest, OptimizeSingleEmbeddingSingleDocument) { PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/2), vector)); + BasicHit(/*section_id=*/0, /*document_id=*/2), vector, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(2); // Before optimize EXPECT_THAT( - GetHits(/*dimension=*/3, /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre(EmbeddingHit( BasicHit(/*section_id=*/0, /*document_id=*/2), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(0.1, 0.2, 0.3)); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.1, 0.2, 0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 2); // Run optimize without deleting any documents, and check that the index is // not changed. - ICING_ASSERT_OK(embedding_index_->Optimize( - /*document_id_old_to_new=*/{0, 1, 2}, - /*new_last_added_document_id=*/2)); + ICING_ASSERT_OK( + embedding_index_->Optimize(document_store_.get(), schema_store_.get(), + /*document_id_old_to_new=*/{0, 1, 2}, + /*new_last_added_document_id=*/2)); EXPECT_THAT( - GetHits(/*dimension=*/3, /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre(EmbeddingHit( BasicHit(/*section_id=*/0, /*document_id=*/2), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(0.1, 0.2, 0.3)); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.1, 0.2, 0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 2); // Run optimize to map document id 2 to 1, and check that the index is // updated correctly. ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), /*document_id_old_to_new=*/{0, kInvalidDocumentId, 1}, /*new_last_added_document_id=*/1)); EXPECT_THAT( - GetHits(/*dimension=*/3, /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre(EmbeddingHit( BasicHit(/*section_id=*/0, /*document_id=*/1), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(0.1, 0.2, 0.3)); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.1, 0.2, 0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 1); // Run optimize to delete the document. ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), /*document_id_old_to_new=*/{0, kInvalidDocumentId}, /*new_last_added_document_id=*/0)); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(IsEmpty())); EXPECT_TRUE(embedding_index_->is_empty()); EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(true)); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); + EXPECT_EQ(embedding_index_->last_added_document_id(), 0); +} + +TEST_F(EmbeddingIndexTest, OptimizeSingleQuantizedEmbeddingSingleDocument) { + PropertyProto::VectorProto vector = CreateVector("model", {0.1, 0.2, 0.3}); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/2), vector, + QUANTIZATION_TYPE_QUANTIZE_8_BIT)); + ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); + embedding_index_->set_last_added_document_id(2); + + // Before optimize + EmbeddingHit hit(BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/2), + /*location=*/0); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre(hit))); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + hit, /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {0.1, 0.2, 0.3}))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); + EXPECT_EQ(embedding_index_->last_added_document_id(), 2); + + // Run optimize without deleting any documents, and check that the index is + // not changed + ICING_ASSERT_OK( + embedding_index_->Optimize(document_store_.get(), schema_store_.get(), + /*document_id_old_to_new=*/{0, 1, 2}, + /*new_last_added_document_id=*/2)); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre(hit))); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + hit, /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {0.1, 0.2, 0.3}))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); + EXPECT_EQ(embedding_index_->last_added_document_id(), 2); + + // Run optimize to map document id 2 to 1, and check that the index is + // updated correctly + ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), + /*document_id_old_to_new=*/{0, kInvalidDocumentId, 1}, + /*new_last_added_document_id=*/1)); + hit = EmbeddingHit(BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/1), + /*location=*/0); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre(hit))); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + hit, /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {0.1, 0.2, 0.3}))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); + EXPECT_EQ(embedding_index_->last_added_document_id(), 1); + + // Run optimize to delete the document + ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), + /*document_id_old_to_new=*/{0, kInvalidDocumentId}, + /*new_last_added_document_id=*/0)); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(IsEmpty())); + EXPECT_TRUE(embedding_index_->is_empty()); + EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(true)); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), Eq(0)); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); } @@ -589,63 +932,108 @@ PropertyProto::VectorProto vector1 = CreateVector("model", {0.1, 0.2, 0.3}); PropertyProto::VectorProto vector2 = CreateVector("model", {-0.1, -0.2, -0.3}); + PropertyProto::VectorProto vector3 = CreateVector("model", {0.4, 0.5, 0.6}); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/2), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/2), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/2), vector2)); + BasicHit(/*section_id=*/0, /*document_id=*/2), vector2, + QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/2), vector3, + QUANTIZATION_TYPE_QUANTIZE_8_BIT)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(2); // Before optimize - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EmbeddingHit quantized_hit( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/2), + /*location=*/0); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/2), /*location=*/0), EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/2), - /*location=*/3)))); - EXPECT_THAT(GetRawEmbeddingData(), + /*location=*/3), + quantized_hit))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, -0.1, -0.2, -0.3)); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), vector3.values()))); EXPECT_EQ(embedding_index_->last_added_document_id(), 2); // Run optimize without deleting any documents, and check that the index is // not changed. - ICING_ASSERT_OK(embedding_index_->Optimize( - /*document_id_old_to_new=*/{0, 1, 2}, - /*new_last_added_document_id=*/2)); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + ICING_ASSERT_OK( + embedding_index_->Optimize(document_store_.get(), schema_store_.get(), + /*document_id_old_to_new=*/{0, 1, 2}, + /*new_last_added_document_id=*/2)); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/2), /*location=*/0), EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/2), - /*location=*/3)))); - EXPECT_THAT(GetRawEmbeddingData(), + /*location=*/3), + quantized_hit))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, -0.1, -0.2, -0.3)); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), vector3.values()))); EXPECT_EQ(embedding_index_->last_added_document_id(), 2); // Run optimize to map document id 2 to 1, and check that the index is // updated correctly. ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), /*document_id_old_to_new=*/{0, kInvalidDocumentId, 1}, /*new_last_added_document_id=*/1)); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + quantized_hit = + EmbeddingHit(BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/1), + /*location=*/0); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/1), /*location=*/0), EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/1), - /*location=*/3)))); - EXPECT_THAT(GetRawEmbeddingData(), + /*location=*/3), + quantized_hit))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, -0.1, -0.2, -0.3)); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), vector3.values()))); EXPECT_EQ(embedding_index_->last_added_document_id(), 1); // Run optimize to delete the document. ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), /*document_id_old_to_new=*/{0, kInvalidDocumentId}, /*new_last_added_document_id=*/0)); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(IsEmpty())); EXPECT_TRUE(embedding_index_->is_empty()); EXPECT_THAT(IndexContainsMetadataOnly(), IsOkAndHolds(true)); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), Eq(0)); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); } @@ -654,26 +1042,46 @@ PropertyProto::VectorProto vector2 = CreateVector("model", {1, 2, 3}); PropertyProto::VectorProto vector3 = CreateVector("model", {-0.1, -0.2, -0.3}); + PropertyProto::VectorProto vector4 = CreateVector("model", {0.4, 0.5, 0.6}); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/1, /*document_id=*/0), vector2)); + BasicHit(/*section_id=*/1, /*document_id=*/0), vector2, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/1), vector3)); + BasicHit(/*section_id=*/0, /*document_id=*/1), vector3, + QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/1), vector4, + QUANTIZATION_TYPE_QUANTIZE_8_BIT)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(1); // Before optimize - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EmbeddingHit quantized_hit( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/1), + /*location=*/0); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/1), /*location=*/6), + quantized_hit, EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0), EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/0), /*location=*/3)))); - EXPECT_THAT(GetRawEmbeddingData(), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 0.3, 1, 2, 3, -0.1, -0.2, -0.3)); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), vector4.values()))); EXPECT_EQ(embedding_index_->last_added_document_id(), 1); // Run optimize without deleting any documents. It is expected to see that the @@ -683,32 +1091,57 @@ // Also keep in mind that once the raw data is rearranged, calling another // Optimize subsequently will not change the raw data again. for (int i = 0; i < 2; i++) { - ICING_ASSERT_OK(embedding_index_->Optimize( - /*document_id_old_to_new=*/{0, 1}, - /*new_last_added_document_id=*/1)); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), - IsOkAndHolds(ElementsAre( - EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/1), - /*location=*/0), - EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), - /*location=*/3), - EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/0), - /*location=*/6)))); - EXPECT_THAT(GetRawEmbeddingData(), + ICING_ASSERT_OK( + embedding_index_->Optimize(document_store_.get(), schema_store_.get(), + /*document_id_old_to_new=*/{0, 1}, + /*new_last_added_document_id=*/1)); + EXPECT_THAT( + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre( + EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/1), + /*location=*/0), + quantized_hit, + EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), + /*location=*/3), + EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/0), + /*location=*/6)))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(-0.1, -0.2, -0.3, 0.1, 0.2, 0.3, 1, 2, 3)); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), vector4.values()))); EXPECT_EQ(embedding_index_->last_added_document_id(), 1); } // Run optimize to delete document 0, and check that the index is // updated correctly. ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), /*document_id_old_to_new=*/{kInvalidDocumentId, 0}, /*new_last_added_document_id=*/0)); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + quantized_hit = + EmbeddingHit(BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/0), + /*location=*/0); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), - /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(-0.1, -0.2, -0.3)); + /*location=*/0), + quantized_hit))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(-0.1, -0.2, -0.3)); + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(3 + sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), vector4.values()))); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); } @@ -718,26 +1151,31 @@ PropertyProto::VectorProto vector3 = CreateVector("model2", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/1), vector2)); + BasicHit(/*section_id=*/0, /*document_id=*/1), vector2, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/1, /*document_id=*/1), vector3)); + BasicHit(/*section_id=*/1, /*document_id=*/1), vector3, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(1); // Before optimize - EXPECT_THAT(GetHits(/*dimension=*/2, /*model_signature=*/"model1"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/2, + /*model_signature=*/"model1"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/1), /*location=*/2), EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model2"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model2"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/1), /*location=*/4)))); - EXPECT_THAT(GetRawEmbeddingData(), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.1, 0.2, 1, 2, -0.1, -0.2, -0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 1); @@ -750,20 +1188,25 @@ // Also keep in mind that once the raw data is rearranged, calling another // Optimize subsequently will not change the raw data again. for (int i = 0; i < 2; i++) { - ICING_ASSERT_OK(embedding_index_->Optimize( - /*document_id_old_to_new=*/{0, 1}, - /*new_last_added_document_id=*/1)); - EXPECT_THAT(GetHits(/*dimension=*/2, /*model_signature=*/"model1"), - IsOkAndHolds(ElementsAre( - EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/1), - /*location=*/0), - EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), - /*location=*/2)))); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model2"), - IsOkAndHolds(ElementsAre( - EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/1), - /*location=*/4)))); - EXPECT_THAT(GetRawEmbeddingData(), + ICING_ASSERT_OK( + embedding_index_->Optimize(document_store_.get(), schema_store_.get(), + /*document_id_old_to_new=*/{0, 1}, + /*new_last_added_document_id=*/1)); + EXPECT_THAT( + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/2, + /*model_signature=*/"model1"), + IsOkAndHolds(ElementsAre( + EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/1), + /*location=*/0), + EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), + /*location=*/2)))); + EXPECT_THAT( + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model2"), + IsOkAndHolds(ElementsAre( + EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/1), + /*location=*/4)))); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(1, 2, 0.1, 0.2, -0.1, -0.2, -0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 1); } @@ -771,15 +1214,19 @@ // Run optimize to delete document 1, and check that the index is // updated correctly. ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), /*document_id_old_to_new=*/{0, kInvalidDocumentId}, /*new_last_added_document_id=*/0)); - EXPECT_THAT(GetHits(/*dimension=*/2, /*model_signature=*/"model1"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/2, + /*model_signature=*/"model1"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model2"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model2"), IsOkAndHolds(IsEmpty())); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(0.1, 0.2)); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.1, 0.2)); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); } @@ -789,36 +1236,45 @@ PropertyProto::VectorProto vector2 = CreateVector("model2", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/0, /*document_id=*/0), vector1)); + BasicHit(/*section_id=*/0, /*document_id=*/0), vector1, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(/*section_id=*/1, /*document_id=*/1), vector2)); + BasicHit(/*section_id=*/1, /*document_id=*/1), vector2, + QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); embedding_index_->set_last_added_document_id(1); // Before optimize - EXPECT_THAT(GetHits(/*dimension=*/2, /*model_signature=*/"model1"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/2, + /*model_signature=*/"model1"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/0, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model2"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model2"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/1), /*location=*/2)))); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(0.1, 0.2, -0.1, -0.2, -0.3)); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.1, 0.2, -0.1, -0.2, -0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 1); // Run optimize to delete document 0, and check that the index is // updated correctly. ICING_ASSERT_OK(embedding_index_->Optimize( + document_store_.get(), schema_store_.get(), /*document_id_old_to_new=*/{kInvalidDocumentId, 0}, /*new_last_added_document_id=*/0)); - EXPECT_THAT(GetHits(/*dimension=*/2, /*model_signature=*/"model1"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/2, + /*model_signature=*/"model1"), IsOkAndHolds(IsEmpty())); - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model2"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model2"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(/*section_id=*/1, /*document_id=*/0), /*location=*/0)))); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(-0.1, -0.2, -0.3)); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(-0.1, -0.2, -0.3)); EXPECT_EQ(embedding_index_->last_added_document_id(), 0); }
diff --git a/icing/index/embed/embedding-scorer.cc b/icing/index/embed/embedding-scorer.cc index 0d84e01..0a3ed6c 100644 --- a/icing/index/embed/embedding-scorer.cc +++ b/icing/index/embed/embedding-scorer.cc
@@ -15,12 +15,15 @@ #include "icing/index/embed/embedding-scorer.h" #include <cmath> +#include <cstdint> #include <memory> #include <string> +#include <type_traits> #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" +#include "icing/index/embed/quantizer.h" #include "icing/proto/search.pb.h" namespace icing { @@ -28,31 +31,55 @@ namespace { -float CalculateDotProduct(int dimension, const float* v1, const float* v2) { +template <typename T> +inline std::enable_if_t<std::is_same<T, float>::value, float> ToFloat(T value) { + return value; +} + +template <typename T> +inline std::enable_if_t<std::is_same<T, float>::value, float> ToFloat( + T value, const Quantizer&) { + return value; +} + +template <typename T> +inline std::enable_if_t<std::is_same<T, uint8_t>::value, float> ToFloat( + T quantized, const Quantizer& quantizer) { + return quantizer.Dequantize(quantized); +} + +template <typename T1, typename T2, typename... Args> +float CalculateDotProduct(int dimension, const T1* v1, const T2* v2, + const Args&... args) { float dot_product = 0.0; for (int i = 0; i < dimension; ++i) { - dot_product += v1[i] * v2[i]; + dot_product += ToFloat(v1[i], args...) * ToFloat(v2[i], args...); } return dot_product; } -float CalculateNorm2(int dimension, const float* v) { - return std::sqrt(CalculateDotProduct(dimension, v, v)); +template <typename T, typename... Args> +float CalculateNorm2(int dimension, const T* v, const Args&... args) { + return std::sqrt(CalculateDotProduct(dimension, v, v, args...)); } -float CalculateCosine(int dimension, const float* v1, const float* v2) { - float divisor = CalculateNorm2(dimension, v1) * CalculateNorm2(dimension, v2); +template <typename T1, typename T2, typename... Args> +float CalculateCosine(int dimension, const T1* v1, const T2* v2, + const Args&... args) { + float divisor = CalculateNorm2(dimension, v1, args...) * + CalculateNorm2(dimension, v2, args...); if (divisor == 0.0) { return 0.0; } - return CalculateDotProduct(dimension, v1, v2) / divisor; + return CalculateDotProduct(dimension, v1, v2, args...) / divisor; } -float CalculateEuclideanDistance(int dimension, const float* v1, - const float* v2) { +template <typename T1, typename T2, typename... Args> +float CalculateEuclideanDistance(int dimension, const T1* v1, const T2* v2, + const Args&... args) { float result = 0.0; for (int i = 0; i < dimension; ++i) { - float diff = v1[i] - v2[i]; + float diff = ToFloat(v1[i], args...) - ToFloat(v2[i], args...); result += diff * diff; } return std::sqrt(result); @@ -91,5 +118,23 @@ return CalculateEuclideanDistance(dimension, v1, v2); } +float CosineEmbeddingScorer::Score(int dimension, const float* v1, + const uint8_t* v2, + const Quantizer& quantizer) const { + return CalculateCosine(dimension, v1, v2, quantizer); +} + +float DotProductEmbeddingScorer::Score(int dimension, const float* v1, + const uint8_t* v2, + const Quantizer& quantizer) const { + return CalculateDotProduct(dimension, v1, v2, quantizer); +} + +float EuclideanDistanceEmbeddingScorer::Score( + int dimension, const float* v1, const uint8_t* v2, + const Quantizer& quantizer) const { + return CalculateEuclideanDistance(dimension, v1, v2, quantizer); +} + } // namespace lib } // namespace icing
diff --git a/icing/index/embed/embedding-scorer.h b/icing/index/embed/embedding-scorer.h index 8caf0bc..b0e0f75 100644 --- a/icing/index/embed/embedding-scorer.h +++ b/icing/index/embed/embedding-scorer.h
@@ -15,9 +15,11 @@ #ifndef ICING_INDEX_EMBED_EMBEDDING_SCORER_H_ #define ICING_INDEX_EMBED_EMBEDDING_SCORER_H_ +#include <cstdint> #include <memory> #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/index/embed/quantizer.h" #include "icing/proto/search.pb.h" namespace icing { @@ -27,8 +29,11 @@ public: static libtextclassifier3::StatusOr<std::unique_ptr<EmbeddingScorer>> Create( SearchSpecProto::EmbeddingQueryMetricType::Code metric_type); + virtual float Score(int dimension, const float* v1, const float* v2) const = 0; + virtual float Score(int dimension, const float* v1, const uint8_t* v2, + const Quantizer& quantizer) const = 0; virtual ~EmbeddingScorer() = default; }; @@ -36,16 +41,22 @@ class CosineEmbeddingScorer : public EmbeddingScorer { public: float Score(int dimension, const float* v1, const float* v2) const override; + float Score(int dimension, const float* v1, const uint8_t* v2, + const Quantizer& quantizer) const override; }; class DotProductEmbeddingScorer : public EmbeddingScorer { public: float Score(int dimension, const float* v1, const float* v2) const override; + float Score(int dimension, const float* v1, const uint8_t* v2, + const Quantizer& quantizer) const override; }; class EuclideanDistanceEmbeddingScorer : public EmbeddingScorer { public: float Score(int dimension, const float* v1, const float* v2) const override; + float Score(int dimension, const float* v1, const uint8_t* v2, + const Quantizer& quantizer) const override; }; } // namespace lib
diff --git a/icing/index/embed/embedding-scorer_test.cc b/icing/index/embed/embedding-scorer_test.cc new file mode 100644 index 0000000..162b969 --- /dev/null +++ b/icing/index/embed/embedding-scorer_test.cc
@@ -0,0 +1,126 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/index/embed/embedding-scorer.h" + +#include <cstdint> +#include <memory> +#include <vector> + +#include "gtest/gtest.h" +#include "icing/index/embed/quantizer.h" +#include "icing/testing/common-matchers.h" + +namespace icing { +namespace lib { + +namespace { + +std::vector<uint8_t> QuantizeVector(std::vector<float> v, + const Quantizer& quantizer) { + std::vector<uint8_t> quantized; + quantized.reserve(v.size()); + for (float value : v) { + quantized.push_back(quantizer.Quantize(value)); + } + return quantized; +} + +TEST(EmbeddingScorerTest, DotProduct) { + constexpr float eps_quantized = 0.01f; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<EmbeddingScorer> embedding_scorer, + EmbeddingScorer::Create( + SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT)); + ICING_ASSERT_OK_AND_ASSIGN( + Quantizer quantizer, + Quantizer::Create(/*float_min=*/-1.0f, /*float_max=*/1.0f)); + + int dimension = 3; + std::vector<float> v1 = {0.1f, 0.2f, 0.3f}; + std::vector<float> v2 = {0.5f, 0.5f, 0.6f}; + std::vector<uint8_t> v2_quantized = QuantizeVector(v2, quantizer); + float expected_dot_product = 0.1f * 0.5f + 0.2f * 0.5f + 0.3f * 0.6f; + + // Test float computation + EXPECT_FLOAT_EQ(embedding_scorer->Score(dimension, v1.data(), v2.data()), + expected_dot_product); + + // Test quantization + EXPECT_NEAR(embedding_scorer->Score(dimension, v1.data(), v2_quantized.data(), + quantizer), + expected_dot_product, eps_quantized); +} + +TEST(EmbeddingScorerTest, Cosine) { + constexpr float eps = 0.001f; + constexpr float eps_quantized = 0.01f; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<EmbeddingScorer> embedding_scorer, + EmbeddingScorer::Create( + SearchSpecProto::EmbeddingQueryMetricType::COSINE)); + ICING_ASSERT_OK_AND_ASSIGN( + Quantizer quantizer, + Quantizer::Create(/*float_min=*/-1.0f, /*float_max=*/1.0f)); + + int dimension = 3; + std::vector<float> v1 = {0.7f, -0.3f, -0.6f}; + std::vector<float> v2 = {-0.5f, 0.1f, -0.2f}; + std::vector<uint8_t> v2_quantized = QuantizeVector(v2, quantizer); + float expected_cosine = -0.4896f; + + // Test float computation + EXPECT_NEAR(embedding_scorer->Score(dimension, v1.data(), v2.data()), + expected_cosine, eps); + + // Test quantization + EXPECT_NEAR(embedding_scorer->Score(dimension, v1.data(), v2_quantized.data(), + quantizer), + expected_cosine, eps_quantized); +} + +TEST(EmbeddingScorerTest, Euclidean) { + constexpr float eps = 0.001f; + constexpr float eps_quantized = 0.01f; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<EmbeddingScorer> embedding_scorer, + EmbeddingScorer::Create( + SearchSpecProto::EmbeddingQueryMetricType::EUCLIDEAN)); + ICING_ASSERT_OK_AND_ASSIGN( + Quantizer quantizer, + Quantizer::Create(/*float_min=*/-1.0f, /*float_max=*/1.0f)); + + int dimension = 3; + std::vector<float> v1 = {0.6f, -0.2f, 0.9f}; + std::vector<float> v2 = {-0.8f, -0.4f, 0.2f}; + std::vector<uint8_t> v2_quantized = QuantizeVector(v2, quantizer); + float expected_euclidean = 1.5780f; + + // Test float computation + EXPECT_NEAR(embedding_scorer->Score(dimension, v1.data(), v2.data()), + expected_euclidean, eps); + + // Test quantization + EXPECT_NEAR(embedding_scorer->Score(dimension, v1.data(), v2_quantized.data(), + quantizer), + expected_euclidean, eps_quantized); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/index/embed/quantizer.h b/icing/index/embed/quantizer.h new file mode 100644 index 0000000..099c96d --- /dev/null +++ b/icing/index/embed/quantizer.h
@@ -0,0 +1,91 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_INDEX_EMBED_QUANTIZER_H_ +#define ICING_INDEX_EMBED_QUANTIZER_H_ + +#include <algorithm> +#include <cmath> +#include <cstdint> +#include <cstring> +#include <limits> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" + +namespace icing { +namespace lib { + +// A class for quantizing and dequantizing floating-point values to and from +// 8-bit unsigned integers. The maximum quantization error is +// (float_max - float_min) / 255 / 2. +class Quantizer { + public: + // Creates a new Quantizer instance based on the specified range. Values + // outside this range will be quantized to the closest boundary. + // + // Returns: + // - An Quantizer instance on success. + // - INVALID_ARGUMENT_ERROR if float_min is greater than or equal to + // float_max. + static libtextclassifier3::StatusOr<Quantizer> Create(float float_min, + float float_max) { + if (float_min > float_max) { + return absl_ports::InvalidArgumentError( + "float_min must be less than or equal to float_max."); + } + float scale_factor = 0.0; + if (float_max - float_min > kEpsilon) { // Not equal. + scale_factor = + static_cast<float>(kMaxQuantizedValue) / (float_max - float_min); + } + return Quantizer(float_min, scale_factor); + } + + // Creates a new Quantizer instance from the serialized data. + explicit Quantizer(const char* data) { + memcpy(this, data, sizeof(Quantizer)); + } + + uint8_t Quantize(float value) const { + double normalized = + (static_cast<double>(value) - float_min_) * scale_factor_; + double quantized = std::round(normalized); + quantized = + std::clamp(quantized, 0.0, static_cast<double>(kMaxQuantizedValue)); + return static_cast<uint8_t>(quantized); + } + + float Dequantize(uint8_t quantized) const { + if (scale_factor_ == 0.0) { + return float_min_; + } + return (quantized / scale_factor_) + float_min_; + } + + private: + static constexpr uint8_t kMaxQuantizedValue = + std::numeric_limits<uint8_t>::max(); + static constexpr float kEpsilon = 1e-6; + + explicit Quantizer(float float_min, float scale_factor) + : float_min_(float_min), scale_factor_(scale_factor) {} + float float_min_; + float scale_factor_; +}; + +} // namespace lib +} // namespace icing + +#endif // ICING_INDEX_EMBED_QUANTIZER_H_
diff --git a/icing/index/embed/quantizer_test.cc b/icing/index/embed/quantizer_test.cc new file mode 100644 index 0000000..8b50bd5 --- /dev/null +++ b/icing/index/embed/quantizer_test.cc
@@ -0,0 +1,171 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/index/embed/quantizer.h" + +#include <cstdint> +#include <limits> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/testing/common-matchers.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::AnyOf; +using ::testing::Eq; + +constexpr float kFloatEps = 1e-5f; + +constexpr float GetMaximumErrorForQuantization(float float_min, + float float_max) { + return (float_max - float_min) / 255 / 2 + kFloatEps; +} + +TEST(QuantizerTest, CreateFailure) { + EXPECT_THAT(Quantizer::Create(/*float_min=*/1.0f, /*float_max=*/0.0f), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST(QuantizerTest, QuantizeAndDequantize) { + constexpr float float_min = -1.0f; + constexpr float float_max = 1.0f; + constexpr float eps = GetMaximumErrorForQuantization(float_min, float_max); + + ICING_ASSERT_OK_AND_ASSIGN(Quantizer quantizer, + Quantizer::Create(float_min, float_max)); + + // float_min + EXPECT_EQ(quantizer.Quantize(float_min), 0); + EXPECT_FLOAT_EQ(quantizer.Dequantize(0), float_min); + + // float_max + EXPECT_EQ(quantizer.Quantize(float_max), 255); + EXPECT_FLOAT_EQ(quantizer.Dequantize(255), float_max); + + // Midpoint + // After scaling the midpoint value, we should get 127.5. Due to float + // precision, the quantized value can be either 127 or 128. + float original_value = 0.0f; + uint8_t quantized_value = quantizer.Quantize(original_value); + EXPECT_THAT(quantized_value, AnyOf(Eq(127), Eq(128))); + EXPECT_NEAR(quantizer.Dequantize(quantized_value), original_value, eps); + + // Other values + original_value = -0.5f; + quantized_value = quantizer.Quantize(original_value); + EXPECT_EQ(quantized_value, 64); + EXPECT_NEAR(quantizer.Dequantize(quantized_value), original_value, eps); + original_value = 0.5f; + quantized_value = quantizer.Quantize(original_value); + EXPECT_EQ(quantized_value, 191); + EXPECT_NEAR(quantizer.Dequantize(quantized_value), original_value, eps); + + // Out of range values + EXPECT_EQ(quantizer.Quantize(-2.0f), 0); + EXPECT_EQ(quantizer.Quantize(2.0f), 255); + EXPECT_EQ(quantizer.Quantize(std::numeric_limits<float>::lowest()), 0); + EXPECT_EQ(quantizer.Quantize(std::numeric_limits<float>::max()), 255); +} + +TEST(QuantizerTest, QuantizeAndDequantizeLargerRange) { + constexpr float float_min = -100.0f; + constexpr float float_max = 100.0f; + constexpr float eps = GetMaximumErrorForQuantization(float_min, float_max); + + ICING_ASSERT_OK_AND_ASSIGN(Quantizer quantizer, + Quantizer::Create(float_min, float_max)); + + // float_min + EXPECT_EQ(quantizer.Quantize(float_min), 0); + EXPECT_FLOAT_EQ(quantizer.Dequantize(0), float_min); + + // float_max + EXPECT_EQ(quantizer.Quantize(float_max), 255); + EXPECT_FLOAT_EQ(quantizer.Dequantize(255), float_max); + + // Midpoint + // After scaling the midpoint value, we should get 127.5. Due to float + // precision, the quantized value can be either 127 or 128. + float original_value = 0.0f; + uint8_t quantized_value = quantizer.Quantize(original_value); + EXPECT_THAT(quantized_value, AnyOf(Eq(127), Eq(128))); + EXPECT_NEAR(quantizer.Dequantize(quantized_value), original_value, eps); + + // Other values + original_value = -50.0f; + quantized_value = quantizer.Quantize(original_value); + EXPECT_EQ(quantized_value, 64); + EXPECT_NEAR(quantizer.Dequantize(quantized_value), original_value, eps); + original_value = 50.0f; + quantized_value = quantizer.Quantize(original_value); + EXPECT_EQ(quantized_value, 191); + EXPECT_NEAR(quantizer.Dequantize(quantized_value), original_value, eps); + + // Out of range values + EXPECT_EQ(quantizer.Quantize(-150.0f), 0); + EXPECT_EQ(quantizer.Quantize(150.0f), 255); + EXPECT_EQ(quantizer.Quantize(std::numeric_limits<float>::lowest()), 0); + EXPECT_EQ(quantizer.Quantize(std::numeric_limits<float>::max()), 255); +} + +// Test when the float range is equal to the uint8_t range. In this case, +// Quantize and Dequantize should be identity functions. +TEST(QuantizerTest, QuantizeAndDequantizeEqualRange) { + ICING_ASSERT_OK_AND_ASSIGN( + Quantizer quantizer, + Quantizer::Create(/*float_min=*/0.0f, /*float_max=*/255.0f)); + + for (int i = 0; i < 256; ++i) { + EXPECT_EQ(quantizer.Quantize(static_cast<float>(i)), i); + EXPECT_FLOAT_EQ(quantizer.Dequantize(i), static_cast<float>(i)); + } + + // Out of range values + EXPECT_EQ(quantizer.Quantize(-300.0f), 0); + EXPECT_EQ(quantizer.Quantize(300.0f), 255); + EXPECT_EQ(quantizer.Quantize(std::numeric_limits<float>::lowest()), 0); + EXPECT_EQ(quantizer.Quantize(std::numeric_limits<float>::max()), 255); +} + +TEST(QuantizerTest, QuantizeAndDequantizeWithMinEqualToMax) { + float constant_values[] = {0.0f, 1.0f, -1.0f, 5.0f, -5.0f, 100.0f, -100.0f}; + for (float constant_value : constant_values) { + ICING_ASSERT_OK_AND_ASSIGN(Quantizer quantizer, + Quantizer::Create(/*float_min=*/constant_value, + /*float_max=*/constant_value)); + + // All values should be quantized to 0. + for (float float_value = constant_value - 5; + float_value <= constant_value + 5; ++float_value) { + EXPECT_EQ(quantizer.Quantize(float_value), 0); + } + + // All quantized values should be dequantized to the constant value. + for (int quantized_value = 0; + quantized_value <= std::numeric_limits<uint8_t>::max(); + ++quantized_value) { + EXPECT_FLOAT_EQ(quantizer.Dequantize(quantized_value), constant_value); + } + } +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/index/embedding-indexing-handler.cc b/icing/index/embedding-indexing-handler.cc index 049e307..28d6568 100644 --- a/icing/index/embedding-indexing-handler.cc +++ b/icing/index/embedding-indexing-handler.cc
@@ -33,17 +33,19 @@ libtextclassifier3::StatusOr<std::unique_ptr<EmbeddingIndexingHandler>> EmbeddingIndexingHandler::Create(const Clock* clock, - EmbeddingIndex* embedding_index) { + EmbeddingIndex* embedding_index, + bool enable_embedding_index) { ICING_RETURN_ERROR_IF_NULL(clock); ICING_RETURN_ERROR_IF_NULL(embedding_index); - return std::unique_ptr<EmbeddingIndexingHandler>( - new EmbeddingIndexingHandler(clock, embedding_index)); + return std::unique_ptr<EmbeddingIndexingHandler>(new EmbeddingIndexingHandler( + clock, embedding_index, enable_embedding_index)); } libtextclassifier3::Status EmbeddingIndexingHandler::Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, - bool recovery_mode, PutDocumentStatsProto* put_document_stats) { + DocumentId /*old_document_id*/ _, bool recovery_mode, + PutDocumentStatsProto* put_document_stats) { std::unique_ptr<Timer> index_timer = clock_.GetNewTimer(); if (!IsDocumentIdValid(document_id)) { @@ -64,18 +66,21 @@ } embedding_index_.set_last_added_document_id(document_id); - for (const Section<PropertyProto::VectorProto>& vector_section : - tokenized_document.vector_sections()) { - BasicHit hit(/*section_id=*/vector_section.metadata.id, document_id); - for (const PropertyProto::VectorProto& vector : vector_section.content) { - ICING_RETURN_IF_ERROR(embedding_index_.BufferEmbedding(hit, vector)); + if (enable_embedding_index_) { + for (const Section<PropertyProto::VectorProto>& vector_section : + tokenized_document.vector_sections()) { + BasicHit hit(/*section_id=*/vector_section.metadata.id, document_id); + for (const PropertyProto::VectorProto& vector : vector_section.content) { + ICING_RETURN_IF_ERROR(embedding_index_.BufferEmbedding( + hit, vector, vector_section.metadata.quantization_type)); + } } - } - ICING_RETURN_IF_ERROR(embedding_index_.CommitBufferToIndex()); + ICING_RETURN_IF_ERROR(embedding_index_.CommitBufferToIndex()); - if (put_document_stats != nullptr) { - put_document_stats->set_embedding_index_latency_ms( - index_timer->GetElapsedMilliseconds()); + if (put_document_stats != nullptr) { + put_document_stats->set_embedding_index_latency_ms( + index_timer->GetElapsedMilliseconds()); + } } return libtextclassifier3::Status::OK;
diff --git a/icing/index/embedding-indexing-handler.h b/icing/index/embedding-indexing-handler.h index f3adf6a..6bbca42 100644 --- a/icing/index/embedding-indexing-handler.h +++ b/icing/index/embedding-indexing-handler.h
@@ -40,11 +40,15 @@ // - An EmbeddingIndexingHandler instance on success // - FAILED_PRECONDITION_ERROR if any of the input pointer is null static libtextclassifier3::StatusOr<std::unique_ptr<EmbeddingIndexingHandler>> - Create(const Clock* clock, EmbeddingIndex* embedding_index); + Create(const Clock* clock, EmbeddingIndex* embedding_index, + bool enable_embedding_index); // Handles the embedding indexing process: add hits into the embedding index // for all contents in tokenized_document.vector_sections. // + // Parameter old_document_id is unused since there is no need to migrate data + // from old_document_id to (new) document_id. + // // Returns: // - OK on success. // - INVALID_ARGUMENT_ERROR if document_id is invalid OR document_id is less @@ -54,14 +58,19 @@ // - Any embedding index errors. libtextclassifier3::Status Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, - bool recovery_mode, PutDocumentStatsProto* put_document_stats) override; + DocumentId /*old_document_id*/ _, bool recovery_mode, + PutDocumentStatsProto* put_document_stats) override; private: explicit EmbeddingIndexingHandler(const Clock* clock, - EmbeddingIndex* embedding_index) - : DataIndexingHandler(clock), embedding_index_(*embedding_index) {} + EmbeddingIndex* embedding_index, + bool enable_embedding_index) + : DataIndexingHandler(clock), + embedding_index_(*embedding_index), + enable_embedding_index_(enable_embedding_index) {} EmbeddingIndex& embedding_index_; + bool enable_embedding_index_; }; } // namespace lib
diff --git a/icing/index/embedding-indexing-handler_test.cc b/icing/index/embedding-indexing-handler_test.cc index a1b3ea8..5cfbc87 100644 --- a/icing/index/embedding-indexing-handler_test.cc +++ b/icing/index/embedding-indexing-handler_test.cc
@@ -14,25 +14,22 @@ #include "icing/index/embedding-indexing-handler.h" -#include <cstdint> #include <initializer_list> #include <memory> #include <string> #include <string_view> #include <utility> -#include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" -#include "icing/text_classifier/lib3/utils/base/statusor.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "icing/absl_ports/canonical_errors.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-hit.h" #include "icing/index/embed/embedding-index.h" -#include "icing/index/embed/posting-list-embedding-hit-accessor.h" +#include "icing/index/embed/quantizer.h" #include "icing/index/hit/hit.h" #include "icing/portable/platform.h" #include "icing/proto/document_wrapper.pb.h" @@ -45,12 +42,12 @@ #include "icing/testing/common-matchers.h" #include "icing/testing/embedding-test-utils.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" -#include "icing/util/status-macros.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" @@ -61,49 +58,60 @@ using ::testing::ElementsAre; using ::testing::Eq; +using ::testing::FloatNear; using ::testing::IsEmpty; using ::testing::IsTrue; +using ::testing::Pointwise; // Indexable properties (section) and section id. Section id is determined by // the lexicographical order of indexable property paths. // Schema type with indexable properties: FakeType // Section id = 0: "body" // Section id = 1: "bodyEmbedding" -// Section id = 2: "title" -// Section id = 3: "titleEmbedding" +// Section id = 2: "quantizedEmbedding" +// Section id = 3: "title" +// Section id = 4: "titleEmbedding" static constexpr std::string_view kFakeType = "FakeType"; static constexpr std::string_view kPropertyBody = "body"; static constexpr std::string_view kPropertyBodyEmbedding = "bodyEmbedding"; +static constexpr std::string_view kPropertyQuantizedEmbedding = + "quantizedEmbedding"; static constexpr std::string_view kPropertyTitle = "title"; static constexpr std::string_view kPropertyTitleEmbedding = "titleEmbedding"; static constexpr std::string_view kPropertyNonIndexableEmbedding = "nonIndexableEmbedding"; static constexpr SectionId kSectionIdBodyEmbedding = 1; -static constexpr SectionId kSectionIdTitleEmbedding = 3; +static constexpr SectionId kSectionIdQuantizedEmbedding = 2; +static constexpr SectionId kSectionIdTitleEmbedding = 4; // Schema type with nested indexable properties: FakeCollectionType // Section id = 0: "collection.body" // Section id = 1: "collection.bodyEmbedding" -// Section id = 2: "collection.title" -// Section id = 3: "collection.titleEmbedding" -// Section id = 4: "fullDocEmbedding" +// Section id = 2: "collection.quantizedEmbedding" +// Section id = 3: "collection.title" +// Section id = 4: "collection.titleEmbedding" +// Section id = 5: "fullDocEmbedding" static constexpr std::string_view kFakeCollectionType = "FakeCollectionType"; static constexpr std::string_view kPropertyCollection = "collection"; static constexpr std::string_view kPropertyFullDocEmbedding = "fullDocEmbedding"; static constexpr SectionId kSectionIdNestedBodyEmbedding = 1; -static constexpr SectionId kSectionIdNestedTitleEmbedding = 3; -static constexpr SectionId kSectionIdFullDocEmbedding = 4; +static constexpr SectionId kSectionIdNestedQuantizedEmbedding = 2; +static constexpr SectionId kSectionIdNestedTitleEmbedding = 4; +static constexpr SectionId kSectionIdFullDocEmbedding = 5; + +constexpr float kEpsQuantized = 0.01f; class EmbeddingIndexingHandlerTest : public ::testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -115,10 +123,6 @@ schema_store_dir_ = base_dir_ + "/schema_store"; document_store_dir_ = base_dir_ + "/document_store"; - ICING_ASSERT_OK_AND_ASSIGN( - embedding_index_, - EmbeddingIndex::Create(&filesystem_, embedding_index_working_path_)); - language_segmenter_factory::SegmenterOptions segmenter_options(ULOC_US); ICING_ASSERT_OK_AND_ASSIGN( lang_segmenter_, @@ -128,8 +132,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), IsTrue()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType( @@ -159,6 +163,14 @@ EmbeddingIndexingConfig::EmbeddingIndexingType:: LINEAR_SEARCH) .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName(kPropertyQuantizedEmbedding) + .SetDataTypeVector( + EmbeddingIndexingConfig::EmbeddingIndexingType:: + LINEAR_SEARCH, + QUANTIZATION_TYPE_QUANTIZE_8_BIT) + .SetCardinality(CARDINALITY_REPEATED)) .AddProperty(PropertyConfigBuilder() .SetName(kPropertyNonIndexableEmbedding) .SetDataType(TYPE_VECTOR) @@ -188,15 +200,19 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult doc_store_create_result, DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); document_store_ = std::move(doc_store_create_result.document_store); + + ICING_ASSERT_OK_AND_ASSIGN( + embedding_index_, + EmbeddingIndex::Create(&filesystem_, embedding_index_working_path_, + &fake_clock_, feature_flags_.get())); } void TearDown() override { @@ -208,43 +224,7 @@ filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); } - libtextclassifier3::StatusOr<std::vector<EmbeddingHit>> GetHits( - uint32_t dimension, std::string_view model_signature) { - std::vector<EmbeddingHit> hits; - - libtextclassifier3::StatusOr< - std::unique_ptr<PostingListEmbeddingHitAccessor>> - pl_accessor_or = - embedding_index_->GetAccessor(dimension, model_signature); - std::unique_ptr<PostingListEmbeddingHitAccessor> pl_accessor; - if (pl_accessor_or.ok()) { - pl_accessor = std::move(pl_accessor_or).ValueOrDie(); - } else if (absl_ports::IsNotFound(pl_accessor_or.status())) { - return hits; - } else { - return std::move(pl_accessor_or).status(); - } - - while (true) { - ICING_ASSIGN_OR_RETURN(std::vector<EmbeddingHit> batch, - pl_accessor->GetNextHitsBatch()); - if (batch.empty()) { - return hits; - } - hits.insert(hits.end(), batch.begin(), batch.end()); - } - } - - std::vector<float> GetRawEmbeddingData() { - auto data_or = embedding_index_->GetRawEmbeddingData(); - if (!data_or.ok()) { - return std::vector<float>(); - } - return std::vector<float>( - data_or.ValueOrDie(), - data_or.ValueOrDie() + embedding_index_->GetTotalVectorSize()); - } - + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; FakeClock fake_clock_; std::string base_dir_; @@ -262,11 +242,13 @@ TEST_F(EmbeddingIndexingHandlerTest, CreationWithNullPointerShouldFail) { EXPECT_THAT(EmbeddingIndexingHandler::Create(/*clock=*/nullptr, - embedding_index_.get()), + embedding_index_.get(), + /*enable_embedding_index=*/true), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT(EmbeddingIndexingHandler::Create(&fake_clock_, - /*embedding_index=*/nullptr), + /*embedding_index=*/nullptr, + /*enable_embedding_index=*/true), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } @@ -282,6 +264,84 @@ .AddVectorProperty(std::string(kPropertyBodyEmbedding), CreateVector("model", {0.4, 0.5, 0.6}), CreateVector("model", {0.7, 0.8, 0.9})) + .AddVectorProperty(std::string(kPropertyQuantizedEmbedding), + CreateVector("model", {0.1, 0.2, 0.3}), + CreateVector("model", {0.4, 0.5, 0.6})) + .AddVectorProperty(std::string(kPropertyNonIndexableEmbedding), + CreateVector("model", {1.1, 1.2, 1.3})) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::PutResult put_result, + document_store_->Put(tokenized_document.document())); + DocumentId document_id = put_result.new_document_id; + + ASSERT_THAT(embedding_index_->last_added_document_id(), + Eq(kInvalidDocumentId)); + // Handle document. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<EmbeddingIndexingHandler> handler, + EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get(), + /*enable_embedding_index=*/true)); + EXPECT_THAT(handler->Handle( + tokenized_document, document_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + IsOk()); + + // Check index + EmbeddingHit hit1(BasicHit(kSectionIdBodyEmbedding, /*document_id=*/0), + /*location=*/0); + EmbeddingHit hit2(BasicHit(kSectionIdBodyEmbedding, /*document_id=*/0), + /*location=*/3); + EmbeddingHit hit3(BasicHit(kSectionIdTitleEmbedding, /*document_id=*/0), + /*location=*/6); + // Quantized embeddings are stored in a different location from unquantized + // embeddings, so the location starts from 0 again. + EmbeddingHit quantized_hit1( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/0), + /*location=*/0); + EmbeddingHit quantized_hit2( + BasicHit(kSectionIdQuantizedEmbedding, /*document_id=*/0), + /*location=*/3 + sizeof(Quantizer)); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre(hit1, hit2, quantized_hit1, + quantized_hit2, hit3))); + // Check unquantized embedding data + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.1, 0.2, 0.3)); + // Check quantized embedding data + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(6 + 2 * sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit1, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {0.1, 0.2, 0.3}))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit2, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {0.4, 0.5, 0.6}))); + + EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(document_id)); +} + +TEST_F(EmbeddingIndexingHandlerTest, EmbeddingShouldNotBeIndexedIfDisabled) { + DocumentProto document = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyTitle), "title") + .AddVectorProperty(std::string(kPropertyTitleEmbedding), + CreateVector("model", {0.1, 0.2, 0.3})) + .AddStringProperty(std::string(kPropertyBody), "body") + .AddVectorProperty(std::string(kPropertyBodyEmbedding), + CreateVector("model", {0.4, 0.5, 0.6}), + CreateVector("model", {0.7, 0.8, 0.9})) .AddVectorProperty(std::string(kPropertyNonIndexableEmbedding), CreateVector("model", {1.1, 1.2, 1.3})) .Build(); @@ -296,27 +356,22 @@ ASSERT_THAT(embedding_index_->last_added_document_id(), Eq(kInvalidDocumentId)); - // Handle document. + // If enable_embedding_index is false, the handler should not index any + // embeddings. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndexingHandler> handler, - EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get())); - EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); + EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get(), + /*enable_embedding_index=*/false)); + EXPECT_THAT(handler->Handle( + tokenized_document, document_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + IsOk()); - // Check index - EXPECT_THAT( - GetHits(/*dimension=*/3, /*model_signature=*/"model"), - IsOkAndHolds(ElementsAre( - EmbeddingHit(BasicHit(kSectionIdBodyEmbedding, /*document_id=*/0), - /*location=*/0), - EmbeddingHit(BasicHit(kSectionIdBodyEmbedding, /*document_id=*/0), - /*location=*/3), - EmbeddingHit(BasicHit(kSectionIdTitleEmbedding, /*document_id=*/0), - /*location=*/6)))); - EXPECT_THAT(GetRawEmbeddingData(), - ElementsAre(0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.1, 0.2, 0.3)); + // Check that the embedding index is empty. + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(IsEmpty())); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(document_id)); } @@ -337,6 +392,9 @@ .AddVectorProperty(std::string(kPropertyBodyEmbedding), CreateVector("model", {0.4, 0.5, 0.6}), CreateVector("model", {0.7, 0.8, 0.9})) + .AddVectorProperty(std::string(kPropertyQuantizedEmbedding), + CreateVector("model", {0.1, 0.2, 0.3}), + CreateVector("model", {0.4, 0.5, 0.6})) .AddVectorProperty( std::string(kPropertyNonIndexableEmbedding), CreateVector("model", {1.1, 1.2, 1.3})) @@ -358,34 +416,57 @@ // Handle document. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndexingHandler> handler, - EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get())); - EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); + EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get(), + /*enable_embedding_index=*/true)); + EXPECT_THAT(handler->Handle( + tokenized_document, document_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + IsOk()); // Check index + EmbeddingHit hit1(BasicHit(kSectionIdNestedBodyEmbedding, /*document_id=*/0), + /*location=*/0); + EmbeddingHit hit2(BasicHit(kSectionIdNestedBodyEmbedding, /*document_id=*/0), + /*location=*/3); + EmbeddingHit hit3(BasicHit(kSectionIdNestedTitleEmbedding, /*document_id=*/0), + /*location=*/6); + EmbeddingHit hit4(BasicHit(kSectionIdFullDocEmbedding, /*document_id=*/0), + /*location=*/9); + // Quantized embeddings are stored in a different location from unquantized + // embeddings, so the location starts from 0 again. + EmbeddingHit quantized_hit1( + BasicHit(kSectionIdNestedQuantizedEmbedding, /*document_id=*/0), + /*location=*/0); + EmbeddingHit quantized_hit2( + BasicHit(kSectionIdNestedQuantizedEmbedding, /*document_id=*/0), + /*location=*/3 + sizeof(Quantizer)); + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), + IsOkAndHolds(ElementsAre(hit1, hit2, quantized_hit1, + quantized_hit2, hit3, hit4))); + // Check unquantized embedding data EXPECT_THAT( - GetHits(/*dimension=*/3, /*model_signature=*/"model"), - IsOkAndHolds(ElementsAre( - EmbeddingHit( - BasicHit(kSectionIdNestedBodyEmbedding, /*document_id=*/0), - /*location=*/0), - EmbeddingHit( - BasicHit(kSectionIdNestedBodyEmbedding, /*document_id=*/0), - /*location=*/3), - EmbeddingHit( - BasicHit(kSectionIdNestedTitleEmbedding, /*document_id=*/0), - /*location=*/6), - EmbeddingHit(BasicHit(kSectionIdFullDocEmbedding, /*document_id=*/0), - /*location=*/9)))); - EXPECT_THAT(GetRawEmbeddingData(), ElementsAre(0.4, 0.5, 0.6, 0.7, 0.8, 0.9, - 0.1, 0.2, 0.3, 2.1, 2.2, 2.3)); + GetRawEmbeddingDataFromIndex(embedding_index_.get()), + ElementsAre(0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.1, 0.2, 0.3, 2.1, 2.2, 2.3)); + // Check quantized embedding data + EXPECT_THAT(embedding_index_->GetTotalQuantizedVectorSize(), + Eq(6 + 2 * sizeof(Quantizer))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit1, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {0.1, 0.2, 0.3}))); + EXPECT_THAT( + GetAndRestoreQuantizedEmbeddingVectorFromIndex(embedding_index_.get(), + quantized_hit2, + /*dimension=*/3), + IsOkAndHolds(Pointwise(FloatNear(kEpsQuantized), {0.4, 0.5, 0.6}))); + EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(document_id)); } TEST_F(EmbeddingIndexingHandlerTest, - HandleInvalidDocumentIdShouldReturnInvalidArgumentError) { + HandleInvalidNewDocumentIdShouldReturnInvalidArgumentError) { DocumentProto document = DocumentBuilder() .SetKey("icing", "fake_type/1") @@ -413,34 +494,39 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndexingHandler> handler, - EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get())); + EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get(), + /*enable_embedding_index=*/true)); // Handling document with kInvalidDocumentId should cause a failure, and both // index data and last_added_document_id should remain unchanged. EXPECT_THAT( handler->Handle(tokenized_document, kInvalidDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(kCurrentDocumentId)); // Check that the embedding index should be empty - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(IsEmpty())); EXPECT_TRUE(embedding_index_->is_empty()); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); // Recovery mode should get the same result. EXPECT_THAT( handler->Handle(tokenized_document, kInvalidDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(kCurrentDocumentId)); // Check that the embedding index should be empty - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(IsEmpty())); EXPECT_TRUE(embedding_index_->is_empty()); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); } TEST_F(EmbeddingIndexingHandlerTest, @@ -470,41 +556,44 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndexingHandler> handler, - EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get())); + EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get(), + /*enable_embedding_index=*/true)); // Handling document with document_id == last_added_document_id should cause a // failure, and both index data and last_added_document_id should remain // unchanged. embedding_index_->set_last_added_document_id(document_id); ASSERT_THAT(embedding_index_->last_added_document_id(), Eq(document_id)); - EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT(handler->Handle( + tokenized_document, document_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(document_id)); // Check that the embedding index should be empty - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(IsEmpty())); EXPECT_TRUE(embedding_index_->is_empty()); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); // Handling document with document_id < last_added_document_id should cause a // failure, and both index data and last_added_document_id should remain // unchanged. embedding_index_->set_last_added_document_id(document_id + 1); ASSERT_THAT(embedding_index_->last_added_document_id(), Eq(document_id + 1)); - EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT(handler->Handle( + tokenized_document, document_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(document_id + 1)); // Check that the embedding index should be empty - EXPECT_THAT(GetHits(/*dimension=*/3, /*model_signature=*/"model"), + EXPECT_THAT(GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(IsEmpty())); EXPECT_TRUE(embedding_index_->is_empty()); - EXPECT_THAT(GetRawEmbeddingData(), IsEmpty()); + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), IsEmpty()); } TEST_F(EmbeddingIndexingHandlerTest, @@ -556,19 +645,22 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<EmbeddingIndexingHandler> handler, - EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get())); + EmbeddingIndexingHandler::Create(&fake_clock_, embedding_index_.get(), + /*enable_embedding_index=*/true)); // Handle document with document_id > last_added_document_id in recovery mode. // The handler should index this document and update last_added_document_id. EXPECT_THAT( - handler->Handle(tokenized_document1, document_id1, /*recovery_mode=*/true, + handler->Handle(tokenized_document1, document_id1, + put_result1.old_document_id, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(document_id1)); // Check index EXPECT_THAT( - GetHits(/*dimension=*/3, /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(kSectionIdBodyEmbedding, /*document_id=*/0), /*location=*/0), @@ -576,7 +668,7 @@ /*location=*/3), EmbeddingHit(BasicHit(kSectionIdTitleEmbedding, /*document_id=*/0), /*location=*/6)))); - EXPECT_THAT(GetRawEmbeddingData(), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.1, 0.2, 0.3)); // Handle document with document_id == last_added_document_id in recovery @@ -586,14 +678,16 @@ embedding_index_->set_last_added_document_id(document_id2); ASSERT_THAT(embedding_index_->last_added_document_id(), Eq(document_id2)); EXPECT_THAT( - handler->Handle(tokenized_document2, document_id2, /*recovery_mode=*/true, + handler->Handle(tokenized_document2, document_id2, + put_result2.old_document_id, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(document_id2)); // Check index EXPECT_THAT( - GetHits(/*dimension=*/3, /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(kSectionIdBodyEmbedding, /*document_id=*/0), /*location=*/0), @@ -601,7 +695,7 @@ /*location=*/3), EmbeddingHit(BasicHit(kSectionIdTitleEmbedding, /*document_id=*/0), /*location=*/6)))); - EXPECT_THAT(GetRawEmbeddingData(), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.1, 0.2, 0.3)); // Handle document with document_id < last_added_document_id in recovery mode. @@ -610,14 +704,16 @@ embedding_index_->set_last_added_document_id(document_id2 + 1); ASSERT_THAT(embedding_index_->last_added_document_id(), Eq(document_id2 + 1)); EXPECT_THAT( - handler->Handle(tokenized_document2, document_id2, /*recovery_mode=*/true, + handler->Handle(tokenized_document2, document_id2, + put_result2.old_document_id, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(embedding_index_->last_added_document_id(), Eq(document_id2 + 1)); // Check index EXPECT_THAT( - GetHits(/*dimension=*/3, /*model_signature=*/"model"), + GetEmbeddingHitsFromIndex(embedding_index_.get(), /*dimension=*/3, + /*model_signature=*/"model"), IsOkAndHolds(ElementsAre( EmbeddingHit(BasicHit(kSectionIdBodyEmbedding, /*document_id=*/0), /*location=*/0), @@ -625,7 +721,7 @@ /*location=*/3), EmbeddingHit(BasicHit(kSectionIdTitleEmbedding, /*document_id=*/0), /*location=*/6)))); - EXPECT_THAT(GetRawEmbeddingData(), + EXPECT_THAT(GetRawEmbeddingDataFromIndex(embedding_index_.get()), ElementsAre(0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.1, 0.2, 0.3)); }
diff --git a/icing/index/index-processor.cc b/icing/index/index-processor.cc index 9a773e8..6ed270f 100644 --- a/icing/index/index-processor.cc +++ b/icing/index/index-processor.cc
@@ -28,12 +28,13 @@ libtextclassifier3::Status IndexProcessor::IndexDocument( const TokenizedDocument& tokenized_document, DocumentId document_id, - PutDocumentStatsProto* put_document_stats) { + DocumentId old_document_id, PutDocumentStatsProto* put_document_stats) { std::unique_ptr<Timer> index_timer = clock_.GetNewTimer(); for (auto& data_indexing_handler : data_indexing_handlers_) { ICING_RETURN_IF_ERROR(data_indexing_handler->Handle( - tokenized_document, document_id, recovery_mode_, put_document_stats)); + tokenized_document, document_id, old_document_id, recovery_mode_, + put_document_stats)); } if (put_document_stats != nullptr) {
diff --git a/icing/index/index-processor.h b/icing/index/index-processor.h index 9b96f00..1eb8b55 100644 --- a/icing/index/index-processor.h +++ b/icing/index/index-processor.h
@@ -15,14 +15,15 @@ #ifndef ICING_INDEX_INDEX_PROCESSOR_H_ #define ICING_INDEX_INDEX_PROCESSOR_H_ -#include <cstdint> #include <memory> +#include <utility> #include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/index/data-indexing-handler.h" #include "icing/proto/logging.pb.h" #include "icing/store/document-id.h" +#include "icing/util/clock.h" #include "icing/util/tokenized-document.h" namespace icing { @@ -42,6 +43,12 @@ // the first max_tokens_per_document will be added to the index. All tokens of // length exceeding max_token_length will be shortened to max_token_length. // + // old_document_id is provided. If valid, then it means the document with + // the same (namespace, uri) exists previously, and it is updated with new + // contents at this round. Each indexing handler should decide whether + // migrating existing data from old_document_id to (new) document_id according + // to each index's data logic. + // // Indexing a document *may* trigger an index merge. If a merge fails, then // all content in the index will be lost. // @@ -53,6 +60,7 @@ // - Any DataIndexingHandler errors. libtextclassifier3::Status IndexDocument( const TokenizedDocument& tokenized_document, DocumentId document_id, + DocumentId old_document_id, PutDocumentStatsProto* put_document_stats = nullptr); private:
diff --git a/icing/index/index-processor_benchmark.cc b/icing/index/index-processor_benchmark.cc index 8f5e319..b849bad 100644 --- a/icing/index/index-processor_benchmark.cc +++ b/icing/index/index-processor_benchmark.cc
@@ -24,6 +24,7 @@ #include "gmock/gmock.h" #include "third_party/absl/flags/flag.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/index/data-indexing-handler.h" #include "icing/index/index-processor.h" @@ -37,14 +38,15 @@ #include "icing/schema/schema-store.h" #include "icing/store/document-id.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/logging.h" #include "icing/util/status-macros.h" #include "icing/util/tokenized-document.h" @@ -169,14 +171,15 @@ .ValueOrDie(); } -std::unique_ptr<SchemaStore> CreateSchemaStore(const Filesystem& filesystem, - const Clock* clock, - const std::string& base_dir) { +std::unique_ptr<SchemaStore> CreateSchemaStore( + const Filesystem& filesystem, const Clock* clock, + const std::string& base_dir, const FeatureFlags& feature_flags) { std::string schema_store_dir = base_dir + "/schema_store_test"; filesystem.CreateDirectoryRecursively(schema_store_dir.c_str()); std::unique_ptr<SchemaStore> schema_store = - SchemaStore::Create(&filesystem, schema_store_dir, clock).ValueOrDie(); + SchemaStore::Create(&filesystem, schema_store_dir, clock, &feature_flags) + .ValueOrDie(); SchemaProto schema; CreateFakeTypeConfig(schema.add_types()); @@ -217,10 +220,11 @@ void BM_IndexDocumentWithOneProperty(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); IcingFilesystem icing_filesystem; Filesystem filesystem; std::string base_dir = GetTestTempDir() + "/index_processor_benchmark"; @@ -244,7 +248,7 @@ std::unique_ptr<Normalizer> normalizer = CreateNormalizer(); Clock clock; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, &clock, base_dir); + CreateSchemaStore(filesystem, &clock, base_dir, feature_flags); ICING_ASSERT_OK_AND_ASSIGN( std::vector<std::unique_ptr<DataIndexingHandler>> handlers, @@ -259,10 +263,12 @@ input_document) .ValueOrDie())); + DocumentId old_document_id = kInvalidDocumentId; DocumentId document_id = 0; for (auto _ : state) { - ICING_ASSERT_OK( - index_processor->IndexDocument(tokenized_document, document_id++)); + ICING_ASSERT_OK(index_processor->IndexDocument( + tokenized_document, document_id, old_document_id)); + old_document_id = document_id++; } index_processor.reset(); @@ -293,10 +299,11 @@ void BM_IndexDocumentWithTenProperties(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); IcingFilesystem icing_filesystem; Filesystem filesystem; std::string base_dir = GetTestTempDir() + "/index_processor_benchmark"; @@ -320,7 +327,7 @@ std::unique_ptr<Normalizer> normalizer = CreateNormalizer(); Clock clock; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, &clock, base_dir); + CreateSchemaStore(filesystem, &clock, base_dir, feature_flags); ICING_ASSERT_OK_AND_ASSIGN( std::vector<std::unique_ptr<DataIndexingHandler>> handlers, @@ -336,10 +343,12 @@ input_document) .ValueOrDie())); + DocumentId old_document_id = kInvalidDocumentId; DocumentId document_id = 0; for (auto _ : state) { - ICING_ASSERT_OK( - index_processor->IndexDocument(tokenized_document, document_id++)); + ICING_ASSERT_OK(index_processor->IndexDocument( + tokenized_document, document_id, old_document_id)); + old_document_id = document_id++; } index_processor.reset(); @@ -370,10 +379,11 @@ void BM_IndexDocumentWithDiacriticLetters(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); IcingFilesystem icing_filesystem; Filesystem filesystem; std::string base_dir = GetTestTempDir() + "/index_processor_benchmark"; @@ -397,7 +407,7 @@ std::unique_ptr<Normalizer> normalizer = CreateNormalizer(); Clock clock; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, &clock, base_dir); + CreateSchemaStore(filesystem, &clock, base_dir, feature_flags); ICING_ASSERT_OK_AND_ASSIGN( std::vector<std::unique_ptr<DataIndexingHandler>> handlers, @@ -413,10 +423,12 @@ input_document) .ValueOrDie())); + DocumentId old_document_id = kInvalidDocumentId; DocumentId document_id = 0; for (auto _ : state) { - ICING_ASSERT_OK( - index_processor->IndexDocument(tokenized_document, document_id++)); + ICING_ASSERT_OK(index_processor->IndexDocument( + tokenized_document, document_id, old_document_id)); + old_document_id = document_id++; } index_processor.reset(); @@ -447,10 +459,11 @@ void BM_IndexDocumentWithHiragana(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); IcingFilesystem icing_filesystem; Filesystem filesystem; std::string base_dir = GetTestTempDir() + "/index_processor_benchmark"; @@ -474,7 +487,7 @@ std::unique_ptr<Normalizer> normalizer = CreateNormalizer(); Clock clock; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, &clock, base_dir); + CreateSchemaStore(filesystem, &clock, base_dir, feature_flags); ICING_ASSERT_OK_AND_ASSIGN( std::vector<std::unique_ptr<DataIndexingHandler>> handlers, @@ -489,10 +502,12 @@ input_document) .ValueOrDie())); + DocumentId old_document_id = kInvalidDocumentId; DocumentId document_id = 0; for (auto _ : state) { - ICING_ASSERT_OK( - index_processor->IndexDocument(tokenized_document, document_id++)); + ICING_ASSERT_OK(index_processor->IndexDocument( + tokenized_document, document_id, old_document_id)); + old_document_id = document_id++; } index_processor.reset();
diff --git a/icing/index/index-processor_test.cc b/icing/index/index-processor_test.cc index 2dce924..6edcd52 100644 --- a/icing/index/index-processor_test.cc +++ b/icing/index/index-processor_test.cc
@@ -29,6 +29,7 @@ #include "icing/absl_ports/str_cat.h" #include "icing/absl_ports/str_join.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/data-indexing-handler.h" @@ -58,15 +59,16 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/random-string.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" #include "icing/util/crc32.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" @@ -154,10 +156,11 @@ class IndexProcessorTest : public Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -203,8 +206,8 @@ ASSERT_TRUE( filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str())); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType( @@ -286,14 +289,14 @@ ASSERT_TRUE(filesystem_.CreateDirectoryRecursively(doc_store_dir_.c_str())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, doc_store_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); doc_store_ = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN( @@ -336,6 +339,7 @@ std::unique_ptr<IcingMockFilesystem> mock_icing_filesystem_; + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; IcingFilesystem icing_filesystem_; FakeClock fake_clock_; @@ -392,8 +396,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); } @@ -408,8 +414,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); } @@ -424,8 +432,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); ICING_ASSERT_OK_AND_ASSIGN( @@ -459,8 +469,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); std::string coffeeRepeatedString = "coffee"; @@ -480,8 +492,10 @@ tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId1), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId1, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId1)); ICING_ASSERT_OK_AND_ASSIGN( @@ -540,8 +554,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); ICING_ASSERT_OK_AND_ASSIGN( @@ -567,8 +583,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); ICING_ASSERT_OK_AND_ASSIGN( @@ -603,9 +621,11 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED, - testing::HasSubstr("Hit buffer is full!"))); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED, + testing::HasSubstr("Hit buffer is full!"))); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); } @@ -625,8 +645,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); } @@ -661,8 +683,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); // "good" should have been indexed normally. @@ -708,8 +732,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(8)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); // Query the lite index. "r" should have 5 matches. @@ -750,8 +776,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); document = @@ -764,8 +792,10 @@ tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId1), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId1, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId1)); // Only document_id 1 should surface in a prefix query for "Rock" @@ -790,8 +820,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); document = @@ -804,8 +836,10 @@ tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId1), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId1, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId1)); ICING_ASSERT_OK_AND_ASSIGN( @@ -833,8 +867,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId1), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId1, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId1)); ICING_ASSERT_OK_AND_ASSIGN(int64_t index_element_size, @@ -855,8 +891,10 @@ tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Verify that both index_ and integer_index_ are unchanged. EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId1)); EXPECT_THAT(index_->GetElementsSize(), IsOkAndHolds(index_element_size)); @@ -865,8 +903,10 @@ IsOkAndHolds(integer_index_crc)); // As should indexing a document document_id == last_added_document_id. - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId1), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId1, + /*old_document_id=*/kInvalidDocumentId), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Verify that both index_ and integer_index_ are unchanged. EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId1)); EXPECT_THAT(index_->GetElementsSize(), IsOkAndHolds(index_element_size)); @@ -909,8 +949,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor.IndexDocument(tokenized_document, kDocumentId1), - IsOk()); + EXPECT_THAT( + index_processor.IndexDocument(tokenized_document, kDocumentId1, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId1)); ICING_ASSERT_OK_AND_ASSIGN(int64_t index_element_size, @@ -932,8 +974,10 @@ tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor.IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor.IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); // Verify that both index_ and integer_index_ are unchanged. EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId1)); EXPECT_THAT(index_->GetElementsSize(), IsOkAndHolds(index_element_size)); @@ -942,8 +986,10 @@ IsOkAndHolds(integer_index_crc)); // As should indexing a document document_id == last_added_document_id. - EXPECT_THAT(index_processor.IndexDocument(tokenized_document, kDocumentId1), - IsOk()); + EXPECT_THAT( + index_processor.IndexDocument(tokenized_document, kDocumentId1, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); // Verify that both index_ and integer_index_ are unchanged. EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId1)); EXPECT_THAT(index_->GetElementsSize(), IsOkAndHolds(index_element_size)); @@ -970,8 +1016,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); ICING_ASSERT_OK_AND_ASSIGN( @@ -1004,8 +1052,10 @@ TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document_one)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); } @@ -1050,12 +1100,18 @@ // empties the LiteIndex. constexpr int kNumDocsLiteIndexExhaustion = 3373; for (; doc_id < kNumDocsLiteIndexExhaustion; ++doc_id) { - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, doc_id), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument( + tokenized_document, doc_id, + /*old_document_id=*/doc_id == 0 ? kInvalidDocumentId : doc_id - 1), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(doc_id)); } - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, doc_id), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument( + tokenized_document, doc_id, + /*old_document_id=*/doc_id == 0 ? kInvalidDocumentId : doc_id - 1), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(doc_id)); } @@ -1112,19 +1168,23 @@ // 3. Index one document. This should fit in the LiteIndex without requiring a // merge. DocumentId doc_id = 0; - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, doc_id), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, doc_id, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(doc_id)); // 4. Add one more document to trigger a merge, which should fail and result // in a Reset. ++doc_id; - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, doc_id), + EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, doc_id, + /*old_document_id=*/doc_id - 1), StatusIs(libtextclassifier3::StatusCode::DATA_LOSS)); EXPECT_THAT(index_->last_added_document_id(), Eq(kInvalidDocumentId)); // 5. Indexing a new document should succeed. - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, doc_id), + EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, doc_id, + /*old_document_id=*/doc_id - 1), IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(doc_id)); } @@ -1143,8 +1203,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(1)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); ICING_ASSERT_OK_AND_ASSIGN( @@ -1175,8 +1237,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(1)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); // We expect to match the document we indexed as "Hello, w" is a prefix @@ -1209,8 +1273,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(1)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); ICING_ASSERT_OK_AND_ASSIGN( @@ -1240,8 +1306,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(7)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); std::unordered_map<SectionId, Hit::TermFrequency> expected_map{ @@ -1289,8 +1357,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(7)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); std::unordered_map<SectionId, Hit::TermFrequency> expected_map{ @@ -1319,8 +1389,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(7)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); // "alexsav@" only matches "alexsav@google.com" @@ -1370,8 +1442,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(7)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); std::unordered_map<SectionId, Hit::TermFrequency> expect_map{{}}; @@ -1401,8 +1475,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(7)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); ICING_ASSERT_OK_AND_ASSIGN( @@ -1459,8 +1535,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(8)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); ICING_ASSERT_OK_AND_ASSIGN( @@ -1501,8 +1579,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(7)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); // "goo" is a prefix of "google" and "google.com" @@ -1553,8 +1633,10 @@ document)); EXPECT_THAT(tokenized_document.num_string_tokens(), Eq(8)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); EXPECT_THAT(index_->last_added_document_id(), Eq(kDocumentId0)); // no token starts with "gle", so we should have no hits @@ -1606,8 +1688,10 @@ // Expected to have 1 integer section. EXPECT_THAT(tokenized_document.integer_sections(), SizeIs(1)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<DocHitInfoIterator> itr, @@ -1636,8 +1720,10 @@ // Expected to have 1 integer section. EXPECT_THAT(tokenized_document.integer_sections(), SizeIs(1)); - EXPECT_THAT(index_processor_->IndexDocument(tokenized_document, kDocumentId0), - IsOk()); + EXPECT_THAT( + index_processor_->IndexDocument(tokenized_document, kDocumentId0, + /*old_document_id=*/kInvalidDocumentId), + IsOk()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<DocHitInfoIterator> itr,
diff --git a/icing/index/index.cc b/icing/index/index.cc index 54031fd..a03f07a 100644 --- a/icing/index/index.cc +++ b/icing/index/index.cc
@@ -19,6 +19,7 @@ #include <cstdint> #include <memory> #include <string> +#include <string_view> #include <utility> #include <vector> @@ -308,7 +309,8 @@ new_last_added_document_id); } -libtextclassifier3::Status Index::Editor::BufferTerm(const char* term) { +libtextclassifier3::Status Index::Editor::BufferTerm( + std::string_view term, TermMatchType::Code match_type) { // Step 1: See if this term is already in the lexicon uint32_t tvi; auto tvi_or = lite_index_->GetTermId(term); @@ -316,10 +318,19 @@ // Step 2: Update the lexicon, either add the term or update its properties if (tvi_or.ok()) { tvi = tvi_or.ValueOrDie(); - if (seen_tokens_.find(tvi) != seen_tokens_.end()) { - ICING_VLOG(1) << "Updating term frequency for term " << term; - if (seen_tokens_[tvi] != Hit::kMaxTermFrequency) { - ++seen_tokens_[tvi]; + auto itr = seen_tokens_.find(tvi); + if (itr != seen_tokens_.end()) { + HitDetails& hit_details = itr->second; + if (hit_details.match_type == TermMatchType::EXACT_ONLY && + match_type == TermMatchType::PREFIX) { + // If the same term is added multiple times, but with different match + // types, we will always update the match type to PREFIX. + ICING_VLOG(1) << "Updating term match type for term " << term; + hit_details.match_type = TermMatchType::PREFIX; + } + if (hit_details.term_frequency != Hit::kMaxTermFrequency) { + ICING_VLOG(1) << "Updating term frequency for term " << term; + ++hit_details.term_frequency; } return libtextclassifier3::Status::OK; } @@ -327,22 +338,23 @@ << " is already present in lexicon. Updating."; // Already in the lexicon. Just update the properties. ICING_RETURN_IF_ERROR(lite_index_->UpdateTermProperties( - tvi, term_match_type_ == TermMatchType::PREFIX, namespace_id_)); + tvi, match_type == TermMatchType::PREFIX, namespace_id_)); } else { ICING_VLOG(1) << "Term " << term << " is not in lexicon. Inserting."; // Haven't seen this term before. Add it to the lexicon. ICING_ASSIGN_OR_RETURN( - tvi, lite_index_->InsertTerm(term, term_match_type_, namespace_id_)); + tvi, lite_index_->InsertTerm(term, match_type, namespace_id_)); } // Token seen for the first time in the current document. - seen_tokens_[tvi] = 1; + seen_tokens_[tvi] = {match_type, /*term_frequency=*/1}; return libtextclassifier3::Status::OK; } libtextclassifier3::Status Index::Editor::IndexAllBufferedTerms() { for (auto itr = seen_tokens_.begin(); itr != seen_tokens_.end(); itr++) { - Hit hit(section_id_, document_id_, /*term_frequency=*/itr->second, - /*is_in_prefix_section=*/term_match_type_ == TermMatchType::PREFIX, + Hit hit(section_id_, document_id_, itr->second.term_frequency, + /*is_in_prefix_section=*/itr->second.match_type == + TermMatchType::PREFIX, /*is_prefix_hit=*/false, /*is_stemmed_hit=*/false); ICING_ASSIGN_OR_RETURN( uint32_t term_id, term_id_codec_->EncodeTvi(itr->first, TviType::LITE));
diff --git a/icing/index/index.h b/icing/index/index.h index 0d436e2..fe3415c 100644 --- a/icing/index/index.h +++ b/icing/index/index.h
@@ -18,6 +18,7 @@ #include <cstdint> #include <memory> #include <string> +#include <string_view> #include <unordered_map> #include <utility> #include <vector> @@ -261,16 +262,16 @@ // TODO(b/141180665): Add nullptr checks for the raw pointers Editor(const TermIdCodec* term_id_codec, LiteIndex* lite_index, DocumentId document_id, SectionId section_id, - TermMatchType::Code term_match_type, NamespaceId namespace_id) + NamespaceId namespace_id) : term_id_codec_(term_id_codec), lite_index_(lite_index), document_id_(document_id), - term_match_type_(term_match_type), namespace_id_(namespace_id), section_id_(section_id) {} // Buffer the term in seen_tokens_. - libtextclassifier3::Status BufferTerm(const char* term); + libtextclassifier3::Status BufferTerm(std::string_view term, + TermMatchType::Code match_type); // Index all the terms stored in seen_tokens_. libtextclassifier3::Status IndexAllBufferedTerms(); @@ -278,18 +279,21 @@ // The Editor is able to store previously seen terms as TermIds. This is // is more efficient than a client doing this externally because TermIds are // not exposed to clients. - std::unordered_map<uint32_t, Hit::TermFrequency> seen_tokens_; + struct HitDetails { + TermMatchType::Code match_type; + Hit::TermFrequency term_frequency; + }; + std::unordered_map<uint32_t, HitDetails> seen_tokens_; const TermIdCodec* term_id_codec_; LiteIndex* lite_index_; DocumentId document_id_; - TermMatchType::Code term_match_type_; NamespaceId namespace_id_; SectionId section_id_; }; Editor Edit(DocumentId document_id, SectionId section_id, - TermMatchType::Code term_match_type, NamespaceId namespace_id) { + NamespaceId namespace_id) { return Editor(term_id_codec_.get(), lite_index_.get(), document_id, - section_id, term_match_type, namespace_id); + section_id, namespace_id); } bool WantsMerge() const { return lite_index_->WantsMerge(); }
diff --git a/icing/index/index_test.cc b/icing/index/index_test.cc index db94e78..7f73590 100644 --- a/icing/index/index_test.cc +++ b/icing/index/index_test.cc
@@ -208,11 +208,11 @@ ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); - ASSERT_THAT(edit.BufferTerm("bar"), IsOk()); - ASSERT_THAT(edit.BufferTerm("baz"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + ASSERT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); + ASSERT_THAT(edit.BufferTerm("baz", TermMatchType::EXACT_ONLY), IsOk()); ASSERT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // Persist and recreate the index with lite_index_sort_at_indexing=true @@ -230,9 +230,9 @@ } TEST_F(IndexTest, AdvancePastEnd) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -257,9 +257,9 @@ } TEST_F(IndexTest, AdvancePastEndAfterMerge) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -286,15 +286,15 @@ } TEST_F(IndexTest, IteratorGetCallStats_mainIndexOnly) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // Merge the index. @@ -348,15 +348,15 @@ } TEST_F(IndexTest, IteratorGetCallStats_liteIndexOnly) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -407,29 +407,29 @@ } TEST_F(IndexTest, IteratorGetCallStats) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // Merge the index. 2 hits for "foo" will be merged into the main index. ICING_ASSERT_OK(index_->Merge()); // Insert 2 more hits for "foo". It will be in the lite index. - edit = index_->Edit(kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId3, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId3, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -508,9 +508,9 @@ } TEST_F(IndexTest, SingleHitSingleTermIndex) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -524,9 +524,9 @@ } TEST_F(IndexTest, SingleHitSingleTermIndexAfterMerge) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -542,9 +542,9 @@ } TEST_F(IndexTest, SingleHitSingleTermIndexAfterOptimize) { - Index::Editor edit = index_->Edit( - kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId2, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId2); @@ -580,9 +580,9 @@ } TEST_F(IndexTest, SingleHitSingleTermIndexAfterMergeAndOptimize) { - Index::Editor edit = index_->Edit( - kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId2, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId2); @@ -620,10 +620,10 @@ } TEST_F(IndexTest, SingleHitMultiTermIndex) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -637,10 +637,10 @@ } TEST_F(IndexTest, SingleHitMultiTermIndexAfterMerge) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -656,19 +656,19 @@ } TEST_F(IndexTest, MultiHitMultiTermIndexAfterOptimize) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId2, kSectionId3, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId2); @@ -721,19 +721,19 @@ } TEST_F(IndexTest, MultiHitMultiTermIndexAfterMergeAndOptimize) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId2, kSectionId3, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId2); @@ -788,10 +788,10 @@ } TEST_F(IndexTest, NoHitMultiTermIndex) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -804,10 +804,10 @@ } TEST_F(IndexTest, NoHitMultiTermIndexAfterMerge) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -822,19 +822,19 @@ } TEST_F(IndexTest, MultiHitMultiTermIndex) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId2, kSectionId3, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -850,19 +850,19 @@ } TEST_F(IndexTest, MultiHitMultiTermIndexAfterMerge) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId2, kSectionId3, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -880,14 +880,14 @@ } TEST_F(IndexTest, MultiHitSectionRestrict) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); SectionIdMask desired_section = 1U << kSectionId2; @@ -902,14 +902,14 @@ } TEST_F(IndexTest, MultiHitSectionRestrictAfterMerge) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -928,12 +928,12 @@ TEST_F(IndexTest, SingleHitDedupeIndex) { ICING_ASSERT_OK_AND_ASSIGN(int64_t size, index_->GetElementsSize()); EXPECT_THAT(size, Eq(0)); - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); ICING_ASSERT_OK_AND_ASSIGN(size, index_->GetElementsSize()); EXPECT_THAT(size, Gt(0)); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); ICING_ASSERT_OK_AND_ASSIGN(int64_t new_size, index_->GetElementsSize()); EXPECT_THAT(new_size, Eq(size)); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); @@ -950,8 +950,8 @@ TEST_F(IndexTest, PrefixHit) { Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -966,8 +966,8 @@ TEST_F(IndexTest, PrefixHitAfterMerge) { Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -984,13 +984,13 @@ TEST_F(IndexTest, MultiPrefixHit) { Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -1007,13 +1007,13 @@ TEST_F(IndexTest, MultiPrefixHitAfterMerge) { Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -1031,14 +1031,14 @@ } TEST_F(IndexTest, NoExactHitInPrefixQuery) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -1052,14 +1052,14 @@ } TEST_F(IndexTest, NoExactHitInPrefixQueryAfterMerge) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -1076,9 +1076,9 @@ TEST_F(IndexTest, PrefixHitDedupe) { Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -1093,9 +1093,9 @@ TEST_F(IndexTest, PrefixHitDedupeAfterMerge) { Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -1174,9 +1174,9 @@ TEST_F(IndexTest, NonAsciiTerms) { Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("ăăăŤăĄăŻ"), IsOk()); - ASSERT_THAT(edit.BufferTerm("ăăŞă"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("ăăăŤăĄăŻ", TermMatchType::PREFIX), IsOk()); + ASSERT_THAT(edit.BufferTerm("ăăŞă", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -1199,9 +1199,9 @@ TEST_F(IndexTest, NonAsciiTermsAfterMerge) { Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("ăăăŤăĄăŻ"), IsOk()); - ASSERT_THAT(edit.BufferTerm("ăăŞă"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("ăăăŤăĄăŻ", TermMatchType::PREFIX), IsOk()); + ASSERT_THAT(edit.BufferTerm("ăăŞă", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -1246,11 +1246,11 @@ std::uniform_int_distribution<size_t> uniform(0u, query_terms.size() - 1); while (status.ok()) { for (int i = 0; i < 100; ++i) { - Index::Editor edit = - index_->Edit(document_id, kSectionId2, TermMatchType::PREFIX, - /*namespace_id=*/0); + Index::Editor edit = index_->Edit(document_id, kSectionId2, + /*namespace_id=*/0); size_t idx = uniform(random); - status = edit.BufferTerm(query_terms.at(idx).c_str()); + status = + edit.BufferTerm(query_terms.at(idx).c_str(), TermMatchType::PREFIX); if (!status.ok()) { break; } @@ -1263,15 +1263,14 @@ } // Adding more hits should fail. - Index::Editor edit = - index_->Edit(document_id + 1, kSectionId2, TermMatchType::PREFIX, - /*namespace_id=*/0); + Index::Editor edit = index_->Edit(document_id + 1, kSectionId2, + /*namespace_id=*/0); std::string term = prefix + "foo"; - EXPECT_THAT(edit.BufferTerm(term.c_str()), IsOk()); + EXPECT_THAT(edit.BufferTerm(term.c_str(), TermMatchType::PREFIX), IsOk()); term = prefix + "bar"; - EXPECT_THAT(edit.BufferTerm(term.c_str()), IsOk()); + EXPECT_THAT(edit.BufferTerm(term.c_str(), TermMatchType::PREFIX), IsOk()); term = prefix + "baz"; - EXPECT_THAT(edit.BufferTerm(term.c_str()), IsOk()); + EXPECT_THAT(edit.BufferTerm(term.c_str(), TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); @@ -1316,11 +1315,11 @@ std::uniform_int_distribution<size_t> uniform(0u, query_terms.size() - 1); while (status.ok()) { for (int i = 0; i < 100; ++i) { - Index::Editor edit = - index_->Edit(document_id, kSectionId2, TermMatchType::PREFIX, - /*namespace_id=*/0); + Index::Editor edit = index_->Edit(document_id, kSectionId2, + /*namespace_id=*/0); size_t idx = uniform(random); - status = edit.BufferTerm(query_terms.at(idx).c_str()); + status = + edit.BufferTerm(query_terms.at(idx).c_str(), TermMatchType::PREFIX); if (!status.ok()) { break; } @@ -1335,15 +1334,14 @@ StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); // Adding more hits should fail. - Index::Editor edit = - index_->Edit(document_id + 1, kSectionId2, TermMatchType::PREFIX, - /*namespace_id=*/0); + Index::Editor edit = index_->Edit(document_id + 1, kSectionId2, + /*namespace_id=*/0); std::string term = prefix + "foo"; - EXPECT_THAT(edit.BufferTerm(term.c_str()), IsOk()); + EXPECT_THAT(edit.BufferTerm(term.c_str(), TermMatchType::PREFIX), IsOk()); term = prefix + "bar"; - EXPECT_THAT(edit.BufferTerm(term.c_str()), IsOk()); + EXPECT_THAT(edit.BufferTerm(term.c_str(), TermMatchType::PREFIX), IsOk()); term = prefix + "baz"; - EXPECT_THAT(edit.BufferTerm(term.c_str()), IsOk()); + EXPECT_THAT(edit.BufferTerm(term.c_str(), TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); ICING_ASSERT_OK_AND_ASSIGN( @@ -1356,13 +1354,13 @@ // After merging with the main index. Adding more hits should succeed now. ICING_ASSERT_OK(index_->Merge()); - edit = index_->Edit(document_id + 1, kSectionId2, TermMatchType::PREFIX, 0); + edit = index_->Edit(document_id + 1, kSectionId2, 0); prefix + "foo"; - EXPECT_THAT(edit.BufferTerm(term.c_str()), IsOk()); + EXPECT_THAT(edit.BufferTerm(term.c_str(), TermMatchType::PREFIX), IsOk()); term = prefix + "bar"; - EXPECT_THAT(edit.BufferTerm(term.c_str()), IsOk()); + EXPECT_THAT(edit.BufferTerm(term.c_str(), TermMatchType::PREFIX), IsOk()); term = prefix + "baz"; - EXPECT_THAT(edit.BufferTerm(term.c_str()), IsOk()); + EXPECT_THAT(edit.BufferTerm(term.c_str(), TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<DocHitInfoIterator> itr, @@ -1420,8 +1418,9 @@ term_match_type = TermMatchType::EXACT_ONLY; } Index::Editor edit = index_->Edit(/*document_id=*/i, section_id, - term_match_type, /*namespace_id=*/0); - ICING_ASSERT_OK(edit.BufferTerm(query_terms.at(i).c_str())); + /*namespace_id=*/0); + ICING_ASSERT_OK( + edit.BufferTerm(query_terms.at(i).c_str(), term_match_type)); ICING_ASSERT_OK(edit.IndexAllBufferedTerms()); } @@ -1478,8 +1477,8 @@ term = prefix + RandomString("abcd", 5, &random); } Index::Editor edit = index_->Edit(/*document_id=*/i, section_id, - term_match_type, /*namespace_id=*/0); - ICING_ASSERT_OK(edit.BufferTerm(term.c_str())); + /*namespace_id=*/0); + ICING_ASSERT_OK(edit.BufferTerm(term.c_str(), term_match_type)); ICING_ASSERT_OK(edit.IndexAllBufferedTerms()); ++lite_index_size; index_->set_last_added_document_id(i); @@ -1542,8 +1541,9 @@ term_match_type = TermMatchType::EXACT_ONLY; } Index::Editor edit = index_->Edit(/*document_id=*/i, section_id, - term_match_type, /*namespace_id=*/0); - ICING_ASSERT_OK(edit.BufferTerm(query_terms.at(i).c_str())); + /*namespace_id=*/0); + ICING_ASSERT_OK( + edit.BufferTerm(query_terms.at(i).c_str(), term_match_type)); ICING_ASSERT_OK(edit.IndexAllBufferedTerms()); index_->set_last_added_document_id(i); } @@ -1641,9 +1641,9 @@ TEST_F(IndexTest, IndexCreateCorruptionFailure) { // Add some content to the index Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); - ASSERT_THAT(edit.BufferTerm("bar"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); + ASSERT_THAT(edit.BufferTerm("bar", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // Close the index. @@ -1674,9 +1674,9 @@ TEST_F(IndexTest, UpdateChecksum) { // Add some content to the index Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); - ASSERT_THAT(edit.BufferTerm("bar"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); + ASSERT_THAT(edit.BufferTerm("bar", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); Crc32 lite_only_crc = index_->GetChecksum(); EXPECT_THAT(index_->UpdateChecksum(), Eq(lite_only_crc)); @@ -1690,10 +1690,10 @@ EXPECT_THAT(index_->GetChecksum(), Eq(main_only_crc)); // Add some more content to the lite index - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("baz"), IsOk()); - ASSERT_THAT(edit.BufferTerm("bat"), IsOk()); + ASSERT_THAT(edit.BufferTerm("baz", TermMatchType::PREFIX), IsOk()); + ASSERT_THAT(edit.BufferTerm("bat", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); Crc32 both_crc = index_->GetChecksum(); EXPECT_THAT(both_crc, Not(Eq(lite_only_crc))); @@ -1705,9 +1705,9 @@ TEST_F(IndexTest, IndexPersistence) { // Add some content to the index Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); - ASSERT_THAT(edit.BufferTerm("bar"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); + ASSERT_THAT(edit.BufferTerm("bar", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); EXPECT_THAT(index_->PersistToDisk(), IsOk()); @@ -1735,9 +1735,9 @@ TEST_F(IndexTest, IndexPersistenceAfterMerge) { // Add some content to the index Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); - ASSERT_THAT(edit.BufferTerm("bar"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); + ASSERT_THAT(edit.BufferTerm("bar", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); EXPECT_THAT(index_->PersistToDisk(), IsOk()); @@ -1773,9 +1773,9 @@ TEST_F(IndexTest, FindTermByPrefixShouldReturnEmpty) { Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); + /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); EXPECT_THAT( @@ -1808,11 +1808,11 @@ } TEST_F(IndexTest, FindTermByPrefixShouldReturnCorrectResult) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("bar"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("bar", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // "b" should only match "bar" but not "foo". @@ -1835,12 +1835,12 @@ } TEST_F(IndexTest, FindTermByPrefixShouldRespectNumToReturn) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit.BufferTerm("fo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // We have 3 results but only 2 should be returned. @@ -1863,23 +1863,20 @@ } TEST_F(IndexTest, FindTermByPrefixShouldReturnTermsInAllNamespaces) { - Index::Editor edit1 = - index_->Edit(kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); + Index::Editor edit1 = index_->Edit(kDocumentId0, kSectionId2, + /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit1.BufferTerm("fo"), IsOk()); + EXPECT_THAT(edit1.BufferTerm("fo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit1.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit2 = - index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/1); - EXPECT_THAT(edit2.BufferTerm("foo"), IsOk()); + Index::Editor edit2 = index_->Edit(kDocumentId1, kSectionId2, + /*namespace_id=*/1); + EXPECT_THAT(edit2.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit2.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit3 = - index_->Edit(kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/2); - EXPECT_THAT(edit3.BufferTerm("fool"), IsOk()); + Index::Editor edit3 = index_->Edit(kDocumentId2, kSectionId2, + /*namespace_id=*/2); + EXPECT_THAT(edit3.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit3.IndexAllBufferedTerms(), IsOk()); // Should return "fo", "foo" and "fool" across all namespaces. @@ -1906,18 +1903,16 @@ } TEST_F(IndexTest, FindTermByPrefixShouldReturnCorrectHitCount) { - Index::Editor edit1 = - index_->Edit(kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); + Index::Editor edit1 = index_->Edit(kDocumentId0, kSectionId2, + /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit1.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit1.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit1.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit1.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit1.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit2 = - index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); - EXPECT_THAT(edit2.BufferTerm("fool"), IsOk()); + Index::Editor edit2 = index_->Edit(kDocumentId1, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit2.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit2.IndexAllBufferedTerms(), IsOk()); // 'foo' has 1 hit, 'fool' has 2 hits. @@ -1944,9 +1939,9 @@ AlwaysTrueSuggestionResultCheckerImpl impl; // Create multiple hit batches. for (int i = 0; i < 4000; i++) { - Index::Editor edit = index_->Edit(i, kSectionId2, TermMatchType::EXACT_ONLY, + Index::Editor edit = index_->Edit(i, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); } @@ -1970,56 +1965,53 @@ TEST_F(IndexTest, FindTermByPrefixShouldReturnInOrder) { // Push 6 term-six, 5 term-five, 4 term-four, 3 term-three, 2 term-two and one // term-one into lite index. - Index::Editor edit1 = - index_->Edit(kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); + Index::Editor edit1 = index_->Edit(kDocumentId0, kSectionId2, + /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit1.BufferTerm("term-one"), IsOk()); - EXPECT_THAT(edit1.BufferTerm("term-two"), IsOk()); - EXPECT_THAT(edit1.BufferTerm("term-three"), IsOk()); - EXPECT_THAT(edit1.BufferTerm("term-four"), IsOk()); - EXPECT_THAT(edit1.BufferTerm("term-five"), IsOk()); - EXPECT_THAT(edit1.BufferTerm("term-six"), IsOk()); + EXPECT_THAT(edit1.BufferTerm("term-one", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit1.BufferTerm("term-two", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit1.BufferTerm("term-three", TermMatchType::EXACT_ONLY), + IsOk()); + EXPECT_THAT(edit1.BufferTerm("term-four", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit1.BufferTerm("term-five", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit1.BufferTerm("term-six", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit1.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit2 = - index_->Edit(kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); - EXPECT_THAT(edit2.BufferTerm("term-two"), IsOk()); - EXPECT_THAT(edit2.BufferTerm("term-three"), IsOk()); - EXPECT_THAT(edit2.BufferTerm("term-four"), IsOk()); - EXPECT_THAT(edit2.BufferTerm("term-five"), IsOk()); - EXPECT_THAT(edit2.BufferTerm("term-six"), IsOk()); + Index::Editor edit2 = index_->Edit(kDocumentId2, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit2.BufferTerm("term-two", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit2.BufferTerm("term-three", TermMatchType::EXACT_ONLY), + IsOk()); + EXPECT_THAT(edit2.BufferTerm("term-four", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit2.BufferTerm("term-five", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit2.BufferTerm("term-six", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit2.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit3 = - index_->Edit(kDocumentId3, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); - EXPECT_THAT(edit3.BufferTerm("term-three"), IsOk()); - EXPECT_THAT(edit3.BufferTerm("term-four"), IsOk()); - EXPECT_THAT(edit3.BufferTerm("term-five"), IsOk()); - EXPECT_THAT(edit3.BufferTerm("term-six"), IsOk()); + Index::Editor edit3 = index_->Edit(kDocumentId3, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit3.BufferTerm("term-three", TermMatchType::EXACT_ONLY), + IsOk()); + EXPECT_THAT(edit3.BufferTerm("term-four", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit3.BufferTerm("term-five", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit3.BufferTerm("term-six", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit3.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit4 = - index_->Edit(kDocumentId4, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); - EXPECT_THAT(edit4.BufferTerm("term-four"), IsOk()); - EXPECT_THAT(edit4.BufferTerm("term-five"), IsOk()); - EXPECT_THAT(edit4.BufferTerm("term-six"), IsOk()); + Index::Editor edit4 = index_->Edit(kDocumentId4, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit4.BufferTerm("term-four", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit4.BufferTerm("term-five", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit4.BufferTerm("term-six", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit4.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit5 = - index_->Edit(kDocumentId5, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); - EXPECT_THAT(edit5.BufferTerm("term-five"), IsOk()); - EXPECT_THAT(edit5.BufferTerm("term-six"), IsOk()); + Index::Editor edit5 = index_->Edit(kDocumentId5, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit5.BufferTerm("term-five", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit5.BufferTerm("term-six", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit5.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit6 = - index_->Edit(kDocumentId6, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); - EXPECT_THAT(edit6.BufferTerm("term-six"), IsOk()); + Index::Editor edit6 = index_->Edit(kDocumentId6, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit6.BufferTerm("term-six", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit6.IndexAllBufferedTerms(), IsOk()); // verify the order in lite index is correct. @@ -2051,20 +2043,20 @@ // keep push terms to the lite index. We will add 2 document to term-five, // term-three and term-one. The output order should be 5-6-3-4-1-2. - Index::Editor edit7 = - index_->Edit(kDocumentId7, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); - EXPECT_THAT(edit7.BufferTerm("term-one"), IsOk()); - EXPECT_THAT(edit7.BufferTerm("term-three"), IsOk()); - EXPECT_THAT(edit7.BufferTerm("term-five"), IsOk()); + Index::Editor edit7 = index_->Edit(kDocumentId7, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit7.BufferTerm("term-one", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit7.BufferTerm("term-three", TermMatchType::EXACT_ONLY), + IsOk()); + EXPECT_THAT(edit7.BufferTerm("term-five", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit7.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit8 = - index_->Edit(kDocumentId8, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); - EXPECT_THAT(edit8.BufferTerm("term-one"), IsOk()); - EXPECT_THAT(edit8.BufferTerm("term-three"), IsOk()); - EXPECT_THAT(edit8.BufferTerm("term-five"), IsOk()); + Index::Editor edit8 = index_->Edit(kDocumentId8, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit8.BufferTerm("term-one", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit8.BufferTerm("term-three", TermMatchType::EXACT_ONLY), + IsOk()); + EXPECT_THAT(edit8.BufferTerm("term-five", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit8.IndexAllBufferedTerms(), IsOk()); // verify the combination of lite index and main index is in correct order. @@ -2091,23 +2083,20 @@ } TEST_F(IndexTest, FindTermByPrefix_InTermMatchTypePrefix_ShouldReturnInOrder) { - Index::Editor edit1 = - index_->Edit(kDocumentId0, kSectionId2, TermMatchType::PREFIX, - /*namespace_id=*/0); + Index::Editor edit1 = index_->Edit(kDocumentId0, kSectionId2, + /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit1.BufferTerm("fo"), IsOk()); + EXPECT_THAT(edit1.BufferTerm("fo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit1.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit2 = - index_->Edit(kDocumentId2, kSectionId2, TermMatchType::PREFIX, - /*namespace_id=*/0); - EXPECT_THAT(edit2.BufferTerm("foo"), IsOk()); + Index::Editor edit2 = index_->Edit(kDocumentId2, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit2.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit2.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit3 = - index_->Edit(kDocumentId3, kSectionId2, TermMatchType::PREFIX, - /*namespace_id=*/0); - EXPECT_THAT(edit3.BufferTerm("fool"), IsOk()); + Index::Editor edit3 = index_->Edit(kDocumentId3, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit3.BufferTerm("fool", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit3.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -2136,41 +2125,40 @@ } TEST_F(IndexTest, FindTermByPrefixShouldReturnHitCountForMain) { - Index::Editor edit = - index_->Edit(kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); + Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, + /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId3, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId3, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId4, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId4, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId5, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId5, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId6, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId6, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId7, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId7, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // 'foo' has 1 hit, 'fool' has 8 hits. @@ -2194,19 +2182,18 @@ } TEST_F(IndexTest, FindTermByPrefixShouldReturnCombinedHitCount) { - Index::Editor edit = - index_->Edit(kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); + Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, + /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); EXPECT_THAT( @@ -2219,21 +2206,19 @@ } TEST_F(IndexTest, FindTermRankComparison) { - Index::Editor edit = - index_->Edit(kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); + Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, + /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit.BufferTerm("fo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("fo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("fo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - Index::Editor edit2 = - index_->Edit(kDocumentId2, kSectionId2, TermMatchType::PREFIX, - /*namespace_id=*/0); - EXPECT_THAT(edit2.BufferTerm("fo"), IsOk()); - EXPECT_THAT(edit2.BufferTerm("foo"), IsOk()); + Index::Editor edit2 = index_->Edit(kDocumentId2, kSectionId2, + /*namespace_id=*/0); + EXPECT_THAT(edit2.BufferTerm("fo", TermMatchType::PREFIX), IsOk()); + EXPECT_THAT(edit2.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit2.IndexAllBufferedTerms(), IsOk()); EXPECT_THAT( @@ -2288,19 +2273,18 @@ } TEST_F(IndexTest, FindTermByPrefixShouldReturnTermsFromBothIndices) { - Index::Editor edit = - index_->Edit(kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); + Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, + /*namespace_id=*/0); AlwaysTrueSuggestionResultCheckerImpl impl; - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // 'foo' has 1 hit in the main index, 'fool' has 1 hit in the lite index. @@ -2319,9 +2303,9 @@ EXPECT_THAT(size, Eq(0)); // Add an element. - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN(size, index_->GetElementsSize()); EXPECT_THAT(size, Gt(0)); @@ -2332,24 +2316,24 @@ } TEST_F(IndexTest, ExactResultsFromLiteAndMain) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foot"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foot", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); - edit = index_->Edit(kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("footer"), IsOk()); + EXPECT_THAT(edit.BufferTerm("footer", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId2, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId2, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -2365,24 +2349,24 @@ } TEST_F(IndexTest, PrefixResultsFromLiteAndMain) { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); - EXPECT_THAT(edit.BufferTerm("fool"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + EXPECT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foot"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foot", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); - edit = index_->Edit(kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId2, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("footer"), IsOk()); + EXPECT_THAT(edit.BufferTerm("footer", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId2, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId2, kSectionId3, /*namespace_id=*/0); - EXPECT_THAT(edit.BufferTerm("foo"), IsOk()); + EXPECT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK_AND_ASSIGN( @@ -2401,26 +2385,26 @@ TEST_F(IndexTest, GetDebugInfo) { // Add two documents to the lite index, merge them into the main index and // then add another doc to the lite index. - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); index_->set_last_added_document_id(kDocumentId1); - ASSERT_THAT(edit.BufferTerm("foot"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foot", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); - edit = index_->Edit(kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId2, /*namespace_id=*/0); index_->set_last_added_document_id(kDocumentId2); - ASSERT_THAT(edit.BufferTerm("footer"), IsOk()); + ASSERT_THAT(edit.BufferTerm("footer", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId2, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId2, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); IndexDebugInfoProto out0 = index_->GetDebugInfo(DebugInfoVerbosity::BASIC); @@ -2438,10 +2422,10 @@ SizeIs(Gt(out0.lite_index_info().size()))); // Add one more doc to the lite index. Debug strings should change. - edit = index_->Edit(kDocumentId3, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId3, kSectionId2, /*namespace_id=*/0); index_->set_last_added_document_id(kDocumentId3); - ASSERT_THAT(edit.BufferTerm("far"), IsOk()); + ASSERT_THAT(edit.BufferTerm("far", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); IndexDebugInfoProto out2 = index_->GetDebugInfo(DebugInfoVerbosity::BASIC); @@ -2468,17 +2452,17 @@ TEST_F(IndexTest, BackfillingMultipleTermsSucceeds) { // Add two documents to the lite index, merge them into the main index and // then add another doc to the lite index. - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId0, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId0, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foot"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foot", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // After this merge the index should have posting lists for @@ -2488,9 +2472,9 @@ ICING_ASSERT_OK(index_->Merge()); // Add one more doc to the lite index. - edit = index_->Edit(kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId2, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("far"), IsOk()); + ASSERT_THAT(edit.BufferTerm("far", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // After this merge the index should add a posting list for "far" and a @@ -2517,14 +2501,14 @@ TEST_F(IndexTest, BackfillingNewTermsSucceeds) { // Add two documents to the lite index, merge them into the main index and // then add another doc to the lite index. - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); - ASSERT_THAT(edit.BufferTerm("fool"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::EXACT_ONLY), IsOk()); + ASSERT_THAT(edit.BufferTerm("fool", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foot"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foot", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // After this merge the index should have posting lists for // "fool" {(doc0,sec2)}, @@ -2532,18 +2516,18 @@ // "foo" {(doc1,sec3),(doc0,sec2)} ICING_ASSERT_OK(index_->Merge()); - edit = index_->Edit(kDocumentId2, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId2, kSectionId2, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("footer"), IsOk()); + ASSERT_THAT(edit.BufferTerm("footer", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId2, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId2, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // Add one more doc to the lite index. Debug strings should change. - edit = index_->Edit(kDocumentId3, kSectionId2, TermMatchType::EXACT_ONLY, + edit = index_->Edit(kDocumentId3, kSectionId2, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("far"), IsOk()); + ASSERT_THAT(edit.BufferTerm("far", TermMatchType::EXACT_ONLY), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // After this merge the index should add posting lists for "far" and "footer" @@ -2582,8 +2566,8 @@ // Add one document to the lite index Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // Clipping to invalid should have no effect. ICING_EXPECT_OK(index_->TruncateTo(kInvalidDocumentId)); @@ -2606,9 +2590,9 @@ ElementsAre(EqualsDocHitInfo( kDocumentId0, std::vector<SectionId>{kSectionId2}))); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foot"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foot", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // Clipping to invalid should still have no effect even if both indices have @@ -2637,8 +2621,8 @@ // Add one document to the lite index Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId0); ICING_EXPECT_OK(index_->TruncateTo(index_->last_added_document_id())); @@ -2662,9 +2646,9 @@ ElementsAre(EqualsDocHitInfo( kDocumentId0, std::vector<SectionId>{kSectionId2}))); - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foot"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foot", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId1); @@ -2685,17 +2669,17 @@ TEST_F(IndexTest, TruncateToThrowsOutLiteIndex) { // Add one document to the lite index and merge it into main. Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId0); ICING_ASSERT_OK(index_->Merge()); // Add another document to the lite index. - edit = index_->Edit(kDocumentId1, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foot"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foot", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId1); @@ -2715,22 +2699,22 @@ TEST_F(IndexTest, TruncateToThrowsOutBothIndices) { // Add two documents to the lite index and merge them into main. Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId0); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foul"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foul", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId1); ICING_ASSERT_OK(index_->Merge()); // Add another document to the lite index. - edit = index_->Edit(kDocumentId2, kSectionId3, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId2, kSectionId3, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foot"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foot", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); index_->set_last_added_document_id(kDocumentId2); @@ -2748,13 +2732,13 @@ TEST_F(IndexTest, IndexStorageInfoProto) { // Add two documents to the lite index and merge them into main. { - Index::Editor edit = index_->Edit( - kDocumentId0, kSectionId2, TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + Index::Editor edit = + index_->Edit(kDocumentId0, kSectionId2, /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foul"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foul", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); ICING_ASSERT_OK(index_->Merge()); @@ -2776,12 +2760,12 @@ TEST_F(IndexTest, PublishQueryStats) { // Add two documents to the lite index without merging. Index::Editor edit = index_->Edit(kDocumentId0, kSectionId2, - TermMatchType::PREFIX, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); - edit = index_->Edit(kDocumentId1, kSectionId2, TermMatchType::PREFIX, + edit = index_->Edit(kDocumentId1, kSectionId2, /*namespace_id=*/0); - ASSERT_THAT(edit.BufferTerm("foul"), IsOk()); + ASSERT_THAT(edit.BufferTerm("foul", TermMatchType::PREFIX), IsOk()); EXPECT_THAT(edit.IndexAllBufferedTerms(), IsOk()); // Verify query stats.
diff --git a/icing/index/integer-section-indexing-handler.cc b/icing/index/integer-section-indexing-handler.cc index 63b09df..bf9cdd8 100644 --- a/icing/index/integer-section-indexing-handler.cc +++ b/icing/index/integer-section-indexing-handler.cc
@@ -47,7 +47,8 @@ libtextclassifier3::Status IntegerSectionIndexingHandler::Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, - bool recovery_mode, PutDocumentStatsProto* put_document_stats) { + DocumentId /*old_document_id*/ _, bool recovery_mode, + PutDocumentStatsProto* put_document_stats) { std::unique_ptr<Timer> index_timer = clock_.GetNewTimer(); if (!IsDocumentIdValid(document_id)) {
diff --git a/icing/index/integer-section-indexing-handler.h b/icing/index/integer-section-indexing-handler.h index 0a501aa..b6e1dc2 100644 --- a/icing/index/integer-section-indexing-handler.h +++ b/icing/index/integer-section-indexing-handler.h
@@ -47,6 +47,9 @@ // Handles the integer indexing process: add hits into the integer index for // all contents in tokenized_document.integer_sections. // + // Parameter old_document_id is unused since there is no need to migrate data + // from old_document_id to (new) document_id. + // // Returns: // - OK on success. // - INVALID_ARGUMENT_ERROR if document_id is invalid OR document_id is less @@ -55,7 +58,8 @@ // - Any NumericIndex<int64_t>::Editor errors. libtextclassifier3::Status Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, - bool recovery_mode, PutDocumentStatsProto* put_document_stats) override; + DocumentId /*old_document_id*/ _, bool recovery_mode, + PutDocumentStatsProto* put_document_stats) override; private: explicit IntegerSectionIndexingHandler(const Clock* clock,
diff --git a/icing/index/integer-section-indexing-handler_test.cc b/icing/index/integer-section-indexing-handler_test.cc index 4da05f9..00b435b 100644 --- a/icing/index/integer-section-indexing-handler_test.cc +++ b/icing/index/integer-section-indexing-handler_test.cc
@@ -14,6 +14,7 @@ #include "icing/index/integer-section-indexing-handler.h" +#include <cstdint> #include <limits> #include <memory> #include <string> @@ -25,6 +26,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator.h" @@ -40,11 +42,12 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" @@ -88,10 +91,11 @@ class IntegerSectionIndexingHandlerTest : public ::testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -118,8 +122,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), IsTrue()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType( @@ -167,13 +171,12 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult doc_store_create_result, DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); document_store_ = std::move(doc_store_create_result.document_store); } @@ -187,6 +190,7 @@ filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; FakeClock fake_clock_; std::string base_dir_; @@ -234,7 +238,6 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result, document_store_->Put(tokenized_document.document())); - DocumentId document_id = put_result.new_document_id; ASSERT_THAT(integer_index_->last_added_document_id(), Eq(kInvalidDocumentId)); // Handle document. @@ -243,10 +246,12 @@ IntegerSectionIndexingHandler::Create(&fake_clock_, integer_index_.get())); EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, + handler->Handle(tokenized_document, put_result.new_document_id, + put_result.old_document_id, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); - EXPECT_THAT(integer_index_->last_added_document_id(), Eq(document_id)); + EXPECT_THAT(integer_index_->last_added_document_id(), + Eq(put_result.new_document_id)); // Query "timestamp". ICING_ASSERT_OK_AND_ASSIGN( @@ -257,7 +262,8 @@ *schema_store_, fake_clock_.GetSystemTimeMilliseconds())); EXPECT_THAT(GetHits(std::move(itr)), ElementsAre(EqualsDocHitInfo( - document_id, std::vector<SectionId>{kSectionIdTimestamp}))); + put_result.new_document_id, + std::vector<SectionId>{kSectionIdTimestamp}))); } TEST_F(IntegerSectionIndexingHandlerTest, HandleNestedIntegerSection) { @@ -284,7 +290,6 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result, document_store_->Put(tokenized_document.document())); - DocumentId document_id = put_result.new_document_id; ASSERT_THAT(integer_index_->last_added_document_id(), Eq(kInvalidDocumentId)); // Handle nested_document. @@ -293,10 +298,12 @@ IntegerSectionIndexingHandler::Create(&fake_clock_, integer_index_.get())); EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, + handler->Handle(tokenized_document, put_result.new_document_id, + put_result.old_document_id, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); - EXPECT_THAT(integer_index_->last_added_document_id(), Eq(document_id)); + EXPECT_THAT(integer_index_->last_added_document_id(), + Eq(put_result.new_document_id)); // Query "nested.timestamp". ICING_ASSERT_OK_AND_ASSIGN( @@ -305,10 +312,10 @@ "nested.timestamp", /*key_lower=*/std::numeric_limits<int64_t>::min(), /*key_upper=*/std::numeric_limits<int64_t>::max(), *document_store_, *schema_store_, fake_clock_.GetSystemTimeMilliseconds())); - EXPECT_THAT( - GetHits(std::move(itr)), - ElementsAre(EqualsDocHitInfo( - document_id, std::vector<SectionId>{kSectionIdNestedTimestamp}))); + EXPECT_THAT(GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo( + put_result.new_document_id, + std::vector<SectionId>{kSectionIdNestedTimestamp}))); // Query "price". ICING_ASSERT_OK_AND_ASSIGN( @@ -317,9 +324,10 @@ kPropertyPrice, /*key_lower=*/std::numeric_limits<int64_t>::min(), /*key_upper=*/std::numeric_limits<int64_t>::max(), *document_store_, *schema_store_, fake_clock_.GetSystemTimeMilliseconds())); - EXPECT_THAT(GetHits(std::move(itr)), - ElementsAre(EqualsDocHitInfo( - document_id, std::vector<SectionId>{kSectionIdPrice}))); + EXPECT_THAT( + GetHits(std::move(itr)), + ElementsAre(EqualsDocHitInfo(put_result.new_document_id, + std::vector<SectionId>{kSectionIdPrice}))); // Query "timestamp". Should get empty result. ICING_ASSERT_OK_AND_ASSIGN( @@ -347,7 +355,6 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result, document_store_->Put(tokenized_document.document())); - DocumentId document_id = put_result.new_document_id; ASSERT_THAT(integer_index_->last_added_document_id(), Eq(kInvalidDocumentId)); // Handle document. Index data should remain unchanged since there is no @@ -357,10 +364,12 @@ IntegerSectionIndexingHandler::Create(&fake_clock_, integer_index_.get())); EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, + handler->Handle(tokenized_document, put_result.new_document_id, + put_result.old_document_id, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); - EXPECT_THAT(integer_index_->last_added_document_id(), Eq(document_id)); + EXPECT_THAT(integer_index_->last_added_document_id(), + Eq(put_result.new_document_id)); // Query "timestamp". Should get empty result. ICING_ASSERT_OK_AND_ASSIGN( @@ -373,7 +382,7 @@ } TEST_F(IntegerSectionIndexingHandlerTest, - HandleInvalidDocumentIdShouldReturnInvalidArgumentError) { + HandleInvalidNewDocumentIdShouldReturnInvalidArgumentError) { DocumentProto document = DocumentBuilder() .SetKey("icing", "fake_type/1") @@ -401,6 +410,7 @@ // index data and last_added_document_id should remain unchanged. EXPECT_THAT( handler->Handle(tokenized_document, kInvalidDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(integer_index_->last_added_document_id(), Eq(kCurrentDocumentId)); @@ -417,6 +427,7 @@ // Recovery mode should get the same result. EXPECT_THAT( handler->Handle(tokenized_document, kInvalidDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(integer_index_->last_added_document_id(), Eq(kCurrentDocumentId)); @@ -448,7 +459,6 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result, document_store_->Put(tokenized_document.document())); - DocumentId document_id = put_result.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerSectionIndexingHandler> handler, @@ -458,13 +468,16 @@ // Handling document with document_id == last_added_document_id should cause a // failure, and both index data and last_added_document_id should remain // unchanged. - integer_index_->set_last_added_document_id(document_id); - ASSERT_THAT(integer_index_->last_added_document_id(), Eq(document_id)); + integer_index_->set_last_added_document_id(put_result.new_document_id); + ASSERT_THAT(integer_index_->last_added_document_id(), + Eq(put_result.new_document_id)); EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, + handler->Handle(tokenized_document, put_result.new_document_id, + put_result.old_document_id, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(integer_index_->last_added_document_id(), Eq(document_id)); + EXPECT_THAT(integer_index_->last_added_document_id(), + Eq(put_result.new_document_id)); // Query "timestamp". Should get empty result. ICING_ASSERT_OK_AND_ASSIGN( @@ -478,13 +491,16 @@ // Handling document with document_id < last_added_document_id should cause a // failure, and both index data and last_added_document_id should remain // unchanged. - integer_index_->set_last_added_document_id(document_id + 1); - ASSERT_THAT(integer_index_->last_added_document_id(), Eq(document_id + 1)); + integer_index_->set_last_added_document_id(put_result.new_document_id + 1); + ASSERT_THAT(integer_index_->last_added_document_id(), + Eq(put_result.new_document_id + 1)); EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, + handler->Handle(tokenized_document, put_result.new_document_id, + put_result.old_document_id, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(integer_index_->last_added_document_id(), Eq(document_id + 1)); + EXPECT_THAT(integer_index_->last_added_document_id(), + Eq(put_result.new_document_id + 1)); // Query "timestamp". Should get empty result. ICING_ASSERT_OK_AND_ASSIGN( @@ -525,11 +541,9 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, document_store_->Put(tokenized_document1.document())); - DocumentId document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, document_store_->Put(tokenized_document2.document())); - DocumentId document_id2 = put_result2.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerSectionIndexingHandler> handler, @@ -539,10 +553,12 @@ // Handle document with document_id > last_added_document_id in recovery mode. // The handler should index this document and update last_added_document_id. EXPECT_THAT( - handler->Handle(tokenized_document1, document_id1, /*recovery_mode=*/true, + handler->Handle(tokenized_document1, put_result1.new_document_id, + put_result1.old_document_id, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); - EXPECT_THAT(integer_index_->last_added_document_id(), Eq(document_id1)); + EXPECT_THAT(integer_index_->last_added_document_id(), + Eq(put_result1.new_document_id)); // Query "timestamp". ICING_ASSERT_OK_AND_ASSIGN( @@ -553,19 +569,23 @@ *schema_store_, fake_clock_.GetSystemTimeMilliseconds())); EXPECT_THAT(GetHits(std::move(itr)), ElementsAre(EqualsDocHitInfo( - document_id1, std::vector<SectionId>{kSectionIdTimestamp}))); + put_result1.new_document_id, + std::vector<SectionId>{kSectionIdTimestamp}))); // Handle document with document_id == last_added_document_id in recovery // mode. We should not get any error, but the handler should ignore the // document, so both index data and last_added_document_id should remain // unchanged. - integer_index_->set_last_added_document_id(document_id2); - ASSERT_THAT(integer_index_->last_added_document_id(), Eq(document_id2)); + integer_index_->set_last_added_document_id(put_result2.new_document_id); + ASSERT_THAT(integer_index_->last_added_document_id(), + Eq(put_result2.new_document_id)); EXPECT_THAT( - handler->Handle(tokenized_document2, document_id2, /*recovery_mode=*/true, + handler->Handle(tokenized_document2, put_result2.new_document_id, + put_result2.old_document_id, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); - EXPECT_THAT(integer_index_->last_added_document_id(), Eq(document_id2)); + EXPECT_THAT(integer_index_->last_added_document_id(), + Eq(put_result2.new_document_id)); // Query "timestamp". Should not get hits for document2. ICING_ASSERT_OK_AND_ASSIGN( @@ -576,18 +596,22 @@ *schema_store_, fake_clock_.GetSystemTimeMilliseconds())); EXPECT_THAT(GetHits(std::move(itr)), ElementsAre(EqualsDocHitInfo( - document_id1, std::vector<SectionId>{kSectionIdTimestamp}))); + put_result1.new_document_id, + std::vector<SectionId>{kSectionIdTimestamp}))); // Handle document with document_id < last_added_document_id in recovery mode. // We should not get any error, but the handler should ignore the document, so // both index data and last_added_document_id should remain unchanged. - integer_index_->set_last_added_document_id(document_id2 + 1); - ASSERT_THAT(integer_index_->last_added_document_id(), Eq(document_id2 + 1)); + integer_index_->set_last_added_document_id(put_result2.new_document_id + 1); + ASSERT_THAT(integer_index_->last_added_document_id(), + Eq(put_result2.new_document_id + 1)); EXPECT_THAT( - handler->Handle(tokenized_document2, document_id2, /*recovery_mode=*/true, + handler->Handle(tokenized_document2, put_result2.new_document_id, + put_result2.old_document_id, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); - EXPECT_THAT(integer_index_->last_added_document_id(), Eq(document_id2 + 1)); + EXPECT_THAT(integer_index_->last_added_document_id(), + Eq(put_result2.new_document_id + 1)); // Query "timestamp". Should not get hits for document2. ICING_ASSERT_OK_AND_ASSIGN( @@ -598,7 +622,8 @@ *schema_store_, fake_clock_.GetSystemTimeMilliseconds())); EXPECT_THAT(GetHits(std::move(itr)), ElementsAre(EqualsDocHitInfo( - document_id1, std::vector<SectionId>{kSectionIdTimestamp}))); + put_result1.new_document_id, + std::vector<SectionId>{kSectionIdTimestamp}))); } } // namespace
diff --git a/icing/index/iterator/doc-hit-info-iterator-by-uri.cc b/icing/index/iterator/doc-hit-info-iterator-by-uri.cc new file mode 100644 index 0000000..c5a9ecc --- /dev/null +++ b/icing/index/iterator/doc-hit-info-iterator-by-uri.cc
@@ -0,0 +1,108 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/index/iterator/doc-hit-info-iterator-by-uri.h" + +#include <algorithm> +#include <functional> +#include <memory> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/absl_ports/str_cat.h" +#include "icing/index/hit/doc-hit-info.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" +#include "icing/util/status-macros.h" + +namespace icing { +namespace lib { + +libtextclassifier3::StatusOr<std::unique_ptr<DocHitInfoIteratorByUri>> +DocHitInfoIteratorByUri::Create(const DocumentStore* document_store, + const SearchSpecProto& search_spec) { + ICING_RETURN_ERROR_IF_NULL(document_store); + + if (search_spec.document_uri_filters().empty()) { + return absl_ports::InvalidArgumentError( + "Cannot create DocHitInfoIteratorByUri with empty " + "document_uri_filters"); + } + + std::unordered_set<std::string> all_namespaces( + search_spec.namespace_filters().begin(), + search_spec.namespace_filters().end()); + std::vector<DocumentId> target_document_ids; + for (const NamespaceDocumentUriGroup& namespace_document_uri_group : + search_spec.document_uri_filters()) { + const std::string& current_namespace = + namespace_document_uri_group.namespace_(); + if (!all_namespaces.empty() && + all_namespaces.count(current_namespace) == 0) { + // The current namespace doesn't appear in the namespace filter. + return absl_ports::InvalidArgumentError(absl_ports::StrCat( + "The namespace : ", current_namespace, + " appears in the document uri filter, but does not appear in the " + "namespace filter.")); + } + + if (namespace_document_uri_group.document_uris().empty()) { + return absl_ports::InvalidArgumentError(absl_ports::StrCat( + "The namespace ", current_namespace, + " has not specified any document in the document uri filter. Please " + "use the namespace filter to exclude a namespace instead.")); + } + for (const std::string& document_uri : + namespace_document_uri_group.document_uris()) { + auto document_id_or = + document_store->GetDocumentId(current_namespace, document_uri); + if (!document_id_or.ok()) { + continue; + } + target_document_ids.push_back(document_id_or.ValueOrDie()); + } + } + // Sort the document ids in descending order, which is a requirement for all + // iterators. + std::sort(target_document_ids.begin(), target_document_ids.end(), + std::greater<DocumentId>()); + // Deduplicate the document ids. + auto last_itr = + std::unique(target_document_ids.begin(), target_document_ids.end()); + target_document_ids.erase(last_itr, target_document_ids.end()); + + return std::unique_ptr<DocHitInfoIteratorByUri>( + new DocHitInfoIteratorByUri(std::move(target_document_ids))); +} + +libtextclassifier3::Status DocHitInfoIteratorByUri::Advance() { + current_document_id_index_++; + + if (current_document_id_index_ >= target_document_ids_.size()) { + doc_hit_info_ = DocHitInfo(kInvalidDocumentId); + return absl_ports::ResourceExhaustedError( + "No more DocHitInfos in iterator"); + } + doc_hit_info_.set_document_id( + target_document_ids_[current_document_id_index_]); + return libtextclassifier3::Status::OK; +} + +} // namespace lib +} // namespace icing
diff --git a/icing/index/iterator/doc-hit-info-iterator-by-uri.h b/icing/index/iterator/doc-hit-info-iterator-by-uri.h new file mode 100644 index 0000000..d136535 --- /dev/null +++ b/icing/index/iterator/doc-hit-info-iterator-by-uri.h
@@ -0,0 +1,82 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_INDEX_ITERATOR_DOC_HIT_INFO_ITERATOR_BY_URI_H_ +#define ICING_INDEX_ITERATOR_DOC_HIT_INFO_ITERATOR_BY_URI_H_ + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/index/iterator/doc-hit-info-iterator.h" +#include "icing/proto/search.pb.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" + +namespace icing { +namespace lib { + +class DocHitInfoIteratorByUri : public DocHitInfoIterator { + public: + // Creates a DocHitInfoIteratorByUri based on the given search_spec. + // + // Returns: + // - A DocHitInfoIteratorByUri instance on success. + // - INVALID_ARGUMENT error if the search_spec's document_uri_filters is + // empty or if any namespace in the document_uri_filters has no document + // URIs specified. + static libtextclassifier3::StatusOr<std::unique_ptr<DocHitInfoIteratorByUri>> + Create(const DocumentStore* document_store, + const SearchSpecProto& search_spec); + + libtextclassifier3::Status Advance() override; + + libtextclassifier3::StatusOr<TrimmedNode> TrimRightMostNode() && override { + return absl_ports::InvalidArgumentError( + "DocHitInfoIteratorByUri should not be used in suggestion."); + } + + void MapChildren(const ChildrenMapper& mapper) override {} + + CallStats GetCallStats() const override { + return CallStats( + /*num_leaf_advance_calls_lite_index_in=*/0, + /*num_leaf_advance_calls_main_index_in=*/0, + /*num_leaf_advance_calls_integer_index_in=*/0, + /*num_leaf_advance_calls_no_index_in=*/num_advance_calls_, + /*num_blocks_inspected_in=*/0); + } + + std::string ToString() const override { return "uri_iterator"; } + + private: + explicit DocHitInfoIteratorByUri(std::vector<DocumentId> target_document_ids) + : target_document_ids_(std::move(target_document_ids)), + current_document_id_index_(-1), + num_advance_calls_(0) {} + + std::vector<DocumentId> target_document_ids_; + int current_document_id_index_; + + int num_advance_calls_; +}; + +} // namespace lib +} // namespace icing + +#endif // ICING_INDEX_ITERATOR_DOC_HIT_INFO_ITERATOR_BY_URI_H_
diff --git a/icing/index/iterator/doc-hit-info-iterator-by-uri_test.cc b/icing/index/iterator/doc-hit-info-iterator-by-uri_test.cc new file mode 100644 index 0000000..a11bf9e --- /dev/null +++ b/icing/index/iterator/doc-hit-info-iterator-by-uri_test.cc
@@ -0,0 +1,368 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/index/iterator/doc-hit-info-iterator-by-uri.h" + +#include <memory> +#include <string> +#include <utility> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/document-builder.h" +#include "icing/feature-flags.h" +#include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" +#include "icing/index/iterator/doc-hit-info-iterator-test-util.h" +#include "icing/proto/document.pb.h" +#include "icing/proto/schema.pb.h" +#include "icing/schema-builder.h" +#include "icing/schema/schema-store.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" +#include "icing/testing/common-matchers.h" +#include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" +#include "icing/testing/tmp-directory.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::ElementsAre; +using ::testing::IsEmpty; + +class DocHitInfoIteratorByUriTest : public ::testing::Test { + protected: + DocHitInfoIteratorByUriTest() : test_dir_(GetTestTempDir() + "/icing") {} + + void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + + filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email")) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN( + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); + ICING_ASSERT_OK(schema_store_->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); + document_store_ = std::move(create_result.document_store); + } + + void TearDown() override { + document_store_.reset(); + schema_store_.reset(); + filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); + } + + std::unique_ptr<FeatureFlags> feature_flags_; + std::unique_ptr<SchemaStore> schema_store_; + std::unique_ptr<DocumentStore> document_store_; + FakeClock fake_clock_; + const Filesystem filesystem_; + const std::string test_dir_; +}; + +TEST_F(DocHitInfoIteratorByUriTest, EmptyFilterIsInvalid) { + // Create a search spec without a uri filter specified. + SearchSpecProto search_spec; + EXPECT_THAT( + DocHitInfoIteratorByUri::Create(document_store_.get(), search_spec), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + + // Add a namespace group with no uris. + NamespaceDocumentUriGroup* namespace_uris = + search_spec.add_document_uri_filters(); + namespace_uris->set_namespace_("namespace"); + EXPECT_THAT( + DocHitInfoIteratorByUri::Create(document_store_.get(), search_spec), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(DocHitInfoIteratorByUriTest, MatchesSomeDocuments) { + // Put documents + DocumentProto document1 = DocumentBuilder() + .SetKey("namespace", "email/1") + .SetSchema("email") + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("namespace", "email/2") + .SetSchema("email") + .Build(); + DocumentProto document3 = DocumentBuilder() + .SetKey("namespace", "email/3") + .SetSchema("email") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document1)); + DocumentId document_id1 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document2)); + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document3)); + DocumentId document_id3 = put_result.new_document_id; + + // Create a search spec with uri filters that only match document1 and + // document3. + SearchSpecProto search_spec; + NamespaceDocumentUriGroup* uris = search_spec.add_document_uri_filters(); + uris->set_namespace_("namespace"); + uris->add_document_uris("email/1"); + uris->add_document_uris("email/3"); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIteratorByUri> iterator, + DocHitInfoIteratorByUri::Create(document_store_.get(), search_spec)); + EXPECT_THAT(GetDocumentIds(iterator.get()), + ElementsAre(document_id3, document_id1)); + EXPECT_FALSE(iterator->Advance().ok()); +} + +TEST_F(DocHitInfoIteratorByUriTest, MatchesAllDocuments) { + // Put documents + DocumentProto document1 = DocumentBuilder() + .SetKey("namespace", "email/1") + .SetSchema("email") + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("namespace", "email/2") + .SetSchema("email") + .Build(); + DocumentProto document3 = DocumentBuilder() + .SetKey("namespace", "email/3") + .SetSchema("email") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document1)); + DocumentId document_id1 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document2)); + DocumentId document_id2 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document3)); + DocumentId document_id3 = put_result.new_document_id; + + // Create a search spec with uri filters that match all documents. + SearchSpecProto search_spec; + NamespaceDocumentUriGroup* uris = search_spec.add_document_uri_filters(); + uris->set_namespace_("namespace"); + uris->add_document_uris("email/1"); + uris->add_document_uris("email/2"); + uris->add_document_uris("email/3"); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIteratorByUri> iterator, + DocHitInfoIteratorByUri::Create(document_store_.get(), search_spec)); + EXPECT_THAT(GetDocumentIds(iterator.get()), + ElementsAre(document_id3, document_id2, document_id1)); + EXPECT_FALSE(iterator->Advance().ok()); +} + +TEST_F(DocHitInfoIteratorByUriTest, NonexistentUriIsOk) { + // Put documents + DocumentProto document1 = DocumentBuilder() + .SetKey("namespace", "email/1") + .SetSchema("email") + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("namespace", "email/2") + .SetSchema("email") + .Build(); + DocumentProto document3 = DocumentBuilder() + .SetKey("namespace", "email/3") + .SetSchema("email") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document1)); + DocumentId document_id1 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document2)); + DocumentId document_id2 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document3)); + DocumentId document_id3 = put_result.new_document_id; + + // Create a search spec with a nonexistent uri in uri filters. + SearchSpecProto search_spec; + NamespaceDocumentUriGroup* uris = search_spec.add_document_uri_filters(); + uris->set_namespace_("namespace"); + uris->add_document_uris("email/1"); + uris->add_document_uris("email/2"); + uris->add_document_uris("email/3"); + uris->add_document_uris("nonexistent_uri"); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIteratorByUri> iterator, + DocHitInfoIteratorByUri::Create(document_store_.get(), search_spec)); + EXPECT_THAT(GetDocumentIds(iterator.get()), + ElementsAre(document_id3, document_id2, document_id1)); + EXPECT_FALSE(iterator->Advance().ok()); +} + +TEST_F(DocHitInfoIteratorByUriTest, AllNonexistentUriShouldReturnEmptyResults) { + // Put documents + DocumentProto document1 = DocumentBuilder() + .SetKey("namespace", "email/1") + .SetSchema("email") + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("namespace", "email/2") + .SetSchema("email") + .Build(); + DocumentProto document3 = DocumentBuilder() + .SetKey("namespace", "email/3") + .SetSchema("email") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document1)); + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document2)); + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document3)); + + // Create a search spec with all nonexistent uris. + SearchSpecProto search_spec; + NamespaceDocumentUriGroup* uris = search_spec.add_document_uri_filters(); + uris->set_namespace_("namespace"); + uris->add_document_uris("nonexistent_uri1"); + uris->add_document_uris("nonexistent_uri2"); + uris->add_document_uris("nonexistent_uri3"); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIteratorByUri> iterator, + DocHitInfoIteratorByUri::Create(document_store_.get(), search_spec)); + EXPECT_THAT(GetDocumentIds(iterator.get()), IsEmpty()); + EXPECT_FALSE(iterator->Advance().ok()); +} + +TEST_F(DocHitInfoIteratorByUriTest, MultipleNamespaces) { + // Put documents + DocumentProto document1 = DocumentBuilder() + .SetKey("namespace1", "email/1") + .SetSchema("email") + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("namespace1", "email/2") + .SetSchema("email") + .Build(); + DocumentProto document3 = DocumentBuilder() + .SetKey("namespace2", "email/3") + .SetSchema("email") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document1)); + DocumentId document_id1 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document2)); + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document3)); + DocumentId document_id3 = put_result.new_document_id; + + // Create a search spec with uri filters that match document1 and document3 in + // different namespaces. + SearchSpecProto search_spec; + NamespaceDocumentUriGroup* namespace1_uris = + search_spec.add_document_uri_filters(); + namespace1_uris->set_namespace_("namespace1"); + namespace1_uris->add_document_uris("email/1"); + NamespaceDocumentUriGroup* namespace2_uris = + search_spec.add_document_uri_filters(); + namespace2_uris->set_namespace_("namespace2"); + namespace2_uris->add_document_uris("email/3"); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIteratorByUri> iterator, + DocHitInfoIteratorByUri::Create(document_store_.get(), search_spec)); + EXPECT_THAT(GetDocumentIds(iterator.get()), + ElementsAre(document_id3, document_id1)); + EXPECT_FALSE(iterator->Advance().ok()); +} + +TEST_F(DocHitInfoIteratorByUriTest, DuplicatedUriIsOk) { + // Put documents + DocumentProto document1 = DocumentBuilder() + .SetKey("namespace", "email/1") + .SetSchema("email") + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("namespace", "email/2") + .SetSchema("email") + .Build(); + DocumentProto document3 = DocumentBuilder() + .SetKey("namespace", "email/3") + .SetSchema("email") + .Build(); + DocumentProto document4 = DocumentBuilder() + .SetKey("namespace", "email/4") + .SetSchema("email") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document1)); + DocumentId document_id1 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document2)); + DocumentId document_id2 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document3)); + DocumentId document_id3 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, document_store_->Put(document4)); + DocumentId document_id4 = put_result.new_document_id; + + // Create a search spec with duplicated uri filters. The result document ids + // should be de-duplicated. + SearchSpecProto search_spec; + NamespaceDocumentUriGroup* uris = search_spec.add_document_uri_filters(); + uris->set_namespace_("namespace"); + uris->add_document_uris("email/1"); + uris->add_document_uris("email/2"); + uris->add_document_uris("email/3"); + uris->add_document_uris("email/3"); + + uris = search_spec.add_document_uri_filters(); + uris->set_namespace_("namespace"); + uris->add_document_uris("email/2"); + uris->add_document_uris("email/4"); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIteratorByUri> iterator, + DocHitInfoIteratorByUri::Create(document_store_.get(), search_spec)); + EXPECT_THAT( + GetDocumentIds(iterator.get()), + ElementsAre(document_id4, document_id3, document_id2, document_id1)); + EXPECT_FALSE(iterator->Advance().ok()); +} + +TEST_F(DocHitInfoIteratorByUriTest, TrimRightMostNodeResultsInError) { + SearchSpecProto search_spec; + NamespaceDocumentUriGroup* uris = search_spec.add_document_uri_filters(); + uris->set_namespace_("namespace"); + uris->add_document_uris("uri"); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIteratorByUri> iterator, + DocHitInfoIteratorByUri::Create(document_store_.get(), search_spec)); + EXPECT_THAT(std::move(*iterator).TrimRightMostNode(), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/index/iterator/doc-hit-info-iterator-filter_test.cc b/icing/index/iterator/doc-hit-info-iterator-filter_test.cc index b088212..38a628c 100644 --- a/icing/index/iterator/doc-hit-info-iterator-filter_test.cc +++ b/icing/index/iterator/doc-hit-info-iterator-filter_test.cc
@@ -25,6 +25,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/hit/doc-hit-info.h" @@ -41,6 +42,7 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/util/clock.h" @@ -55,13 +57,13 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, const SchemaStore* schema_store) { + const Clock* clock, const SchemaStore* schema_store, + const FeatureFlags& feature_flags) { return DocumentStore::Create( - filesystem, base_dir, clock, schema_store, + filesystem, base_dir, clock, schema_store, &feature_flags, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr); } @@ -71,6 +73,7 @@ : test_dir_(GetTestTempDir() + "/icing") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); test_document1_ = DocumentBuilder().SetKey("icing", "email/1").SetSchema("email").Build(); @@ -84,8 +87,8 @@ .AddType(SchemaTypeConfigBuilder().SetType("email")) .Build(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK(schema_store_->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -93,7 +96,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); document_store_ = std::move(create_result.document_store); } @@ -105,6 +108,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; std::unique_ptr<SchemaStore> schema_store_; std::unique_ptr<DocumentStore> document_store_; FakeClock fake_clock_; @@ -238,6 +242,7 @@ : test_dir_(GetTestTempDir() + "/icing") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); document1_namespace1_ = DocumentBuilder() .SetKey(namespace1_, "email/1") @@ -261,8 +266,8 @@ .AddType(SchemaTypeConfigBuilder().SetType("email")) .Build(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK(schema_store_->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -270,7 +275,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); document_store_ = std::move(create_result.document_store); } @@ -282,6 +287,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; std::unique_ptr<SchemaStore> schema_store_; std::unique_ptr<DocumentStore> document_store_; FakeClock fake_clock_; @@ -428,6 +434,7 @@ : test_dir_(GetTestTempDir() + "/icing") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); document1_schema1_ = DocumentBuilder() .SetKey("namespace", "1") @@ -459,8 +466,8 @@ .AddParentType(kSchema2)) .Build(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK(schema_store_->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -468,7 +475,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); document_store_ = std::move(create_result.document_store); } @@ -480,6 +487,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; std::unique_ptr<SchemaStore> schema_store_; std::unique_ptr<DocumentStore> document_store_; FakeClock fake_clock_; @@ -735,6 +743,7 @@ : test_dir_(GetTestTempDir() + "/icing") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); SchemaProto schema = @@ -742,8 +751,8 @@ .AddType(SchemaTypeConfigBuilder().SetType(email_schema_)) .Build(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK(schema_store_->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -751,7 +760,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); document_store_ = std::move(create_result.document_store); } @@ -763,6 +772,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; std::unique_ptr<SchemaStore> schema_store_; std::unique_ptr<DocumentStore> document_store_; FakeClock fake_clock_; @@ -779,7 +789,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -812,7 +822,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -845,7 +855,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -879,7 +889,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -910,6 +920,7 @@ DocHitInfoIteratorFilterTest() : test_dir_(GetTestTempDir() + "/icing") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); document1_namespace1_schema1_ = DocumentBuilder() .SetKey(namespace1_, "1") @@ -948,8 +959,8 @@ .AddType(SchemaTypeConfigBuilder().SetType(schema2_)) .Build(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK(schema_store_->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -957,7 +968,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); document_store_ = std::move(create_result.document_store); } @@ -969,6 +980,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; std::unique_ptr<SchemaStore> schema_store_; std::unique_ptr<DocumentStore> document_store_; FakeClock fake_clock_; @@ -992,7 +1004,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store);
diff --git a/icing/index/iterator/doc-hit-info-iterator-match-score-expression.cc b/icing/index/iterator/doc-hit-info-iterator-match-score-expression.cc new file mode 100644 index 0000000..8c94b16 --- /dev/null +++ b/icing/index/iterator/doc-hit-info-iterator-match-score-expression.cc
@@ -0,0 +1,54 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/index/iterator/doc-hit-info-iterator-match-score-expression.h" + +#include <memory> +#include <string_view> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/index/hit/doc-hit-info.h" +#include "icing/index/iterator/doc-hit-info-iterator.h" +#include "icing/scoring/advanced_scoring/score-expression.h" +#include "icing/store/document-id.h" + +namespace icing { +namespace lib { + +libtextclassifier3::Status DocHitInfoIteratorMatchScoreExpression::Advance() { + while (delegate_->Advance().ok()) { + libtextclassifier3::StatusOr<double> score_or = + scoring_expression_->EvaluateDouble(delegate_->doc_hit_info(), + /*query_it=*/nullptr); + if (!score_or.ok()) { + continue; + } + double score = score_or.ValueOrDie(); + if (score < score_low_ || score > score_high_) { + // Score is outside of the desired range, skip this result. + continue; + } + doc_hit_info_ = delegate_->doc_hit_info(); + return libtextclassifier3::Status::OK; + } + + // Didn't find anything on the delegate iterator. + doc_hit_info_ = DocHitInfo(kInvalidDocumentId); + return absl_ports::ResourceExhaustedError("No more DocHitInfos in iterator"); +} + +} // namespace lib +} // namespace icing
diff --git a/icing/index/iterator/doc-hit-info-iterator-match-score-expression.h b/icing/index/iterator/doc-hit-info-iterator-match-score-expression.h new file mode 100644 index 0000000..f46b8da --- /dev/null +++ b/icing/index/iterator/doc-hit-info-iterator-match-score-expression.h
@@ -0,0 +1,99 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_INDEX_ITERATOR_DOC_HIT_INFO_ITERATOR_MATCH_SCORE_EXPRESSION_H_ +#define ICING_INDEX_ITERATOR_DOC_HIT_INFO_ITERATOR_MATCH_SCORE_EXPRESSION_H_ + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/absl_ports/str_cat.h" +#include "icing/index/iterator/doc-hit-info-iterator-all-document-id.h" +#include "icing/index/iterator/doc-hit-info-iterator.h" +#include "icing/schema/section.h" +#include "icing/scoring/advanced_scoring/score-expression.h" +#include "icing/store/document-id.h" + +namespace icing { +namespace lib { + +class DocHitInfoIteratorMatchScoreExpression : public DocHitInfoIterator { + public: + explicit DocHitInfoIteratorMatchScoreExpression( + DocumentId last_added_document_id, + std::unique_ptr<ScoreExpression> scoring_expression, double score_low, + double score_high) + : delegate_(std::make_unique<DocHitInfoIteratorAllDocumentId>( + last_added_document_id)), + scoring_expression_(std::move(scoring_expression)), + score_low_(score_low), + score_high_(score_high) {}; + + libtextclassifier3::Status Advance() override; + + libtextclassifier3::StatusOr<TrimmedNode> TrimRightMostNode() && override { + // TODO(b/377215223): Decide on the correct behavior for this function when + // supporting query language optimizations and the delegate is not + // DocHitInfoIteratorAllDocumentId. + return absl_ports::InvalidArgumentError( + "Query suggestions for the matchScoreExpression function are not " + "supported"); + } + + void MapChildren(const ChildrenMapper& mapper) override { + delegate_ = mapper(std::move(delegate_)); + } + + CallStats GetCallStats() const override { return delegate_->GetCallStats(); } + + std::string ToString() const override { + // TODO(b/377215223): Consider having a ToString method for ScoreExpression, + // so that it can be displayed here. + return absl_ports::StrCat("(matchScoreExpression: ", delegate_->ToString(), + ")"); + } + + void PopulateMatchedTermsStats( + std::vector<TermMatchInfo>* matched_terms_stats, + SectionIdMask filtering_section_mask = kSectionIdMaskAll) const override { + // TODO(b/377215223): Decide on the correct behavior for this function when + // supporting query language optimizations and the delegate is not + // DocHitInfoIteratorAllDocumentId. + } + + private: + // TODO(b/377215223): Currently, the delegate can only be + // DocHitInfoIteratorAllDocumentId, but filtering on all documents is + // inefficient. Consider the following optimizations: + // 1. Implement a query language optimization so that queries like "foo AND + // matchScoreExpression(...)" can be optimized by letting + // DocHitInfoIteratorMatchScoreExpression filter on "foo". + // 2. Apply other filter iterators at the bottom level instead of the top + // level, similar to how section restrictions are handled. + std::unique_ptr<DocHitInfoIterator> delegate_; + + std::unique_ptr<ScoreExpression> scoring_expression_; + double score_low_; + double score_high_; +}; + +} // namespace lib +} // namespace icing + +#endif // ICING_INDEX_ITERATOR_DOC_HIT_INFO_ITERATOR_MATCH_SCORE_EXPRESSION_H_
diff --git a/icing/index/iterator/doc-hit-info-iterator-property-in-schema_test.cc b/icing/index/iterator/doc-hit-info-iterator-property-in-schema_test.cc index 17bd55d..6fb84ce 100644 --- a/icing/index/iterator/doc-hit-info-iterator-property-in-schema_test.cc +++ b/icing/index/iterator/doc-hit-info-iterator-property-in-schema_test.cc
@@ -22,7 +22,9 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator-all-document-id.h" #include "icing/index/iterator/doc-hit-info-iterator-test-util.h" @@ -36,6 +38,7 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" namespace icing { @@ -53,6 +56,7 @@ : test_dir_(GetTestTempDir() + "/icing") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); document1_ = DocumentBuilder() .SetKey("namespace", "uri1") @@ -89,22 +93,22 @@ .Build(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK(schema_store_->SetSchema( schema_, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } @@ -114,6 +118,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; std::unique_ptr<SchemaStore> schema_store_; std::unique_ptr<DocumentStore> document_store_; const Filesystem filesystem_;
diff --git a/icing/index/iterator/doc-hit-info-iterator-section-restrict_test.cc b/icing/index/iterator/doc-hit-info-iterator-section-restrict_test.cc index 94e9b5e..34e206e 100644 --- a/icing/index/iterator/doc-hit-info-iterator-section-restrict_test.cc +++ b/icing/index/iterator/doc-hit-info-iterator-section-restrict_test.cc
@@ -23,7 +23,9 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator-and.h" #include "icing/index/iterator/doc-hit-info-iterator-test-util.h" @@ -38,6 +40,7 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" namespace icing { @@ -58,6 +61,7 @@ : test_dir_(GetTestTempDir() + "/icing") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); document1_ = DocumentBuilder() .SetKey("namespace", "uri1") @@ -94,22 +98,22 @@ .Build(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK(schema_store_->SetSchema( schema_, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } @@ -119,6 +123,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; std::unique_ptr<SchemaStore> schema_store_; std::unique_ptr<DocumentStore> document_store_; const Filesystem filesystem_;
diff --git a/icing/index/lite/lite-index.cc b/icing/index/lite/lite-index.cc index 98a34c4..cd03dd8 100644 --- a/icing/index/lite/lite-index.cc +++ b/icing/index/lite/lite-index.cc
@@ -308,7 +308,7 @@ } libtextclassifier3::StatusOr<uint32_t> LiteIndex::InsertTerm( - const std::string& term, TermMatchType::Code term_match_type, + std::string_view term, TermMatchType::Code term_match_type, NamespaceId namespace_id) { absl_ports::unique_lock l(&mutex_); uint32_t tvi; @@ -366,7 +366,7 @@ } libtextclassifier3::StatusOr<uint32_t> LiteIndex::GetTermId( - const std::string& term) const { + std::string_view term) const { absl_ports::shared_lock l(&mutex_); char dummy; uint32_t tvi; @@ -686,8 +686,11 @@ // below if there are any valid hits pointing to that termid. tvi_to_delete.insert(curr_tvi); } + DocumentId old_document_id = term_id_hit_pair.hit().document_id(); DocumentId new_document_id = - document_id_old_to_new[term_id_hit_pair.hit().document_id()]; + old_document_id >= 0 && old_document_id < document_id_old_to_new.size() + ? document_id_old_to_new[old_document_id] + : kInvalidDocumentId; if (new_document_id == kInvalidDocumentId) { continue; } @@ -704,6 +707,15 @@ // allocated region of hit_buffer_. TermIdHitPair::Value* valp = hit_buffer_.GetMutableMem<TermIdHitPair::Value>(new_size++, 1); + if (valp == nullptr) { + // This really shouldn't happen since we are only writing to the already + // allocated region of hit_buffer_. But just in case, we log and return an + // error here. + ICING_LOG(ERROR) + << "GetMutableMem failed in Optimize. This should never happen."; + return absl_ports::ResourceExhaustedError( + "Allocating more space in hit buffer failed!"); + } *valp = new_term_id_hit_pair.value(); } header_->set_cur_size(new_size);
diff --git a/icing/index/lite/lite-index.h b/icing/index/lite/lite-index.h index 149fbea..280df88 100644 --- a/icing/index/lite/lite-index.h +++ b/icing/index/lite/lite-index.h
@@ -96,8 +96,8 @@ libtextclassifier3::Status PersistToDisk() ICING_LOCKS_EXCLUDED(mutex_); // Returns term_id if term found, NOT_FOUND otherwise. - libtextclassifier3::StatusOr<uint32_t> GetTermId( - const std::string& term) const ICING_LOCKS_EXCLUDED(mutex_); + libtextclassifier3::StatusOr<uint32_t> GetTermId(std::string_view term) const + ICING_LOCKS_EXCLUDED(mutex_); // Returns an iterator for all terms for which 'prefix' is a prefix. class PrefixIterator { @@ -130,7 +130,7 @@ // A value index on success // RESOURCE_EXHAUSTED if lexicon is full or no disk space is available libtextclassifier3::StatusOr<uint32_t> InsertTerm( - const std::string& term, TermMatchType::Code term_match_type, + std::string_view term, TermMatchType::Code term_match_type, NamespaceId namespace_id) ICING_LOCKS_EXCLUDED(mutex_); // Updates term properties by setting hasPrefixHits and namespace id of the
diff --git a/icing/index/numeric/integer-index-storage.cc b/icing/index/numeric/integer-index-storage.cc index 22ed842..6a00bf1 100644 --- a/icing/index/numeric/integer-index-storage.cc +++ b/icing/index/numeric/integer-index-storage.cc
@@ -716,9 +716,11 @@ old_pl_accessor->GetNextDataBatch()); while (!batch_old_data.empty()) { for (const IntegerIndexData& old_data : batch_old_data) { + DocumentId old_document_id = old_data.basic_hit().document_id(); DocumentId new_document_id = - old_data.basic_hit().document_id() < document_id_old_to_new.size() - ? document_id_old_to_new[old_data.basic_hit().document_id()] + old_document_id >= 0 && + old_document_id < document_id_old_to_new.size() + ? document_id_old_to_new[old_document_id] : kInvalidDocumentId; // Transfer the document id of the hit if the document is not deleted // or outdated.
diff --git a/icing/index/numeric/integer-index_test.cc b/icing/index/numeric/integer-index_test.cc index 4563b5b..7bc684b 100644 --- a/icing/index/numeric/integer-index_test.cc +++ b/icing/index/numeric/integer-index_test.cc
@@ -27,7 +27,9 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/index/numeric/dummy-numeric-index.h" @@ -37,9 +39,12 @@ #include "icing/proto/document.pb.h" #include "icing/proto/schema.pb.h" #include "icing/schema-builder.h" +#include "icing/schema/schema-store.h" #include "icing/schema/section.h" #include "icing/store/document-id.h" +#include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" namespace icing { @@ -68,6 +73,7 @@ class NumericIndexIntegerTest : public ::testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); base_dir_ = GetTestTempDir() + "/icing"; ASSERT_THAT(filesystem_.CreateDirectoryRecursively(base_dir_.c_str()), IsTrue()); @@ -76,22 +82,24 @@ std::string schema_dir = base_dir_ + "/schema_test"; ASSERT_TRUE(filesystem_.CreateDirectoryRecursively(schema_dir.c_str())); + ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, SchemaStore::Create(&filesystem_, schema_dir, &clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_dir, &clock_, + feature_flags_.get())); std::string document_store_dir = base_dir_ + "/doc_store_test"; ASSERT_TRUE( filesystem_.CreateDirectoryRecursively(document_store_dir.c_str())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult doc_store_create_result, - DocumentStore::Create( - &filesystem_, document_store_dir, &clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, document_store_dir, &clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); doc_store_ = std::move(doc_store_create_result.document_store); } @@ -159,14 +167,14 @@ ICING_ASSIGN_OR_RETURN( DocumentStore::CreateResult doc_store_create_result, - DocumentStore::Create( - &filesystem_, document_store_dir, &clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, document_store_dir, &clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); doc_store_ = std::move(doc_store_create_result.document_store); return std::move(doc_store_optimize_result.document_id_old_to_new); } @@ -187,6 +195,7 @@ return result; } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; std::string base_dir_; std::string working_path_;
diff --git a/icing/index/property-existence-indexing-handler.cc b/icing/index/property-existence-indexing-handler.cc index 504f380..0b4d87f 100644 --- a/icing/index/property-existence-indexing-handler.cc +++ b/icing/index/property-existence-indexing-handler.cc
@@ -81,6 +81,7 @@ libtextclassifier3::Status PropertyExistenceIndexingHandler::Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, + DocumentId /*old_document_id*/ _, PutDocumentStatsProto* put_document_stats) { std::unique_ptr<Timer> index_timer = clock_.GetNewTimer(); @@ -88,13 +89,12 @@ // Section id is irrelevant to metadata tokens that is used to support // property existence check. Index::Editor editor = - index_.Edit(document_id, /*section_id=*/0, TermMatchType::EXACT_ONLY, - /*namespace_id=*/0); + index_.Edit(document_id, /*section_id=*/0, /*namespace_id=*/0); std::unordered_set<std::string> meta_tokens; ConstructPropertyExistenceMetaToken( /*current_path=*/"", tokenized_document.document(), meta_tokens); for (const std::string& meta_token : meta_tokens) { - status = editor.BufferTerm(meta_token.c_str()); + status = editor.BufferTerm(meta_token, TermMatchType::EXACT_ONLY); if (!status.ok()) { // We've encountered a failure. Bail out. We'll mark this doc as deleted // and signal a failure to the client.
diff --git a/icing/index/property-existence-indexing-handler.h b/icing/index/property-existence-indexing-handler.h index 55c0bb4..cfdc3d8 100644 --- a/icing/index/property-existence-indexing-handler.h +++ b/icing/index/property-existence-indexing-handler.h
@@ -63,6 +63,9 @@ // - kPropertyExistenceTokenPrefix + "propC" // - kPropertyExistenceTokenPrefix + "propC.propD" // + // Parameter old_document_id is unused since there is no need to migrate data + // from old_document_id to (new) document_id. + // /// Returns: // - OK on success // - RESOURCE_EXHAUSTED_ERROR if the index is full and can't add anymore @@ -70,6 +73,7 @@ // - INTERNAL_ERROR if any other errors occur. libtextclassifier3::Status Handle(const TokenizedDocument& tokenized_document, DocumentId document_id, + DocumentId /*old_document_id*/ _, PutDocumentStatsProto* put_document_stats); private:
diff --git a/icing/index/property-existence-indexing-handler_test.cc b/icing/index/property-existence-indexing-handler_test.cc index 3228734..653bf0c 100644 --- a/icing/index/property-existence-indexing-handler_test.cc +++ b/icing/index/property-existence-indexing-handler_test.cc
@@ -28,6 +28,7 @@ #include "gtest/gtest.h" #include "icing/absl_ports/str_cat.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/hit/doc-hit-info.h" @@ -46,13 +47,14 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" @@ -78,10 +80,11 @@ class PropertyExistenceIndexingHandlerTest : public Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -106,9 +109,10 @@ ASSERT_THAT( filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), IsTrue()); + ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType( @@ -157,13 +161,12 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult doc_store_create_result, DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); document_store_ = std::move(doc_store_create_result.document_store); } @@ -177,6 +180,7 @@ filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; IcingFilesystem icing_filesystem_; FakeClock fake_clock_; @@ -274,12 +278,15 @@ // Handle all docs EXPECT_THAT(handler->Handle(tokenized_document0, document_id0, + put_result0.old_document_id, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(handler->Handle(tokenized_document1, document_id1, + put_result1.old_document_id, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(handler->Handle(tokenized_document2, document_id2, + put_result0.old_document_id, /*put_document_stats=*/nullptr), IsOk()); @@ -383,6 +390,7 @@ std::unique_ptr<PropertyExistenceIndexingHandler> handler, PropertyExistenceIndexingHandler::Create(&fake_clock_, index.get())); EXPECT_THAT(handler->Handle(tokenized_root_document, document_id, + put_result.old_document_id, /*put_document_stats=*/nullptr), IsOk()); @@ -507,12 +515,15 @@ // Handle all docs EXPECT_THAT(handler->Handle(tokenized_document0, document_id0, + put_result0.old_document_id, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(handler->Handle(tokenized_document1, document_id1, + put_result1.old_document_id, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(handler->Handle(tokenized_document2, document_id2, + put_result2.old_document_id, /*put_document_stats=*/nullptr), IsOk());
diff --git a/icing/index/string-section-indexing-handler.cc b/icing/index/string-section-indexing-handler.cc index 8b20d04..438b51f 100644 --- a/icing/index/string-section-indexing-handler.cc +++ b/icing/index/string-section-indexing-handler.cc
@@ -16,7 +16,6 @@ #include <cstdint> #include <memory> -#include <string> #include <string_view> #include "icing/text_classifier/lib3/utils/base/status.h" @@ -47,6 +46,7 @@ libtextclassifier3::Status StringSectionIndexingHandler::Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, + DocumentId /*old_document_id*/ _, PutDocumentStatsProto* put_document_stats) { uint32_t num_tokens = 0; libtextclassifier3::Status status; @@ -59,17 +59,13 @@ } // TODO(b/152934343): pass real namespace ids in Index::Editor editor = - index_.Edit(document_id, section.metadata.id, - section.metadata.term_match_type, /*namespace_id=*/0); + index_.Edit(document_id, section.metadata.id, /*namespace_id=*/0); for (std::string_view token : section.token_sequence) { ++num_tokens; switch (section.metadata.tokenizer) { case StringIndexingConfig::TokenizerType::VERBATIM: - // data() is safe to use here because a token created from the - // VERBATIM tokenizer is the entire string value. The character at - // data() + token.length() is guaranteed to be a null char. - status = editor.BufferTerm(token.data()); + status = editor.BufferTerm(token, section.metadata.term_match_type); break; case StringIndexingConfig::TokenizerType::NONE: [[fallthrough]]; @@ -78,8 +74,10 @@ case StringIndexingConfig::TokenizerType::URL: [[fallthrough]]; case StringIndexingConfig::TokenizerType::PLAIN: - std::string normalized_term = normalizer_.NormalizeTerm(token); - status = editor.BufferTerm(normalized_term.c_str()); + Normalizer::NormalizedTerm normalized_term = + normalizer_.NormalizeTerm(token); + status = editor.BufferTerm(normalized_term.text, + section.metadata.term_match_type); } if (!status.ok()) {
diff --git a/icing/index/string-section-indexing-handler.h b/icing/index/string-section-indexing-handler.h index 8452e9f..b644c65 100644 --- a/icing/index/string-section-indexing-handler.h +++ b/icing/index/string-section-indexing-handler.h
@@ -52,6 +52,9 @@ // all contents in tokenized_document.tokenized_string_sections and merge lite // index into main index if necessary. // + // Parameter old_document_id is unused since there is no need to migrate data + // from old_document_id to (new) document_id. + // /// Returns: // - OK on success // - RESOURCE_EXHAUSTED_ERROR if the index is full and can't add anymore @@ -60,6 +63,7 @@ // - Any main/lite index errors. libtextclassifier3::Status Handle(const TokenizedDocument& tokenized_document, DocumentId document_id, + DocumentId /*old_document_id*/ _, PutDocumentStatsProto* put_document_stats); private:
diff --git a/icing/index/term-indexing-handler.cc b/icing/index/term-indexing-handler.cc index 7eb9dda..24b57a1 100644 --- a/icing/index/term-indexing-handler.cc +++ b/icing/index/term-indexing-handler.cc
@@ -65,7 +65,8 @@ libtextclassifier3::Status TermIndexingHandler::Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, - bool recovery_mode, PutDocumentStatsProto* put_document_stats) { + DocumentId old_document_id, bool recovery_mode, + PutDocumentStatsProto* put_document_stats) { std::unique_ptr<Timer> index_timer = clock_.GetNewTimer(); if (index_.last_added_document_id() != kInvalidDocumentId && @@ -84,11 +85,11 @@ libtextclassifier3::Status status = libtextclassifier3::Status::OK; if (property_existence_indexing_handler_ != nullptr) { status = property_existence_indexing_handler_->Handle( - tokenized_document, document_id, put_document_stats); + tokenized_document, document_id, old_document_id, put_document_stats); } if (status.ok()) { status = string_section_indexing_handler_->Handle( - tokenized_document, document_id, put_document_stats); + tokenized_document, document_id, old_document_id, put_document_stats); } if (put_document_stats != nullptr) {
diff --git a/icing/index/term-indexing-handler.h b/icing/index/term-indexing-handler.h index c055bbf..79e407a 100644 --- a/icing/index/term-indexing-handler.h +++ b/icing/index/term-indexing-handler.h
@@ -56,7 +56,7 @@ // - Sorts the lite index if necessary. // - Merges the lite index into the main index if necessary. // - /// Returns: + // Returns: // - OK on success // - INVALID_ARGUMENT_ERROR if document_id is less than or equal to the // document_id of a previously indexed document in non recovery mode. @@ -68,7 +68,8 @@ // - Any main/lite index errors. libtextclassifier3::Status Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, - bool recovery_mode, PutDocumentStatsProto* put_document_stats) override; + DocumentId old_document_id, bool recovery_mode, + PutDocumentStatsProto* put_document_stats) override; private: explicit TermIndexingHandler(const Clock* clock, Index* index,
diff --git a/icing/index/term-indexing-handler_test.cc b/icing/index/term-indexing-handler_test.cc index 7b97f88..a82541f 100644 --- a/icing/index/term-indexing-handler_test.cc +++ b/icing/index/term-indexing-handler_test.cc
@@ -29,6 +29,7 @@ #include "gtest/gtest.h" #include "icing/absl_ports/str_cat.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/hit/doc-hit-info.h" @@ -50,13 +51,14 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" @@ -99,10 +101,11 @@ class TermIndexingHandlerTest : public Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -128,8 +131,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), IsTrue()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType( @@ -174,13 +177,12 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult doc_store_create_result, DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); document_store_ = std::move(doc_store_create_result.document_store); } @@ -194,6 +196,7 @@ filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; IcingFilesystem icing_filesystem_; FakeClock fake_clock_; @@ -273,10 +276,10 @@ TermIndexingHandler::Create( &fake_clock_, normalizer_.get(), index.get(), /*build_property_existence_metadata_hits=*/true)); - EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); + EXPECT_THAT(handler->Handle( + tokenized_document, document_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + IsOk()); EXPECT_THAT(index->last_added_document_id(), Eq(document_id)); @@ -338,10 +341,10 @@ TermIndexingHandler::Create( &fake_clock_, normalizer_.get(), index.get(), /*build_property_existence_metadata_hits=*/true)); - EXPECT_THAT( - handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); + EXPECT_THAT(handler->Handle( + tokenized_document, document_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + IsOk()); EXPECT_THAT(index->last_added_document_id(), Eq(document_id)); @@ -436,22 +439,25 @@ // Handle doc0 and doc1. The LiteIndex should sort and merge after adding // these - EXPECT_THAT(handler->Handle(tokenized_document0, document_id0, - /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); - EXPECT_THAT(handler->Handle(tokenized_document1, document_id1, - /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); + EXPECT_THAT( + handler->Handle(tokenized_document0, document_id0, + put_result0.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT( + handler->Handle(tokenized_document1, document_id1, + put_result1.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); EXPECT_THAT(index->last_added_document_id(), Eq(document_id1)); EXPECT_THAT(index->LiteIndexNeedSort(), IsFalse()); // Handle doc2. The LiteIndex should have an unsorted portion after adding - EXPECT_THAT(handler->Handle(tokenized_document2, document_id2, - /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); + EXPECT_THAT( + handler->Handle(tokenized_document2, document_id2, + put_result2.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); EXPECT_THAT(index->last_added_document_id(), Eq(document_id2)); // Hits in the hit buffer: @@ -580,18 +586,21 @@ /*build_property_existence_metadata_hits=*/true)); // Handle all docs - EXPECT_THAT(handler->Handle(tokenized_document0, document_id0, - /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); - EXPECT_THAT(handler->Handle(tokenized_document1, document_id1, - /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); - EXPECT_THAT(handler->Handle(tokenized_document2, document_id2, - /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); + EXPECT_THAT( + handler->Handle(tokenized_document0, document_id0, + put_result0.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT( + handler->Handle(tokenized_document1, document_id1, + put_result1.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT( + handler->Handle(tokenized_document2, document_id2, + put_result2.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); EXPECT_THAT(index->last_added_document_id(), Eq(document_id2)); // We've disabled sorting during indexing so the HitBuffer's unsorted section
diff --git a/icing/jni/icing-search-engine-jni.cc b/icing/jni/icing-search-engine-jni.cc index afba2a1..de431f5 100644 --- a/icing/jni/icing-search-engine-jni.cc +++ b/icing/jni/icing-search-engine-jni.cc
@@ -15,6 +15,7 @@ #include <jni.h> #include <string> +#include <string_view> #include <utility> #include "icing/icing-search-engine.h" @@ -137,6 +138,23 @@ return SerializeProtoToJniByteArray(env, get_schema_result_proto); } +// TODO : b/337913932 - pre-register this API once Jetpack build is dropped back +// into g3 +JNIEXPORT jbyteArray JNICALL +Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetSchemaForDatabase( + JNIEnv* env, jclass clazz, jobject object, jstring database) { + icing::lib::IcingSearchEngine* icing = + GetIcingSearchEnginePointer(env, object); + + icing::lib::ScopedUtfChars scoped_database_chars(env, database); + std::string_view database_str_view(scoped_database_chars.c_str(), + scoped_database_chars.size()); + icing::lib::GetSchemaResultProto get_schema_result_proto = + icing->GetSchema(database_str_view); + + return SerializeProtoToJniByteArray(env, get_schema_result_proto); +} + jbyteArray nativeGetSchemaType(JNIEnv* env, jclass clazz, jobject object, jstring schema_type) { icing::lib::IcingSearchEngine* icing = @@ -249,8 +267,9 @@ return; } -jbyteArray nativeOpenWriteBlob( - JNIEnv* env, jclass clazz, jobject object, jbyteArray blob_handle_bytes) { +// TODO(b/273591938): Change this API back to the pre-registered API. +jbyteArray nativeOpenWriteBlob(JNIEnv* env, jclass clazz, jobject object, + jbyteArray blob_handle_bytes) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -266,8 +285,25 @@ return SerializeProtoToJniByteArray(env, blob_result_proto); } -jbyteArray nativeOpenReadBlob( - JNIEnv* env, jclass clazz, jobject object, jbyteArray blob_handle_bytes) { +jbyteArray nativeRemoveBlob(JNIEnv* env, jclass clazz, jobject object, + jbyteArray blob_handle_bytes) { + icing::lib::IcingSearchEngine* icing = + GetIcingSearchEnginePointer(env, object); + + icing::lib::PropertyProto::BlobHandleProto blob_handle; + if (!ParseProtoFromJniByteArray(env, blob_handle_bytes, &blob_handle)) { + ICING_LOG(icing::lib::ERROR) + << "Failed to parse BlobHandle in nativeRemoveBlob"; + return nullptr; + } + + icing::lib::BlobProto blob_result_proto = icing->RemoveBlob(blob_handle); + + return SerializeProtoToJniByteArray(env, blob_result_proto); +} + +jbyteArray nativeOpenReadBlob(JNIEnv* env, jclass clazz, jobject object, + jbyteArray blob_handle_bytes) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -283,8 +319,8 @@ return SerializeProtoToJniByteArray(env, blob_result_proto); } -jbyteArray nativeCommitBlob( - JNIEnv* env, jclass clazz, jobject object, jbyteArray blob_handle_bytes) { +jbyteArray nativeCommitBlob(JNIEnv* env, jclass clazz, jobject object, + jbyteArray blob_handle_bytes) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -578,6 +614,9 @@ {"nativeOpenWriteBlob", "(Lcom/google/android/icing/IcingSearchEngineImpl;[B)[B", reinterpret_cast<void*>(nativeOpenWriteBlob)}, + {"nativeRemoveBlob", + "(Lcom/google/android/icing/IcingSearchEngineImpl;[B)[B", + reinterpret_cast<void*>(nativeRemoveBlob)}, {"nativeOpenReadBlob", "(Lcom/google/android/icing/IcingSearchEngineImpl;[B)[B", reinterpret_cast<void*>(nativeOpenReadBlob)},
diff --git a/icing/join/doc-join-info.cc b/icing/join/document-join-id-pair.cc similarity index 83% rename from icing/join/doc-join-info.cc rename to icing/join/document-join-id-pair.cc index 3b06f01..2bd9141 100644 --- a/icing/join/doc-join-info.cc +++ b/icing/join/document-join-id-pair.cc
@@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "icing/join/doc-join-info.h" - -#include <cstdint> +#include "icing/join/document-join-id-pair.h" #include "icing/schema/joinable-property.h" #include "icing/store/document-id.h" @@ -23,8 +21,8 @@ namespace icing { namespace lib { -DocJoinInfo::DocJoinInfo(DocumentId document_id, - JoinablePropertyId joinable_property_id) { +DocumentJoinIdPair::DocumentJoinIdPair( + DocumentId document_id, JoinablePropertyId joinable_property_id) { Value temp_value = 0; bit_util::BitfieldSet(/*new_value=*/document_id, /*lsb_offset=*/kJoinablePropertyIdBits, @@ -35,12 +33,12 @@ value_ = temp_value; } -DocumentId DocJoinInfo::document_id() const { +DocumentId DocumentJoinIdPair::document_id() const { return bit_util::BitfieldGet(value_, /*lsb_offset=*/kJoinablePropertyIdBits, /*len=*/kDocumentIdBits); } -JoinablePropertyId DocJoinInfo::joinable_property_id() const { +JoinablePropertyId DocumentJoinIdPair::joinable_property_id() const { return bit_util::BitfieldGet(value_, /*lsb_offset=*/0, /*len=*/kJoinablePropertyIdBits); }
diff --git a/icing/join/doc-join-info.h b/icing/join/document-join-id-pair.h similarity index 62% rename from icing/join/doc-join-info.h rename to icing/join/document-join-id-pair.h index 7696b92..603b1fc 100644 --- a/icing/join/doc-join-info.h +++ b/icing/join/document-join-id-pair.h
@@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ICING_JOIN_DOC_JOIN_INFO -#define ICING_JOIN_DOC_JOIN_INFO +#ifndef ICING_JOIN_DOCUMENT_JOIN_ID_PAIR_H_ +#define ICING_JOIN_DOCUMENT_JOIN_ID_PAIR_H_ +#include <cstddef> #include <cstdint> +#include <functional> #include <limits> #include "icing/schema/joinable-property.h" @@ -24,16 +26,23 @@ namespace icing { namespace lib { -// DocJoinInfo is composed of document_id and joinable_property_id. -class DocJoinInfo { +// DocumentJoinIdPair is composed of document_id and joinable_property_id. +class DocumentJoinIdPair { public: - // The datatype used to encode DocJoinInfo information: the document_id and - // joinable_property_id. + // The datatype used to encode DocumentJoinIdPair information: the document_id + // and joinable_property_id. using Value = uint32_t; + struct Hasher { + std::size_t operator()( + const DocumentJoinIdPair& document_join_id_pair) const { + return std::hash<Value>()(document_join_id_pair.value()); + } + }; + static_assert(kDocumentIdBits + kJoinablePropertyIdBits <= sizeof(Value) * 8, "Cannot encode document id and joinable property id in " - "DocJoinInfo::Value"); + "DocumentJoinIdPair::Value"); // All bits of kInvalidValue are 1, and it contains: // - 0b1 for 4 unused bits. @@ -44,10 +53,17 @@ // it doesn't matter what JoinablePropertyId we set for kInvalidValue. static constexpr Value kInvalidValue = std::numeric_limits<Value>::max(); - explicit DocJoinInfo(DocumentId document_id, - JoinablePropertyId joinable_property_id); + // Default constexpr constructor to construct an invalid DocumentJoinIdPair. + constexpr DocumentJoinIdPair() : value_(kInvalidValue) {} - explicit DocJoinInfo(Value value = kInvalidValue) : value_(value) {} + explicit DocumentJoinIdPair(DocumentId document_id, + JoinablePropertyId joinable_property_id); + + explicit DocumentJoinIdPair(Value value) : value_(value) {} + + bool operator==(const DocumentJoinIdPair& other) const { + return value_ == other.value_; + } bool is_valid() const { return value_ != kInvalidValue; } Value value() const { return value_; } @@ -58,9 +74,9 @@ // Value bits layout: 4 unused + 22 document_id + 6 joinable_property_id. Value value_; } __attribute__((packed)); -static_assert(sizeof(DocJoinInfo) == 4, ""); +static_assert(sizeof(DocumentJoinIdPair) == 4, ""); } // namespace lib } // namespace icing -#endif // ICING_JOIN_DOC_JOIN_INFO +#endif // ICING_JOIN_DOCUMENT_JOIN_ID_PAIR_H_
diff --git a/icing/join/doc-join-info_test.cc b/icing/join/document-join-id-pair_test.cc similarity index 72% rename from icing/join/doc-join-info_test.cc rename to icing/join/document-join-id-pair_test.cc index 7025473..2f76c88 100644 --- a/icing/join/doc-join-info_test.cc +++ b/icing/join/document-join-id-pair_test.cc
@@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "icing/join/doc-join-info.h" +#include "icing/join/document-join-id-pair.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -31,59 +31,62 @@ static constexpr DocumentId kSomeDocumentId = 24; static constexpr JoinablePropertyId kSomeJoinablePropertyId = 5; -TEST(DocJoinInfoTest, Accessors) { - DocJoinInfo doc_join_info(kSomeDocumentId, kSomeJoinablePropertyId); +TEST(DocumentJoinIdPairTest, Accessors) { + DocumentJoinIdPair doc_join_info(kSomeDocumentId, kSomeJoinablePropertyId); EXPECT_THAT(doc_join_info.document_id(), Eq(kSomeDocumentId)); EXPECT_THAT(doc_join_info.joinable_property_id(), Eq(kSomeJoinablePropertyId)); } -TEST(DocJoinInfoTest, Invalid) { - DocJoinInfo default_invalid; +TEST(DocumentJoinIdPairTest, Invalid) { + DocumentJoinIdPair default_invalid; EXPECT_THAT(default_invalid.is_valid(), IsFalse()); - // Also make sure the invalid DocJoinInfo contains an invalid document id. + // Also make sure the invalid DocumentJoinIdPair contains an invalid document + // id. EXPECT_THAT(default_invalid.document_id(), Eq(kInvalidDocumentId)); EXPECT_THAT(default_invalid.joinable_property_id(), Eq(kMaxJoinablePropertyId)); } -TEST(DocJoinInfoTest, Valid) { - DocJoinInfo maximum_document_id_info(kMaxDocumentId, kSomeJoinablePropertyId); +TEST(DocumentJoinIdPairTest, Valid) { + DocumentJoinIdPair maximum_document_id_info(kMaxDocumentId, + kSomeJoinablePropertyId); EXPECT_THAT(maximum_document_id_info.is_valid(), IsTrue()); EXPECT_THAT(maximum_document_id_info.document_id(), Eq(kMaxDocumentId)); EXPECT_THAT(maximum_document_id_info.joinable_property_id(), Eq(kSomeJoinablePropertyId)); - DocJoinInfo maximum_joinable_property_id_info(kSomeDocumentId, - kMaxJoinablePropertyId); + DocumentJoinIdPair maximum_joinable_property_id_info(kSomeDocumentId, + kMaxJoinablePropertyId); EXPECT_THAT(maximum_joinable_property_id_info.is_valid(), IsTrue()); EXPECT_THAT(maximum_joinable_property_id_info.document_id(), Eq(kSomeDocumentId)); EXPECT_THAT(maximum_joinable_property_id_info.joinable_property_id(), Eq(kMaxJoinablePropertyId)); - DocJoinInfo minimum_document_id_info(kMinDocumentId, kSomeJoinablePropertyId); + DocumentJoinIdPair minimum_document_id_info(kMinDocumentId, + kSomeJoinablePropertyId); EXPECT_THAT(minimum_document_id_info.is_valid(), IsTrue()); EXPECT_THAT(minimum_document_id_info.document_id(), Eq(kMinDocumentId)); EXPECT_THAT(minimum_document_id_info.joinable_property_id(), Eq(kSomeJoinablePropertyId)); - DocJoinInfo minimum_joinable_property_id_info(kSomeDocumentId, - kMinJoinablePropertyId); + DocumentJoinIdPair minimum_joinable_property_id_info(kSomeDocumentId, + kMinJoinablePropertyId); EXPECT_THAT(minimum_joinable_property_id_info.is_valid(), IsTrue()); EXPECT_THAT(minimum_joinable_property_id_info.document_id(), Eq(kSomeDocumentId)); EXPECT_THAT(minimum_joinable_property_id_info.joinable_property_id(), Eq(kMinJoinablePropertyId)); - DocJoinInfo all_maximum_info(kMaxDocumentId, kMaxJoinablePropertyId); + DocumentJoinIdPair all_maximum_info(kMaxDocumentId, kMaxJoinablePropertyId); EXPECT_THAT(all_maximum_info.is_valid(), IsTrue()); EXPECT_THAT(all_maximum_info.document_id(), Eq(kMaxDocumentId)); EXPECT_THAT(all_maximum_info.joinable_property_id(), Eq(kMaxJoinablePropertyId)); - DocJoinInfo all_minimum_info(kMinDocumentId, kMinJoinablePropertyId); + DocumentJoinIdPair all_minimum_info(kMinDocumentId, kMinJoinablePropertyId); EXPECT_THAT(all_minimum_info.is_valid(), IsTrue()); EXPECT_THAT(all_minimum_info.document_id(), Eq(kMinDocumentId)); EXPECT_THAT(all_minimum_info.joinable_property_id(),
diff --git a/icing/join/join-children-fetcher-impl-deprecated.cc b/icing/join/join-children-fetcher-impl-deprecated.cc new file mode 100644 index 0000000..34391d5 --- /dev/null +++ b/icing/join/join-children-fetcher-impl-deprecated.cc
@@ -0,0 +1,59 @@ +// Copyright (C) 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/join/join-children-fetcher-impl-deprecated.h" + +#include <memory> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/absl_ports/str_cat.h" +#include "icing/scoring/scored-document-hit.h" +#include "icing/store/document-id.h" + +namespace icing { +namespace lib { + +/* static */ libtextclassifier3::StatusOr< + std::unique_ptr<JoinChildrenFetcherImplDeprecated>> +JoinChildrenFetcherImplDeprecated::Create( + const JoinSpecProto& join_spec, + std::unordered_map<DocumentId, std::vector<ScoredDocumentHit>>&& + map_joinable_qualified_id) { + // TODO(b/256022027): So far we only support kQualifiedIdExpr for + // parent_property_expression, we could support more. + if (join_spec.parent_property_expression() != kQualifiedIdExpr) { + return absl_ports::UnimplementedError(absl_ports::StrCat( + "Parent property expression must be ", kQualifiedIdExpr)); + } + + return std::unique_ptr<JoinChildrenFetcherImplDeprecated>( + new JoinChildrenFetcherImplDeprecated( + join_spec, std::move(map_joinable_qualified_id))); +} + +libtextclassifier3::StatusOr<std::vector<ScoredDocumentHit>> +JoinChildrenFetcherImplDeprecated::GetChildren(DocumentId parent_doc_id) const { + if (auto iter = map_joinable_qualified_id_.find(parent_doc_id); + iter != map_joinable_qualified_id_.end()) { + return iter->second; + } + return std::vector<ScoredDocumentHit>(); +} + +} // namespace lib +} // namespace icing
diff --git a/icing/join/join-children-fetcher-impl-deprecated.h b/icing/join/join-children-fetcher-impl-deprecated.h new file mode 100644 index 0000000..a80dbc7 --- /dev/null +++ b/icing/join/join-children-fetcher-impl-deprecated.h
@@ -0,0 +1,76 @@ +// Copyright (C) 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_JOIN_JOIN_CHILDREN_FETCHER_IMPL_DEPRECATED_H_ +#define ICING_JOIN_JOIN_CHILDREN_FETCHER_IMPL_DEPRECATED_H_ + +#include <memory> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/join/join-children-fetcher.h" +#include "icing/proto/search.pb.h" +#include "icing/scoring/scored-document-hit.h" +#include "icing/store/document-id.h" + +namespace icing { +namespace lib { + +// A class that provides the GetChildren method for joins to fetch all children +// documents given a parent document id. Only QualifiedIdJoinIndexImplV1 and +// QualifiedIdJoinIndexImplV2 will use this class, since we can construct the +// map without parent document id available. +// +// Internally, the class maintains a map for each joinable value type that +// groups children according to the joinable values. Currently we only support +// QUALIFIED_ID joining, in which the joinable value type is document id. +class JoinChildrenFetcherImplDeprecated : public JoinChildrenFetcher { + public: + // Creates JoinChildrenFetcherImplDeprecated. + // + // Returns: + // - A JoinChildrenFetcherImplDeprecated instance on success. + // - UNIMPLEMENTED_ERROR if the join type specified by join_spec is not + // supported. + static libtextclassifier3::StatusOr< + std::unique_ptr<JoinChildrenFetcherImplDeprecated>> + Create(const JoinSpecProto& join_spec, + std::unordered_map<DocumentId, std::vector<ScoredDocumentHit>>&& + map_joinable_qualified_id); + + ~JoinChildrenFetcherImplDeprecated() override = default; + + libtextclassifier3::StatusOr<std::vector<ScoredDocumentHit>> GetChildren( + DocumentId parent_doc_id) const override; + + private: + explicit JoinChildrenFetcherImplDeprecated( + const JoinSpecProto& join_spec, + std::unordered_map<DocumentId, std::vector<ScoredDocumentHit>>&& + map_joinable_qualified_id) + : JoinChildrenFetcher(join_spec), + map_joinable_qualified_id_(std::move(map_joinable_qualified_id)) {} + + // The map that groups children by qualified id used to support QualifiedId + // joining. The joining type is document id. + std::unordered_map<DocumentId, std::vector<ScoredDocumentHit>> + map_joinable_qualified_id_; +}; + +} // namespace lib +} // namespace icing + +#endif // ICING_JOIN_JOIN_CHILDREN_FETCHER_IMPL_DEPRECATED_H_
diff --git a/icing/join/join-children-fetcher_test.cc b/icing/join/join-children-fetcher-impl-deprecated_test.cc similarity index 67% rename from icing/join/join-children-fetcher_test.cc rename to icing/join/join-children-fetcher-impl-deprecated_test.cc index 92a7a81..0c40531 100644 --- a/icing/join/join-children-fetcher_test.cc +++ b/icing/join/join-children-fetcher-impl-deprecated_test.cc
@@ -12,15 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "icing/join/join-children-fetcher.h" +#include "icing/join/join-children-fetcher-impl-deprecated.h" +#include <memory> +#include <string> #include <unordered_map> +#include <utility> +#include <vector> +#include "icing/text_classifier/lib3/utils/base/status.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/join/join-processor.h" #include "icing/proto/search.pb.h" #include "icing/schema/section.h" +#include "icing/scoring/scored-document-hit.h" +#include "icing/store/document-id.h" #include "icing/testing/common-matchers.h" namespace icing { @@ -31,7 +38,7 @@ using ::testing::ElementsAre; using ::testing::IsEmpty; -TEST(JoinChildrenFetcherTest, FetchQualifiedIdJoinChildren) { +TEST(JoinChildrenFetcherImplDeprecatedTest, FetchQualifiedIdJoinChildren) { JoinSpecProto join_spec; join_spec.set_parent_property_expression( std::string(JoinProcessor::kQualifiedIdExpr)); @@ -47,14 +54,17 @@ map_joinable_qualified_id[parent_doc_id].push_back(child1); map_joinable_qualified_id[parent_doc_id].push_back(child2); - JoinChildrenFetcher fetcher(join_spec, std::move(map_joinable_qualified_id)); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplDeprecated> fetcher, + JoinChildrenFetcherImplDeprecated::Create( + join_spec, std::move(map_joinable_qualified_id))); ICING_ASSERT_OK_AND_ASSIGN(std::vector<ScoredDocumentHit> children, - fetcher.GetChildren(parent_doc_id)); + fetcher->GetChildren(parent_doc_id)); EXPECT_THAT(children, ElementsAre(EqualsScoredDocumentHit(child1), EqualsScoredDocumentHit(child2))); } -TEST(JoinChildrenFetcherTest, FetchJoinEmptyChildren) { +TEST(JoinChildrenFetcherImplDeprecatedTest, FetchJoinEmptyChildren) { JoinSpecProto join_spec; join_spec.set_parent_property_expression( std::string(JoinProcessor::kQualifiedIdExpr)); @@ -62,18 +72,23 @@ DocumentId parent_doc_id = 0; - JoinChildrenFetcher fetcher(join_spec, /*map_joinable_qualified_id=*/{}); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplDeprecated> fetcher, + JoinChildrenFetcherImplDeprecated::Create( + join_spec, + /*map_joinable_qualified_id=*/{})); ICING_ASSERT_OK_AND_ASSIGN(std::vector<ScoredDocumentHit> children, - fetcher.GetChildren(parent_doc_id)); + fetcher->GetChildren(parent_doc_id)); EXPECT_THAT(children, IsEmpty()); } -TEST(JoinChildrenFetcherTest, UnsupportedJoin) { +TEST(JoinChildrenFetcherImplDeprecatedTest, UnsupportedJoin) { JoinSpecProto join_spec; join_spec.set_parent_property_expression("name"); join_spec.set_child_property_expression("sender"); - JoinChildrenFetcher fetcher(join_spec, /*map_joinable_qualified_id=*/{}); - EXPECT_THAT(fetcher.GetChildren(0), + + EXPECT_THAT(JoinChildrenFetcherImplDeprecated::Create( + join_spec, /*map_joinable_qualified_id=*/{}), StatusIs(libtextclassifier3::StatusCode::UNIMPLEMENTED)); }
diff --git a/icing/join/join-children-fetcher-impl-v3.cc b/icing/join/join-children-fetcher-impl-v3.cc new file mode 100644 index 0000000..456c3ba --- /dev/null +++ b/icing/join/join-children-fetcher-impl-v3.cc
@@ -0,0 +1,166 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/join/join-children-fetcher-impl-v3.h" + +#include <algorithm> +#include <cstdint> +#include <memory> +#include <optional> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/absl_ports/str_cat.h" +#include "icing/join/document-join-id-pair.h" +#include "icing/join/qualified-id-join-index.h" +#include "icing/proto/scoring.pb.h" +#include "icing/proto/search.pb.h" +#include "icing/schema/joinable-property.h" +#include "icing/schema/schema-store.h" +#include "icing/scoring/scored-document-hit.h" +#include "icing/store/document-filter-data.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" +#include "icing/util/status-macros.h" + +namespace icing { +namespace lib { + +/* static */ libtextclassifier3::StatusOr< + std::unique_ptr<JoinChildrenFetcherImplV3>> +JoinChildrenFetcherImplV3::Create( + const JoinSpecProto& join_spec, const SchemaStore* schema_store, + const DocumentStore* doc_store, + const QualifiedIdJoinIndex* qualified_id_join_index, + int64_t current_time_ms, + std::vector<ScoredDocumentHit>&& child_scored_document_hits) { + ICING_RETURN_ERROR_IF_NULL(schema_store); + ICING_RETURN_ERROR_IF_NULL(doc_store); + ICING_RETURN_ERROR_IF_NULL(qualified_id_join_index); + + // TODO(b/256022027): So far we only support kQualifiedIdExpr for + // parent_property_expression, we could support more. + if (join_spec.parent_property_expression() != kQualifiedIdExpr) { + return absl_ports::UnimplementedError(absl_ports::StrCat( + "Parent property expression must be ", kQualifiedIdExpr)); + } + + if (qualified_id_join_index->version() != + QualifiedIdJoinIndex::Version::kV3) { + return absl_ports::InvalidArgumentError( + "QualifiedIdJoinIndex version must be v3 with " + "JoinChildrenFetcherImplV3."); + } + + // For each child scored document hit, we need to find the joinable property + // id that matches the joinable property specified in the join spec. + std::unordered_map<DocumentJoinIdPair, ScoredDocumentHit, + DocumentJoinIdPair::Hasher> + child_join_id_pair_to_scored_document_hit_map; + for (ScoredDocumentHit& child_scored_document_hit : + child_scored_document_hits) { + DocumentId child_doc_id = child_scored_document_hit.document_id(); + + // Get the document filter data for the child document. + std::optional<DocumentFilterData> child_doc_filter_data = + doc_store->GetAliveDocumentFilterData(child_doc_id, current_time_ms); + if (!child_doc_filter_data) { + continue; + } + + // Get the joinable property metadata. + ICING_ASSIGN_OR_RETURN(const JoinablePropertyMetadata* metadata, + schema_store->GetJoinablePropertyMetadata( + child_doc_filter_data->schema_type_id(), + join_spec.child_property_expression())); + if (metadata == nullptr) { + continue; + } + + child_join_id_pair_to_scored_document_hit_map.insert( + {DocumentJoinIdPair(child_doc_id, metadata->id), + std::move(child_scored_document_hit)}); + } + + return std::unique_ptr<JoinChildrenFetcherImplV3>( + new JoinChildrenFetcherImplV3( + join_spec, qualified_id_join_index, + std::move(child_join_id_pair_to_scored_document_hit_map))); +} + +libtextclassifier3::StatusOr<std::vector<ScoredDocumentHit>> +JoinChildrenFetcherImplV3::GetChildren(DocumentId parent_doc_id) const { + // If the parent_doc_id and its children are already in the cache, return the + // cached value. + if (auto cache_itr = cached_parent_to_children_map_.find(parent_doc_id); + cache_itr != cached_parent_to_children_map_.end()) { + return cache_itr->second; + } + + // Otherwise, fetch the children from the qualified id join index and cache + // the result. + ICING_ASSIGN_OR_RETURN( + std::vector<DocumentJoinIdPair> child_document_join_id_pairs, + qualified_id_join_index_.Get(parent_doc_id)); + + // Filter and construct child_scored_document_hits for the given parent doc + // id. + // - child_join_id_pair_to_scored_document_hit_map_ contains (DocumentId, + // JoinablePropertyId) of the child documents in the child result set and + // the desired joinable property id which matches the joinable property + // specified in the join spec. + // - The DocumentJoinIdPair list fetched from the join index may contain child + // documents that are not in the child result set, or the joinable property + // id is not the desired one. For example: + // - Parent document 1 may have 4 child DocumentJoinIdPairs: + // - (doc_id=100, joinable_property_id=1) + // - (doc_id=100, joinable_property_id=2) + // - (doc_id=101, joinable_property_id=1) + // - (doc_id=102, joinable_property_id=1) + // - All of these 4 pairs will be fetched from the join index. + // - Now, the child query obtains result doc ids = [100, 102, 103], and the + // client wants to join the child document by the joinable property "foo", + // which has joinable property id 1 in its schema. + // - In this case, when handling parent document 1, we should filter out + // children [(100, 2), (101, 1)] since they are not in the child result + // set, or the joinable property does not match the one specified in the + // join spec. + std::vector<ScoredDocumentHit> child_scored_document_hits; + for (const DocumentJoinIdPair& child_document_join_id_pair : + child_document_join_id_pairs) { + if (auto filter_itr = child_join_id_pair_to_scored_document_hit_map_.find( + child_document_join_id_pair); + filter_itr != child_join_id_pair_to_scored_document_hit_map_.end()) { + child_scored_document_hits.push_back(filter_itr->second); + } + } + + // Sort child_scored_document_hits according to the child scoring spec. + ScoredDocumentHitComparator score_comparator( + /*is_descending=*/join_spec_.nested_spec().scoring_spec().order_by() == + ScoringSpecProto::Order::DESC); + std::sort(child_scored_document_hits.begin(), + child_scored_document_hits.end(), score_comparator); + + cached_parent_to_children_map_.insert( + {parent_doc_id, child_scored_document_hits}); + + return child_scored_document_hits; +} + +} // namespace lib +} // namespace icing
diff --git a/icing/join/join-children-fetcher-impl-v3.h b/icing/join/join-children-fetcher-impl-v3.h new file mode 100644 index 0000000..37afbc4 --- /dev/null +++ b/icing/join/join-children-fetcher-impl-v3.h
@@ -0,0 +1,97 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_JOIN_JOIN_CHILDREN_FETCHER_IMPL_V3_H_ +#define ICING_JOIN_JOIN_CHILDREN_FETCHER_IMPL_V3_H_ + +#include <cstdint> +#include <memory> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/join/document-join-id-pair.h" +#include "icing/join/join-children-fetcher.h" +#include "icing/join/qualified-id-join-index.h" +#include "icing/proto/search.pb.h" +#include "icing/schema/schema-store.h" +#include "icing/scoring/scored-document-hit.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" + +namespace icing { +namespace lib { + +// A class that provides the GetChildren method for joins to fetch all children +// documents given a parent document id. +// - It does lazy lookup and caches the results after lookup and filter. +// - Only QualifiedIdJoinIndexImplV3 will use this class. +class JoinChildrenFetcherImplV3 : public JoinChildrenFetcher { + public: + // Creates JoinChildrenFetcherImplV3. + // + // Returns: + // - A JoinChildrenFetcherImplV3 instance on success. + // - FAILED_PRECONDITION_ERROR if any of the input pointer is null + // - UNIMPLEMENTED_ERROR if the join type specified by join_spec is not + // supported. + // - INVALID_ARGUMENT_ERROR if qualified_id_join_index is not v3. + static libtextclassifier3::StatusOr< + std::unique_ptr<JoinChildrenFetcherImplV3>> + Create(const JoinSpecProto& join_spec, const SchemaStore* schema_store, + const DocumentStore* doc_store, + const QualifiedIdJoinIndex* qualified_id_join_index, + int64_t current_time_ms, + std::vector<ScoredDocumentHit>&& child_scored_document_hits); + + ~JoinChildrenFetcherImplV3() override = default; + + libtextclassifier3::StatusOr<std::vector<ScoredDocumentHit>> GetChildren( + DocumentId parent_doc_id) const override; + + private: + explicit JoinChildrenFetcherImplV3( + const JoinSpecProto& join_spec, + const QualifiedIdJoinIndex* qualified_id_join_index, + std::unordered_map<DocumentJoinIdPair, ScoredDocumentHit, + DocumentJoinIdPair::Hasher>&& + child_join_id_pair_to_scored_document_hit_map) + : JoinChildrenFetcher(join_spec), + qualified_id_join_index_(*qualified_id_join_index), + child_join_id_pair_to_scored_document_hit_map_( + std::move(child_join_id_pair_to_scored_document_hit_map)) {} + + const QualifiedIdJoinIndex& qualified_id_join_index_; // Does not own. + + // Map the child join id pair to its scored document hit. It will be used to + // filter the child join id pairs. See GetChildren() implementation for more + // details. + std::unordered_map<DocumentJoinIdPair, ScoredDocumentHit, + DocumentJoinIdPair::Hasher> + child_join_id_pair_to_scored_document_hit_map_; + + // The cache for storing parent document id to its children's scored document + // hits. GetChildren() will first check the cache to see if the parent + // document id is already in the cache. If not, it will do a lookup and + // filter the children (via child_join_id_pair_to_scored_document_hit_map_) + // and then store the result in the cache. + mutable std::unordered_map<DocumentId, std::vector<ScoredDocumentHit>> + cached_parent_to_children_map_; +}; + +} // namespace lib +} // namespace icing + +#endif // ICING_JOIN_JOIN_CHILDREN_FETCHER_IMPL_V3_H_
diff --git a/icing/join/join-children-fetcher-impl-v3_test.cc b/icing/join/join-children-fetcher-impl-v3_test.cc new file mode 100644 index 0000000..e6e3251 --- /dev/null +++ b/icing/join/join-children-fetcher-impl-v3_test.cc
@@ -0,0 +1,918 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/join/join-children-fetcher-impl-v3.h" + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/document-builder.h" +#include "icing/feature-flags.h" +#include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" +#include "icing/join/document-join-id-pair.h" +#include "icing/join/join-processor.h" +#include "icing/join/qualified-id-join-index-impl-v1.h" +#include "icing/join/qualified-id-join-index-impl-v2.h" +#include "icing/join/qualified-id-join-index-impl-v3.h" +#include "icing/join/qualified-id-join-indexing-handler.h" +#include "icing/portable/platform.h" +#include "icing/proto/document.pb.h" +#include "icing/proto/document_wrapper.pb.h" +#include "icing/proto/schema.pb.h" +#include "icing/proto/scoring.pb.h" +#include "icing/proto/search.pb.h" +#include "icing/schema-builder.h" +#include "icing/schema/schema-store.h" +#include "icing/schema/section.h" +#include "icing/scoring/scored-document-hit.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" +#include "icing/testing/common-matchers.h" +#include "icing/testing/fake-clock.h" +#include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" +#include "icing/testing/tmp-directory.h" +#include "icing/tokenization/language-segmenter-factory.h" +#include "icing/tokenization/language-segmenter.h" +#include "icing/util/icu-data-file-helper.h" +#include "icing/util/status-macros.h" +#include "icing/util/tokenized-document.h" +#include "unicode/uloc.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::IsTrue; + +class JoinChildrenFetcherImplV3Test : public ::testing::Test { + protected: + void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + base_dir_ = GetTestTempDir() + "/icing_test"; + ASSERT_THAT(filesystem_.CreateDirectoryRecursively(base_dir_.c_str()), + IsTrue()); + + schema_store_dir_ = base_dir_ + "/schema_store"; + doc_store_dir_ = base_dir_ + "/doc_store"; + qualified_id_join_index_dir_ = base_dir_ + "/qualified_id_join_index"; + + if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { + ICING_ASSERT_OK( + // File generated via icu_data_file rule in //icing/BUILD. + icu_data_file_helper::SetUpIcuDataFile( + GetTestFilePath("icing/icu.dat"))); + } + + language_segmenter_factory::SegmenterOptions options(ULOC_US); + ICING_ASSERT_OK_AND_ASSIGN( + lang_segmenter_, + language_segmenter_factory::Create(std::move(options))); + + ASSERT_THAT( + filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), + IsTrue()); + ICING_ASSERT_OK_AND_ASSIGN( + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("Name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("Message") + .AddProperty(PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("receiver") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder().SetType("Email").AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeJoinableString(JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL))) + + .Build(); + ASSERT_THAT(schema_store_->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOk()); + + ASSERT_THAT(filesystem_.CreateDirectoryRecursively(doc_store_dir_.c_str()), + IsTrue()); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); + doc_store_ = std::move(create_result.document_store); + + ICING_ASSERT_OK_AND_ASSIGN( + qualified_id_join_index_, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, qualified_id_join_index_dir_, *feature_flags_)); + } + + void TearDown() override { + qualified_id_join_index_.reset(); + doc_store_.reset(); + schema_store_.reset(); + lang_segmenter_.reset(); + + filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); + } + + libtextclassifier3::StatusOr<DocumentId> PutAndIndexDocument( + const DocumentProto& document) { + ICING_ASSIGN_OR_RETURN(DocumentStore::PutResult put_result, + doc_store_->Put(document)); + ICING_ASSIGN_OR_RETURN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + document)); + + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + ICING_RETURN_IF_ERROR( + handler->Handle(tokenized_document, put_result.new_document_id, + put_result.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr)); + return put_result.new_document_id; + } + + std::unique_ptr<FeatureFlags> feature_flags_; + Filesystem filesystem_; + FakeClock fake_clock_; + + std::string base_dir_; + std::string schema_store_dir_; + std::string doc_store_dir_; + std::string qualified_id_join_index_dir_; + + std::unique_ptr<LanguageSegmenter> lang_segmenter_; + std::unique_ptr<SchemaStore> schema_store_; + std::unique_ptr<DocumentStore> doc_store_; + std::unique_ptr<QualifiedIdJoinIndexImplV3> qualified_id_join_index_; +}; + +TEST_F(JoinChildrenFetcherImplV3Test, CreationWithNullPointerShouldFail) { + JoinSpecProto join_spec; + join_spec.set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec.set_child_property_expression("sender"); + join_spec.mutable_nested_spec()->mutable_scoring_spec()->set_order_by( + ScoringSpecProto::Order::ASC); + + EXPECT_THAT(JoinChildrenFetcherImplV3::Create( + join_spec, /*schema_store=*/nullptr, doc_store_.get(), + qualified_id_join_index_.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/{}), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + + EXPECT_THAT(JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), /*doc_store=*/nullptr, + qualified_id_join_index_.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/{}), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + + EXPECT_THAT(JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + /*qualified_id_join_index=*/nullptr, + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/{}), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); +} + +TEST_F(JoinChildrenFetcherImplV3Test, + CreationWithWrongVersionJoinIndexShouldFail) { + JoinSpecProto join_spec; + join_spec.set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec.set_child_property_expression("sender"); + join_spec.mutable_nested_spec()->mutable_scoring_spec()->set_order_by( + ScoringSpecProto::Order::ASC); + + std::string qualified_id_join_index_dir_v1 = + qualified_id_join_index_dir_ + "_v1"; + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV1> qualified_id_join_index_v1, + QualifiedIdJoinIndexImplV1::Create( + filesystem_, std::move(qualified_id_join_index_dir_v1), + /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true)); + + std::string qualified_id_join_index_dir_v2 = + qualified_id_join_index_dir_ + "_v2"; + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV2> qualified_id_join_index_v2, + QualifiedIdJoinIndexImplV2::Create( + filesystem_, std::move(qualified_id_join_index_dir_v2), + /*pre_mapping_fbv=*/false)); + + EXPECT_THAT(JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + qualified_id_join_index_v1.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/{}), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + + EXPECT_THAT(JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + qualified_id_join_index_v2.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/{}), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(JoinChildrenFetcherImplV3Test, GetChildren) { + // Simulate the following scenario: + // - Email child schema has 1 joinable property: "sender" with joinable + // property id 0. + // - email1 joins with person1 by property "sender". + // - email2 joins with person1 by property "sender". + // - email3 joins with person2 by property "sender". + // - email1, email2 and email3 are in the child result set. + DocumentProto person1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person1") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + DocumentProto person2 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person2") + .SetSchema("Person") + .AddStringProperty("Name", "Bob") + .Build(); + + DocumentProto email1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email1") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email2") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email3") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person2") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person1_id, + PutAndIndexDocument(person1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person2_id, + PutAndIndexDocument(person2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email1_id, PutAndIndexDocument(email1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email2_id, PutAndIndexDocument(email2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email3_id, PutAndIndexDocument(email3)); + + ScoredDocumentHit scored_doc_hit_email1(email1_id, kSectionIdMaskNone, + /*score=*/1.0); + ScoredDocumentHit scored_doc_hit_email2(email2_id, kSectionIdMaskNone, + /*score=*/2.0); + ScoredDocumentHit scored_doc_hit_email3(email3_id, kSectionIdMaskNone, + /*score=*/3.0); + + // Join by property "sender". + JoinSpecProto join_spec; + join_spec.set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec.set_child_property_expression("sender"); + join_spec.mutable_nested_spec()->mutable_scoring_spec()->set_order_by( + ScoringSpecProto::Order::ASC); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplV3> fetcher, + JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + qualified_id_join_index_.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/ + {scored_doc_hit_email1, scored_doc_hit_email2, + scored_doc_hit_email3})); + + EXPECT_THAT(fetcher->GetChildren(person1_id), + IsOkAndHolds( + ElementsAre(EqualsScoredDocumentHit(scored_doc_hit_email1), + EqualsScoredDocumentHit(scored_doc_hit_email2)))); + EXPECT_THAT(fetcher->GetChildren(person2_id), + IsOkAndHolds( + ElementsAre(EqualsScoredDocumentHit(scored_doc_hit_email3)))); +} + +TEST_F(JoinChildrenFetcherImplV3Test, + GetChildrenJoinChildrenFromMultipleSchemas) { + // Simulate the following scenario: + // - Email child schema has 1 joinable property: "sender" with joinable + // property id 0. + // - email1 joins with person1 by property "sender". + // - email2 joins with person2 by property "sender". + // - Message child schema has 2 joinable properties: "receiver", "sender" with + // joinable property ids 0, 1 respectively. + // - message1 joins with person2 by property "sender". + // - message2 joins with person2 by property "sender". + // - message3 joins with person1 by property "sender". + // - email1, email2, message1, message2 and message3 are in the child result + // set. + DocumentProto person1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person1") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + DocumentProto person2 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person2") + .SetSchema("Person") + .AddStringProperty("Name", "Bob") + .Build(); + + DocumentProto email1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email1") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email2") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person2") + .Build(); + + DocumentProto message1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message1") + .SetSchema("Message") + .AddStringProperty("sender", "pkg$db/namespace#person2") + .Build(); + DocumentProto message2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message2") + .SetSchema("Message") + .AddStringProperty("sender", "pkg$db/namespace#person2") + .Build(); + DocumentProto message3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message3") + .SetSchema("Message") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person1_id, + PutAndIndexDocument(person1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person2_id, + PutAndIndexDocument(person2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email1_id, PutAndIndexDocument(email1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email2_id, PutAndIndexDocument(email2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message1_id, + PutAndIndexDocument(message1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message2_id, + PutAndIndexDocument(message2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message3_id, + PutAndIndexDocument(message3)); + + ScoredDocumentHit scored_doc_hit_email1(email1_id, kSectionIdMaskNone, + /*score=*/1.0); + ScoredDocumentHit scored_doc_hit_email2(email2_id, kSectionIdMaskNone, + /*score=*/2.0); + ScoredDocumentHit scored_doc_hit_message1(message1_id, kSectionIdMaskNone, + /*score=*/3.0); + ScoredDocumentHit scored_doc_hit_message2(message2_id, kSectionIdMaskNone, + /*score=*/4.0); + ScoredDocumentHit scored_doc_hit_message3(message3_id, kSectionIdMaskNone, + /*score=*/5.0); + + // Join by property "sender". + JoinSpecProto join_spec; + join_spec.set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec.set_child_property_expression("sender"); + join_spec.mutable_nested_spec()->mutable_scoring_spec()->set_order_by( + ScoringSpecProto::Order::ASC); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplV3> fetcher, + JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + qualified_id_join_index_.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/ + {scored_doc_hit_email1, scored_doc_hit_email2, + scored_doc_hit_message1, scored_doc_hit_message2, + scored_doc_hit_message3})); + + EXPECT_THAT(fetcher->GetChildren(person1_id), + IsOkAndHolds(ElementsAre( + EqualsScoredDocumentHit(scored_doc_hit_email1), + EqualsScoredDocumentHit(scored_doc_hit_message3)))); + EXPECT_THAT(fetcher->GetChildren(person2_id), + IsOkAndHolds(ElementsAre( + EqualsScoredDocumentHit(scored_doc_hit_email2), + EqualsScoredDocumentHit(scored_doc_hit_message1), + EqualsScoredDocumentHit(scored_doc_hit_message2)))); +} + +TEST_F(JoinChildrenFetcherImplV3Test, + GetChildrenShouldFilterOutUndesiredJoinablePropertyIds) { + // Simulate the following scenario: + // - Message child schema has 2 joinable properties: "receiver", "sender" with + // joinable property ids 0, 1 respectively. + // - message1 joins with person1 by property "receiver". + // - message2 joins with person1 by property "sender". + // - message2 joins with person2 by property "receiver". + // - message3 joins with person1 by property "receiver". + // - message1, message2 and message3 are in the child result set. + DocumentProto person1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person1") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + DocumentProto person2 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person2") + .SetSchema("Person") + .AddStringProperty("Name", "Bob") + .Build(); + + DocumentProto message1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message1") + .SetSchema("Message") + .AddStringProperty("receiver", "pkg$db/namespace#person1") + .Build(); + DocumentProto message2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message2") + .SetSchema("Message") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .AddStringProperty("receiver", "pkg$db/namespace#person2") + .Build(); + DocumentProto message3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message3") + .SetSchema("Message") + .AddStringProperty("receiver", "pkg$db/namespace#person1") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person1_id, + PutAndIndexDocument(person1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person2_id, + PutAndIndexDocument(person2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message1_id, + PutAndIndexDocument(message1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message2_id, + PutAndIndexDocument(message2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message3_id, + PutAndIndexDocument(message3)); + // Sanity check for the join index. + ASSERT_THAT(qualified_id_join_index_->Get(person1_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair(message1_id, 0), + DocumentJoinIdPair(message2_id, 1), + DocumentJoinIdPair(message3_id, 0)))); + ASSERT_THAT(qualified_id_join_index_->Get(person2_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair(message2_id, 0)))); + + ScoredDocumentHit scored_doc_hit_message1(message1_id, kSectionIdMaskNone, + /*score=*/1.0); + ScoredDocumentHit scored_doc_hit_message2(message2_id, kSectionIdMaskNone, + /*score=*/2.0); + ScoredDocumentHit scored_doc_hit_message3(message3_id, kSectionIdMaskNone, + /*score=*/3.0); + + // Join by property "receiver". + JoinSpecProto join_spec; + join_spec.set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec.set_child_property_expression("receiver"); + join_spec.mutable_nested_spec()->mutable_scoring_spec()->set_order_by( + ScoringSpecProto::Order::ASC); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplV3> fetcher, + JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + qualified_id_join_index_.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/ + {scored_doc_hit_message1, scored_doc_hit_message2, + scored_doc_hit_message3})); + + // Person1 should get message1 and message3, but not message2 since message2 + // joins with person1 by property "sender", but join spec uses "receiver" to + // join. + EXPECT_THAT(fetcher->GetChildren(person1_id), + IsOkAndHolds(ElementsAre( + EqualsScoredDocumentHit(scored_doc_hit_message1), + EqualsScoredDocumentHit(scored_doc_hit_message3)))); + // Person2 should get message2. + EXPECT_THAT(fetcher->GetChildren(person2_id), + IsOkAndHolds(ElementsAre( + EqualsScoredDocumentHit(scored_doc_hit_message2)))); +} + +TEST_F(JoinChildrenFetcherImplV3Test, + GetChildrenShouldFilterOutUndesiredChildDocumentIds) { + // Simulate the following scenario: + // - Email child schema has 1 joinable property: "sender" with joinable + // property id 0. + // - email1 joins with person1 by property "sender". + // - email2 joins with person1 by property "sender". + // - email3 joins with person2 by property "sender". + // - email4 joins with person3 by property "sender". + // - Only email1 and email4 are in the child result set. + DocumentProto person1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person1") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + DocumentProto person2 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person2") + .SetSchema("Person") + .AddStringProperty("Name", "Bob") + .Build(); + DocumentProto person3 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person3") + .SetSchema("Person") + .AddStringProperty("Name", "Eve") + .Build(); + + DocumentProto email1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email1") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email2") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email3") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person2") + .Build(); + DocumentProto email4 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email4") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person3") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person1_id, + PutAndIndexDocument(person1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person2_id, + PutAndIndexDocument(person2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person3_id, + PutAndIndexDocument(person3)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email1_id, PutAndIndexDocument(email1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email2_id, PutAndIndexDocument(email2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email3_id, PutAndIndexDocument(email3)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email4_id, PutAndIndexDocument(email4)); + // Sanity check for the join index. + ASSERT_THAT(qualified_id_join_index_->Get(person1_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair(email1_id, 0), + DocumentJoinIdPair(email2_id, 0)))); + ASSERT_THAT(qualified_id_join_index_->Get(person2_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair(email3_id, 0)))); + ASSERT_THAT(qualified_id_join_index_->Get(person3_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair(email4_id, 0)))); + + ScoredDocumentHit scored_doc_hit_email1(email1_id, kSectionIdMaskNone, + /*score=*/1.0); + ScoredDocumentHit scored_doc_hit_email4(email4_id, kSectionIdMaskNone, + /*score=*/4.0); + + // Join by property "sender". + JoinSpecProto join_spec; + join_spec.set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec.set_child_property_expression("sender"); + join_spec.mutable_nested_spec()->mutable_scoring_spec()->set_order_by( + ScoringSpecProto::Order::ASC); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplV3> fetcher, + JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + qualified_id_join_index_.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/ + {scored_doc_hit_email1, scored_doc_hit_email4})); + + // Person1 should get email1, but not email2 since email2 is not in the result + // set. + EXPECT_THAT(fetcher->GetChildren(person1_id), + IsOkAndHolds( + ElementsAre(EqualsScoredDocumentHit(scored_doc_hit_email1)))); + // Person2 should get empty result set since email3 is not in the result set. + EXPECT_THAT(fetcher->GetChildren(person2_id), IsOkAndHolds(IsEmpty())); + // Person3 should get email4. + EXPECT_THAT(fetcher->GetChildren(person3_id), + IsOkAndHolds( + ElementsAre(EqualsScoredDocumentHit(scored_doc_hit_email4)))); +} + +TEST_F(JoinChildrenFetcherImplV3Test, + GetChildrenShouldSkipNonExistingJoinableProperty) { + // Simulate the following scenario: + // - Email child schema has 1 joinable property: "sender" with joinable + // property id 0. + // - email1 joins with person1 by property "sender". + // - email2 joins with person1 by property "sender". + // - email3 joins with person2 by property "sender". + // - email1, email2 and email3 are in the child result set, but email3 is + // expired. + DocumentProto person1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person1") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + DocumentProto person2 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person2") + .SetSchema("Person") + .AddStringProperty("Name", "Bob") + .Build(); + + DocumentProto email1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email1") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email2") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email3") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person2") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person1_id, + PutAndIndexDocument(person1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person2_id, + PutAndIndexDocument(person2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email1_id, PutAndIndexDocument(email1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email2_id, PutAndIndexDocument(email2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email3_id, PutAndIndexDocument(email3)); + + ScoredDocumentHit scored_doc_hit_email1(email1_id, kSectionIdMaskNone, + /*score=*/1.0); + ScoredDocumentHit scored_doc_hit_email2(email2_id, kSectionIdMaskNone, + /*score=*/2.0); + ScoredDocumentHit scored_doc_hit_email3(email3_id, kSectionIdMaskNone, + /*score=*/3.0); + + // Join by a non-existing property "foo". + JoinSpecProto join_spec; + join_spec.set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec.set_child_property_expression("foo"); + join_spec.mutable_nested_spec()->mutable_scoring_spec()->set_order_by( + ScoringSpecProto::Order::ASC); + + // Create and GetChildren should not fail. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplV3> fetcher, + JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + qualified_id_join_index_.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/ + {scored_doc_hit_email1, scored_doc_hit_email2, + scored_doc_hit_email3})); + + EXPECT_THAT(fetcher->GetChildren(person1_id), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(fetcher->GetChildren(person2_id), IsOkAndHolds(IsEmpty())); +} + +TEST_F(JoinChildrenFetcherImplV3Test, + GetChildrenShouldSkipExpiredChildDocuments) { + fake_clock_.SetSystemTimeMilliseconds(1000); + + // Simulate the following scenario: + // - Email child schema has 1 joinable property: "sender" with joinable + // property id 0. + // - email1 joins with person1 by property "sender". + // - email2 joins with person1 by property "sender". + // - email3 joins with person1 by property "sender". + // - email1, email2 and email3 are in the child result set. + DocumentProto person1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person1") + .SetSchema("Person") + .SetTtlMs(5000) + .AddStringProperty("Name", "Alice") + .Build(); + + DocumentProto email1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email1") + .SetSchema("Email") + .SetTtlMs(5000) + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email2") + .SetSchema("Email") + .SetTtlMs(5000) + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email3") + .SetSchema("Email") + .SetTtlMs(1000) + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person1_id, + PutAndIndexDocument(person1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email1_id, PutAndIndexDocument(email1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email2_id, PutAndIndexDocument(email2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email3_id, PutAndIndexDocument(email3)); + + ScoredDocumentHit scored_doc_hit_email1(email1_id, kSectionIdMaskNone, + /*score=*/1.0); + ScoredDocumentHit scored_doc_hit_email2(email2_id, kSectionIdMaskNone, + /*score=*/2.0); + ScoredDocumentHit scored_doc_hit_email3(email3_id, kSectionIdMaskNone, + /*score=*/3.0); + + // Join by property "sender". + JoinSpecProto join_spec; + join_spec.set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec.set_child_property_expression("sender"); + join_spec.mutable_nested_spec()->mutable_scoring_spec()->set_order_by( + ScoringSpecProto::Order::ASC); + + // Adjust the clock to make email3 expired. + fake_clock_.SetSystemTimeMilliseconds(3000); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplV3> fetcher, + JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + qualified_id_join_index_.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/ + {scored_doc_hit_email1, scored_doc_hit_email2, + scored_doc_hit_email3})); + + // Email3 should be skipped since it is expired. + // + // This scenario is unlikely to happen in practice since expired documents + // will mostly not be included in the child result set, but we still want to + // make sure an invalid DocumentFilterData is handled correctly. + EXPECT_THAT(fetcher->GetChildren(person1_id), + IsOkAndHolds( + ElementsAre(EqualsScoredDocumentHit(scored_doc_hit_email1), + EqualsScoredDocumentHit(scored_doc_hit_email2)))); +} + +TEST_F(JoinChildrenFetcherImplV3Test, GetChildrenShouldDoLazyLookupAndCache) { + // Simulate the following scenario: + // - Email child schema has 1 joinable property: "sender" with joinable + // property id 0. + // - email1 joins with person1 by property "sender". + // - email2 joins with person1 by property "sender". + // - email3 joins with person2 by property "sender". + // - email1, email2 and email3 are in the child result set. + DocumentProto person1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person1") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + DocumentProto person2 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person2") + .SetSchema("Person") + .AddStringProperty("Name", "Bob") + .Build(); + + DocumentProto email1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email1") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email2") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .Build(); + DocumentProto email3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "email3") + .SetSchema("Email") + .AddStringProperty("sender", "pkg$db/namespace#person2") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person1_id, + PutAndIndexDocument(person1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person2_id, + PutAndIndexDocument(person2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email1_id, PutAndIndexDocument(email1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email2_id, PutAndIndexDocument(email2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email3_id, PutAndIndexDocument(email3)); + + ScoredDocumentHit scored_doc_hit_email1(email1_id, kSectionIdMaskNone, + /*score=*/1.0); + ScoredDocumentHit scored_doc_hit_email2(email2_id, kSectionIdMaskNone, + /*score=*/2.0); + ScoredDocumentHit scored_doc_hit_email3(email3_id, kSectionIdMaskNone, + /*score=*/3.0); + + // Join by property "sender". + JoinSpecProto join_spec; + join_spec.set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec.set_child_property_expression("sender"); + join_spec.mutable_nested_spec()->mutable_scoring_spec()->set_order_by( + ScoringSpecProto::Order::ASC); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplV3> fetcher, + JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_.get(), doc_store_.get(), + qualified_id_join_index_.get(), + /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds(), + /*child_scored_document_hits=*/ + {scored_doc_hit_email1, scored_doc_hit_email2, + scored_doc_hit_email3})); + + EXPECT_THAT(fetcher->GetChildren(person1_id), + IsOkAndHolds( + ElementsAre(EqualsScoredDocumentHit(scored_doc_hit_email1), + EqualsScoredDocumentHit(scored_doc_hit_email2)))); + EXPECT_THAT(fetcher->GetChildren(person2_id), + IsOkAndHolds( + ElementsAre(EqualsScoredDocumentHit(scored_doc_hit_email3)))); + + // Intentionally clear the join index. + ICING_ASSERT_OK(qualified_id_join_index_->Clear()); + + // GetChildren again. The last round lookup should have cached the results, so + // even though the join index is cleared, we should still get the same result. + EXPECT_THAT(fetcher->GetChildren(person1_id), + IsOkAndHolds( + ElementsAre(EqualsScoredDocumentHit(scored_doc_hit_email1), + EqualsScoredDocumentHit(scored_doc_hit_email2)))); + EXPECT_THAT(fetcher->GetChildren(person2_id), + IsOkAndHolds( + ElementsAre(EqualsScoredDocumentHit(scored_doc_hit_email3)))); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/join/join-children-fetcher.cc b/icing/join/join-children-fetcher.cc deleted file mode 100644 index c6d1b97..0000000 --- a/icing/join/join-children-fetcher.cc +++ /dev/null
@@ -1,39 +0,0 @@ -// Copyright (C) 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "icing/join/join-children-fetcher.h" - -#include "icing/absl_ports/canonical_errors.h" -#include "icing/absl_ports/str_cat.h" - -namespace icing { -namespace lib { - -libtextclassifier3::StatusOr<std::vector<ScoredDocumentHit>> -JoinChildrenFetcher::GetChildren(DocumentId parent_doc_id) const { - if (join_spec_.parent_property_expression() == kQualifiedIdExpr) { - if (auto iter = map_joinable_qualified_id_.find(parent_doc_id); - iter != map_joinable_qualified_id_.end()) { - return iter->second; - } - return std::vector<ScoredDocumentHit>(); - } - // TODO(b/256022027): So far we only support kQualifiedIdExpr for - // parent_property_expression, we could support more. - return absl_ports::UnimplementedError(absl_ports::StrCat( - "Parent property expression must be ", kQualifiedIdExpr)); -} - -} // namespace lib -} // namespace icing
diff --git a/icing/join/join-children-fetcher.h b/icing/join/join-children-fetcher.h index 1b875bc..4f9870f 100644 --- a/icing/join/join-children-fetcher.h +++ b/icing/join/join-children-fetcher.h
@@ -1,4 +1,4 @@ -// Copyright (C) 2023 Google LLC +// Copyright (C) 2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ #ifndef ICING_JOIN_JOIN_CHILDREN_FETCHER_H_ #define ICING_JOIN_JOIN_CHILDREN_FETCHER_H_ -#include <unordered_map> +#include <string_view> #include <vector> #include "icing/text_classifier/lib3/utils/base/statusor.h" @@ -26,22 +26,13 @@ namespace icing { namespace lib { -// A class that provides the GetChildren method for joins to fetch all children -// documents given a parent document id. -// -// Internally, the class maintains a map for each joinable value type that -// groups children according to the joinable values. Currently we only support -// QUALIFIED_ID joining, in which the joinable value type is document id. +// A virtual class that provides the GetChildren method for joins to fetch all +// children documents given a parent document id. class JoinChildrenFetcher { public: - explicit JoinChildrenFetcher( - const JoinSpecProto& join_spec, - std::unordered_map<DocumentId, std::vector<ScoredDocumentHit>>&& - map_joinable_qualified_id) - : join_spec_(join_spec), - map_joinable_qualified_id_(std::move(map_joinable_qualified_id)) {} + virtual ~JoinChildrenFetcher() = default; - // Get a vector of children ScoredDocumentHit by parent document id. + // Gets a vector of children ScoredDocumentHit by parent document id. // // TODO(b/256022027): Implement property value joins with types of string and // int. In these cases, GetChildren should look up join index to fetch @@ -50,21 +41,18 @@ // corresponding map in this class using the joinable property value. // // Returns: - // The vector of results on success. - // UNIMPLEMENTED_ERROR if the join type specified by join_spec is not - // supported. - libtextclassifier3::StatusOr<std::vector<ScoredDocumentHit>> GetChildren( - DocumentId parent_doc_id) const; + // - The vector of ScoredDocumentHits for its children on success. + // - Other errors, depending on the implementation. + virtual libtextclassifier3::StatusOr<std::vector<ScoredDocumentHit>> + GetChildren(DocumentId parent_doc_id) const = 0; - private: + protected: + explicit JoinChildrenFetcher(const JoinSpecProto& join_spec) + : join_spec_(join_spec) {} + static constexpr std::string_view kQualifiedIdExpr = "this.qualifiedId()"; const JoinSpecProto& join_spec_; // Does not own! - - // The map that groups children by qualified id used to support QualifiedId - // joining. The joining type is document id. - std::unordered_map<DocumentId, std::vector<ScoredDocumentHit>> - map_joinable_qualified_id_; }; } // namespace lib
diff --git a/icing/join/join-processor.cc b/icing/join/join-processor.cc index 1b7ca0d..38846a4 100644 --- a/icing/join/join-processor.cc +++ b/icing/join/join-processor.cc
@@ -15,19 +15,25 @@ #include "icing/join/join-processor.h" #include <algorithm> +#include <deque> #include <memory> #include <optional> +#include <queue> #include <string> #include <string_view> #include <unordered_map> +#include <unordered_set> #include <utility> #include <vector> +#include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" #include "icing/join/aggregation-scorer.h" -#include "icing/join/doc-join-info.h" +#include "icing/join/document-join-id-pair.h" +#include "icing/join/join-children-fetcher-impl-deprecated.h" +#include "icing/join/join-children-fetcher-impl-v3.h" #include "icing/join/join-children-fetcher.h" #include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id.h" @@ -38,13 +44,14 @@ #include "icing/scoring/scored-document-hit.h" #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" +#include "icing/util/logging.h" #include "icing/util/status-macros.h" namespace icing { namespace lib { -libtextclassifier3::StatusOr<JoinChildrenFetcher> +libtextclassifier3::StatusOr<std::unique_ptr<JoinChildrenFetcher>> JoinProcessor::GetChildrenFetcher( const JoinSpecProto& join_spec, std::vector<ScoredDocumentHit>&& child_scored_document_hits) { @@ -55,119 +62,27 @@ "Parent property expression must be ", kQualifiedIdExpr)); } + switch (qualified_id_join_index_->version()) { + case QualifiedIdJoinIndex::Version::kV1: + return GetChildrenFetcherV1(join_spec, + std::move(child_scored_document_hits)); + case QualifiedIdJoinIndex::Version::kV2: + return GetChildrenFetcherV2(join_spec, + std::move(child_scored_document_hits)); + case QualifiedIdJoinIndex::Version::kV3: + return JoinChildrenFetcherImplV3::Create( + join_spec, schema_store_, doc_store_, qualified_id_join_index_, + current_time_ms_, std::move(child_scored_document_hits)); + } +} + +libtextclassifier3::StatusOr<std::unique_ptr<JoinChildrenFetcher>> +JoinProcessor::GetChildrenFetcherV1( + const JoinSpecProto& join_spec, + std::vector<ScoredDocumentHit>&& child_scored_document_hits) { ScoredDocumentHitComparator score_comparator( /*is_descending=*/join_spec.nested_spec().scoring_spec().order_by() == ScoringSpecProto::Order::DESC); - - if (qualified_id_join_index_->is_v2()) { - // v2 - // Step 1a: sort child ScoredDocumentHits in document id descending order. - std::sort(child_scored_document_hits.begin(), - child_scored_document_hits.end(), - [](const ScoredDocumentHit& lhs, const ScoredDocumentHit& rhs) { - return lhs.document_id() > rhs.document_id(); - }); - - // Step 1b: group all child ScoredDocumentHits by the document's - // schema_type_id. - std::unordered_map<SchemaTypeId, std::vector<ScoredDocumentHit>> - schema_to_child_scored_doc_hits_map; - for (const ScoredDocumentHit& child_scored_document_hit : - child_scored_document_hits) { - std::optional<DocumentFilterData> child_doc_filter_data = - doc_store_->GetAliveDocumentFilterData( - child_scored_document_hit.document_id(), current_time_ms_); - if (!child_doc_filter_data) { - continue; - } - - schema_to_child_scored_doc_hits_map[child_doc_filter_data - ->schema_type_id()] - .push_back(child_scored_document_hit); - } - - // Step 1c: for each schema_type_id, lookup QualifiedIdJoinIndexImplV2 to - // fetch all child join data from posting list(s). Convert all - // child join data to referenced parent document ids and bucketize - // child ScoredDocumentHits by it. - std::unordered_map<DocumentId, std::vector<ScoredDocumentHit>> - parent_to_child_docs_map; - for (auto& [schema_type_id, grouped_child_scored_doc_hits] : - schema_to_child_scored_doc_hits_map) { - // Get joinable_property_id of this schema. - ICING_ASSIGN_OR_RETURN( - const JoinablePropertyMetadata* metadata, - schema_store_->GetJoinablePropertyMetadata( - schema_type_id, join_spec.child_property_expression())); - if (metadata == nullptr || - metadata->value_type != JoinableConfig::ValueType::QUALIFIED_ID) { - // Currently we only support qualified id, so skip other types. - continue; - } - - // Lookup QualifiedIdJoinIndexImplV2. - ICING_ASSIGN_OR_RETURN( - std::unique_ptr<QualifiedIdJoinIndex::JoinDataIteratorBase> - join_index_iter, - qualified_id_join_index_->GetIterator( - schema_type_id, /*joinable_property_id=*/metadata->id)); - - // - Join index contains all join data of schema_type_id and - // join_index_iter will return all of them in (child) document id - // descending order. - // - But we only need join data of child document ids which appear in - // grouped_child_scored_doc_hits. Also grouped_child_scored_doc_hits - // contain ScoredDocumentHits in (child) document id descending order. - // - Therefore, we advance 2 iterators to intersect them and get desired - // join data. - auto child_scored_doc_hits_iter = grouped_child_scored_doc_hits.cbegin(); - while (join_index_iter->Advance().ok() && - child_scored_doc_hits_iter != - grouped_child_scored_doc_hits.cend()) { - // Advance child_scored_doc_hits_iter until it points to a - // ScoredDocumentHit with document id <= the one pointed by - // join_index_iter. - while (child_scored_doc_hits_iter != - grouped_child_scored_doc_hits.cend() && - child_scored_doc_hits_iter->document_id() > - join_index_iter->GetCurrent().document_id()) { - ++child_scored_doc_hits_iter; - } - - if (child_scored_doc_hits_iter != - grouped_child_scored_doc_hits.cend() && - child_scored_doc_hits_iter->document_id() == - join_index_iter->GetCurrent().document_id()) { - // We get a join data whose child document id exists in both join - // index and grouped_child_scored_doc_hits. Convert its join info to - // referenced parent document ids and bucketize ScoredDocumentHits by - // it (putting into parent_to_child_docs_map). - const NamespaceFingerprintIdentifier& ref_ns_id = - join_index_iter->GetCurrent().join_info(); - libtextclassifier3::StatusOr<DocumentId> ref_parent_doc_id_or = - doc_store_->GetDocumentId(ref_ns_id); - if (ref_parent_doc_id_or.ok()) { - parent_to_child_docs_map[std::move(ref_parent_doc_id_or) - .ValueOrDie()] - .push_back(*child_scored_doc_hits_iter); - } - } - } - } - - // Step 1d: finally, sort each parent's joined child ScoredDocumentHits by - // score. - for (auto& [parent_doc_id, bucketized_child_scored_hits] : - parent_to_child_docs_map) { - std::sort(bucketized_child_scored_hits.begin(), - bucketized_child_scored_hits.end(), score_comparator); - } - - return JoinChildrenFetcher(join_spec, std::move(parent_to_child_docs_map)); - } - - // v1 - // TODO(b/275121148): deprecate this part after rollout v2. std::sort(child_scored_document_hits.begin(), child_scored_document_hits.end(), score_comparator); @@ -191,7 +106,117 @@ map_joinable_qualified_id[ref_doc_id].push_back(child); } - return JoinChildrenFetcher(join_spec, std::move(map_joinable_qualified_id)); + return JoinChildrenFetcherImplDeprecated::Create( + join_spec, std::move(map_joinable_qualified_id)); +} + +libtextclassifier3::StatusOr<std::unique_ptr<JoinChildrenFetcher>> +JoinProcessor::GetChildrenFetcherV2( + const JoinSpecProto& join_spec, + std::vector<ScoredDocumentHit>&& child_scored_document_hits) { + // Step 1a: sort child ScoredDocumentHits in document id descending order. + std::sort(child_scored_document_hits.begin(), + child_scored_document_hits.end(), + [](const ScoredDocumentHit& lhs, const ScoredDocumentHit& rhs) { + return lhs.document_id() > rhs.document_id(); + }); + + // Step 1b: group all child ScoredDocumentHits by the document's + // schema_type_id. + std::unordered_map<SchemaTypeId, std::vector<ScoredDocumentHit>> + schema_to_child_scored_doc_hits_map; + for (const ScoredDocumentHit& child_scored_document_hit : + child_scored_document_hits) { + std::optional<DocumentFilterData> child_doc_filter_data = + doc_store_->GetAliveDocumentFilterData( + child_scored_document_hit.document_id(), current_time_ms_); + if (!child_doc_filter_data) { + continue; + } + + schema_to_child_scored_doc_hits_map[child_doc_filter_data->schema_type_id()] + .push_back(child_scored_document_hit); + } + + // Step 1c: for each schema_type_id, lookup QualifiedIdJoinIndexImplV2 to + // fetch all child join data from posting list(s). Convert all + // child join data to referenced parent document ids and bucketize + // child ScoredDocumentHits by it. + std::unordered_map<DocumentId, std::vector<ScoredDocumentHit>> + parent_to_child_docs_map; + for (auto& [schema_type_id, grouped_child_scored_doc_hits] : + schema_to_child_scored_doc_hits_map) { + // Get joinable_property_id of this schema. + ICING_ASSIGN_OR_RETURN( + const JoinablePropertyMetadata* metadata, + schema_store_->GetJoinablePropertyMetadata( + schema_type_id, join_spec.child_property_expression())); + if (metadata == nullptr || + metadata->value_type != JoinableConfig::ValueType::QUALIFIED_ID) { + // Currently we only support qualified id, so skip other types. + continue; + } + + // Lookup QualifiedIdJoinIndexImplV2. + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<QualifiedIdJoinIndex::JoinDataIteratorBase> + join_index_iter, + qualified_id_join_index_->GetIterator( + schema_type_id, /*joinable_property_id=*/metadata->id)); + + // - Join index contains all join data of schema_type_id and + // join_index_iter will return all of them in (child) document id + // descending order. + // - But we only need join data of child document ids which appear in + // grouped_child_scored_doc_hits. Also grouped_child_scored_doc_hits + // contain ScoredDocumentHits in (child) document id descending order. + // - Therefore, we advance 2 iterators to intersect them and get desired + // join data. + auto child_scored_doc_hits_iter = grouped_child_scored_doc_hits.cbegin(); + while (join_index_iter->Advance().ok() && + child_scored_doc_hits_iter != grouped_child_scored_doc_hits.cend()) { + // Advance child_scored_doc_hits_iter until it points to a + // ScoredDocumentHit with document id <= the one pointed by + // join_index_iter. + while (child_scored_doc_hits_iter != + grouped_child_scored_doc_hits.cend() && + child_scored_doc_hits_iter->document_id() > + join_index_iter->GetCurrent().document_id()) { + ++child_scored_doc_hits_iter; + } + + if (child_scored_doc_hits_iter != grouped_child_scored_doc_hits.cend() && + child_scored_doc_hits_iter->document_id() == + join_index_iter->GetCurrent().document_id()) { + // We get a join data whose child document id exists in both join + // index and grouped_child_scored_doc_hits. Convert its join info to + // referenced parent document ids and bucketize ScoredDocumentHits by + // it (putting into parent_to_child_docs_map). + const NamespaceIdFingerprint& ref_doc_nsid_uri_fingerprint = + join_index_iter->GetCurrent().join_info(); + libtextclassifier3::StatusOr<DocumentId> ref_parent_doc_id_or = + doc_store_->GetDocumentId(ref_doc_nsid_uri_fingerprint); + if (ref_parent_doc_id_or.ok()) { + parent_to_child_docs_map[std::move(ref_parent_doc_id_or).ValueOrDie()] + .push_back(*child_scored_doc_hits_iter); + } + } + } + } + + // Step 1d: finally, sort each parent's joined child ScoredDocumentHits by + // score. + ScoredDocumentHitComparator score_comparator( + /*is_descending=*/join_spec.nested_spec().scoring_spec().order_by() == + ScoringSpecProto::Order::DESC); + for (auto& [parent_doc_id, bucketized_child_scored_hits] : + parent_to_child_docs_map) { + std::sort(bucketized_child_scored_hits.begin(), + bucketized_child_scored_hits.end(), score_comparator); + } + + return JoinChildrenFetcherImplDeprecated::Create( + join_spec, std::move(parent_to_child_docs_map)); } libtextclassifier3::StatusOr<std::vector<JoinedScoredDocumentHit>> @@ -221,25 +246,103 @@ return joined_scored_document_hits; } +libtextclassifier3::StatusOr<std::unordered_set<DocumentId>> +JoinProcessor::GetPropagatedChildDocumentsToDelete( + const std::unordered_set<DocumentId>& deleted_document_ids) { + // Sanity check: join index should be V3. + if (qualified_id_join_index_->version() != + QualifiedIdJoinIndex::Version::kV3) { + return absl_ports::UnimplementedError( + "QualifiedIdJoinIndex version must be V3 to support delete " + "propagation."); + } + + // BFS traverse to find all child documents to propagate delete. + std::queue<DocumentId> que( + std::deque(deleted_document_ids.begin(), deleted_document_ids.end())); + std::unordered_set<DocumentId> child_documents_to_delete; + while (!que.empty()) { + DocumentId doc_id_to_expand = que.front(); + que.pop(); + + ICING_ASSIGN_OR_RETURN(std::vector<DocumentJoinIdPair> child_join_id_pairs, + qualified_id_join_index_->Get(doc_id_to_expand)); + for (const DocumentJoinIdPair& child_join_id_pair : child_join_id_pairs) { + if (child_documents_to_delete.find(child_join_id_pair.document_id()) != + child_documents_to_delete.end() || + deleted_document_ids.find(child_join_id_pair.document_id()) != + deleted_document_ids.end()) { + // Already added into the set to delete or already deleted (happens only + // when there is a cycle back to the deleted or traversed document in + // the join relation). Skip it. + continue; + } + + // Get DocumentFilterData of the child document to look up its schema type + // id. + // - Skip if the child document has been deleted, since delete propagation + // should've been done to all its children when deleting it previously. + // - Otherwise, we have to handle this child document and propagate delete + // to the grandchildren, even if it is expired. + std::optional<DocumentFilterData> child_filter_data = + doc_store_->GetNonDeletedDocumentFilterData( + child_join_id_pair.document_id()); + if (!child_filter_data) { + // The child document has been deleted. Skip. + continue; + } + + libtextclassifier3::StatusOr<const JoinablePropertyMetadata*> + metadata_or = schema_store_->GetJoinablePropertyMetadata( + child_filter_data->schema_type_id(), + child_join_id_pair.joinable_property_id()); + if (!metadata_or.ok() || metadata_or.ValueOrDie() == nullptr) { + // This shouldn't happen because we've validated it during indexing and + // only put valid DocumentJoinIdPair into qualified id join index. + // Log and skip it. + ICING_LOG(ERROR) << "Failed to get metadata for schema type id " + << child_filter_data->schema_type_id() + << ", joinable property id " + << static_cast<int>( + child_join_id_pair.joinable_property_id()); + continue; + } + const JoinablePropertyMetadata* metadata = metadata_or.ValueOrDie(); + + if (metadata->value_type == JoinableConfig::ValueType::QUALIFIED_ID && + metadata->delete_propagation_type == + JoinableConfig::DeletePropagationType::PROPAGATE_FROM) { + child_documents_to_delete.insert(child_join_id_pair.document_id()); + que.push(child_join_id_pair.document_id()); + } + } + } + + return child_documents_to_delete; +} + libtextclassifier3::StatusOr<DocumentId> JoinProcessor::FetchReferencedQualifiedId( - const DocumentId& document_id, const std::string& property_path) const { - std::optional<DocumentFilterData> filter_data = - doc_store_->GetAliveDocumentFilterData(document_id, current_time_ms_); - if (!filter_data) { + const DocumentId& child_document_id, + const std::string& property_path) const { + std::optional<DocumentFilterData> child_filter_data = + doc_store_->GetAliveDocumentFilterData(child_document_id, + current_time_ms_); + if (!child_filter_data) { return kInvalidDocumentId; } - ICING_ASSIGN_OR_RETURN(const JoinablePropertyMetadata* metadata, - schema_store_->GetJoinablePropertyMetadata( - filter_data->schema_type_id(), property_path)); + ICING_ASSIGN_OR_RETURN( + const JoinablePropertyMetadata* metadata, + schema_store_->GetJoinablePropertyMetadata( + child_filter_data->schema_type_id(), property_path)); if (metadata == nullptr || metadata->value_type != JoinableConfig::ValueType::QUALIFIED_ID) { // Currently we only support qualified id. return kInvalidDocumentId; } - DocJoinInfo info(document_id, metadata->id); + DocumentJoinIdPair info(child_document_id, metadata->id); libtextclassifier3::StatusOr<std::string_view> ref_qualified_id_str_or = qualified_id_join_index_->Get(info); if (!ref_qualified_id_str_or.ok()) {
diff --git a/icing/join/join-processor.h b/icing/join/join-processor.h index 517e9db..8174693 100644 --- a/icing/join/join-processor.h +++ b/icing/join/join-processor.h
@@ -16,16 +16,20 @@ #define ICING_JOIN_JOIN_PROCESSOR_H_ #include <cstdint> +#include <memory> #include <string> #include <string_view> +#include <unordered_set> #include <vector> +#include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/join/join-children-fetcher.h" #include "icing/join/qualified-id-join-index.h" #include "icing/proto/search.pb.h" #include "icing/schema/schema-store.h" #include "icing/scoring/scored-document-hit.h" +#include "icing/store/document-id.h" #include "icing/store/document-store.h" namespace icing { @@ -44,14 +48,15 @@ qualified_id_join_index_(qualified_id_join_index), current_time_ms_(current_time_ms) {} - // Get a JoinChildrenFetcher used to fetch all children documents by a parent + // Gets a JoinChildrenFetcher used to fetch all children documents by a parent // document id. // // Returns: - // A JoinChildrenFetcher instance on success. + // std::unique_ptr<JoinChildrenFetcher> instance on success. // UNIMPLEMENTED_ERROR if the join type specified by join_spec is not // supported. - libtextclassifier3::StatusOr<JoinChildrenFetcher> GetChildrenFetcher( + libtextclassifier3::StatusOr<std::unique_ptr<JoinChildrenFetcher>> + GetChildrenFetcher( const JoinSpecProto& join_spec, std::vector<ScoredDocumentHit>&& child_scored_document_hits); @@ -60,7 +65,37 @@ std::vector<ScoredDocumentHit>&& parent_scored_document_hits, const JoinChildrenFetcher& join_children_fetcher); + // Gets all child documents to delete, propagated from the given deleted + // documents. + // + // Returns: + // - On success, a set of child document ids to delete. + // - Any other errors. + libtextclassifier3::StatusOr<std::unordered_set<DocumentId>> + GetPropagatedChildDocumentsToDelete( + const std::unordered_set<DocumentId>& deleted_document_ids); + private: + // TODO(b/275121148): deprecate v1, v2 after rollout v3. + + // Helper function to construct JoinChildrenFetcher for + // QualfiedIdJoinIndexImplV1. + // + // Note: JoinChildrenFetcherImplDeprecated will be returned. + libtextclassifier3::StatusOr<std::unique_ptr<JoinChildrenFetcher>> + GetChildrenFetcherV1( + const JoinSpecProto& join_spec, + std::vector<ScoredDocumentHit>&& child_scored_document_hits); + + // Helper function to construct JoinChildrenFetcher for + // QualfiedIdJoinIndexImplV2. + // + // Note: JoinChildrenFetcherImplDeprecated will be returned. + libtextclassifier3::StatusOr<std::unique_ptr<JoinChildrenFetcher>> + GetChildrenFetcherV2( + const JoinSpecProto& join_spec, + std::vector<ScoredDocumentHit>&& child_scored_document_hits); + // Fetches referenced document id of the given document under the given // property path. //
diff --git a/icing/join/join-processor_test.cc b/icing/join/join-processor_test.cc index a205125..2cc1e25 100644 --- a/icing/join/join-processor_test.cc +++ b/icing/join/join-processor_test.cc
@@ -14,21 +14,28 @@ #include "icing/join/join-processor.h" +#include <cstdint> #include <memory> +#include <optional> #include <string> +#include <unordered_set> #include <utility> #include <vector> +#include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" +#include "icing/join/document-join-id-pair.h" #include "icing/join/join-children-fetcher.h" #include "icing/join/qualified-id-join-index-impl-v1.h" #include "icing/join/qualified-id-join-index-impl-v2.h" +#include "icing/join/qualified-id-join-index-impl-v3.h" #include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id-join-indexing-handler.h" #include "icing/portable/platform.h" @@ -45,11 +52,12 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/status-macros.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" @@ -60,14 +68,22 @@ namespace { using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::IsEmpty; using ::testing::IsTrue; +using ::testing::Ne; +using ::testing::UnorderedElementsAre; // TODO(b/275121148): remove template after deprecating // QualifiedIdJoinIndexImplV1. template <typename T> class JoinProcessorTest : public ::testing::Test { protected: + using JOIN_INDEX_TYPE = T; + void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); test_dir_ = GetTestTempDir() + "/icing_join_processor_test"; ASSERT_THAT(filesystem_.CreateDirectoryRecursively(test_dir_.c_str()), IsTrue()); @@ -79,7 +95,7 @@ if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -92,8 +108,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), IsTrue()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() @@ -124,14 +140,42 @@ TOKENIZER_PLAIN) .SetCardinality(CARDINALITY_OPTIONAL)) .AddProperty(PropertyConfigBuilder() - .SetName("sender") - .SetDataTypeJoinableString( - JOINABLE_VALUE_TYPE_QUALIFIED_ID) - .SetCardinality(CARDINALITY_OPTIONAL)) - .AddProperty(PropertyConfigBuilder() .SetName("receiver") .SetDataTypeJoinableString( - JOINABLE_VALUE_TYPE_QUALIFIED_ID) + JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_NONE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("reporter") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("Label") + .AddProperty(PropertyConfigBuilder() + .SetName("text") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("object") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("softLink") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); @@ -144,14 +188,14 @@ IsTrue()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, doc_store_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); doc_store_ = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN(qualified_id_join_index_, @@ -188,11 +232,17 @@ /*pre_mapping_fbv=*/false); } + template <> + libtextclassifier3::StatusOr<std::unique_ptr<QualifiedIdJoinIndex>> + CreateQualifiedIdJoinIndex<QualifiedIdJoinIndexImplV3>() { + return QualifiedIdJoinIndexImplV3::Create( + filesystem_, qualified_id_join_index_dir_, *feature_flags_); + } + libtextclassifier3::StatusOr<DocumentId> PutAndIndexDocument( const DocumentProto& document) { ICING_ASSIGN_OR_RETURN(DocumentStore::PutResult put_result, doc_store_->Put(document)); - DocumentId document_id = put_result.new_document_id; ICING_ASSIGN_OR_RETURN( TokenizedDocument tokenized_document, TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), @@ -202,10 +252,11 @@ std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), qualified_id_join_index_.get())); - ICING_RETURN_IF_ERROR(handler->Handle(tokenized_document, document_id, - /*recovery_mode=*/false, - /*put_document_stats=*/nullptr)); - return document_id; + ICING_RETURN_IF_ERROR( + handler->Handle(tokenized_document, put_result.new_document_id, + put_result.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr)); + return put_result.new_document_id; } libtextclassifier3::StatusOr<std::vector<JoinedScoredDocumentHit>> Join( @@ -216,14 +267,15 @@ doc_store_.get(), schema_store_.get(), qualified_id_join_index_.get(), /*current_time_ms=*/fake_clock_.GetSystemTimeMilliseconds()); ICING_ASSIGN_OR_RETURN( - JoinChildrenFetcher join_children_fetcher, + std::unique_ptr<JoinChildrenFetcher> join_children_fetcher, join_processor.GetChildrenFetcher( join_spec, std::move(child_scored_document_hits))); return join_processor.Join(join_spec, std::move(parent_scored_document_hits), - join_children_fetcher); + *join_children_fetcher); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; std::string test_dir_; std::string schema_store_dir_; @@ -239,7 +291,8 @@ }; using TestTypes = - ::testing::Types<QualifiedIdJoinIndexImplV1, QualifiedIdJoinIndexImplV2>; + ::testing::Types<QualifiedIdJoinIndexImplV1, QualifiedIdJoinIndexImplV2, + QualifiedIdJoinIndexImplV3>; TYPED_TEST_SUITE(JoinProcessorTest, TestTypes); TYPED_TEST(JoinProcessorTest, JoinByQualifiedId_allDocuments) { @@ -921,6 +974,679 @@ /*child_scored_document_hits=*/{scored_doc_hit6})))); } +TYPED_TEST(JoinProcessorTest, GetPropagatedChildDocumentsToDelete) { + DocumentProto person = DocumentBuilder() + .SetKey("pkg$db/namespace", "person") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + + // "sender" (joinable property id 2) has + // DELETE_PROPAGATION_TYPE_PROPAGATE_FROM. + DocumentProto message = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message") + .SetSchema("Message") + .AddStringProperty("content", "test content") + .AddStringProperty("sender", "pkg$db/namespace#person") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person_doc_id, + this->PutAndIndexDocument(person)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message_doc_id, + this->PutAndIndexDocument(message)); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({person_doc_id}), + StatusIs(libtextclassifier3::StatusCode::UNIMPLEMENTED, + HasSubstr("QualifiedIdJoinIndex version must be V3 to " + "support delete propagation"))); + } else { + // Sanity check. + ASSERT_THAT(this->qualified_id_join_index_->Get(person_doc_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + message_doc_id, /*joinable_property_id=*/2)))); + + // Deleting the parent document should propagate the delete to its child + // document via the joinable property with + // DELETE_PROPAGATION_TYPE_PROPAGATE_FROM. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({person_doc_id}), + IsOkAndHolds(UnorderedElementsAre(message_doc_id))); + } +} + +TYPED_TEST( + JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_shouldNotPropagateToChildDocumentsWithPropagateDeleteDisabled) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + DocumentProto person = DocumentBuilder() + .SetKey("pkg$db/namespace", "person") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + + // "receiver" (joinable property id 0) has DELETE_PROPAGATION_TYPE_NONE. + DocumentProto message = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message") + .SetSchema("Message") + .AddStringProperty("content", "test content") + .AddStringProperty("receiver", "pkg$db/namespace#person") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person_doc_id, + this->PutAndIndexDocument(person)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message_doc_id, + this->PutAndIndexDocument(message)); + + // Sanity check. + ASSERT_THAT(this->qualified_id_join_index_->Get(person_doc_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + message_doc_id, /*joinable_property_id=*/0)))); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // Deleting the parent document should not propagate the delete to its child + // document via the joinable property with DELETE_PROPAGATION_TYPE_NONE. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({person_doc_id}), + IsOkAndHolds(IsEmpty())); +} + +TYPED_TEST( + JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_shouldNotPropagateToNonJoinableChildDocuments) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + DocumentProto person = DocumentBuilder() + .SetKey("pkg$db/namespace", "person") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + + // Put person's qualified id string in a non-joinable property. + DocumentProto message = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message") + .SetSchema("Message") + .AddStringProperty("content", "pkg$db/namespace#person") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person_doc_id, + this->PutAndIndexDocument(person)); + ICING_ASSERT_OK(this->PutAndIndexDocument(message)); + + // Sanity check. + ASSERT_THAT(this->qualified_id_join_index_->Get(person_doc_id), + IsOkAndHolds(IsEmpty())); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // Deleting the parent document should not propagate the delete to + // non-joinable child documents. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({person_doc_id}), + IsOkAndHolds(IsEmpty())); +} + +TYPED_TEST(JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_propagateViaMultipleProperties) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + DocumentProto person = DocumentBuilder() + .SetKey("pkg$db/namespace", "person") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + + // - "sender" (joinable property id 2) has + // DELETE_PROPAGATION_TYPE_PROPAGATE_FROM. + // - "receiver" (joinable property id 0) has DELETE_PROPAGATION_TYPE_NONE. + DocumentProto message = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message") + .SetSchema("Message") + .AddStringProperty("content", "test content") + .AddStringProperty("sender", "pkg$db/namespace#person") + .AddStringProperty("receiver", "pkg$db/namespace#person") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person_doc_id, + this->PutAndIndexDocument(person)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message_doc_id, + this->PutAndIndexDocument(message)); + + // Sanity check. + ASSERT_THAT( + this->qualified_id_join_index_->Get(person_doc_id), + IsOkAndHolds(ElementsAre( + DocumentJoinIdPair(message_doc_id, /*joinable_property_id=*/0), + DocumentJoinIdPair(message_doc_id, /*joinable_property_id=*/2)))); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // Deleting the parent document should propagate the delete to its child + // document when there is at least one property with + // DELETE_PROPAGATION_TYPE_PROPAGATE_FROM. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({person_doc_id}), + IsOkAndHolds(UnorderedElementsAre(message_doc_id))); +} + +TYPED_TEST(JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_propagateToMultipleChildren) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + DocumentProto person = DocumentBuilder() + .SetKey("pkg$db/namespace", "person") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + + // "sender" (joinable property id 2) has + // DELETE_PROPAGATION_TYPE_PROPAGATE_FROM. + DocumentProto message1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message1") + .SetSchema("Message") + .AddStringProperty("content", "test content") + .AddStringProperty("sender", "pkg$db/namespace#person") + .Build(); + DocumentProto message2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message2") + .SetSchema("Message") + .AddStringProperty("content", "test content") + .AddStringProperty("sender", "pkg$db/namespace#person") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person_doc_id, + this->PutAndIndexDocument(person)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message1_doc_id, + this->PutAndIndexDocument(message1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message2_doc_id, + this->PutAndIndexDocument(message2)); + + // Sanity check. + ASSERT_THAT( + this->qualified_id_join_index_->Get(person_doc_id), + IsOkAndHolds(ElementsAre( + DocumentJoinIdPair(message1_doc_id, /*joinable_property_id=*/2), + DocumentJoinIdPair(message2_doc_id, /*joinable_property_id=*/2)))); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // Deleting the parent document should propagate the delete to all of its + // child documents via the joinable property with + // DELETE_PROPAGATION_TYPE_PROPAGATE_FROM. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({person_doc_id}), + IsOkAndHolds(UnorderedElementsAre(message1_doc_id, message2_doc_id))); +} + +TYPED_TEST( + JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_propagateFromMultipleProperties) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + DocumentProto person1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person1") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + DocumentProto person2 = DocumentBuilder() + .SetKey("pkg$db/namespace", "person2") + .SetSchema("Person") + .AddStringProperty("Name", "Bob") + .Build(); + + // "sender" (joinable property id 2) and "reporter" (joinable property id 1) + // have DELETE_PROPAGATION_TYPE_PROPAGATE_FROM. + DocumentProto message = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message") + .SetSchema("Message") + .AddStringProperty("content", "test content") + .AddStringProperty("sender", "pkg$db/namespace#person1") + .AddStringProperty("reporter", "pkg$db/namespace#person2") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person1_doc_id, + this->PutAndIndexDocument(person1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person2_doc_id, + this->PutAndIndexDocument(person2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message_doc_id, + this->PutAndIndexDocument(message)); + + // Sanity check. + ASSERT_THAT(this->qualified_id_join_index_->Get(person1_doc_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + message_doc_id, /*joinable_property_id=*/2)))); + ASSERT_THAT(this->qualified_id_join_index_->Get(person2_doc_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + message_doc_id, /*joinable_property_id=*/1)))); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // message document should be propagated to be deleted from both person1 and + // person2 via "sender" and "reporter" properties respectively. + EXPECT_THAT(join_processor.GetPropagatedChildDocumentsToDelete( + {person1_doc_id, person2_doc_id}), + IsOkAndHolds(UnorderedElementsAre(message_doc_id))); +} + +TYPED_TEST(JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_propagateToGrandChildren) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + // Create the following relations: + // + // ("object") - label1 + // / + // message1 <- + // / \ + // ("sender") ("softLink") - label2 + // / + // person <- + // \ + // ("receiver") ("object") - label3 + // \ / + // message2 <- + // \ + // ("softLink") - label4 + // + // Note: "sender" and "object" have DELETE_PROPAGATION_TYPE_PROPAGATE_FROM, + // while "receiver" and "softLink" have DELETE_PROPAGATION_TYPE_NONE. + DocumentProto person = DocumentBuilder() + .SetKey("pkg$db/namespace", "person") + .SetSchema("Person") + .AddStringProperty("Name", "Alice") + .Build(); + + DocumentProto message1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message1") + .SetSchema("Message") + .AddStringProperty("content", "test content") + .AddStringProperty("sender", "pkg$db/namespace#person") + .Build(); + DocumentProto message2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "message2") + .SetSchema("Message") + .AddStringProperty("content", "test content") + .AddStringProperty("receiver", "pkg$db/namespace#person") + .Build(); + + DocumentProto label1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label1") + .SetSchema("Label") + .AddStringProperty("text", " label1") + .AddStringProperty("object", "pkg$db/namespace#message1") + .Build(); + DocumentProto label2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label2") + .SetSchema("Label") + .AddStringProperty("text", " label2") + .AddStringProperty("softLink", "pkg$db/namespace#message1") + .Build(); + DocumentProto label3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label3") + .SetSchema("Label") + .AddStringProperty("text", " label3") + .AddStringProperty("object", "pkg$db/namespace#message2") + .Build(); + DocumentProto label4 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label4") + .SetSchema("Label") + .AddStringProperty("text", " label4") + .AddStringProperty("softLink", "pkg$db/namespace#message2") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId person_doc_id, + this->PutAndIndexDocument(person)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message1_doc_id, + this->PutAndIndexDocument(message1)); + ICING_ASSERT_OK(this->PutAndIndexDocument(message2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label1_doc_id, + this->PutAndIndexDocument(label1)); + ICING_ASSERT_OK(this->PutAndIndexDocument(label2)); + ICING_ASSERT_OK(this->PutAndIndexDocument(label3)); + ICING_ASSERT_OK(this->PutAndIndexDocument(label4)); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // - For children with type "Message", only message1 should be propagated to + // be deleted. + // - For grand children with type "Label", only label1 should be propagated to + // be deleted. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({person_doc_id}), + IsOkAndHolds(UnorderedElementsAre(message1_doc_id, label1_doc_id))); +} + +TYPED_TEST(JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_cycleReference) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + // Create the following relations: + // + // label1 <- label2 <- label3 + // | ^ + // | | + // +-------------------+ + DocumentProto label1 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label1") + .SetSchema("Label") + .AddStringProperty("text", " label1") + .AddStringProperty("object", "pkg$db/namespace#label3") + .Build(); + DocumentProto label2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label2") + .SetSchema("Label") + .AddStringProperty("text", " label2") + .AddStringProperty("object", "pkg$db/namespace#label1") + .Build(); + DocumentProto label3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label3") + .SetSchema("Label") + .AddStringProperty("text", " label3") + .AddStringProperty("object", "pkg$db/namespace#label2") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label1_doc_id_old, + this->PutAndIndexDocument(label1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label2_doc_id, + this->PutAndIndexDocument(label2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label3_doc_id, + this->PutAndIndexDocument(label3)); + // Put label1 again, due to the requirement of the join index: parent document + // must be present before the child document. This will make the relation data + // "label1 -> label3" present in the join index. + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label1_doc_id_new, + this->PutAndIndexDocument(label1)); + + // Sanity check for migration: put label1 2nd time. + ASSERT_THAT(label1_doc_id_new, Ne(label1_doc_id_old)); + // Old label1's doc id should get children = []. + ASSERT_THAT(this->qualified_id_join_index_->Get(label1_doc_id_old), + IsOkAndHolds(IsEmpty())); + // New label1's doc id should get children = [label2_doc_id]. + ASSERT_THAT(this->qualified_id_join_index_->Get(label1_doc_id_new), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + label2_doc_id, /*joinable_property_id=*/0)))); + // label2 should get children = [label3_doc_id]. + ASSERT_THAT(this->qualified_id_join_index_->Get(label2_doc_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + label3_doc_id, /*joinable_property_id=*/0)))); + // label3 should get children = [label1_doc_id_new]. label1_doc_id_old will + // not be returned because when putting label1 for the 1st time, label3 was + // not present yet, and label1_doc_id_old will not be added to label3's + // children list. + ASSERT_THAT(this->qualified_id_join_index_->Get(label3_doc_id), + IsOkAndHolds( + ElementsAre(DocumentJoinIdPair(label1_doc_id_new, + /*joinable_property_id=*/0)))); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // Call GetPropagatedChildDocumentsToDelete with {label1_doc_id_new}: + // - Propagate to label2_doc_id from label1_doc_id_new. + // - Propagate to label3_doc_id from label2_doc_id. + // - When trying to propage label3_doc_id to its children = + // [label1_doc_id_new]: + // - label1_doc_id_new is already deleted, so it should not be propagated + // again. + // - There should be no infinite propagation loop. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({label1_doc_id_new}), + IsOkAndHolds(UnorderedElementsAre(label2_doc_id, label3_doc_id))); + + // Call GetPropagatedChildDocumentsToDelete with {label2_doc_id}. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({label2_doc_id}), + IsOkAndHolds(UnorderedElementsAre(label1_doc_id_new, label3_doc_id))); + + // Call GetPropagatedChildDocumentsToDelete with {label3_doc_id}. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({label3_doc_id}), + IsOkAndHolds(UnorderedElementsAre(label1_doc_id_new, label2_doc_id))); +} + +TYPED_TEST(JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_selfCycleReference) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + DocumentProto label = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label") + .SetSchema("Label") + .AddStringProperty("text", " label") + .AddStringProperty("object", "pkg$db/namespace#label") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label_doc_id, + this->PutAndIndexDocument(label)); + + // Sanity check. + ASSERT_THAT(this->qualified_id_join_index_->Get(label_doc_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + label_doc_id, /*joinable_property_id=*/0)))); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // Call GetPropagatedChildDocumentsToDelete with {label_doc_id}: should get + // nothing since label_doc_id is already in the deleted set. Also there + // should be no infinite propagation loop. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({label_doc_id}), + IsOkAndHolds(IsEmpty())); +} + +TYPED_TEST( + JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_shouldPropagateToExpiredDocuments) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + constexpr int64_t kCreationTimestampMs = 1000; + constexpr int64_t kShortTtlMs = 1000; + constexpr int64_t kLongTtlMs = 10000; + + // Create the following relations: + // + // label1 <- label2 <- label3 + DocumentProto label1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "label1") + .SetSchema("Label") + .AddStringProperty("text", " label1") + .SetCreationTimestampMs(kCreationTimestampMs) + .SetTtlMs(kLongTtlMs) + .Build(); + DocumentProto label2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label2") + .SetSchema("Label") + .AddStringProperty("text", " label2") + .AddStringProperty("object", "pkg$db/namespace#label1") + .SetCreationTimestampMs(kCreationTimestampMs) + .SetTtlMs(kShortTtlMs) + .Build(); + DocumentProto label3 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label3") + .SetSchema("Label") + .AddStringProperty("text", " label3") + .AddStringProperty("object", "pkg$db/namespace#label2") + .SetCreationTimestampMs(kCreationTimestampMs) + .SetTtlMs(kLongTtlMs) + .Build(); + + this->fake_clock_.SetSystemTimeMilliseconds(kCreationTimestampMs); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label1_doc_id, + this->PutAndIndexDocument(label1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label2_doc_id, + this->PutAndIndexDocument(label2)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label3_doc_id, + this->PutAndIndexDocument(label3)); + + // Adjust the clock to expire label2. + int64_t current_time_ms = kCreationTimestampMs + kShortTtlMs + 100; + this->fake_clock_.SetSystemTimeMilliseconds(current_time_ms); + + // Sanity check: label2 is expired, and label1 and label3 are not expired. + ASSERT_THAT(this->doc_store_->GetAliveDocumentFilterData(label1_doc_id, + current_time_ms), + Ne(std::nullopt)); + ASSERT_THAT(this->doc_store_->GetAliveDocumentFilterData(label2_doc_id, + current_time_ms), + Eq(std::nullopt)); + ASSERT_THAT(this->doc_store_->GetAliveDocumentFilterData(label3_doc_id, + current_time_ms), + Ne(std::nullopt)); + ASSERT_THAT(this->qualified_id_join_index_->Get(label1_doc_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + label2_doc_id, /*joinable_property_id=*/0)))); + ASSERT_THAT(this->qualified_id_join_index_->Get(label2_doc_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + label3_doc_id, /*joinable_property_id=*/0)))); + ASSERT_THAT(this->qualified_id_join_index_->Get(label3_doc_id), + IsOkAndHolds(IsEmpty())); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // Call GetPropagatedChildDocumentsToDelete with {label1_doc_id}: should still + // propagate to label2_doc_id and label3_doc_id even though label2_doc_id is + // expired. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({label1_doc_id}), + IsOkAndHolds(UnorderedElementsAre(label2_doc_id, label3_doc_id))); +} + +TYPED_TEST( + JoinProcessorTest, + GetPropagatedChildDocumentsToDelete_shouldNotPropagateToDeletedDocuments) { + if (!std::is_same_v<typename TestFixture::JOIN_INDEX_TYPE, + QualifiedIdJoinIndexImplV3>) { + GTEST_SKIP() << "Skipping test for non v3 QualifiedIdJoinIndex"; + } + + // Create the following relations: + // + // label1 <- label2 + DocumentProto label1 = DocumentBuilder() + .SetKey("pkg$db/namespace", "label1") + .SetSchema("Label") + .AddStringProperty("text", " label1") + .Build(); + DocumentProto label2 = + DocumentBuilder() + .SetKey("pkg$db/namespace", "label2") + .SetSchema("Label") + .AddStringProperty("text", " label2") + .AddStringProperty("object", "pkg$db/namespace#label1") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label1_doc_id, + this->PutAndIndexDocument(label1)); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId label2_doc_id, + this->PutAndIndexDocument(label2)); + + // Delete label2. + ICING_ASSERT_OK(this->doc_store_->Delete( + label2_doc_id, this->fake_clock_.GetSystemTimeMilliseconds())); + + // Sanity check: label2 is deleted, but join data is still present. + ASSERT_THAT(this->doc_store_->GetAliveDocumentFilterData( + label2_doc_id, this->fake_clock_.GetSystemTimeMilliseconds()), + Eq(std::nullopt)); + ASSERT_THAT(this->qualified_id_join_index_->Get(label1_doc_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + label2_doc_id, /*joinable_property_id=*/0)))); + ASSERT_THAT(this->qualified_id_join_index_->Get(label2_doc_id), + IsOkAndHolds(IsEmpty())); + + JoinProcessor join_processor( + this->doc_store_.get(), this->schema_store_.get(), + this->qualified_id_join_index_.get(), + /*current_time_ms=*/this->fake_clock_.GetSystemTimeMilliseconds()); + + // Call GetPropagatedChildDocumentsToDelete with {label1_doc_id}: should not + // propagate to label2_doc_id. + EXPECT_THAT( + join_processor.GetPropagatedChildDocumentsToDelete({label1_doc_id}), + IsOkAndHolds(IsEmpty())); +} + // TODO(b/256022027): add unit tests for non-joinable property. If joinable // value type is unset, then qualifed id join should not // include the child document even if it contains a valid
diff --git a/icing/join/posting-list-join-data-accessor_test.cc b/icing/join/posting-list-join-data-accessor_test.cc index ddc2d32..1775074 100644 --- a/icing/join/posting-list-join-data-accessor_test.cc +++ b/icing/join/posting-list-join-data-accessor_test.cc
@@ -31,7 +31,7 @@ #include "icing/join/document-id-to-join-info.h" #include "icing/join/posting-list-join-data-serializer.h" #include "icing/store/document-id.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/testing/common-matchers.h" #include "icing/testing/tmp-directory.h" @@ -48,7 +48,7 @@ using ::testing::Ne; using ::testing::SizeIs; -using JoinDataType = DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>; +using JoinDataType = DocumentIdToJoinInfo<NamespaceIdFingerprint>; static constexpr NamespaceId kDefaultNamespaceId = 1; @@ -92,8 +92,8 @@ for (int i = 0; i < num_data; ++i) { data.push_back(JoinDataType( start_document_id, - NamespaceFingerprintIdentifier(ref_namespace_id, - /*fingerprint=*/start_ref_hash_uri))); + NamespaceIdFingerprint(ref_namespace_id, + /*fingerprint=*/start_ref_hash_uri))); ++start_document_id; ++start_ref_hash_uri; @@ -137,7 +137,7 @@ // Add a single data. This will fit in a min-sized posting list. JoinDataType data1( /*document_id=*/1, - NamespaceFingerprintIdentifier(kDefaultNamespaceId, /*fingerprint=*/123)); + NamespaceIdFingerprint(kDefaultNamespaceId, /*fingerprint=*/123)); ICING_ASSERT_OK(pl_accessor->PrependData(data1)); PostingListAccessor::FinalizeResult result1 = std::move(*pl_accessor).Finalize(); @@ -154,7 +154,7 @@ flash_index_storage_.get(), serializer_.get(), result1.id)); JoinDataType data2( /*document_id=*/2, - NamespaceFingerprintIdentifier(kDefaultNamespaceId, /*fingerprint=*/456)); + NamespaceIdFingerprint(kDefaultNamespaceId, /*fingerprint=*/456)); ICING_ASSERT_OK(pl_accessor->PrependData(data2)); PostingListAccessor::FinalizeResult result2 = std::move(*pl_accessor).Finalize(); @@ -372,24 +372,24 @@ flash_index_storage_.get(), serializer_.get())); JoinDataType data1( /*document_id=*/1, - NamespaceFingerprintIdentifier(kDefaultNamespaceId, /*fingerprint=*/819)); + NamespaceIdFingerprint(kDefaultNamespaceId, /*fingerprint=*/819)); ICING_ASSERT_OK(pl_accessor->PrependData(data1)); JoinDataType data2( /*document_id=*/1, - NamespaceFingerprintIdentifier(kDefaultNamespaceId, /*fingerprint=*/818)); + NamespaceIdFingerprint(kDefaultNamespaceId, /*fingerprint=*/818)); EXPECT_THAT(pl_accessor->PrependData(data2), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - JoinDataType data3(/*document_id=*/1, - NamespaceFingerprintIdentifier(kDefaultNamespaceId - 1, - /*fingerprint=*/820)); + JoinDataType data3( + /*document_id=*/1, + NamespaceIdFingerprint(kDefaultNamespaceId - 1, /*fingerprint=*/820)); EXPECT_THAT(pl_accessor->PrependData(data3), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - JoinDataType data4(/*document_id=*/0, - NamespaceFingerprintIdentifier(kDefaultNamespaceId + 1, - /*fingerprint=*/820)); + JoinDataType data4( + /*document_id=*/0, + NamespaceIdFingerprint(kDefaultNamespaceId + 1, /*fingerprint=*/820)); EXPECT_THAT(pl_accessor->PrependData(data4), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -414,7 +414,7 @@ flash_index_storage_.get(), serializer_.get())); JoinDataType data1( /*document_id=*/1, - NamespaceFingerprintIdentifier(kDefaultNamespaceId, /*fingerprint=*/819)); + NamespaceIdFingerprint(kDefaultNamespaceId, /*fingerprint=*/819)); ICING_ASSERT_OK(pl_accessor1->PrependData(data1)); PostingListAccessor::FinalizeResult result1 = std::move(*pl_accessor1).Finalize();
diff --git a/icing/join/posting-list-join-data-serializer.h b/icing/join/posting-list-join-data-serializer.h index 9f39dca..6778f61 100644 --- a/icing/join/posting-list-join-data-serializer.h +++ b/icing/join/posting-list-join-data-serializer.h
@@ -33,7 +33,7 @@ namespace lib { // A serializer class to serialize JoinDataType to PostingListUsed. Usually -// JoinDataType is DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>, +// JoinDataType is DocumentIdToJoinInfo<NamespaceIdFingerprint>, // DocumentIdToJoinInfo<TermId>, or DocumentIdToJoinInfo<int64_t>. // // REQUIRES:
diff --git a/icing/join/posting-list-join-data-serializer_test.cc b/icing/join/posting-list-join-data-serializer_test.cc index 20137b6..893cc68 100644 --- a/icing/join/posting-list-join-data-serializer_test.cc +++ b/icing/join/posting-list-join-data-serializer_test.cc
@@ -23,7 +23,7 @@ #include "gtest/gtest.h" #include "icing/file/posting_list/posting-list-used.h" #include "icing/join/document-id-to-join-info.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/testing/common-matchers.h" using testing::ElementsAre; @@ -38,152 +38,135 @@ namespace { TEST(PostingListJoinDataSerializerTest, GetMinPostingListSizeToFitNotNull) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; - int size = - 2551 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int size = 2551 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - ASSERT_THAT( - serializer.PrependData( - &pl_used, - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/2))), - IsOk()); - EXPECT_THAT( - serializer.GetMinPostingListSizeToFit(&pl_used), - Eq(2 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>))); + ASSERT_THAT(serializer.PrependData( + &pl_used, DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, + NamespaceIdFingerprint(/*namespace_id=*/1, + /*fingerprint=*/2))), + IsOk()); + EXPECT_THAT(serializer.GetMinPostingListSizeToFit(&pl_used), + Eq(2 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>))); - ASSERT_THAT( - serializer.PrependData( - &pl_used, - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/5))), - IsOk()); - EXPECT_THAT( - serializer.GetMinPostingListSizeToFit(&pl_used), - Eq(3 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>))); + ASSERT_THAT(serializer.PrependData( + &pl_used, DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, + NamespaceIdFingerprint(/*namespace_id=*/1, + /*fingerprint=*/5))), + IsOk()); + EXPECT_THAT(serializer.GetMinPostingListSizeToFit(&pl_used), + Eq(3 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>))); } TEST(PostingListJoinDataSerializerTest, GetMinPostingListSizeToFitAlmostFull) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; - int size = 3 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int size = 3 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - ASSERT_THAT( - serializer.PrependData( - &pl_used, - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/2))), - IsOk()); - ASSERT_THAT( - serializer.PrependData( - &pl_used, - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/5))), - IsOk()); + ASSERT_THAT(serializer.PrependData( + &pl_used, DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, + NamespaceIdFingerprint(/*namespace_id=*/1, + /*fingerprint=*/2))), + IsOk()); + ASSERT_THAT(serializer.PrependData( + &pl_used, DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, + NamespaceIdFingerprint(/*namespace_id=*/1, + /*fingerprint=*/5))), + IsOk()); EXPECT_THAT(serializer.GetMinPostingListSizeToFit(&pl_used), Eq(size)); } TEST(PostingListJoinDataSerializerTest, GetMinPostingListSizeToFitFull) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; - int size = 3 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int size = 3 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - ASSERT_THAT( - serializer.PrependData( - &pl_used, - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/2))), - IsOk()); - ASSERT_THAT( - serializer.PrependData( - &pl_used, - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/5))), - IsOk()); - ASSERT_THAT( - serializer.PrependData( - &pl_used, - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/2, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/10))), - IsOk()); + ASSERT_THAT(serializer.PrependData( + &pl_used, DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, + NamespaceIdFingerprint(/*namespace_id=*/1, + /*fingerprint=*/2))), + IsOk()); + ASSERT_THAT(serializer.PrependData( + &pl_used, DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, + NamespaceIdFingerprint(/*namespace_id=*/1, + /*fingerprint=*/5))), + IsOk()); + ASSERT_THAT(serializer.PrependData( + &pl_used, DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/2, + NamespaceIdFingerprint(/*namespace_id=*/1, + /*fingerprint=*/10))), + IsOk()); EXPECT_THAT(serializer.GetMinPostingListSizeToFit(&pl_used), Eq(size)); } TEST(PostingListJoinDataSerializerTest, PrependDataNotFull) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; - int size = - 2551 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int size = 2551 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); // Make used. - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data0( + DocumentIdToJoinInfo<NamespaceIdFingerprint> data0( /*document_id=*/0, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/2)); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2)); EXPECT_THAT(serializer.PrependData(&pl_used, data0), IsOk()); // Size = sizeof(uncompressed data0) - int expected_size = - sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int expected_size = sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(expected_size)); EXPECT_THAT(serializer.GetData(&pl_used), IsOkAndHolds(ElementsAre(data0))); - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data1( + DocumentIdToJoinInfo<NamespaceIdFingerprint> data1( /*document_id=*/1, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/5)); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/5)); EXPECT_THAT(serializer.PrependData(&pl_used, data1), IsOk()); // Size = sizeof(uncompressed data1) // + sizeof(uncompressed data0) - expected_size += sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + expected_size += sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(expected_size)); EXPECT_THAT(serializer.GetData(&pl_used), IsOkAndHolds(ElementsAre(data1, data0))); - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data2( - /*document_id=*/2, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/10)); + DocumentIdToJoinInfo<NamespaceIdFingerprint> data2( + /*document_id=*/2, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/10)); EXPECT_THAT(serializer.PrependData(&pl_used, data2), IsOk()); // Size = sizeof(uncompressed data2) // + sizeof(uncompressed data1) // + sizeof(uncompressed data0) - expected_size += sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + expected_size += sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(expected_size)); EXPECT_THAT(serializer.GetData(&pl_used), IsOkAndHolds(ElementsAre(data2, data1, data0))); } TEST(PostingListJoinDataSerializerTest, PrependDataAlmostFull) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; - int size = 4 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int size = 4 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); @@ -192,64 +175,61 @@ // Transitions: // Adding data0: EMPTY -> NOT_FULL // Adding data1: NOT_FULL -> NOT_FULL - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data0( + DocumentIdToJoinInfo<NamespaceIdFingerprint> data0( /*document_id=*/0, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/2)); - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data1( + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2)); + DocumentIdToJoinInfo<NamespaceIdFingerprint> data1( /*document_id=*/1, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/5)); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/5)); EXPECT_THAT(serializer.PrependData(&pl_used, data0), IsOk()); EXPECT_THAT(serializer.PrependData(&pl_used, data1), IsOk()); - int expected_size = - 2 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int expected_size = 2 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(expected_size)); EXPECT_THAT(serializer.GetData(&pl_used), IsOkAndHolds(ElementsAre(data1, data0))); // Add one more data to transition NOT_FULL -> ALMOST_FULL - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data2( - /*document_id=*/2, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/10)); + DocumentIdToJoinInfo<NamespaceIdFingerprint> data2( + /*document_id=*/2, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/10)); EXPECT_THAT(serializer.PrependData(&pl_used, data2), IsOk()); - expected_size = - 3 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + expected_size = 3 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(expected_size)); EXPECT_THAT(serializer.GetData(&pl_used), IsOkAndHolds(ElementsAre(data2, data1, data0))); // Add one more data to transition ALMOST_FULL -> FULL - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data3( - /*document_id=*/3, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/0)); + DocumentIdToJoinInfo<NamespaceIdFingerprint> data3( + /*document_id=*/3, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/0)); EXPECT_THAT(serializer.PrependData(&pl_used, data3), IsOk()); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(size)); EXPECT_THAT(serializer.GetData(&pl_used), IsOkAndHolds(ElementsAre(data3, data2, data1, data0))); // The posting list is FULL. Adding another data should fail. - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data4( - /*document_id=*/4, NamespaceFingerprintIdentifier( - /*namespace_id=*/0, /*fingerprint=*/1234)); + DocumentIdToJoinInfo<NamespaceIdFingerprint> data4( + /*document_id=*/4, + NamespaceIdFingerprint(/*namespace_id=*/0, /*fingerprint=*/1234)); EXPECT_THAT(serializer.PrependData(&pl_used, data4), StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); } TEST(PostingListJoinDataSerializerTest, PrependSmallerDataShouldFail) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; - int size = 4 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int size = 4 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data( + DocumentIdToJoinInfo<NamespaceIdFingerprint> data( /*document_id=*/100, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/2)); - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> smaller_data( + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2)); + DocumentIdToJoinInfo<NamespaceIdFingerprint> smaller_data( /*document_id=*/99, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/2)); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2)); // NOT_FULL -> NOT_FULL ASSERT_THAT(serializer.PrependData(&pl_used, data), IsOk()); @@ -268,8 +248,7 @@ } TEST(PostingListJoinDataSerializerTest, PrependDataPostingListUsedMinSize) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; int size = serializer.GetMinPostingListSize(); @@ -282,59 +261,57 @@ EXPECT_THAT(serializer.GetData(&pl_used), IsOkAndHolds(IsEmpty())); // Add a data. PL should shift to ALMOST_FULL state - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data0( + DocumentIdToJoinInfo<NamespaceIdFingerprint> data0( /*document_id=*/0, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/2)); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2)); EXPECT_THAT(serializer.PrependData(&pl_used, data0), IsOk()); // Size = sizeof(uncompressed data0) - int expected_size = - sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int expected_size = sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(expected_size)); EXPECT_THAT(serializer.GetData(&pl_used), IsOkAndHolds(ElementsAre(data0))); // Add another data. PL should shift to FULL state. - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data1( + DocumentIdToJoinInfo<NamespaceIdFingerprint> data1( /*document_id=*/1, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/5)); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/5)); EXPECT_THAT(serializer.PrependData(&pl_used, data1), IsOk()); // Size = sizeof(uncompressed data1) + sizeof(uncompressed data0) - expected_size += sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + expected_size += sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(expected_size)); EXPECT_THAT(serializer.GetData(&pl_used), IsOkAndHolds(ElementsAre(data1, data0))); // The posting list is FULL. Adding another data should fail. - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier> data2( - /*document_id=*/2, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/10)); + DocumentIdToJoinInfo<NamespaceIdFingerprint> data2( + /*document_id=*/2, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/10)); EXPECT_THAT(serializer.PrependData(&pl_used, data2), StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); } TEST(PostingListJoinDataSerializerTest, PrependDataArrayDoNotKeepPrepended) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; - int size = 6 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int size = 6 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_in; - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_pushed; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_in; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_pushed; // Add 3 data. The PL is in the empty state and should be able to fit all 3 // data without issue, transitioning the PL from EMPTY -> NOT_FULL. - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( /*document_id=*/0, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/2))); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( /*document_id=*/1, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/5))); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/5))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( /*document_id=*/2, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/10))); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/10))); EXPECT_THAT( serializer.PrependDataArray(&pl_used, data_in.data(), data_in.size(), /*keep_prepended=*/false), @@ -342,19 +319,19 @@ std::move(data_in.begin(), data_in.end(), std::back_inserter(data_pushed)); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(data_pushed.size() * - sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>))); + sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>))); EXPECT_THAT( serializer.GetData(&pl_used), IsOkAndHolds(ElementsAreArray(data_pushed.rbegin(), data_pushed.rend()))); // Add 2 data. The PL should transition from NOT_FULL to ALMOST_FULL. data_in.clear(); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( /*document_id=*/3, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/0))); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/4, NamespaceFingerprintIdentifier(/*namespace_id=*/0, - /*fingerprint=*/1234))); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/0))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/4, + NamespaceIdFingerprint(/*namespace_id=*/0, /*fingerprint=*/1234))); EXPECT_THAT( serializer.PrependDataArray(&pl_used, data_in.data(), data_in.size(), /*keep_prepended=*/false), @@ -362,7 +339,7 @@ std::move(data_in.begin(), data_in.end(), std::back_inserter(data_pushed)); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(data_pushed.size() * - sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>))); + sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>))); EXPECT_THAT( serializer.GetData(&pl_used), IsOkAndHolds(ElementsAreArray(data_pushed.rbegin(), data_pushed.rend()))); @@ -370,19 +347,19 @@ // Add 2 data. The PL should remain ALMOST_FULL since the remaining space can // only fit 1 data. data_in.clear(); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, NamespaceFingerprintIdentifier(/*namespace_id=*/2, - /*fingerprint=*/99))); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/6, NamespaceFingerprintIdentifier(/*namespace_id=*/1, - /*fingerprint=*/63))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, + NamespaceIdFingerprint(/*namespace_id=*/2, /*fingerprint=*/99))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/6, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/63))); EXPECT_THAT( serializer.PrependDataArray(&pl_used, data_in.data(), data_in.size(), /*keep_prepended=*/false), IsOkAndHolds(0)); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(data_pushed.size() * - sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>))); + sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>))); EXPECT_THAT( serializer.GetData(&pl_used), IsOkAndHolds(ElementsAreArray(data_pushed.rbegin(), data_pushed.rend()))); @@ -397,36 +374,35 @@ std::move(data_in.begin(), data_in.end(), std::back_inserter(data_pushed)); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(data_pushed.size() * - sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>))); + sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>))); EXPECT_THAT( serializer.GetData(&pl_used), IsOkAndHolds(ElementsAreArray(data_pushed.rbegin(), data_pushed.rend()))); } TEST(PostingListJoinDataSerializerTest, PrependDataArrayKeepPrepended) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; - int size = 6 * sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>); + int size = 6 * sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_in; - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_pushed; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_in; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_pushed; // Add 3 data. The PL is in the empty state and should be able to fit all 3 // data without issue, transitioning the PL from EMPTY -> NOT_FULL. - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( /*document_id=*/0, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/2))); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( /*document_id=*/1, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/5))); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/5))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( /*document_id=*/2, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/10))); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/10))); EXPECT_THAT( serializer.PrependDataArray(&pl_used, data_in.data(), data_in.size(), /*keep_prepended=*/true), @@ -434,7 +410,7 @@ std::move(data_in.begin(), data_in.end(), std::back_inserter(data_pushed)); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(data_pushed.size() * - sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>))); + sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>))); EXPECT_THAT( serializer.GetData(&pl_used), IsOkAndHolds(ElementsAreArray(data_pushed.rbegin(), data_pushed.rend()))); @@ -442,18 +418,18 @@ // Add 4 data. The PL should prepend 3 data and transition from NOT_FULL to // FULL. data_in.clear(); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( /*document_id=*/3, - NamespaceFingerprintIdentifier(/*namespace_id=*/1, /*fingerprint=*/0))); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/4, NamespaceFingerprintIdentifier(/*namespace_id=*/0, - /*fingerprint=*/1234))); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, NamespaceFingerprintIdentifier(/*namespace_id=*/2, - /*fingerprint=*/99))); - data_in.push_back(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/6, NamespaceFingerprintIdentifier(/*namespace_id=*/1, - /*fingerprint=*/63))); + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/0))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/4, + NamespaceIdFingerprint(/*namespace_id=*/0, /*fingerprint=*/1234))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, + NamespaceIdFingerprint(/*namespace_id=*/2, /*fingerprint=*/99))); + data_in.push_back(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/6, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/63))); EXPECT_THAT( serializer.PrependDataArray(&pl_used, data_in.data(), data_in.size(), /*keep_prepended=*/true), @@ -463,15 +439,14 @@ std::move(data_in.begin(), data_in.end(), std::back_inserter(data_pushed)); EXPECT_THAT(serializer.GetBytesUsed(&pl_used), Eq(data_pushed.size() * - sizeof(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>))); + sizeof(DocumentIdToJoinInfo<NamespaceIdFingerprint>))); EXPECT_THAT( serializer.GetData(&pl_used), IsOkAndHolds(ElementsAreArray(data_pushed.rbegin(), data_pushed.rend()))); } TEST(PostingListJoinDataSerializerTest, MoveFrom) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; int size = 3 * serializer.GetMinPostingListSize(); @@ -479,13 +454,13 @@ PostingListUsed pl_used1, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_arr1 = - {DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/2)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/5))}; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_arr1 = { + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/5))}; ASSERT_THAT( serializer.PrependDataArray(&pl_used1, data_arr1.data(), data_arr1.size(), /*keep_prepended=*/false), @@ -494,21 +469,19 @@ ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used2, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_arr2 = - {DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/2, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/10)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/3, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/0)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/4, - NamespaceFingerprintIdentifier(/*namespace_id=*/0, - /*fingerprint=*/1234)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, - NamespaceFingerprintIdentifier(/*namespace_id=*/2, - /*fingerprint=*/99))}; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_arr2 = { + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/2, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/10)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/3, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/0)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/4, + NamespaceIdFingerprint(/*namespace_id=*/0, /*fingerprint=*/1234)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, + NamespaceIdFingerprint(/*namespace_id=*/2, /*fingerprint=*/99))}; ASSERT_THAT( serializer.PrependDataArray(&pl_used2, data_arr2.data(), data_arr2.size(), /*keep_prepended=*/false), @@ -523,21 +496,20 @@ } TEST(PostingListJoinDataSerializerTest, MoveToNullReturnsFailedPrecondition) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; int size = 3 * serializer.GetMinPostingListSize(); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_arr = { - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/2)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/5))}; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_arr = { + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/5))}; ASSERT_THAT( serializer.PrependDataArray(&pl_used, data_arr.data(), data_arr.size(), /*keep_prepended=*/false), @@ -557,31 +529,29 @@ } TEST(PostingListJoinDataSerializerTest, MoveToPostingListTooSmall) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; int size1 = 3 * serializer.GetMinPostingListSize(); ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used1, PostingListUsed::CreateFromUnitializedRegion(&serializer, size1)); - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_arr1 = - {DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/2)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/5)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/2, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/10)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/3, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/0)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/4, - NamespaceFingerprintIdentifier(/*namespace_id=*/0, - /*fingerprint=*/1234))}; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_arr1 = { + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/5)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/2, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/10)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/3, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/0)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/4, + NamespaceIdFingerprint(/*namespace_id=*/0, /*fingerprint=*/1234))}; ASSERT_THAT( serializer.PrependDataArray(&pl_used1, data_arr1.data(), data_arr1.size(), /*keep_prepended=*/false), @@ -591,10 +561,10 @@ ICING_ASSERT_OK_AND_ASSIGN( PostingListUsed pl_used2, PostingListUsed::CreateFromUnitializedRegion(&serializer, size2)); - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_arr2 = - {DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, NamespaceFingerprintIdentifier( - /*namespace_id=*/2, /*fingerprint=*/99))}; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_arr2 = { + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, + NamespaceIdFingerprint(/*namespace_id=*/2, /*fingerprint=*/99))}; ASSERT_THAT( serializer.PrependDataArray(&pl_used2, data_arr2.data(), data_arr2.size(), /*keep_prepended=*/false), @@ -611,8 +581,7 @@ } TEST(PostingListJoinDataSerializerTest, PopFrontData) { - PostingListJoinDataSerializer< - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> + PostingListJoinDataSerializer<DocumentIdToJoinInfo<NamespaceIdFingerprint>> serializer; int size = 2 * serializer.GetMinPostingListSize(); @@ -620,16 +589,16 @@ PostingListUsed pl_used, PostingListUsed::CreateFromUnitializedRegion(&serializer, size)); - std::vector<DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>> data_arr = { - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/2)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/5)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/2, NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/10))}; + std::vector<DocumentIdToJoinInfo<NamespaceIdFingerprint>> data_arr = { + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/2)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/5)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/2, + NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/10))}; ASSERT_THAT( serializer.PrependDataArray(&pl_used, data_arr.data(), data_arr.size(), /*keep_prepended=*/false),
diff --git a/icing/join/qualified-id-join-index-impl-v1.cc b/icing/join/qualified-id-join-index-impl-v1.cc index a9be0c1..da45e00 100644 --- a/icing/join/qualified-id-join-index-impl-v1.cc +++ b/icing/join/qualified-id-join-index-impl-v1.cc
@@ -14,6 +14,7 @@ #include "icing/join/qualified-id-join-index-impl-v1.h" +#include <cstdint> #include <cstring> #include <memory> #include <string> @@ -29,7 +30,7 @@ #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" #include "icing/file/memory-mapped-file.h" -#include "icing/join/doc-join-info.h" +#include "icing/join/document-join-id-pair.h" #include "icing/join/qualified-id-join-index.h" #include "icing/store/document-id.h" #include "icing/store/dynamic-trie-key-mapper.h" @@ -48,10 +49,10 @@ // Set 1M for max # of qualified id entries and 10 bytes for key-value bytes. // This will take at most 23 MiB disk space and mmap for persistent hash map. -static constexpr int32_t kDocJoinInfoMapperMaxNumEntries = 1 << 20; -static constexpr int32_t kDocJoinInfoMapperAverageKVByteSize = 10; +static constexpr int32_t kDocumentJoinIdPairMapperMaxNumEntries = 1 << 20; +static constexpr int32_t kDocumentJoinIdPairMapperAverageKVByteSize = 10; -static constexpr int32_t kDocJoinInfoMapperDynamicTrieMaxSize = +static constexpr int32_t kDocumentJoinIdPairMapperDynamicTrieMaxSize = 128 * 1024 * 1024; // 128 MiB DocumentId GetNewDocumentId( @@ -67,7 +68,10 @@ return absl_ports::StrCat(working_path, "/metadata"); } -std::string GetDocJoinInfoMapperPath(std::string_view working_path) { +std::string GetDocumentJoinIdPairMapperPath(std::string_view working_path) { + // Use the old directory name "doc_join_info_mapper" to avoid rebuild. We just + // changed the class, variable and function name, so there is no need to + // change the directory name or rebuild the index. return absl_ports::StrCat(working_path, "/doc_join_info_mapper"); } @@ -85,7 +89,7 @@ bool use_persistent_hash_map) { if (!filesystem.FileExists(GetMetadataFilePath(working_path).c_str()) || !filesystem.DirectoryExists( - GetDocJoinInfoMapperPath(working_path).c_str()) || + GetDocumentJoinIdPairMapperPath(working_path).c_str()) || !filesystem.FileExists(GetQualifiedIdStoragePath(working_path).c_str())) { // Discard working_path if any file/directory is missing, and reinitialize. if (filesystem.DirectoryExists(working_path.c_str())) { @@ -108,12 +112,13 @@ } libtextclassifier3::Status QualifiedIdJoinIndexImplV1::Put( - const DocJoinInfo& doc_join_info, std::string_view ref_qualified_id_str) { + const DocumentJoinIdPair& document_join_id_pair, + std::string_view ref_qualified_id_str) { SetDirty(); - if (!doc_join_info.is_valid()) { + if (!document_join_id_pair.is_valid()) { return absl_ports::InvalidArgumentError( - "Cannot put data for an invalid DocJoinInfo"); + "Cannot put data for an invalid DocumentJoinIdPair"); } int32_t qualified_id_index = qualified_id_storage_->num_elements(); @@ -125,8 +130,8 @@ mutable_arr.SetArray(/*idx=*/ref_qualified_id_str.size(), /*arr=*/"\0", /*arr_len=*/1); - ICING_RETURN_IF_ERROR(doc_join_info_mapper_->Put( - encode_util::EncodeIntToCString(doc_join_info.value()), + ICING_RETURN_IF_ERROR(document_join_id_pair_mapper_->Put( + encode_util::EncodeIntToCString(document_join_id_pair.value()), qualified_id_index)); // TODO(b/268521214): add data into delete propagation storage @@ -135,16 +140,16 @@ } libtextclassifier3::StatusOr<std::string_view> QualifiedIdJoinIndexImplV1::Get( - const DocJoinInfo& doc_join_info) const { - if (!doc_join_info.is_valid()) { + const DocumentJoinIdPair& document_join_id_pair) const { + if (!document_join_id_pair.is_valid()) { return absl_ports::InvalidArgumentError( - "Cannot get data for an invalid DocJoinInfo"); + "Cannot get data for an invalid DocumentJoinIdPair"); } ICING_ASSIGN_OR_RETURN( int32_t qualified_id_index, - doc_join_info_mapper_->Get( - encode_util::EncodeIntToCString(doc_join_info.value()))); + document_join_id_pair_mapper_->Get( + encode_util::EncodeIntToCString(document_join_id_pair.value()))); const char* data = qualified_id_storage_->array() + qualified_id_index; return std::string_view(data, strlen(data)); @@ -182,7 +187,7 @@ // Destruct current index's storage instances to safely swap directories. // TODO(b/268521214): handle delete propagation storage - doc_join_info_mapper_.reset(); + document_join_id_pair_mapper_.reset(); qualified_id_storage_.reset(); if (!filesystem_.SwapFiles(temp_working_path_ddir.dir().c_str(), @@ -200,18 +205,19 @@ } if (use_persistent_hash_map_) { ICING_ASSIGN_OR_RETURN( - doc_join_info_mapper_, + document_join_id_pair_mapper_, PersistentHashMapKeyMapper<int32_t>::Create( - filesystem_, GetDocJoinInfoMapperPath(working_path_), + filesystem_, GetDocumentJoinIdPairMapperPath(working_path_), pre_mapping_fbv_, - /*max_num_entries=*/kDocJoinInfoMapperMaxNumEntries, - /*average_kv_byte_size=*/kDocJoinInfoMapperAverageKVByteSize)); + /*max_num_entries=*/kDocumentJoinIdPairMapperMaxNumEntries, + /*average_kv_byte_size=*/ + kDocumentJoinIdPairMapperAverageKVByteSize)); } else { ICING_ASSIGN_OR_RETURN( - doc_join_info_mapper_, + document_join_id_pair_mapper_, DynamicTrieKeyMapper<int32_t>::Create( - filesystem_, GetDocJoinInfoMapperPath(working_path_), - kDocJoinInfoMapperDynamicTrieMaxSize)); + filesystem_, GetDocumentJoinIdPairMapperPath(working_path_), + kDocumentJoinIdPairMapperDynamicTrieMaxSize)); } ICING_ASSIGN_OR_RETURN( @@ -228,26 +234,28 @@ libtextclassifier3::Status QualifiedIdJoinIndexImplV1::Clear() { SetDirty(); - doc_join_info_mapper_.reset(); + document_join_id_pair_mapper_.reset(); // Discard and reinitialize doc join info mapper. - std::string doc_join_info_mapper_path = - GetDocJoinInfoMapperPath(working_path_); + std::string document_join_id_pair_mapper_path = + GetDocumentJoinIdPairMapperPath(working_path_); if (use_persistent_hash_map_) { ICING_RETURN_IF_ERROR(PersistentHashMapKeyMapper<int32_t>::Delete( - filesystem_, doc_join_info_mapper_path)); + filesystem_, document_join_id_pair_mapper_path)); ICING_ASSIGN_OR_RETURN( - doc_join_info_mapper_, + document_join_id_pair_mapper_, PersistentHashMapKeyMapper<int32_t>::Create( - filesystem_, std::move(doc_join_info_mapper_path), pre_mapping_fbv_, - /*max_num_entries=*/kDocJoinInfoMapperMaxNumEntries, - /*average_kv_byte_size=*/kDocJoinInfoMapperAverageKVByteSize)); + filesystem_, std::move(document_join_id_pair_mapper_path), + pre_mapping_fbv_, + /*max_num_entries=*/kDocumentJoinIdPairMapperMaxNumEntries, + /*average_kv_byte_size=*/ + kDocumentJoinIdPairMapperAverageKVByteSize)); } else { ICING_RETURN_IF_ERROR(DynamicTrieKeyMapper<int32_t>::Delete( - filesystem_, doc_join_info_mapper_path)); - ICING_ASSIGN_OR_RETURN(doc_join_info_mapper_, + filesystem_, document_join_id_pair_mapper_path)); + ICING_ASSIGN_OR_RETURN(document_join_id_pair_mapper_, DynamicTrieKeyMapper<int32_t>::Create( - filesystem_, doc_join_info_mapper_path, - kDocJoinInfoMapperDynamicTrieMaxSize)); + filesystem_, document_join_id_pair_mapper_path, + kDocumentJoinIdPairMapperDynamicTrieMaxSize)); } // Clear qualified_id_storage_. @@ -273,22 +281,24 @@ absl_ports::StrCat("Failed to create directory: ", working_path)); } - // Initialize doc_join_info_mapper - std::unique_ptr<KeyMapper<int32_t>> doc_join_info_mapper; + // Initialize document_join_id_pair_mapper + std::unique_ptr<KeyMapper<int32_t>> document_join_id_pair_mapper; if (use_persistent_hash_map) { // TODO(b/263890397): decide PersistentHashMapKeyMapper size ICING_ASSIGN_OR_RETURN( - doc_join_info_mapper, + document_join_id_pair_mapper, PersistentHashMapKeyMapper<int32_t>::Create( - filesystem, GetDocJoinInfoMapperPath(working_path), pre_mapping_fbv, - /*max_num_entries=*/kDocJoinInfoMapperMaxNumEntries, - /*average_kv_byte_size=*/kDocJoinInfoMapperAverageKVByteSize)); + filesystem, GetDocumentJoinIdPairMapperPath(working_path), + pre_mapping_fbv, + /*max_num_entries=*/kDocumentJoinIdPairMapperMaxNumEntries, + /*average_kv_byte_size=*/ + kDocumentJoinIdPairMapperAverageKVByteSize)); } else { ICING_ASSIGN_OR_RETURN( - doc_join_info_mapper, + document_join_id_pair_mapper, DynamicTrieKeyMapper<int32_t>::Create( - filesystem, GetDocJoinInfoMapperPath(working_path), - kDocJoinInfoMapperDynamicTrieMaxSize)); + filesystem, GetDocumentJoinIdPairMapperPath(working_path), + kDocumentJoinIdPairMapperDynamicTrieMaxSize)); } // Initialize qualified_id_storage @@ -305,8 +315,9 @@ new QualifiedIdJoinIndexImplV1( filesystem, std::move(working_path), /*metadata_buffer=*/std::make_unique<uint8_t[]>(kMetadataFileSize), - std::move(doc_join_info_mapper), std::move(qualified_id_storage), - pre_mapping_fbv, use_persistent_hash_map)); + std::move(document_join_id_pair_mapper), + std::move(qualified_id_storage), pre_mapping_fbv, + use_persistent_hash_map)); // Initialize info content. new_index->info().magic = Info::kMagic; new_index->info().last_added_document_id = kInvalidDocumentId; @@ -331,9 +342,9 @@ return absl_ports::InternalError("Fail to read metadata file"); } - // Initialize doc_join_info_mapper + // Initialize document_join_id_pair_mapper bool dynamic_trie_key_mapper_dir_exists = filesystem.DirectoryExists( - absl_ports::StrCat(GetDocJoinInfoMapperPath(working_path), + absl_ports::StrCat(GetDocumentJoinIdPairMapperPath(working_path), "/key_mapper_dir") .c_str()); if ((use_persistent_hash_map && dynamic_trie_key_mapper_dir_exists) || @@ -343,20 +354,22 @@ return absl_ports::FailedPreconditionError("Key mapper type mismatch"); } - std::unique_ptr<KeyMapper<int32_t>> doc_join_info_mapper; + std::unique_ptr<KeyMapper<int32_t>> document_join_id_pair_mapper; if (use_persistent_hash_map) { ICING_ASSIGN_OR_RETURN( - doc_join_info_mapper, + document_join_id_pair_mapper, PersistentHashMapKeyMapper<int32_t>::Create( - filesystem, GetDocJoinInfoMapperPath(working_path), pre_mapping_fbv, - /*max_num_entries=*/kDocJoinInfoMapperMaxNumEntries, - /*average_kv_byte_size=*/kDocJoinInfoMapperAverageKVByteSize)); + filesystem, GetDocumentJoinIdPairMapperPath(working_path), + pre_mapping_fbv, + /*max_num_entries=*/kDocumentJoinIdPairMapperMaxNumEntries, + /*average_kv_byte_size=*/ + kDocumentJoinIdPairMapperAverageKVByteSize)); } else { ICING_ASSIGN_OR_RETURN( - doc_join_info_mapper, + document_join_id_pair_mapper, DynamicTrieKeyMapper<int32_t>::Create( - filesystem, GetDocJoinInfoMapperPath(working_path), - kDocJoinInfoMapperDynamicTrieMaxSize)); + filesystem, GetDocumentJoinIdPairMapperPath(working_path), + kDocumentJoinIdPairMapperDynamicTrieMaxSize)); } // Initialize qualified_id_storage @@ -370,10 +383,11 @@ // Create instance. auto type_joinable_index = std::unique_ptr<QualifiedIdJoinIndexImplV1>( - new QualifiedIdJoinIndexImplV1( - filesystem, std::move(working_path), std::move(metadata_buffer), - std::move(doc_join_info_mapper), std::move(qualified_id_storage), - pre_mapping_fbv, use_persistent_hash_map)); + new QualifiedIdJoinIndexImplV1(filesystem, std::move(working_path), + std::move(metadata_buffer), + std::move(document_join_id_pair_mapper), + std::move(qualified_id_storage), + pre_mapping_fbv, use_persistent_hash_map)); // Initialize existing PersistentStorage. Checksums will be validated. ICING_RETURN_IF_ERROR(type_joinable_index->InitializeExistingStorage()); @@ -390,9 +404,9 @@ const std::vector<DocumentId>& document_id_old_to_new, QualifiedIdJoinIndexImplV1* new_index) const { std::unique_ptr<KeyMapper<int32_t>::Iterator> iter = - doc_join_info_mapper_->GetIterator(); + document_join_id_pair_mapper_->GetIterator(); while (iter->Advance()) { - DocJoinInfo old_doc_join_info( + DocumentJoinIdPair old_document_join_id_pair( encode_util::DecodeIntFromCString(iter->GetKey())); int32_t qualified_id_index = iter->GetValue(); @@ -401,13 +415,13 @@ // Translate to new doc id. DocumentId new_document_id = GetNewDocumentId( - document_id_old_to_new, old_doc_join_info.document_id()); + document_id_old_to_new, old_document_join_id_pair.document_id()); if (new_document_id != kInvalidDocumentId) { - ICING_RETURN_IF_ERROR( - new_index->Put(DocJoinInfo(new_document_id, - old_doc_join_info.joinable_property_id()), - ref_qualified_id_str)); + ICING_RETURN_IF_ERROR(new_index->Put( + DocumentJoinIdPair(new_document_id, + old_document_join_id_pair.joinable_property_id()), + ref_qualified_id_str)); } } @@ -435,7 +449,7 @@ return libtextclassifier3::Status::OK; } - ICING_RETURN_IF_ERROR(doc_join_info_mapper_->PersistToDisk()); + ICING_RETURN_IF_ERROR(document_join_id_pair_mapper_->PersistToDisk()); ICING_RETURN_IF_ERROR(qualified_id_storage_->PersistToDisk()); is_storage_dirty_ = false; return libtextclassifier3::Status::OK; @@ -469,11 +483,12 @@ return Crc32(crcs().component_crcs.storages_crc); } - ICING_ASSIGN_OR_RETURN(Crc32 doc_join_info_mapper_crc, - doc_join_info_mapper_->UpdateChecksum()); + ICING_ASSIGN_OR_RETURN(Crc32 document_join_id_pair_mapper_crc, + document_join_id_pair_mapper_->UpdateChecksum()); ICING_ASSIGN_OR_RETURN(Crc32 qualified_id_storage_crc, qualified_id_storage_->UpdateChecksum()); - return Crc32(doc_join_info_mapper_crc.Get() ^ qualified_id_storage_crc.Get()); + return Crc32(document_join_id_pair_mapper_crc.Get() ^ + qualified_id_storage_crc.Get()); } libtextclassifier3::StatusOr<Crc32> @@ -491,10 +506,11 @@ return Crc32(crcs().component_crcs.storages_crc); } - ICING_ASSIGN_OR_RETURN(Crc32 doc_join_info_mapper_crc, - doc_join_info_mapper_->GetChecksum()); + ICING_ASSIGN_OR_RETURN(Crc32 document_join_id_pair_mapper_crc, + document_join_id_pair_mapper_->GetChecksum()); Crc32 qualified_id_storage_crc = qualified_id_storage_->GetChecksum(); - return Crc32(doc_join_info_mapper_crc.Get() ^ qualified_id_storage_crc.Get()); + return Crc32(document_join_id_pair_mapper_crc.Get() ^ + qualified_id_storage_crc.Get()); } } // namespace lib
diff --git a/icing/join/qualified-id-join-index-impl-v1.h b/icing/join/qualified-id-join-index-impl-v1.h index 0d8da56..9a4a6fb 100644 --- a/icing/join/qualified-id-join-index-impl-v1.h +++ b/icing/join/qualified-id-join-index-impl-v1.h
@@ -28,21 +28,21 @@ #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" #include "icing/file/persistent-storage.h" -#include "icing/join/doc-join-info.h" +#include "icing/join/document-join-id-pair.h" #include "icing/join/qualified-id-join-index.h" #include "icing/schema/joinable-property.h" #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" #include "icing/store/key-mapper.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/util/crc32.h" namespace icing { namespace lib { -// QualifiedIdJoinIndexImplV1: a class to maintain data mapping DocJoinInfo to -// joinable qualified ids and delete propagation info. +// QualifiedIdJoinIndexImplV1: a class to maintain data mapping +// DocumentJoinIdPair to joinable qualified ids and delete propagation info. class QualifiedIdJoinIndexImplV1 : public QualifiedIdJoinIndex { public: struct Info { @@ -110,11 +110,11 @@ ~QualifiedIdJoinIndexImplV1() override; // v2 only API. Returns UNIMPLEMENTED_ERROR. - libtextclassifier3::Status Put(SchemaTypeId schema_type_id, - JoinablePropertyId joinable_property_id, - DocumentId document_id, - std::vector<NamespaceFingerprintIdentifier>&& - ref_namespace_fingerprint_ids) override { + libtextclassifier3::Status Put( + SchemaTypeId schema_type_id, JoinablePropertyId joinable_property_id, + DocumentId document_id, + std::vector<NamespaceIdFingerprint>&& ref_namespace_id_uri_fingerprints) + override { return absl_ports::UnimplementedError("This API is not supported in V1"); } @@ -125,12 +125,32 @@ return absl_ports::UnimplementedError("This API is not supported in V1"); } + // v3 only API. Returns UNIMPLEMENTED_ERROR. libtextclassifier3::Status Put( - const DocJoinInfo& doc_join_info, + const DocumentJoinIdPair& child_document_join_id_pair, + std::vector<DocumentId>&& parent_document_ids) override { + return absl_ports::UnimplementedError("This API is not supported in V1"); + } + + // v3 only API. Returns UNIMPLEMENTED_ERROR. + libtextclassifier3::StatusOr<std::vector<DocumentJoinIdPair>> Get( + DocumentId parent_document_id) const override { + return absl_ports::UnimplementedError("This API is not supported in V1"); + } + + libtextclassifier3::Status Put( + const DocumentJoinIdPair& document_join_id_pair, std::string_view ref_qualified_id_str) override; libtextclassifier3::StatusOr<std::string_view> Get( - const DocJoinInfo& doc_join_info) const override; + const DocumentJoinIdPair& document_join_id_pair) const override; + + // No-op since v1 stores parent information in raw qualified id string format + // and does not require parent migration. + libtextclassifier3::Status MigrateParent( + DocumentId old_document_id, DocumentId new_document_id) override { + return libtextclassifier3::Status::OK; + } libtextclassifier3::Status Optimize( const std::vector<DocumentId>& document_id_old_to_new, @@ -139,9 +159,13 @@ libtextclassifier3::Status Clear() override; - bool is_v2() const override { return false; } + QualifiedIdJoinIndex::Version version() const override { + return QualifiedIdJoinIndex::Version::kV1; + } - int32_t size() const override { return doc_join_info_mapper_->num_keys(); } + int32_t size() const override { + return document_join_id_pair_mapper_->num_keys(); + } bool empty() const override { return size() == 0; } @@ -163,12 +187,12 @@ explicit QualifiedIdJoinIndexImplV1( const Filesystem& filesystem, std::string&& working_path, std::unique_ptr<uint8_t[]> metadata_buffer, - std::unique_ptr<KeyMapper<int32_t>> doc_join_info_mapper, + std::unique_ptr<KeyMapper<int32_t>> document_join_id_pair_mapper, std::unique_ptr<FileBackedVector<char>> qualified_id_storage, bool pre_mapping_fbv, bool use_persistent_hash_map) : QualifiedIdJoinIndex(filesystem, std::move(working_path)), metadata_buffer_(std::move(metadata_buffer)), - doc_join_info_mapper_(std::move(doc_join_info_mapper)), + document_join_id_pair_mapper_(std::move(document_join_id_pair_mapper)), qualified_id_storage_(std::move(qualified_id_storage)), pre_mapping_fbv_(pre_mapping_fbv), use_persistent_hash_map_(use_persistent_hash_map), @@ -246,10 +270,10 @@ // Metadata buffer std::unique_ptr<uint8_t[]> metadata_buffer_; - // Persistent KeyMapper for mapping (encoded) DocJoinInfo (DocumentId, + // Persistent KeyMapper for mapping (encoded) DocumentJoinIdPair (DocumentId, // JoinablePropertyId) to another referenced document's qualified id string // index in qualified_id_storage_. - std::unique_ptr<KeyMapper<int32_t>> doc_join_info_mapper_; + std::unique_ptr<KeyMapper<int32_t>> document_join_id_pair_mapper_; // Storage for qualified id strings. std::unique_ptr<FileBackedVector<char>> qualified_id_storage_;
diff --git a/icing/join/qualified-id-join-index-impl-v1_test.cc b/icing/join/qualified-id-join-index-impl-v1_test.cc index bde9066..5af86fc 100644 --- a/icing/join/qualified-id-join-index-impl-v1_test.cc +++ b/icing/join/qualified-id-join-index-impl-v1_test.cc
@@ -25,7 +25,7 @@ #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" #include "icing/file/persistent-storage.h" -#include "icing/join/doc-join-info.h" +#include "icing/join/document-join-id-pair.h" #include "icing/store/document-id.h" #include "icing/store/dynamic-trie-key-mapper.h" #include "icing/store/key-mapper.h" @@ -158,16 +158,16 @@ param.use_persistent_hash_map)); // Insert some data. - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); ICING_ASSERT_OK(index->PersistToDisk()); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriB")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/5, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriC")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/3, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriB")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/5, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriC")); // GetChecksum should succeed without updating the checksum. ICING_EXPECT_OK(index->GetChecksum()); @@ -193,15 +193,15 @@ param.use_persistent_hash_map)); // Insert some data. - ICING_ASSERT_OK( - index1->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); - ICING_ASSERT_OK( - index1->Put(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriB")); - ICING_ASSERT_OK( - index1->Put(DocJoinInfo(/*document_id=*/5, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriC")); + ICING_ASSERT_OK(index1->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index1->Put( + DocumentJoinIdPair(/*document_id=*/3, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriB")); + ICING_ASSERT_OK(index1->Put( + DocumentJoinIdPair(/*document_id=*/5, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriC")); ASSERT_THAT(index1, Pointee(SizeIs(3))); // After calling UpdateChecksums, all checksums should be recomputed and @@ -217,15 +217,15 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); EXPECT_THAT(index2, Pointee(SizeIs(3))); - EXPECT_THAT( - index2->Get(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20)), - IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriA")); - EXPECT_THAT( - index2->Get(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/20)), - IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriB")); - EXPECT_THAT( - index2->Get(DocJoinInfo(/*document_id=*/5, /*joinable_property_id=*/20)), - IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriC")); + EXPECT_THAT(index2->Get(DocumentJoinIdPair(/*document_id=*/1, + /*joinable_property_id=*/20)), + IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriA")); + EXPECT_THAT(index2->Get(DocumentJoinIdPair(/*document_id=*/3, + /*joinable_property_id=*/20)), + IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriB")); + EXPECT_THAT(index2->Get(DocumentJoinIdPair(/*document_id=*/5, + /*joinable_property_id=*/20)), + IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriC")); } TEST_P(QualifiedIdJoinIndexImplV1Test, @@ -240,15 +240,15 @@ param.use_persistent_hash_map)); // Insert some data. - ICING_ASSERT_OK( - index1->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); - ICING_ASSERT_OK( - index1->Put(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriB")); - ICING_ASSERT_OK( - index1->Put(DocJoinInfo(/*document_id=*/5, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriC")); + ICING_ASSERT_OK(index1->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index1->Put( + DocumentJoinIdPair(/*document_id=*/3, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriB")); + ICING_ASSERT_OK(index1->Put( + DocumentJoinIdPair(/*document_id=*/5, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriC")); ASSERT_THAT(index1, Pointee(SizeIs(3))); // After calling PersistToDisk, all checksums should be recomputed and synced @@ -262,15 +262,15 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); EXPECT_THAT(index2, Pointee(SizeIs(3))); - EXPECT_THAT( - index2->Get(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20)), - IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriA")); - EXPECT_THAT( - index2->Get(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/20)), - IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriB")); - EXPECT_THAT( - index2->Get(DocJoinInfo(/*document_id=*/5, /*joinable_property_id=*/20)), - IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriC")); + EXPECT_THAT(index2->Get(DocumentJoinIdPair(/*document_id=*/1, + /*joinable_property_id=*/20)), + IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriA")); + EXPECT_THAT(index2->Get(DocumentJoinIdPair(/*document_id=*/3, + /*joinable_property_id=*/20)), + IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriB")); + EXPECT_THAT(index2->Get(DocumentJoinIdPair(/*document_id=*/5, + /*joinable_property_id=*/20)), + IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriC")); } TEST_P(QualifiedIdJoinIndexImplV1Test, @@ -286,15 +286,15 @@ param.use_persistent_hash_map)); // Insert some data. - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriB")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/5, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriC")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/3, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriB")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/5, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriC")); ASSERT_THAT(index, Pointee(SizeIs(3))); } @@ -309,14 +309,14 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); EXPECT_THAT(index, Pointee(SizeIs(3))); - EXPECT_THAT(index->Get(DocJoinInfo(/*document_id=*/1, - /*joinable_property_id=*/20)), + EXPECT_THAT(index->Get(DocumentJoinIdPair(/*document_id=*/1, + /*joinable_property_id=*/20)), IsOkAndHolds("namespace#uriA")); - EXPECT_THAT(index->Get(DocJoinInfo(/*document_id=*/3, - /*joinable_property_id=*/20)), + EXPECT_THAT(index->Get(DocumentJoinIdPair(/*document_id=*/3, + /*joinable_property_id=*/20)), IsOkAndHolds("namespace#uriB")); - EXPECT_THAT(index->Get(DocJoinInfo(/*document_id=*/5, - /*joinable_property_id=*/20)), + EXPECT_THAT(index->Get(DocumentJoinIdPair(/*document_id=*/5, + /*joinable_property_id=*/20)), IsOkAndHolds("namespace#uriC")); } } @@ -332,9 +332,9 @@ QualifiedIdJoinIndexImplV1::Create(filesystem_, working_path_, param.pre_mapping_fbv, param.use_persistent_hash_map)); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -389,9 +389,9 @@ QualifiedIdJoinIndexImplV1::Create(filesystem_, working_path_, param.pre_mapping_fbv, param.use_persistent_hash_map)); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -441,9 +441,9 @@ QualifiedIdJoinIndexImplV1::Create(filesystem_, working_path_, param.pre_mapping_fbv, param.use_persistent_hash_map)); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -484,7 +484,7 @@ } TEST_P(QualifiedIdJoinIndexImplV1Test, - InitializeExistingFilesWithCorruptedDocJoinInfoMapperShouldFail) { + InitializeExistingFilesWithCorruptedDocumentJoinIdPairMapperShouldFail) { const QualifiedIdJoinIndexImplV1TestParam& param = GetParam(); { @@ -494,9 +494,9 @@ QualifiedIdJoinIndexImplV1::Create(filesystem_, working_path_, param.pre_mapping_fbv, param.use_persistent_hash_map)); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -544,9 +544,9 @@ QualifiedIdJoinIndexImplV1::Create(filesystem_, working_path_, param.pre_mapping_fbv, param.use_persistent_hash_map)); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -589,7 +589,7 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); - DocJoinInfo default_invalid; + DocumentJoinIdPair default_invalid; EXPECT_THAT( index->Put(default_invalid, /*ref_qualified_id_str=*/"namespace#uriA"), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); @@ -605,7 +605,7 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); - DocJoinInfo default_invalid; + DocumentJoinIdPair default_invalid; EXPECT_THAT(index->Get(default_invalid), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -613,13 +613,16 @@ TEST_P(QualifiedIdJoinIndexImplV1Test, PutAndGet) { const QualifiedIdJoinIndexImplV1TestParam& param = GetParam(); - DocJoinInfo target_info1(/*document_id=*/1, /*joinable_property_id=*/20); + DocumentJoinIdPair target_id_pair1(/*document_id=*/1, + /*joinable_property_id=*/20); std::string_view ref_qualified_id_str_a = "namespace#uriA"; - DocJoinInfo target_info2(/*document_id=*/3, /*joinable_property_id=*/13); + DocumentJoinIdPair target_id_pair2(/*document_id=*/3, + /*joinable_property_id=*/13); std::string_view ref_qualified_id_str_b = "namespace#uriB"; - DocJoinInfo target_info3(/*document_id=*/4, /*joinable_property_id=*/4); + DocumentJoinIdPair target_id_pair3(/*document_id=*/4, + /*joinable_property_id=*/4); std::string_view ref_qualified_id_str_c = "namespace#uriC"; { @@ -630,14 +633,17 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); - EXPECT_THAT(index->Put(target_info1, ref_qualified_id_str_a), IsOk()); - EXPECT_THAT(index->Put(target_info2, ref_qualified_id_str_b), IsOk()); - EXPECT_THAT(index->Put(target_info3, ref_qualified_id_str_c), IsOk()); + EXPECT_THAT(index->Put(target_id_pair1, ref_qualified_id_str_a), IsOk()); + EXPECT_THAT(index->Put(target_id_pair2, ref_qualified_id_str_b), IsOk()); + EXPECT_THAT(index->Put(target_id_pair3, ref_qualified_id_str_c), IsOk()); EXPECT_THAT(index, Pointee(SizeIs(3))); - EXPECT_THAT(index->Get(target_info1), IsOkAndHolds(ref_qualified_id_str_a)); - EXPECT_THAT(index->Get(target_info2), IsOkAndHolds(ref_qualified_id_str_b)); - EXPECT_THAT(index->Get(target_info3), IsOkAndHolds(ref_qualified_id_str_c)); + EXPECT_THAT(index->Get(target_id_pair1), + IsOkAndHolds(ref_qualified_id_str_a)); + EXPECT_THAT(index->Get(target_id_pair2), + IsOkAndHolds(ref_qualified_id_str_b)); + EXPECT_THAT(index->Get(target_id_pair3), + IsOkAndHolds(ref_qualified_id_str_c)); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -649,15 +655,19 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); EXPECT_THAT(index, Pointee(SizeIs(3))); - EXPECT_THAT(index->Get(target_info1), IsOkAndHolds(ref_qualified_id_str_a)); - EXPECT_THAT(index->Get(target_info2), IsOkAndHolds(ref_qualified_id_str_b)); - EXPECT_THAT(index->Get(target_info3), IsOkAndHolds(ref_qualified_id_str_c)); + EXPECT_THAT(index->Get(target_id_pair1), + IsOkAndHolds(ref_qualified_id_str_a)); + EXPECT_THAT(index->Get(target_id_pair2), + IsOkAndHolds(ref_qualified_id_str_b)); + EXPECT_THAT(index->Get(target_id_pair3), + IsOkAndHolds(ref_qualified_id_str_c)); } TEST_P(QualifiedIdJoinIndexImplV1Test, GetShouldReturnNotFoundErrorIfNotExist) { const QualifiedIdJoinIndexImplV1TestParam& param = GetParam(); - DocJoinInfo target_info(/*document_id=*/1, /*joinable_property_id=*/20); + DocumentJoinIdPair target_id_pair(/*document_id=*/1, + /*joinable_property_id=*/20); std::string_view ref_qualified_id_str = "namespace#uriA"; // Create new qualified id join index @@ -668,16 +678,16 @@ param.use_persistent_hash_map)); // Verify entry is not found in the beginning. - EXPECT_THAT(index->Get(target_info), + EXPECT_THAT(index->Get(target_id_pair), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); - ICING_ASSERT_OK(index->Put(target_info, ref_qualified_id_str)); - ASSERT_THAT(index->Get(target_info), IsOkAndHolds(ref_qualified_id_str)); + ICING_ASSERT_OK(index->Put(target_id_pair, ref_qualified_id_str)); + ASSERT_THAT(index->Get(target_id_pair), IsOkAndHolds(ref_qualified_id_str)); // Get another non-existing entry. This should get NOT_FOUND_ERROR. - DocJoinInfo another_target_info(/*document_id=*/2, - /*joinable_property_id=*/20); - EXPECT_THAT(index->Get(another_target_info), + DocumentJoinIdPair another_target_id_pair(/*document_id=*/2, + /*joinable_property_id=*/20); + EXPECT_THAT(index->Get(another_target_id_pair), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } @@ -732,21 +742,21 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/10), - /*ref_qualified_id_str=*/"namespace#uriA")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/5, /*joinable_property_id=*/3), - /*ref_qualified_id_str=*/"namespace#uriA")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/8, /*joinable_property_id=*/9), - /*ref_qualified_id_str=*/"namespace#uriB")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/13, /*joinable_property_id=*/4), - /*ref_qualified_id_str=*/"namespace#uriC")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/21, /*joinable_property_id=*/12), - /*ref_qualified_id_str=*/"namespace#uriC")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/3, /*joinable_property_id=*/10), + /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/5, /*joinable_property_id=*/3), + /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/8, /*joinable_property_id=*/9), + /*ref_qualified_id_str=*/"namespace#uriB")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/13, /*joinable_property_id=*/4), + /*ref_qualified_id_str=*/"namespace#uriC")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/21, /*joinable_property_id=*/12), + /*ref_qualified_id_str=*/"namespace#uriC")); index->set_last_added_document_id(21); ASSERT_THAT(index, Pointee(SizeIs(5))); @@ -769,39 +779,40 @@ // (old_doc_id=3, joinable_property_id=10), which is now (doc_id=0, // joinable_property_id=10), has referenced qualified id str = // "namespace#uriA". - EXPECT_THAT( - index->Get(DocJoinInfo(/*document_id=*/0, /*joinable_property_id=*/10)), - IsOkAndHolds("namespace#uriA")); + EXPECT_THAT(index->Get(DocumentJoinIdPair(/*document_id=*/0, + /*joinable_property_id=*/10)), + IsOkAndHolds("namespace#uriA")); // (old_doc_id=5, joinable_property_id=3) and (old_doc_id=8, // joinable_property_id=9) are now not found since we've deleted old_doc_id = // 5, 8. It is not testable via Get() because there is no valid doc_id mapping - // for old_doc_id = 5, 8 and we cannot generate a valid DocJoinInfo for it. + // for old_doc_id = 5, 8 and we cannot generate a valid DocumentJoinIdPair for + // it. // (old_doc_id=13, joinable_property_id=4), which is now (doc_id=1, // joinable_property_id=4), has referenced qualified id str = // "namespace#uriC". - EXPECT_THAT( - index->Get(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/4)), - IsOkAndHolds("namespace#uriC")); + EXPECT_THAT(index->Get(DocumentJoinIdPair(/*document_id=*/1, + /*joinable_property_id=*/4)), + IsOkAndHolds("namespace#uriC")); // (old_doc_id=21, joinable_property_id=12), which is now (doc_id=2, // joinable_property_id=12), has referenced qualified id str = // "namespace#uriC". - EXPECT_THAT( - index->Get(DocJoinInfo(/*document_id=*/2, /*joinable_property_id=*/12)), - IsOkAndHolds("namespace#uriC")); + EXPECT_THAT(index->Get(DocumentJoinIdPair(/*document_id=*/2, + /*joinable_property_id=*/12)), + IsOkAndHolds("namespace#uriC")); // Joinable index should be able to work normally after Optimize(). - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/99, /*joinable_property_id=*/2), - /*ref_qualified_id_str=*/"namespace#uriD")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/99, /*joinable_property_id=*/2), + /*ref_qualified_id_str=*/"namespace#uriD")); index->set_last_added_document_id(99); EXPECT_THAT(index, Pointee(SizeIs(4))); EXPECT_THAT(index->last_added_document_id(), Eq(99)); - EXPECT_THAT(index->Get(DocJoinInfo(/*document_id=*/99, - /*joinable_property_id=*/2)), + EXPECT_THAT(index->Get(DocumentJoinIdPair(/*document_id=*/99, + /*joinable_property_id=*/2)), IsOkAndHolds("namespace#uriD")); } @@ -814,9 +825,9 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/99, /*joinable_property_id=*/10), - /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/99, /*joinable_property_id=*/10), + /*ref_qualified_id_str=*/"namespace#uriA")); index->set_last_added_document_id(99); // Create document_id_old_to_new with size = 1. Optimize should handle out of @@ -843,21 +854,21 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/10), - /*ref_qualified_id_str=*/"namespace#uriA")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/5, /*joinable_property_id=*/3), - /*ref_qualified_id_str=*/"namespace#uriA")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/8, /*joinable_property_id=*/9), - /*ref_qualified_id_str=*/"namespace#uriB")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/13, /*joinable_property_id=*/4), - /*ref_qualified_id_str=*/"namespace#uriC")); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/21, /*joinable_property_id=*/12), - /*ref_qualified_id_str=*/"namespace#uriC")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/3, /*joinable_property_id=*/10), + /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/5, /*joinable_property_id=*/3), + /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/8, /*joinable_property_id=*/9), + /*ref_qualified_id_str=*/"namespace#uriB")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/13, /*joinable_property_id=*/4), + /*ref_qualified_id_str=*/"namespace#uriC")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/21, /*joinable_property_id=*/12), + /*ref_qualified_id_str=*/"namespace#uriC")); index->set_last_added_document_id(21); // Delete all documents. @@ -876,9 +887,12 @@ TEST_P(QualifiedIdJoinIndexImplV1Test, Clear) { const QualifiedIdJoinIndexImplV1TestParam& param = GetParam(); - DocJoinInfo target_info1(/*document_id=*/1, /*joinable_property_id=*/20); - DocJoinInfo target_info2(/*document_id=*/3, /*joinable_property_id=*/5); - DocJoinInfo target_info3(/*document_id=*/6, /*joinable_property_id=*/13); + DocumentJoinIdPair target_id_pair1(/*document_id=*/1, + /*joinable_property_id=*/20); + DocumentJoinIdPair target_id_pair2(/*document_id=*/3, + /*joinable_property_id=*/5); + DocumentJoinIdPair target_id_pair3(/*document_id=*/6, + /*joinable_property_id=*/13); // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( @@ -887,11 +901,11 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); ICING_ASSERT_OK( - index->Put(target_info1, /*ref_qualified_id_str=*/"namespace#uriA")); + index->Put(target_id_pair1, /*ref_qualified_id_str=*/"namespace#uriA")); ICING_ASSERT_OK( - index->Put(target_info2, /*ref_qualified_id_str=*/"namespace#uriB")); + index->Put(target_id_pair2, /*ref_qualified_id_str=*/"namespace#uriB")); ICING_ASSERT_OK( - index->Put(target_info3, /*ref_qualified_id_str=*/"namespace#uriC")); + index->Put(target_id_pair3, /*ref_qualified_id_str=*/"namespace#uriC")); ASSERT_THAT(index, Pointee(SizeIs(3))); index->set_last_added_document_id(6); ASSERT_THAT(index->last_added_document_id(), Eq(6)); @@ -901,21 +915,22 @@ EXPECT_THAT(index->Clear(), IsOk()); EXPECT_THAT(index, Pointee(IsEmpty())); EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId)); - EXPECT_THAT(index->Get(target_info1), + EXPECT_THAT(index->Get(target_id_pair1), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); - EXPECT_THAT(index->Get(target_info2), + EXPECT_THAT(index->Get(target_id_pair2), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); - EXPECT_THAT(index->Get(target_info3), + EXPECT_THAT(index->Get(target_id_pair3), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); // Join index should be able to work normally after Clear(). - DocJoinInfo target_info4(/*document_id=*/2, /*joinable_property_id=*/19); + DocumentJoinIdPair target_id_pair4(/*document_id=*/2, + /*joinable_property_id=*/19); ICING_ASSERT_OK( - index->Put(target_info4, /*ref_qualified_id_str=*/"namespace#uriD")); + index->Put(target_id_pair4, /*ref_qualified_id_str=*/"namespace#uriD")); index->set_last_added_document_id(2); EXPECT_THAT(index->last_added_document_id(), Eq(2)); - EXPECT_THAT(index->Get(target_info4), IsOkAndHolds("namespace#uriD")); + EXPECT_THAT(index->Get(target_id_pair4), IsOkAndHolds("namespace#uriD")); ICING_ASSERT_OK(index->PersistToDisk()); index.reset(); @@ -926,13 +941,13 @@ param.pre_mapping_fbv, param.use_persistent_hash_map)); EXPECT_THAT(index->last_added_document_id(), Eq(2)); - EXPECT_THAT(index->Get(target_info1), + EXPECT_THAT(index->Get(target_id_pair1), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); - EXPECT_THAT(index->Get(target_info2), + EXPECT_THAT(index->Get(target_id_pair2), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); - EXPECT_THAT(index->Get(target_info3), + EXPECT_THAT(index->Get(target_id_pair3), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); - EXPECT_THAT(index->Get(target_info4), IsOkAndHolds("namespace#uriD")); + EXPECT_THAT(index->Get(target_id_pair4), IsOkAndHolds("namespace#uriD")); } TEST_P(QualifiedIdJoinIndexImplV1Test, SwitchKeyMapperTypeShouldReturnError) { @@ -945,9 +960,9 @@ QualifiedIdJoinIndexImplV1::Create(filesystem_, working_path_, param.pre_mapping_fbv, param.use_persistent_hash_map)); - ICING_ASSERT_OK( - index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), - /*ref_qualified_id_str=*/"namespace#uriA")); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/1, /*joinable_property_id=*/20), + /*ref_qualified_id_str=*/"namespace#uriA")); ICING_ASSERT_OK(index->PersistToDisk()); }
diff --git a/icing/join/qualified-id-join-index-impl-v2.cc b/icing/join/qualified-id-join-index-impl-v2.cc index 04a535c..0402be2 100644 --- a/icing/join/qualified-id-join-index-impl-v2.cc +++ b/icing/join/qualified-id-join-index-impl-v2.cc
@@ -39,7 +39,7 @@ #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" #include "icing/store/key-mapper.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/store/persistent-hash-map-key-mapper.h" #include "icing/util/crc32.h" @@ -62,7 +62,7 @@ inline DocumentId GetNewDocumentId( const std::vector<DocumentId>& document_id_old_to_new, DocumentId old_document_id) { - if (old_document_id >= document_id_old_to_new.size()) { + if (old_document_id < 0 || old_document_id >= document_id_old_to_new.size()) { return kInvalidDocumentId; } return document_id_old_to_new[old_document_id]; @@ -71,7 +71,7 @@ inline NamespaceId GetNewNamespaceId( const std::vector<NamespaceId>& namespace_id_old_to_new, NamespaceId namespace_id) { - if (namespace_id >= namespace_id_old_to_new.size()) { + if (namespace_id < 0 || namespace_id >= namespace_id_old_to_new.size()) { return kInvalidNamespaceId; } return namespace_id_old_to_new[namespace_id]; @@ -224,17 +224,16 @@ libtextclassifier3::Status QualifiedIdJoinIndexImplV2::Put( SchemaTypeId schema_type_id, JoinablePropertyId joinable_property_id, DocumentId document_id, - std::vector<NamespaceFingerprintIdentifier>&& - ref_namespace_fingerprint_ids) { - std::sort(ref_namespace_fingerprint_ids.begin(), - ref_namespace_fingerprint_ids.end()); + std::vector<NamespaceIdFingerprint>&& ref_namespace_id_uri_fingerprints) { + std::sort(ref_namespace_id_uri_fingerprints.begin(), + ref_namespace_id_uri_fingerprints.end()); // Dedupe. - auto last = std::unique(ref_namespace_fingerprint_ids.begin(), - ref_namespace_fingerprint_ids.end()); - ref_namespace_fingerprint_ids.erase(last, - ref_namespace_fingerprint_ids.end()); - if (ref_namespace_fingerprint_ids.empty()) { + auto last = std::unique(ref_namespace_id_uri_fingerprints.begin(), + ref_namespace_id_uri_fingerprints.end()); + ref_namespace_id_uri_fingerprints.erase( + last, ref_namespace_id_uri_fingerprints.end()); + if (ref_namespace_id_uri_fingerprints.empty()) { return libtextclassifier3::Status::OK; } @@ -262,11 +261,11 @@ } // Prepend join data into posting list. - for (const NamespaceFingerprintIdentifier& ref_namespace_fingerprint_id : - ref_namespace_fingerprint_ids) { - ICING_RETURN_IF_ERROR(pl_accessor->PrependData( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - document_id, ref_namespace_fingerprint_id))); + for (const NamespaceIdFingerprint& ref_namespace_id_uri_fingerprint : + ref_namespace_id_uri_fingerprints) { + ICING_RETURN_IF_ERROR( + pl_accessor->PrependData(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + document_id, ref_namespace_id_uri_fingerprint))); } // Finalize the posting list and update mapper. @@ -282,7 +281,7 @@ encoded_schema_type_joinable_property_id_str, result.id)); // Update info. - info().num_data += ref_namespace_fingerprint_ids.size(); + info().num_data += ref_namespace_id_uri_fingerprints.size(); return libtextclassifier3::Status::OK; } @@ -565,9 +564,9 @@ // We can reuse the fingerprint from old_join_data, since document uri // (and its fingerprint) will never change. new_join_data_vec.push_back(JoinDataType( - new_document_id, NamespaceFingerprintIdentifier( - new_ref_namespace_id, - old_join_data.join_info().fingerprint()))); + new_document_id, + NamespaceIdFingerprint(new_ref_namespace_id, + old_join_data.join_info().fingerprint()))); } } ICING_ASSIGN_OR_RETURN(batch_old_join_data,
diff --git a/icing/join/qualified-id-join-index-impl-v2.h b/icing/join/qualified-id-join-index-impl-v2.h index d45ca41..5b402cb 100644 --- a/icing/join/qualified-id-join-index-impl-v2.h +++ b/icing/join/qualified-id-join-index-impl-v2.h
@@ -29,8 +29,8 @@ #include "icing/file/persistent-storage.h" #include "icing/file/posting_list/flash-index-storage.h" #include "icing/file/posting_list/posting-list-identifier.h" -#include "icing/join/doc-join-info.h" #include "icing/join/document-id-to-join-info.h" +#include "icing/join/document-join-id-pair.h" #include "icing/join/posting-list-join-data-accessor.h" #include "icing/join/posting-list-join-data-serializer.h" #include "icing/join/qualified-id-join-index.h" @@ -38,7 +38,7 @@ #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" #include "icing/store/key-mapper.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/util/crc32.h" @@ -46,11 +46,11 @@ namespace lib { // QualifiedIdJoinIndexImplV2: a class to maintain join data (DocumentId to -// referenced NamespaceFingerprintIdentifier). It stores join data in posting -// lists and bucketizes them by (schema_type_id, joinable_property_id). +// referenced NamespaceIdFingerprint). It stores join data in posting lists +// and bucketizes them by (schema_type_id, joinable_property_id). class QualifiedIdJoinIndexImplV2 : public QualifiedIdJoinIndex { public: - using JoinDataType = DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>; + using JoinDataType = DocumentIdToJoinInfo<NamespaceIdFingerprint>; class JoinDataIterator : public JoinDataIteratorBase { public: @@ -109,10 +109,10 @@ WorkingPathType::kDirectory; // Creates a QualifiedIdJoinIndexImplV2 instance to store join data - // (DocumentId to referenced NamespaceFingerPrintIdentifier) for future - // joining search. If any of the underlying file is missing, then delete the - // whole working_path and (re)initialize with new ones. Otherwise initialize - // and create the instance by existing files. + // (DocumentId to referenced NamespaceIdFingerprint) for future joining + // search. If any of the underlying file is missing, then delete the whole + // working_path and (re)initialize with new ones. Otherwise initialize and + // create the instance by existing files. // // filesystem: Object to make system level calls // working_path: Specifies the working path for PersistentStorage. @@ -152,27 +152,47 @@ // v1 only API. Returns UNIMPLEMENTED_ERROR. libtextclassifier3::Status Put( - const DocJoinInfo& doc_join_info, + const DocumentJoinIdPair& document_join_id_pair, std::string_view ref_qualified_id_str) override { return absl_ports::UnimplementedError("This API is not supported in V2"); } // v1 only API. Returns UNIMPLEMENTED_ERROR. libtextclassifier3::StatusOr<std::string_view> Get( - const DocJoinInfo& doc_join_info) const override { + const DocumentJoinIdPair& document_join_id_pair) const override { return absl_ports::UnimplementedError("This API is not supported in V2"); } - libtextclassifier3::Status Put(SchemaTypeId schema_type_id, - JoinablePropertyId joinable_property_id, - DocumentId document_id, - std::vector<NamespaceFingerprintIdentifier>&& - ref_namespace_fingerprint_ids) override; + // v3 only API. Returns UNIMPLEMENTED_ERROR. + libtextclassifier3::Status Put( + const DocumentJoinIdPair& child_document_join_id_pair, + std::vector<DocumentId>&& parent_document_ids) override { + return absl_ports::UnimplementedError("This API is not supported in V2"); + } + + // v3 only API. Returns UNIMPLEMENTED_ERROR. + libtextclassifier3::StatusOr<std::vector<DocumentJoinIdPair>> Get( + DocumentId parent_document_id) const override { + return absl_ports::UnimplementedError("This API is not supported in V2"); + } + + libtextclassifier3::Status Put( + SchemaTypeId schema_type_id, JoinablePropertyId joinable_property_id, + DocumentId document_id, + std::vector<NamespaceIdFingerprint>&& ref_namespace_id_uri_fingerprints) + override; libtextclassifier3::StatusOr<std::unique_ptr<JoinDataIteratorBase>> GetIterator(SchemaTypeId schema_type_id, JoinablePropertyId joinable_property_id) const override; + // No-op since v2 stores parent information in (namespace_id, + // fingerprint(uri)) format and does not require parent migration. + libtextclassifier3::Status MigrateParent( + DocumentId old_document_id, DocumentId new_document_id) override { + return libtextclassifier3::Status::OK; + } + libtextclassifier3::Status Optimize( const std::vector<DocumentId>& document_id_old_to_new, const std::vector<NamespaceId>& namespace_id_old_to_new, @@ -180,7 +200,9 @@ libtextclassifier3::Status Clear() override; - bool is_v2() const override { return true; } + QualifiedIdJoinIndex::Version version() const override { + return QualifiedIdJoinIndex::Version::kV2; + } int32_t size() const override { return info().num_data; } @@ -296,7 +318,7 @@ schema_joinable_id_to_posting_list_mapper_; // Posting list related members. Use posting list to store join data - // (document id to referenced NamespaceFingerprintIdentifier). + // (document id to referenced NamespaceIdFingerprint). std::unique_ptr<PostingListJoinDataSerializer<JoinDataType>> posting_list_serializer_; std::unique_ptr<FlashIndexStorage> flash_index_storage_;
diff --git a/icing/join/qualified-id-join-index-impl-v2_test.cc b/icing/join/qualified-id-join-index-impl-v2_test.cc index d1d1b69..fdfa8bc 100644 --- a/icing/join/qualified-id-join-index-impl-v2_test.cc +++ b/icing/join/qualified-id-join-index-impl-v2_test.cc
@@ -36,7 +36,7 @@ #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" #include "icing/store/key-mapper.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/store/persistent-hash-map-key-mapper.h" #include "icing/testing/common-matchers.h" @@ -161,10 +161,10 @@ TEST_F(QualifiedIdJoinIndexImplV2Test, InitializationShouldFailWithoutPersistToDiskOrDestruction) { - NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/12); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/56); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/78); + NamespaceIdFingerprint id1(/*namespace_id=*/1, /*fingerprint=*/12); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/34); + NamespaceIdFingerprint id3(/*namespace_id=*/1, /*fingerprint=*/56); + NamespaceIdFingerprint id4(/*namespace_id=*/1, /*fingerprint=*/78); // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( @@ -175,14 +175,14 @@ // Insert some data. ICING_ASSERT_OK(index->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id2, id1})); + /*ref_namespace_id_uri_fingerprints=*/{id2, id1})); ICING_ASSERT_OK(index->PersistToDisk()); ICING_ASSERT_OK(index->Put( /*schema_type_id=*/3, /*joinable_property_id=*/10, /*document_id=*/6, - /*ref_namespace_fingerprint_ids=*/{id3})); + /*ref_namespace_id_uri_fingerprints=*/{id3})); ICING_ASSERT_OK(index->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/12, - /*ref_namespace_fingerprint_ids=*/{id4})); + /*ref_namespace_id_uri_fingerprints=*/{id4})); // GetChecksum should succeed without updating the checksum. ICING_EXPECT_OK(index->GetChecksum()); @@ -195,10 +195,10 @@ TEST_F(QualifiedIdJoinIndexImplV2Test, InitializationShouldSucceedWithUpdateChecksums) { - NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/12); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/56); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/78); + NamespaceIdFingerprint id1(/*namespace_id=*/1, /*fingerprint=*/12); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/34); + NamespaceIdFingerprint id3(/*namespace_id=*/1, /*fingerprint=*/56); + NamespaceIdFingerprint id4(/*namespace_id=*/1, /*fingerprint=*/78); // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( @@ -209,13 +209,13 @@ // Insert some data. ICING_ASSERT_OK(index1->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id2, id1})); + /*ref_namespace_id_uri_fingerprints=*/{id2, id1})); ICING_ASSERT_OK(index1->Put( /*schema_type_id=*/3, /*joinable_property_id=*/10, /*document_id=*/6, - /*ref_namespace_fingerprint_ids=*/{id3})); + /*ref_namespace_id_uri_fingerprints=*/{id3})); ICING_ASSERT_OK(index1->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/12, - /*ref_namespace_fingerprint_ids=*/{id4})); + /*ref_namespace_id_uri_fingerprints=*/{id4})); ASSERT_THAT(index1, Pointee(SizeIs(4))); // After calling UpdateChecksums, all checksums should be recomputed and @@ -232,26 +232,24 @@ EXPECT_THAT(index2, Pointee(SizeIs(4))); EXPECT_THAT( GetJoinData(*index2, /*schema_type_id=*/2, /*joinable_property_id=*/1), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/12, /*join_info=*/id4), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id2), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id1)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/12, /*join_info=*/id4), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id2), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id1)))); EXPECT_THAT( GetJoinData(*index2, /*schema_type_id=*/3, /*joinable_property_id=*/10), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/6, /*join_info=*/id3)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/6, /*join_info=*/id3)))); } TEST_F(QualifiedIdJoinIndexImplV2Test, InitializationShouldSucceedWithPersistToDisk) { - NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/12); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/56); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/78); + NamespaceIdFingerprint id1(/*namespace_id=*/1, /*fingerprint=*/12); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/34); + NamespaceIdFingerprint id3(/*namespace_id=*/1, /*fingerprint=*/56); + NamespaceIdFingerprint id4(/*namespace_id=*/1, /*fingerprint=*/78); // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( @@ -262,13 +260,13 @@ // Insert some data. ICING_ASSERT_OK(index1->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id2, id1})); + /*ref_namespace_id_uri_fingerprints=*/{id2, id1})); ICING_ASSERT_OK(index1->Put( /*schema_type_id=*/3, /*joinable_property_id=*/10, /*document_id=*/6, - /*ref_namespace_fingerprint_ids=*/{id3})); + /*ref_namespace_id_uri_fingerprints=*/{id3})); ICING_ASSERT_OK(index1->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/12, - /*ref_namespace_fingerprint_ids=*/{id4})); + /*ref_namespace_id_uri_fingerprints=*/{id4})); ASSERT_THAT(index1, Pointee(SizeIs(4))); // After calling PersistToDisk, all checksums should be recomputed and synced @@ -283,26 +281,24 @@ EXPECT_THAT(index2, Pointee(SizeIs(4))); EXPECT_THAT( GetJoinData(*index2, /*schema_type_id=*/2, /*joinable_property_id=*/1), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/12, /*join_info=*/id4), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id2), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id1)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/12, /*join_info=*/id4), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id2), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id1)))); EXPECT_THAT( GetJoinData(*index2, /*schema_type_id=*/3, /*joinable_property_id=*/10), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/6, /*join_info=*/id3)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/6, /*join_info=*/id3)))); } TEST_F(QualifiedIdJoinIndexImplV2Test, InitializationShouldSucceedAfterDestruction) { - NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/12); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/56); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/78); + NamespaceIdFingerprint id1(/*namespace_id=*/1, /*fingerprint=*/12); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/34); + NamespaceIdFingerprint id3(/*namespace_id=*/1, /*fingerprint=*/56); + NamespaceIdFingerprint id4(/*namespace_id=*/1, /*fingerprint=*/78); { // Create new qualified id join index @@ -314,13 +310,13 @@ // Insert some data. ICING_ASSERT_OK(index->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id2, id1})); + /*ref_namespace_id_uri_fingerprints=*/{id2, id1})); ICING_ASSERT_OK(index->Put( /*schema_type_id=*/3, /*joinable_property_id=*/10, /*document_id=*/6, - /*ref_namespace_fingerprint_ids=*/{id3})); + /*ref_namespace_id_uri_fingerprints=*/{id3})); ICING_ASSERT_OK(index->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/12, - /*ref_namespace_fingerprint_ids=*/{id4})); + /*ref_namespace_id_uri_fingerprints=*/{id4})); ASSERT_THAT(index, Pointee(SizeIs(4))); } @@ -336,18 +332,16 @@ EXPECT_THAT(index, Pointee(SizeIs(4))); EXPECT_THAT( GetJoinData(*index, /*schema_type_id=*/2, /*joinable_property_id=*/1), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/12, /*join_info=*/id4), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id2), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id1)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/12, /*join_info=*/id4), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id2), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id1)))); EXPECT_THAT( GetJoinData(*index, /*schema_type_id=*/3, /*joinable_property_id=*/10), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/6, /*join_info=*/id3)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/6, /*join_info=*/id3)))); } } @@ -361,9 +355,8 @@ /*pre_mapping_fbv=*/false)); ICING_ASSERT_OK(index->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/ - {NamespaceFingerprintIdentifier(/*namespace_id=*/1, - /*fingerprint=*/12)})); + /*ref_namespace_id_uri_fingerprints=*/ + {NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/12)})); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -414,10 +407,9 @@ QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_, /*pre_mapping_fbv=*/false)); ICING_ASSERT_OK(index->Put( - /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/ - {NamespaceFingerprintIdentifier(/*namespace_id=*/1, - /*fingerprint=*/12)})); + /*schema_type_id=*/2, /*joinable_property_id=*/1, + /*document_id=*/5, /*ref_namespace_id_uri_fingerprints=*/ + {NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/12)})); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -464,10 +456,9 @@ QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_, /*pre_mapping_fbv=*/false)); ICING_ASSERT_OK(index->Put( - /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/ - {NamespaceFingerprintIdentifier(/*namespace_id=*/1, - /*fingerprint=*/12)})); + /*schema_type_id=*/2, /*joinable_property_id=*/1, + /*document_id=*/5, /*ref_namespace_id_uri_fingerprints=*/ + {NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/12)})); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -516,10 +507,9 @@ QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_, /*pre_mapping_fbv=*/false)); ICING_ASSERT_OK(index->Put( - /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/ - {NamespaceFingerprintIdentifier(/*namespace_id=*/1, - /*fingerprint=*/12)})); + /*schema_type_id=*/2, /*joinable_property_id=*/1, + /*document_id=*/5, /*ref_namespace_id_uri_fingerprints=*/ + {NamespaceIdFingerprint(/*namespace_id=*/1, /*fingerprint=*/12)})); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -549,7 +539,7 @@ } TEST_F(QualifiedIdJoinIndexImplV2Test, InvalidPut) { - NamespaceFingerprintIdentifier id(/*namespace_id=*/1, /*fingerprint=*/12); + NamespaceIdFingerprint id(/*namespace_id=*/1, /*fingerprint=*/12); // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( @@ -559,15 +549,15 @@ EXPECT_THAT( index->Put(/*schema_type_id=*/-1, /*joinable_property_id=*/1, - /*document_id=*/5, /*ref_namespace_fingerprint_ids=*/{id}), + /*document_id=*/5, /*ref_namespace_id_uri_fingerprints=*/{id}), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT( index->Put(/*schema_type_id=*/2, /*joinable_property_id=*/-1, - /*document_id=*/5, /*ref_namespace_fingerprint_ids=*/{id}), + /*document_id=*/5, /*ref_namespace_id_uri_fingerprints=*/{id}), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(index->Put(/*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/kInvalidDocumentId, - /*ref_namespace_fingerprint_ids=*/{id}), + /*ref_namespace_id_uri_fingerprints=*/{id}), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -599,7 +589,7 @@ EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{}), + /*ref_namespace_id_uri_fingerprints=*/{}), IsOk()); EXPECT_THAT(index, Pointee(IsEmpty())); @@ -616,10 +606,10 @@ SchemaTypeId schema_type_id = 2; JoinablePropertyId joinable_property_id = 1; - NamespaceFingerprintIdentifier id1(/*namespace_id=*/3, /*fingerprint=*/12); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/2, /*fingerprint=*/56); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/0, /*fingerprint=*/78); + NamespaceIdFingerprint id1(/*namespace_id=*/3, /*fingerprint=*/12); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/34); + NamespaceIdFingerprint id3(/*namespace_id=*/2, /*fingerprint=*/56); + NamespaceIdFingerprint id4(/*namespace_id=*/0, /*fingerprint=*/78); { // Create new qualified id join index @@ -630,28 +620,28 @@ EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id2, id1}), + /*ref_namespace_id_uri_fingerprints=*/{id2, id1}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/6, - /*ref_namespace_fingerprint_ids=*/{id3}), + /*ref_namespace_id_uri_fingerprints=*/{id3}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/12, - /*ref_namespace_fingerprint_ids=*/{id4}), + /*ref_namespace_id_uri_fingerprints=*/{id4}), IsOk()); EXPECT_THAT(index, Pointee(SizeIs(4))); - EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/12, /*join_info=*/id4), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/6, /*join_info=*/id3), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id1), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id2)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id, joinable_property_id), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/12, /*join_info=*/id4), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/6, /*join_info=*/id3), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id1), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id2)))); EXPECT_THAT(GetJoinData(*index, schema_type_id + 1, joinable_property_id), IsOkAndHolds(IsEmpty())); EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id + 1), @@ -666,16 +656,16 @@ QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_, /*pre_mapping_fbv=*/false)); EXPECT_THAT(index, Pointee(SizeIs(4))); - EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/12, /*join_info=*/id4), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/6, /*join_info=*/id3), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id1), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id2)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id, joinable_property_id), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/12, /*join_info=*/id4), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/6, /*join_info=*/id3), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id1), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id2)))); EXPECT_THAT(GetJoinData(*index, schema_type_id + 1, joinable_property_id), IsOkAndHolds(IsEmpty())); EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id + 1), @@ -690,10 +680,10 @@ JoinablePropertyId joinable_property_id1 = 1; JoinablePropertyId joinable_property_id2 = 10; - NamespaceFingerprintIdentifier id1(/*namespace_id=*/3, /*fingerprint=*/12); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/2, /*fingerprint=*/56); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/0, /*fingerprint=*/78); + NamespaceIdFingerprint id1(/*namespace_id=*/3, /*fingerprint=*/12); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/34); + NamespaceIdFingerprint id3(/*namespace_id=*/2, /*fingerprint=*/56); + NamespaceIdFingerprint id4(/*namespace_id=*/0, /*fingerprint=*/78); { // Create new qualified id join index @@ -704,38 +694,38 @@ EXPECT_THAT( index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id1}), + /*ref_namespace_id_uri_fingerprints=*/{id1}), IsOk()); EXPECT_THAT( index->Put(schema_type_id1, joinable_property_id2, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id2}), + /*ref_namespace_id_uri_fingerprints=*/{id2}), IsOk()); EXPECT_THAT( index->Put(schema_type_id2, joinable_property_id1, /*document_id=*/12, - /*ref_namespace_fingerprint_ids=*/{id3}), + /*ref_namespace_id_uri_fingerprints=*/{id3}), IsOk()); EXPECT_THAT( index->Put(schema_type_id2, joinable_property_id2, /*document_id=*/12, - /*ref_namespace_fingerprint_ids=*/{id4}), + /*ref_namespace_id_uri_fingerprints=*/{id4}), IsOk()); EXPECT_THAT(index, Pointee(SizeIs(4))); - EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id1), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id1)))); - EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id2), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id2)))); - EXPECT_THAT(GetJoinData(*index, schema_type_id2, joinable_property_id1), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/12, /*join_info=*/id3)))); - EXPECT_THAT(GetJoinData(*index, schema_type_id2, joinable_property_id2), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/12, /*join_info=*/id4)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id1, joinable_property_id1), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id1)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id1, joinable_property_id2), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id2)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id2, joinable_property_id1), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/12, /*join_info=*/id3)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id2, joinable_property_id2), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/12, /*join_info=*/id4)))); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -746,22 +736,22 @@ QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_, /*pre_mapping_fbv=*/false)); EXPECT_THAT(index, Pointee(SizeIs(4))); - EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id1), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id1)))); - EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id2), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/5, /*join_info=*/id2)))); - EXPECT_THAT(GetJoinData(*index, schema_type_id2, joinable_property_id1), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/12, /*join_info=*/id3)))); - EXPECT_THAT(GetJoinData(*index, schema_type_id2, joinable_property_id2), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/12, /*join_info=*/id4)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id1, joinable_property_id1), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id1)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id1, joinable_property_id2), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/5, /*join_info=*/id2)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id2, joinable_property_id1), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/12, /*join_info=*/id3)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id2, joinable_property_id2), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/12, /*join_info=*/id4)))); } TEST_F(QualifiedIdJoinIndexImplV2Test, SetLastAddedDocumentId) { @@ -813,34 +803,34 @@ JoinablePropertyId joinable_property_id1 = 11; JoinablePropertyId joinable_property_id2 = 15; - NamespaceFingerprintIdentifier id1(/*namespace_id=*/2, /*fingerprint=*/101); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/3, /*fingerprint=*/102); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/4, /*fingerprint=*/103); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/0, /*fingerprint=*/104); - NamespaceFingerprintIdentifier id5(/*namespace_id=*/0, /*fingerprint=*/105); - NamespaceFingerprintIdentifier id6(/*namespace_id=*/1, /*fingerprint=*/106); - NamespaceFingerprintIdentifier id7(/*namespace_id=*/3, /*fingerprint=*/107); - NamespaceFingerprintIdentifier id8(/*namespace_id=*/2, /*fingerprint=*/108); + NamespaceIdFingerprint id1(/*namespace_id=*/2, /*fingerprint=*/101); + NamespaceIdFingerprint id2(/*namespace_id=*/3, /*fingerprint=*/102); + NamespaceIdFingerprint id3(/*namespace_id=*/4, /*fingerprint=*/103); + NamespaceIdFingerprint id4(/*namespace_id=*/0, /*fingerprint=*/104); + NamespaceIdFingerprint id5(/*namespace_id=*/0, /*fingerprint=*/105); + NamespaceIdFingerprint id6(/*namespace_id=*/1, /*fingerprint=*/106); + NamespaceIdFingerprint id7(/*namespace_id=*/3, /*fingerprint=*/107); + NamespaceIdFingerprint id8(/*namespace_id=*/2, /*fingerprint=*/108); EXPECT_THAT( index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/3, - /*ref_namespace_fingerprint_ids=*/{id1, id2, id3}), + /*ref_namespace_id_uri_fingerprints=*/{id1, id2, id3}), IsOk()); EXPECT_THAT( index->Put(schema_type_id2, joinable_property_id2, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id4}), + /*ref_namespace_id_uri_fingerprints=*/{id4}), IsOk()); EXPECT_THAT( index->Put(schema_type_id2, joinable_property_id2, /*document_id=*/8, - /*ref_namespace_fingerprint_ids=*/{id5, id6}), + /*ref_namespace_id_uri_fingerprints=*/{id5, id6}), IsOk()); EXPECT_THAT( index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/13, - /*ref_namespace_fingerprint_ids=*/{id7}), + /*ref_namespace_id_uri_fingerprints=*/{id7}), IsOk()); EXPECT_THAT( index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/21, - /*ref_namespace_fingerprint_ids=*/{id8}), + /*ref_namespace_id_uri_fingerprints=*/{id8}), IsOk()); index->set_last_added_document_id(21); @@ -877,15 +867,14 @@ // - old_doc_id=3, old_ref_namespace_id=2: NOT FOUND // // For new_doc_id=0, it should reorder due to posting list restriction. - EXPECT_THAT( - GetJoinData(*index, schema_type_id1, joinable_property_id1), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/2, /*fingerprint=*/102)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/0, /*fingerprint=*/103))))); + EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id1), + IsOkAndHolds(ElementsAre( + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/2, /*fingerprint=*/102)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/0, /*fingerprint=*/103))))); // 2) schema_type_id2, joinable_property_id2: // - old_doc_id=8, old_ref_namespace_id=1: NOT FOUND @@ -894,32 +883,30 @@ // - old_doc_id=5, old_ref_namespace_id=0: NOT FOUND EXPECT_THAT( GetJoinData(*index, schema_type_id2, joinable_property_id2), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/105))))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/1, /*fingerprint=*/105))))); // Verify Put API should work normally after Optimize(). - NamespaceFingerprintIdentifier id9(/*namespace_id=*/1, /*fingerprint=*/109); + NamespaceIdFingerprint id9(/*namespace_id=*/1, /*fingerprint=*/109); EXPECT_THAT( index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/99, - /*ref_namespace_fingerprint_ids=*/{id9}), + /*ref_namespace_id_uri_fingerprints=*/{id9}), IsOk()); index->set_last_added_document_id(99); EXPECT_THAT(index, Pointee(SizeIs(4))); EXPECT_THAT(index->last_added_document_id(), Eq(99)); - EXPECT_THAT( - GetJoinData(*index, schema_type_id1, joinable_property_id1), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/99, /*join_info=*/id9), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/2, /*fingerprint=*/102)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/0, /*fingerprint=*/103))))); + EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id1), + IsOkAndHolds(ElementsAre( + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/99, /*join_info=*/id9), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/2, /*fingerprint=*/102)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/0, /*fingerprint=*/103))))); } TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeDocumentIdChange) { @@ -933,32 +920,32 @@ SchemaTypeId schema_type_id = 2; JoinablePropertyId joinable_property_id = 1; - NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/101); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/102); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/103); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/104); - NamespaceFingerprintIdentifier id5(/*namespace_id=*/1, /*fingerprint=*/105); - NamespaceFingerprintIdentifier id6(/*namespace_id=*/1, /*fingerprint=*/106); + NamespaceIdFingerprint id1(/*namespace_id=*/1, /*fingerprint=*/101); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/102); + NamespaceIdFingerprint id3(/*namespace_id=*/1, /*fingerprint=*/103); + NamespaceIdFingerprint id4(/*namespace_id=*/1, /*fingerprint=*/104); + NamespaceIdFingerprint id5(/*namespace_id=*/1, /*fingerprint=*/105); + NamespaceIdFingerprint id6(/*namespace_id=*/1, /*fingerprint=*/106); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/3, - /*ref_namespace_fingerprint_ids=*/{id1, id2}), + /*ref_namespace_id_uri_fingerprints=*/{id1, id2}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id3}), + /*ref_namespace_id_uri_fingerprints=*/{id3}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/8, - /*ref_namespace_fingerprint_ids=*/{id4}), + /*ref_namespace_id_uri_fingerprints=*/{id4}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/13, - /*ref_namespace_fingerprint_ids=*/{id5}), + /*ref_namespace_id_uri_fingerprints=*/{id5}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/21, - /*ref_namespace_fingerprint_ids=*/{id6}), + /*ref_namespace_id_uri_fingerprints=*/{id6}), IsOk()); index->set_last_added_document_id(21); @@ -987,39 +974,39 @@ // - old_doc_id=5, join_info=id3: NOT FOUND // - old_doc_id=3, join_info=id2: become doc_id=0, join_info=id2 // - old_doc_id=3, join_info=id1: become doc_id=0, join_info=id1 - EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/2, /*join_info=*/id6), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, /*join_info=*/id5), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/id2), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/id1)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id, joinable_property_id), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/2, /*join_info=*/id6), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, /*join_info=*/id5), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/id2), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/id1)))); // Verify Put API should work normally after Optimize(). - NamespaceFingerprintIdentifier id7(/*namespace_id=*/1, /*fingerprint=*/107); + NamespaceIdFingerprint id7(/*namespace_id=*/1, /*fingerprint=*/107); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/99, - /*ref_namespace_fingerprint_ids=*/{id7}), + /*ref_namespace_id_uri_fingerprints=*/{id7}), IsOk()); index->set_last_added_document_id(99); EXPECT_THAT(index, Pointee(SizeIs(5))); EXPECT_THAT(index->last_added_document_id(), Eq(99)); - EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/99, /*join_info=*/id7), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/2, /*join_info=*/id6), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, /*join_info=*/id5), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/id2), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/id1)))); + EXPECT_THAT( + GetJoinData(*index, schema_type_id, joinable_property_id), + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/99, /*join_info=*/id7), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/2, /*join_info=*/id6), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, /*join_info=*/id5), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/id2), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/id1)))); } TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeOutOfRangeDocumentId) { @@ -1032,11 +1019,11 @@ SchemaTypeId schema_type_id = 2; JoinablePropertyId joinable_property_id = 1; - NamespaceFingerprintIdentifier id(/*namespace_id=*/1, /*fingerprint=*/101); + NamespaceIdFingerprint id(/*namespace_id=*/1, /*fingerprint=*/101); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/99, - /*ref_namespace_fingerprint_ids=*/{id}), + /*ref_namespace_id_uri_fingerprints=*/{id}), IsOk()); index->set_last_added_document_id(99); @@ -1069,32 +1056,32 @@ SchemaTypeId schema_type_id = 2; JoinablePropertyId joinable_property_id = 1; - NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/101); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/102); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/103); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/104); - NamespaceFingerprintIdentifier id5(/*namespace_id=*/1, /*fingerprint=*/105); - NamespaceFingerprintIdentifier id6(/*namespace_id=*/1, /*fingerprint=*/106); + NamespaceIdFingerprint id1(/*namespace_id=*/1, /*fingerprint=*/101); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/102); + NamespaceIdFingerprint id3(/*namespace_id=*/1, /*fingerprint=*/103); + NamespaceIdFingerprint id4(/*namespace_id=*/1, /*fingerprint=*/104); + NamespaceIdFingerprint id5(/*namespace_id=*/1, /*fingerprint=*/105); + NamespaceIdFingerprint id6(/*namespace_id=*/1, /*fingerprint=*/106); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/3, - /*ref_namespace_fingerprint_ids=*/{id1, id2}), + /*ref_namespace_id_uri_fingerprints=*/{id1, id2}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id3}), + /*ref_namespace_id_uri_fingerprints=*/{id3}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/8, - /*ref_namespace_fingerprint_ids=*/{id4}), + /*ref_namespace_id_uri_fingerprints=*/{id4}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/13, - /*ref_namespace_fingerprint_ids=*/{id5}), + /*ref_namespace_id_uri_fingerprints=*/{id5}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/21, - /*ref_namespace_fingerprint_ids=*/{id6}), + /*ref_namespace_id_uri_fingerprints=*/{id6}), IsOk()); index->set_last_added_document_id(21); @@ -1129,36 +1116,36 @@ SchemaTypeId schema_type_id = 2; JoinablePropertyId joinable_property_id = 1; - NamespaceFingerprintIdentifier id1(/*namespace_id=*/3, /*fingerprint=*/101); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/5, /*fingerprint=*/102); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/4, /*fingerprint=*/103); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/0, /*fingerprint=*/104); - NamespaceFingerprintIdentifier id5(/*namespace_id=*/2, /*fingerprint=*/105); - NamespaceFingerprintIdentifier id6(/*namespace_id=*/1, /*fingerprint=*/106); + NamespaceIdFingerprint id1(/*namespace_id=*/3, /*fingerprint=*/101); + NamespaceIdFingerprint id2(/*namespace_id=*/5, /*fingerprint=*/102); + NamespaceIdFingerprint id3(/*namespace_id=*/4, /*fingerprint=*/103); + NamespaceIdFingerprint id4(/*namespace_id=*/0, /*fingerprint=*/104); + NamespaceIdFingerprint id5(/*namespace_id=*/2, /*fingerprint=*/105); + NamespaceIdFingerprint id6(/*namespace_id=*/1, /*fingerprint=*/106); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/2, - /*ref_namespace_fingerprint_ids=*/{id1}), + /*ref_namespace_id_uri_fingerprints=*/{id1}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/3, - /*ref_namespace_fingerprint_ids=*/{id2}), + /*ref_namespace_id_uri_fingerprints=*/{id2}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id3}), + /*ref_namespace_id_uri_fingerprints=*/{id3}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/8, - /*ref_namespace_fingerprint_ids=*/{id4}), + /*ref_namespace_id_uri_fingerprints=*/{id4}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/13, - /*ref_namespace_fingerprint_ids=*/{id5}), + /*ref_namespace_id_uri_fingerprints=*/{id5}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/21, - /*ref_namespace_fingerprint_ids=*/{id6}), + /*ref_namespace_id_uri_fingerprints=*/{id6}), IsOk()); index->set_last_added_document_id(21); @@ -1190,49 +1177,47 @@ // - id3 (old_namespace_id=4): NOT FOUND // - id2 (old_namespace_id=5): new_namespace_id=0 (document_id = 3) // - id1 (old_namespace_id=3): new_namespace_id=1 (document_id = 2) - EXPECT_THAT( - GetJoinData(*index, schema_type_id, joinable_property_id), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/21, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/3, /*fingerprint=*/106)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/8, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/2, /*fingerprint=*/104)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/3, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/0, /*fingerprint=*/102)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/2, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/101))))); + EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id), + IsOkAndHolds(ElementsAre( + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/21, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/3, /*fingerprint=*/106)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/8, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/2, /*fingerprint=*/104)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/3, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/0, /*fingerprint=*/102)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/2, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/1, /*fingerprint=*/101))))); // Verify Put API should work normally after Optimize(). - NamespaceFingerprintIdentifier id7(/*namespace_id=*/1, /*fingerprint=*/107); + NamespaceIdFingerprint id7(/*namespace_id=*/1, /*fingerprint=*/107); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/99, - /*ref_namespace_fingerprint_ids=*/{id7}), + /*ref_namespace_id_uri_fingerprints=*/{id7}), IsOk()); index->set_last_added_document_id(99); EXPECT_THAT(index, Pointee(SizeIs(5))); EXPECT_THAT(index->last_added_document_id(), Eq(99)); - EXPECT_THAT( - GetJoinData(*index, schema_type_id, joinable_property_id), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/99, /*join_info=*/id7), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/21, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/3, /*fingerprint=*/106)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/8, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/2, /*fingerprint=*/104)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/3, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/0, /*fingerprint=*/102)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/2, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/101))))); + EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id), + IsOkAndHolds(ElementsAre( + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/99, /*join_info=*/id7), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/21, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/3, /*fingerprint=*/106)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/8, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/2, /*fingerprint=*/104)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/3, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/0, /*fingerprint=*/102)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/2, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/1, /*fingerprint=*/101))))); } TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeNamespaceIdChangeShouldReorder) { @@ -1246,18 +1231,18 @@ SchemaTypeId schema_type_id = 2; JoinablePropertyId joinable_property_id = 1; - NamespaceFingerprintIdentifier id1(/*namespace_id=*/0, /*fingerprint=*/101); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/102); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/2, /*fingerprint=*/103); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/104); + NamespaceIdFingerprint id1(/*namespace_id=*/0, /*fingerprint=*/101); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/102); + NamespaceIdFingerprint id3(/*namespace_id=*/2, /*fingerprint=*/103); + NamespaceIdFingerprint id4(/*namespace_id=*/1, /*fingerprint=*/104); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/0, - /*ref_namespace_fingerprint_ids=*/{id1, id2, id3}), + /*ref_namespace_id_uri_fingerprints=*/{id1, id2, id3}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/1, - /*ref_namespace_fingerprint_ids=*/{id4}), + /*ref_namespace_id_uri_fingerprints=*/{id4}), IsOk()); index->set_last_added_document_id(1); @@ -1283,21 +1268,20 @@ // - id1 (old_namespace_id=0): new_namespace_id=2 (document_id = 0) // // Should reorder to [id4, id1, id3, id2] due to posting list restriction. - EXPECT_THAT( - GetJoinData(*index, schema_type_id, joinable_property_id), - IsOkAndHolds(ElementsAre( - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/1, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/0, /*fingerprint=*/104)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/2, /*fingerprint=*/101)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/1, /*fingerprint=*/103)), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier( - /*namespace_id=*/0, /*fingerprint=*/102))))); + EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id), + IsOkAndHolds(ElementsAre( + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/1, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/0, /*fingerprint=*/104)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/2, /*fingerprint=*/101)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/1, /*fingerprint=*/103)), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/0, /*join_info=*/NamespaceIdFingerprint( + /*namespace_id=*/0, /*fingerprint=*/102))))); } TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeOutOfRangeNamespaceId) { @@ -1310,11 +1294,11 @@ SchemaTypeId schema_type_id = 2; JoinablePropertyId joinable_property_id = 1; - NamespaceFingerprintIdentifier id(/*namespace_id=*/99, /*fingerprint=*/101); + NamespaceIdFingerprint id(/*namespace_id=*/99, /*fingerprint=*/101); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/0, - /*ref_namespace_fingerprint_ids=*/{id}), + /*ref_namespace_id_uri_fingerprints=*/{id}), IsOk()); index->set_last_added_document_id(0); @@ -1347,21 +1331,21 @@ SchemaTypeId schema_type_id = 2; JoinablePropertyId joinable_property_id = 1; - NamespaceFingerprintIdentifier id1(/*namespace_id=*/0, /*fingerprint=*/101); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/102); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/2, /*fingerprint=*/103); + NamespaceIdFingerprint id1(/*namespace_id=*/0, /*fingerprint=*/101); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/102); + NamespaceIdFingerprint id3(/*namespace_id=*/2, /*fingerprint=*/103); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/0, - /*ref_namespace_fingerprint_ids=*/{id1}), + /*ref_namespace_id_uri_fingerprints=*/{id1}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/1, - /*ref_namespace_fingerprint_ids=*/{id2}), + /*ref_namespace_id_uri_fingerprints=*/{id2}), IsOk()); EXPECT_THAT( index->Put(schema_type_id, joinable_property_id, /*document_id=*/2, - /*ref_namespace_fingerprint_ids=*/{id3}), + /*ref_namespace_id_uri_fingerprints=*/{id3}), IsOk()); index->set_last_added_document_id(3); @@ -1386,10 +1370,10 @@ } TEST_F(QualifiedIdJoinIndexImplV2Test, Clear) { - NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/12); - NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34); - NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/56); - NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/78); + NamespaceIdFingerprint id1(/*namespace_id=*/1, /*fingerprint=*/12); + NamespaceIdFingerprint id2(/*namespace_id=*/1, /*fingerprint=*/34); + NamespaceIdFingerprint id3(/*namespace_id=*/1, /*fingerprint=*/56); + NamespaceIdFingerprint id4(/*namespace_id=*/1, /*fingerprint=*/78); // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( @@ -1399,13 +1383,13 @@ // Insert some data. ICING_ASSERT_OK(index->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5, - /*ref_namespace_fingerprint_ids=*/{id2, id1})); + /*ref_namespace_id_uri_fingerprints=*/{id2, id1})); ICING_ASSERT_OK(index->Put( /*schema_type_id=*/3, /*joinable_property_id=*/10, /*document_id=*/6, - /*ref_namespace_fingerprint_ids=*/{id3})); + /*ref_namespace_id_uri_fingerprints=*/{id3})); ICING_ASSERT_OK(index->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/12, - /*ref_namespace_fingerprint_ids=*/{id4})); + /*ref_namespace_id_uri_fingerprints=*/{id4})); ASSERT_THAT(index, Pointee(SizeIs(4))); index->set_last_added_document_id(12); ASSERT_THAT(index->last_added_document_id(), Eq(12)); @@ -1425,22 +1409,21 @@ // Join index should be able to work normally after Clear(). ICING_ASSERT_OK(index->Put( /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/20, - /*ref_namespace_fingerprint_ids=*/{id4, id2, id1, id3})); + /*ref_namespace_id_uri_fingerprints=*/{id4, id2, id1, id3})); index->set_last_added_document_id(20); EXPECT_THAT(index, Pointee(SizeIs(4))); EXPECT_THAT(index->last_added_document_id(), Eq(20)); EXPECT_THAT( GetJoinData(*index, /*schema_type_id=*/2, /*joinable_property_id=*/1), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/20, /*join_info=*/id4), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/20, /*join_info=*/id3), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/20, /*join_info=*/id2), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/20, /*join_info=*/id1)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/20, /*join_info=*/id4), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/20, /*join_info=*/id3), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/20, /*join_info=*/id2), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/20, /*join_info=*/id1)))); ICING_ASSERT_OK(index->PersistToDisk()); index.reset(); @@ -1452,15 +1435,14 @@ EXPECT_THAT(index->last_added_document_id(), Eq(20)); EXPECT_THAT( GetJoinData(*index, /*schema_type_id=*/2, /*joinable_property_id=*/1), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/20, /*join_info=*/id4), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/20, /*join_info=*/id3), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/20, /*join_info=*/id2), - DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/20, /*join_info=*/id1)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/20, /*join_info=*/id4), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/20, /*join_info=*/id3), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/20, /*join_info=*/id2), + DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/20, /*join_info=*/id1)))); } } // namespace
diff --git a/icing/join/qualified-id-join-index-impl-v3.cc b/icing/join/qualified-id-join-index-impl-v3.cc new file mode 100644 index 0000000..bab844d --- /dev/null +++ b/icing/join/qualified-id-join-index-impl-v3.cc
@@ -0,0 +1,794 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/join/qualified-id-join-index-impl-v3.h" + +#include <algorithm> +#include <cstdint> +#include <cstring> +#include <memory> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/absl_ports/str_cat.h" +#include "icing/feature-flags.h" +#include "icing/file/destructible-directory.h" +#include "icing/file/file-backed-vector.h" +#include "icing/file/filesystem.h" +#include "icing/file/memory-mapped-file.h" +#include "icing/join/document-join-id-pair.h" +#include "icing/join/qualified-id-join-index.h" +#include "icing/store/document-id.h" +#include "icing/store/namespace-id.h" +#include "icing/util/crc32.h" +#include "icing/util/logging.h" +#include "icing/util/math-util.h" +#include "icing/util/status-macros.h" + +namespace icing { +namespace lib { + +static constexpr DocumentJoinIdPair kInvalidDocumentJoinIdPair; +static constexpr QualifiedIdJoinIndexImplV3::ArrayInfo kInvalidArrayInfo; + +namespace { + +std::string MakeMetadataFilePath(std::string_view working_path) { + return absl_ports::StrCat(working_path, "/metadata"); +} + +std::string MakeParentDocumentIdToChildArrayInfoFilePath( + std::string_view working_path) { + return absl_ports::StrCat(working_path, + "/parent_document_id_to_child_array_info"); +} + +std::string MakeChildDocumentJoinIdPairArrayFilePath( + std::string_view working_path) { + return absl_ports::StrCat(working_path, "/child_document_join_id_pair_array"); +} + +} // namespace + +/* static */ libtextclassifier3::StatusOr< + std::unique_ptr<QualifiedIdJoinIndexImplV3>> +QualifiedIdJoinIndexImplV3::Create(const Filesystem& filesystem, + std::string working_path, + const FeatureFlags& feature_flags) { + bool metadata_file_exists = + filesystem.FileExists(MakeMetadataFilePath(working_path).c_str()); + bool parent_document_id_to_child_array_info_file_exists = + filesystem.FileExists( + MakeParentDocumentIdToChildArrayInfoFilePath(working_path).c_str()); + bool child_document_join_id_pair_array_file_exists = filesystem.FileExists( + MakeChildDocumentJoinIdPairArrayFilePath(working_path).c_str()); + + // If all files exist, initialize from existing files. + if (metadata_file_exists && + parent_document_id_to_child_array_info_file_exists && + child_document_join_id_pair_array_file_exists) { + return InitializeExistingFiles(filesystem, std::move(working_path), + feature_flags); + } + + // If all files don't exist, initialize new files. + if (!metadata_file_exists && + !parent_document_id_to_child_array_info_file_exists && + !child_document_join_id_pair_array_file_exists) { + return InitializeNewFiles(filesystem, std::move(working_path), + feature_flags); + } + + // Otherwise, some files are missing, and the join index is somehow corrupted. + // Return error to let the caller discard and rebuild. + return absl_ports::FailedPreconditionError( + "Inconsistent state of qualified id join index (v3)"); +} + +QualifiedIdJoinIndexImplV3::~QualifiedIdJoinIndexImplV3() { + if (!PersistToDisk().ok()) { + ICING_LOG(WARNING) << "Failed to persist qualified id join index (v3) to " + "disk while destructing " + << working_path_; + } +} + +libtextclassifier3::Status QualifiedIdJoinIndexImplV3::Put( + const DocumentJoinIdPair& child_document_join_id_pair, + std::vector<DocumentId>&& parent_document_ids) { + if (!child_document_join_id_pair.is_valid()) { + return absl_ports::InvalidArgumentError("Invalid child DocumentJoinIdPair"); + } + + if (parent_document_ids.empty()) { + return libtextclassifier3::Status::OK; + } + + SetDirty(); + + // Sort and dedupe parent document ids. It will be more efficient to access + // parent_document_id_to_child_array_info_ in the order of ids. + std::sort(parent_document_ids.begin(), parent_document_ids.end()); + auto last = + std::unique(parent_document_ids.begin(), parent_document_ids.end()); + parent_document_ids.erase(last, parent_document_ids.end()); + + // Append child_document_join_id_pair to each parent's DocumentJoinIdPair + // array. + for (DocumentId parent_document_id : parent_document_ids) { + if (parent_document_id < 0 || parent_document_id == kInvalidDocumentId) { + // Skip invalid parent document id. + continue; + } + + // Insert child_document_join_id_pair into the parent's DocumentJoinIdPair + // array. + ICING_RETURN_IF_ERROR(AppendChildDocumentJoinIdPairsForParent( + parent_document_id, {child_document_join_id_pair})); + } + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::StatusOr<std::vector<DocumentJoinIdPair>> +QualifiedIdJoinIndexImplV3::Get(DocumentId parent_document_id) const { + if (parent_document_id < 0 || parent_document_id == kInvalidDocumentId) { + return absl_ports::InvalidArgumentError("Invalid parent document id"); + } + + if (parent_document_id >= + parent_document_id_to_child_array_info_->num_elements()) { + return std::vector<DocumentJoinIdPair>(); + } + + // Get the child array info for the parent. + ICING_ASSIGN_OR_RETURN( + const ArrayInfo* array_info, + parent_document_id_to_child_array_info_->Get(parent_document_id)); + if (!array_info->IsValid()) { + return std::vector<DocumentJoinIdPair>(); + } + + // Safe check to avoid out-of-bound access. This should never happen unless + // the internal data structure is corrupted. + if (array_info->index + array_info->length > + child_document_join_id_pair_array_->num_elements()) { + return absl_ports::InternalError(absl_ports::StrCat( + "Invalid array info: ", std::to_string(array_info->index), " + ", + std::to_string(array_info->length), " > ", + std::to_string(child_document_join_id_pair_array_->num_elements()))); + } + + // Get the DocumentJoinIdPair array and return the child DocumentJoinIdPairs. + ICING_ASSIGN_OR_RETURN( + const DocumentJoinIdPair* ptr, + child_document_join_id_pair_array_->Get(array_info->index)); + return std::vector<DocumentJoinIdPair>(ptr, ptr + array_info->used_length); +} + +libtextclassifier3::Status QualifiedIdJoinIndexImplV3::MigrateParent( + DocumentId old_document_id, DocumentId new_document_id) { + if (!IsDocumentIdValid(old_document_id) || + !IsDocumentIdValid(new_document_id)) { + return absl_ports::InvalidArgumentError( + "Invalid parent document id to migrate"); + } + + if (old_document_id >= + parent_document_id_to_child_array_info_->num_elements()) { + // It means the parent document doesn't have any children at this moment. No + // need to migrate. + return libtextclassifier3::Status::OK; + } + + ICING_ASSIGN_OR_RETURN( + FileBackedVector<ArrayInfo>::MutableView mutable_old_array_info, + parent_document_id_to_child_array_info_->GetMutable(old_document_id)); + if (!mutable_old_array_info.Get().IsValid()) { + // It means the parent document doesn't have any children at this moment. No + // need to migrate. + return libtextclassifier3::Status::OK; + } + + ICING_RETURN_IF_ERROR( + ExtendParentDocumentIdToChildArrayInfoIfNecessary(new_document_id)); + ICING_RETURN_IF_ERROR(parent_document_id_to_child_array_info_->Set( + new_document_id, mutable_old_array_info.Get())); + mutable_old_array_info.Get() = kInvalidArrayInfo; + + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::Status QualifiedIdJoinIndexImplV3::Optimize( + const std::vector<DocumentId>& document_id_old_to_new, + const std::vector<NamespaceId>& namespace_id_old_to_new, + DocumentId new_last_added_document_id) { + std::string temp_working_path = working_path_ + "_temp"; + ICING_RETURN_IF_ERROR( + QualifiedIdJoinIndex::Discard(filesystem_, temp_working_path)); + + DestructibleDirectory temp_working_path_ddir(&filesystem_, + std::move(temp_working_path)); + if (!temp_working_path_ddir.is_valid()) { + return absl_ports::InternalError( + "Unable to create temp directory to build new qualified id join index " + "(v3)"); + } + + { + // Transfer all data from the current to new qualified id join index. Also + // PersistToDisk and destruct the instance after finishing, so we can safely + // swap directories later. + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> new_index, + Create(filesystem_, temp_working_path_ddir.dir(), feature_flags_)); + ICING_RETURN_IF_ERROR( + TransferIndex(document_id_old_to_new, new_index.get())); + new_index->set_last_added_document_id(new_last_added_document_id); + new_index->SetDirty(); + + ICING_RETURN_IF_ERROR(new_index->PersistToDisk()); + } + + // Destruct current index's storage instances to safely swap directories. + child_document_join_id_pair_array_.reset(); + parent_document_id_to_child_array_info_.reset(); + metadata_mmapped_file_.reset(); + if (!filesystem_.SwapFiles(temp_working_path_ddir.dir().c_str(), + working_path_.c_str())) { + return absl_ports::InternalError( + "Unable to apply new qualified id join index (v3) due to failed swap"); + } + + // Reinitialize qualified id join index. + ICING_ASSIGN_OR_RETURN( + MemoryMappedFile metadata_mmapped_file, + MemoryMappedFile::Create(filesystem_, MakeMetadataFilePath(working_path_), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + /*max_file_size=*/kMetadataFileSize, + /*pre_mapping_file_offset=*/0, + /*pre_mapping_mmap_size=*/kMetadataFileSize)); + metadata_mmapped_file_ = + std::make_unique<MemoryMappedFile>(std::move(metadata_mmapped_file)); + + ICING_ASSIGN_OR_RETURN( + parent_document_id_to_child_array_info_, + FileBackedVector<ArrayInfo>::Create( + filesystem_, + MakeParentDocumentIdToChildArrayInfoFilePath(working_path_), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<ArrayInfo>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + + ICING_ASSIGN_OR_RETURN( + child_document_join_id_pair_array_, + FileBackedVector<DocumentJoinIdPair>::Create( + filesystem_, MakeChildDocumentJoinIdPairArrayFilePath(working_path_), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<DocumentJoinIdPair>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::Status QualifiedIdJoinIndexImplV3::Clear() { + SetDirty(); + + parent_document_id_to_child_array_info_.reset(); + // Discard and reinitialize parent_document_id_to_child_array_info. + std::string parent_document_id_to_child_array_info_file_path = + MakeParentDocumentIdToChildArrayInfoFilePath(working_path_); + if (!filesystem_.DeleteFile( + parent_document_id_to_child_array_info_file_path.c_str())) { + return absl_ports::InternalError(absl_ports::StrCat( + "Failed to clear parent document id to child array info: ", + parent_document_id_to_child_array_info_file_path)); + } + ICING_ASSIGN_OR_RETURN( + parent_document_id_to_child_array_info_, + FileBackedVector<ArrayInfo>::Create( + filesystem_, parent_document_id_to_child_array_info_file_path, + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<ArrayInfo>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + + child_document_join_id_pair_array_.reset(); + // Discard and reinitialize child_document_join_id_pair_array. + std::string child_document_join_id_pair_array_file_path = + MakeChildDocumentJoinIdPairArrayFilePath(working_path_); + if (!filesystem_.DeleteFile( + child_document_join_id_pair_array_file_path.c_str())) { + return absl_ports::InternalError(absl_ports::StrCat( + "Failed to clear child document join id pair array: ", + child_document_join_id_pair_array_file_path)); + } + ICING_ASSIGN_OR_RETURN( + child_document_join_id_pair_array_, + FileBackedVector<DocumentJoinIdPair>::Create( + filesystem_, child_document_join_id_pair_array_file_path, + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<DocumentJoinIdPair>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + + info().num_data = 0; + info().last_added_document_id = kInvalidDocumentId; + return libtextclassifier3::Status::OK; +} + +/* static */ libtextclassifier3::StatusOr< + std::unique_ptr<QualifiedIdJoinIndexImplV3>> +QualifiedIdJoinIndexImplV3::InitializeNewFiles( + const Filesystem& filesystem, std::string working_path, + const FeatureFlags& feature_flags) { + // Create working directory. + if (!filesystem.CreateDirectoryRecursively(working_path.c_str())) { + return absl_ports::InternalError( + absl_ports::StrCat("Failed to create directory: ", working_path)); + } + + // Initialize metadata file. + // - Create MemoryMappedFile with pre-mapping and call GrowAndRemapIfNecessary + // to grow the underlying file. + // - Initialize metadata content by writing 0 bytes to the memory region + // directly. + ICING_ASSIGN_OR_RETURN( + MemoryMappedFile metadata_mmapped_file, + MemoryMappedFile::Create(filesystem, MakeMetadataFilePath(working_path), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + /*max_file_size=*/kMetadataFileSize, + /*pre_mapping_file_offset=*/0, + /*pre_mapping_mmap_size=*/kMetadataFileSize)); + ICING_RETURN_IF_ERROR(metadata_mmapped_file.GrowAndRemapIfNecessary( + /*file_offset=*/0, /*mmap_size=*/kMetadataFileSize)); + memset(metadata_mmapped_file.mutable_region(), 0, kMetadataFileSize); + + // Initialize parent_document_id_to_child_array_info. + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<FileBackedVector<ArrayInfo>> + parent_document_id_to_child_array_info, + FileBackedVector<ArrayInfo>::Create( + filesystem, + MakeParentDocumentIdToChildArrayInfoFilePath(working_path), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<ArrayInfo>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + + // Initialize child_document_join_id_pair_array. + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<FileBackedVector<DocumentJoinIdPair>> + child_document_join_id_pair_array, + FileBackedVector<DocumentJoinIdPair>::Create( + filesystem, MakeChildDocumentJoinIdPairArrayFilePath(working_path), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<DocumentJoinIdPair>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + + // Create instance. + auto new_join_index = std::unique_ptr<QualifiedIdJoinIndexImplV3>( + new QualifiedIdJoinIndexImplV3( + filesystem, std::move(working_path), + std::make_unique<MemoryMappedFile>(std::move(metadata_mmapped_file)), + std::move(parent_document_id_to_child_array_info), + std::move(child_document_join_id_pair_array), feature_flags)); + // Initialize info content. + new_join_index->info().magic = Info::kMagic; + new_join_index->info().num_data = 0; + new_join_index->info().last_added_document_id = kInvalidDocumentId; + + // Initialize new PersistentStorage. The initial checksums will be computed + // and set via InitializeNewStorage. + ICING_RETURN_IF_ERROR(new_join_index->InitializeNewStorage()); + + return new_join_index; +} + +/* static */ libtextclassifier3::StatusOr< + std::unique_ptr<QualifiedIdJoinIndexImplV3>> +QualifiedIdJoinIndexImplV3::InitializeExistingFiles( + const Filesystem& filesystem, std::string working_path, + const FeatureFlags& feature_flags) { + // Initialize metadata file. + ICING_ASSIGN_OR_RETURN( + MemoryMappedFile metadata_mmapped_file, + MemoryMappedFile::Create(filesystem, MakeMetadataFilePath(working_path), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + /*max_file_size=*/kMetadataFileSize, + /*pre_mapping_file_offset=*/0, + /*pre_mapping_mmap_size=*/kMetadataFileSize)); + if (metadata_mmapped_file.available_size() != kMetadataFileSize) { + return absl_ports::FailedPreconditionError("Incorrect metadata file size"); + } + + // Initialize parent_document_id_to_child_array_info. Set mmap pre-mapping + // size to 0, but MemoryMappedFile will still mmap to the file size. + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<FileBackedVector<ArrayInfo>> + parent_document_id_to_child_array_info, + FileBackedVector<ArrayInfo>::Create( + filesystem, + MakeParentDocumentIdToChildArrayInfoFilePath(working_path), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<ArrayInfo>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + + // Initialize child_document_join_id_pair_array. Set mmap pre-mapping size to + // 0, but MemoryMappedFile will still mmap to the file size. + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<FileBackedVector<DocumentJoinIdPair>> + child_document_join_id_pair_array, + FileBackedVector<DocumentJoinIdPair>::Create( + filesystem, MakeChildDocumentJoinIdPairArrayFilePath(working_path), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<DocumentJoinIdPair>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + + // Create instance. + auto join_index = std::unique_ptr<QualifiedIdJoinIndexImplV3>( + new QualifiedIdJoinIndexImplV3( + filesystem, std::move(working_path), + std::make_unique<MemoryMappedFile>(std::move(metadata_mmapped_file)), + std::move(parent_document_id_to_child_array_info), + std::move(child_document_join_id_pair_array), feature_flags)); + + // Initialize existing PersistentStorage. Checksums will be validated. + ICING_RETURN_IF_ERROR(join_index->InitializeExistingStorage()); + + // Validate magic. + if (join_index->info().magic != Info::kMagic) { + return absl_ports::FailedPreconditionError("Incorrect magic value"); + } + + return join_index; +} + +libtextclassifier3::Status +QualifiedIdJoinIndexImplV3::AppendChildDocumentJoinIdPairsForParent( + DocumentId parent_document_id, + std::vector<DocumentJoinIdPair>&& child_document_join_id_pairs) { + if (child_document_join_id_pairs.empty()) { + return libtextclassifier3::Status::OK; + } + + // Step 1: extend parent_document_id_to_child_array_info_ if necessary. + ICING_RETURN_IF_ERROR( + ExtendParentDocumentIdToChildArrayInfoIfNecessary(parent_document_id)); + + // Step 2: get the parent's child DocumentJoinIdPair mutable array (extend if + // necessary), append new child_document_join_id_pairs to the array, + // and assign the new ArrayInfo to + // parent_document_id_to_child_array_info_. + ICING_ASSIGN_OR_RETURN( + const ArrayInfo* array_info, + parent_document_id_to_child_array_info_->Get(parent_document_id)); + ICING_ASSIGN_OR_RETURN( + GetMutableAndExtendResult result, + GetMutableAndExtendChildDocumentJoinIdPairArrayIfNecessary( + *array_info, /*num_to_add=*/child_document_join_id_pairs.size())); + // - [0, result.array_info.used_length) contain valid elements. + // - We will write new elements starting from index + // result.array_info.used_length, and update the used_length. + result.mutable_arr.SetArray(/*idx=*/result.array_info.used_length, + /*arr=*/child_document_join_id_pairs.data(), + /*len=*/child_document_join_id_pairs.size()); + result.array_info.used_length += child_document_join_id_pairs.size(); + + // Set ArrayInfo back to parent_document_id_to_child_array_info_. + ICING_RETURN_IF_ERROR(parent_document_id_to_child_array_info_->Set( + parent_document_id, result.array_info)); + + // Update header. + info().num_data += child_document_join_id_pairs.size(); + + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::Status +QualifiedIdJoinIndexImplV3::ExtendParentDocumentIdToChildArrayInfoIfNecessary( + DocumentId parent_document_id) { + if (parent_document_id >= + parent_document_id_to_child_array_info_->num_elements()) { + int32_t num_to_extend = + (parent_document_id + 1) - + parent_document_id_to_child_array_info_->num_elements(); + ICING_ASSIGN_OR_RETURN( + FileBackedVector<ArrayInfo>::MutableArrayView mutable_arr, + parent_document_id_to_child_array_info_->Allocate(num_to_extend)); + mutable_arr.Fill(/*idx=*/0, /*len=*/num_to_extend, kInvalidArrayInfo); + } + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::StatusOr< + QualifiedIdJoinIndexImplV3::GetMutableAndExtendResult> +QualifiedIdJoinIndexImplV3:: + GetMutableAndExtendChildDocumentJoinIdPairArrayIfNecessary( + const ArrayInfo& array_info, int32_t num_to_add) { + if (num_to_add > kMaxNumChildrenPerParent) { + return absl_ports::ResourceExhaustedError( + absl_ports::StrCat("Too many children to add for a single parent: ", + std::to_string(num_to_add))); + } + + if (!array_info.IsValid()) { + // If the array info is invalid, then it means the array is not allocated + // yet. Allocate a new array and fill it with invalid values. + ArrayInfo new_array_info( + /*index_in=*/child_document_join_id_pair_array_->num_elements(), + /*length_in=*/math_util::NextPowerOf2(num_to_add), + /*used_length_in=*/0); + ICING_ASSIGN_OR_RETURN( + FileBackedVector<DocumentJoinIdPair>::MutableArrayView new_mutable_arr, + child_document_join_id_pair_array_->Allocate(new_array_info.length)); + new_mutable_arr.Fill(/*idx=*/0, /*len=*/new_array_info.length, + kInvalidDocumentJoinIdPair); + + return GetMutableAndExtendResult{.array_info = new_array_info, + .mutable_arr = std::move(new_mutable_arr)}; + } + + // The original array is valid. Get it and check if it needs to be extended. + ICING_ASSIGN_OR_RETURN(FileBackedVector<DocumentJoinIdPair>::MutableArrayView + original_mutable_arr, + child_document_join_id_pair_array_->GetMutable( + array_info.index, array_info.length)); + // - [0, array_info.used_length) contain valid elements. + // - [array_info.used_length, array_info.length) are invalid elements and + // available for new elements. + if (array_info.length - array_info.used_length >= num_to_add) { + // Available space is enough to fit num_to_add. Return the original array. + return GetMutableAndExtendResult{ + .array_info = array_info, + .mutable_arr = std::move(original_mutable_arr)}; + } + + // Need to extend. The new array length should be able to fit all existing + // elements and the newly added elements. + // + // Check the new # of elements should not exceed the max limit. Use + // substraction to avoid overflow. + if (num_to_add > kMaxNumChildrenPerParent - array_info.used_length) { + return absl_ports::ResourceExhaustedError(absl_ports::StrCat( + "Too many children to add for an existing parent: ", + std::to_string(num_to_add), + "; existing: ", std::to_string(array_info.used_length))); + } + + if (array_info.index + array_info.length == + child_document_join_id_pair_array_->num_elements()) { + // The original array is at the end of the FBV. We can extend and reshape it + // directly without copying elements. + // + // (Note: # means data out of the array_info we're dealing with.) + // Original FBV and array_info layout: + // +-------------------------+---------------.------+ + // |#########################| used |unused| + // +-------------------------+---------------.------+ + // ^ ^ + // index FBV_END + // |<-used_length->| + // |<- length ->| + // + // New FBV and new_array_info layout after extension: + // +-------------------------+---------------.------.------------------+ + // |#########################| used |unused| extended slice | + // +-------------------------+---------------.------.------------------+ + // ^ ^ + // index FBV_END + // |<-used_length->| + // |<- new length ->| + ArrayInfo new_array_info = array_info; + new_array_info.length = + math_util::NextPowerOf2(array_info.used_length + num_to_add); + + // Extend the original array by allocating a new slice. + ICING_ASSIGN_OR_RETURN( + FileBackedVector<DocumentJoinIdPair>::MutableArrayView extended_slice, + child_document_join_id_pair_array_->Allocate(new_array_info.length - + array_info.length)); + // Fill the extended slice with invalid values. + extended_slice.Fill(/*idx=*/0, + /*len=*/new_array_info.length - array_info.length, + kInvalidDocumentJoinIdPair); + + // Construct the mutable array view for new_array_info and return. + ICING_ASSIGN_OR_RETURN( + FileBackedVector<DocumentJoinIdPair>::MutableArrayView mutable_arr, + child_document_join_id_pair_array_->GetMutable(new_array_info.index, + new_array_info.length)); + + return GetMutableAndExtendResult{.array_info = std::move(new_array_info), + .mutable_arr = std::move(mutable_arr)}; + } + + // The original array is not at the end of the file. We allocate a new array + // starting at the end of the FBV, copy all existing elements to the new + // array, and fill the remaining with invalid values. + // + // Original FBV and array_info layout: + // +---+---------------.------+------+ + // |###| used |unused|######| + // +---+---------------.------+------+ + // ^ ^ + // index FBV_END + // |<-used_length->| + // |<- length ->| + // + // New FBV and new_array_info layout after extension and moving: + // +---+----------------------+------+---------------.-----------------------+ + // |###| <INVALID> |######| used | | + // +---+----------------------+------+---------------.-----------------------+ + // ^ + // new_index + // |<-used_length->| + // |<- new_length ->| + ArrayInfo new_array_info( + /*index_in=*/child_document_join_id_pair_array_->num_elements(), + /*length_in=*/ + math_util::NextPowerOf2(array_info.used_length + num_to_add), + /*used_length_in=*/array_info.used_length); + ICING_ASSIGN_OR_RETURN( + FileBackedVector<DocumentJoinIdPair>::MutableArrayView new_mutable_arr, + child_document_join_id_pair_array_->Allocate(new_array_info.length)); + + // It is possible that Allocate() causes FBV to grow and remap. In this case, + // original_mutable_arr will point to an invalid memory region, so we need to + // refresh it. + // + // Note: original_mutable_arr has length = array_info.length and only contains + // valid elements with length = array_info.used_length. We could've + // constructed original_mutable_arr with length = array_info.used_length + // here, but let's make it consistent with all other cases (i.e. + // constructing the array view object for the entire extensible array). + ICING_ASSIGN_OR_RETURN(original_mutable_arr, + child_document_join_id_pair_array_->GetMutable( + array_info.index, array_info.length)); + + // Move all existing elements to the new array. + new_mutable_arr.SetArray(/*idx=*/0, original_mutable_arr.data(), + array_info.used_length); + // Fill the remaining with invalid values. + new_mutable_arr.Fill(/*idx=*/array_info.used_length, + /*len=*/new_array_info.length - array_info.used_length, + kInvalidDocumentJoinIdPair); + // Invalidate the original array. + original_mutable_arr.Fill( + /*idx=*/0, /*len=*/array_info.length, kInvalidDocumentJoinIdPair); + return GetMutableAndExtendResult{.array_info = new_array_info, + .mutable_arr = std::move(new_mutable_arr)}; +} + +libtextclassifier3::Status QualifiedIdJoinIndexImplV3::TransferIndex( + const std::vector<DocumentId>& document_id_old_to_new, + QualifiedIdJoinIndexImplV3* new_index) const { + for (DocumentId old_parent_doc_id = 0; + old_parent_doc_id < + parent_document_id_to_child_array_info_->num_elements(); + ++old_parent_doc_id) { + if (old_parent_doc_id >= document_id_old_to_new.size() || + document_id_old_to_new[old_parent_doc_id] == kInvalidDocumentId) { + // Skip if the old parent document id is invalid after optimization. + continue; + } + + ICING_ASSIGN_OR_RETURN( + const ArrayInfo* array_info, + parent_document_id_to_child_array_info_->Get(old_parent_doc_id)); + if (!array_info->IsValid()) { + continue; + } + ICING_ASSIGN_OR_RETURN( + const DocumentJoinIdPair* ptr, + child_document_join_id_pair_array_->Get(array_info->index)); + + // Get all child DocumentJoinIdPairs and assign new child document ids. + std::vector<DocumentJoinIdPair> new_child_doc_join_id_pairs; + new_child_doc_join_id_pairs.reserve(array_info->length); + for (int i = 0; i < array_info->used_length; ++i) { + DocumentId old_child_doc_id = ptr[i].document_id(); + DocumentId new_child_doc_id = + old_child_doc_id >= 0 && + old_child_doc_id < document_id_old_to_new.size() + ? document_id_old_to_new[old_child_doc_id] + : kInvalidDocumentId; + if (new_child_doc_id == kInvalidDocumentId) { + continue; + } + + new_child_doc_join_id_pairs.push_back( + DocumentJoinIdPair(new_child_doc_id, ptr[i].joinable_property_id())); + } + + ICING_RETURN_IF_ERROR(new_index->AppendChildDocumentJoinIdPairsForParent( + document_id_old_to_new[old_parent_doc_id], + std::move(new_child_doc_join_id_pairs))); + } + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::Status QualifiedIdJoinIndexImplV3::PersistMetadataToDisk() { + // We can skip persisting metadata to disk only if both info and storage are + // clean. + if (is_initialized_ && !is_info_dirty() && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + + // Changes should have been applied to the underlying file when using + // MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, but call msync() as an + // extra safety step to ensure they are written out. + ICING_RETURN_IF_ERROR(metadata_mmapped_file_->PersistToDisk()); + is_info_dirty_ = false; + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::Status QualifiedIdJoinIndexImplV3::PersistStoragesToDisk() { + if (is_initialized_ && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + + ICING_RETURN_IF_ERROR( + parent_document_id_to_child_array_info_->PersistToDisk()); + ICING_RETURN_IF_ERROR(child_document_join_id_pair_array_->PersistToDisk()); + + is_storage_dirty_ = false; + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::StatusOr<Crc32> +QualifiedIdJoinIndexImplV3::UpdateStoragesChecksum() { + if (is_initialized_ && !is_storage_dirty()) { + return Crc32(crcs().component_crcs.storages_crc); + } + + // Compute crcs + ICING_ASSIGN_OR_RETURN( + Crc32 parent_document_id_to_child_array_info_crc, + parent_document_id_to_child_array_info_->UpdateChecksum()); + ICING_ASSIGN_OR_RETURN(Crc32 child_document_join_id_pair_array_crc, + child_document_join_id_pair_array_->UpdateChecksum()); + + return Crc32(parent_document_id_to_child_array_info_crc.Get() ^ + child_document_join_id_pair_array_crc.Get()); +} + +libtextclassifier3::StatusOr<Crc32> +QualifiedIdJoinIndexImplV3::GetInfoChecksum() const { + if (is_initialized_ && !is_info_dirty()) { + return Crc32(crcs().component_crcs.info_crc); + } + + return info().GetChecksum(); +} + +libtextclassifier3::StatusOr<Crc32> +QualifiedIdJoinIndexImplV3::GetStoragesChecksum() const { + if (is_initialized_ && !is_storage_dirty()) { + return Crc32(crcs().component_crcs.storages_crc); + } + + // Get checksums for all components. + Crc32 parent_document_id_to_child_array_info_crc = + parent_document_id_to_child_array_info_->GetChecksum(); + Crc32 child_document_join_id_pair_array_crc = + child_document_join_id_pair_array_->GetChecksum(); + + return Crc32(parent_document_id_to_child_array_info_crc.Get() ^ + child_document_join_id_pair_array_crc.Get()); +} + +} // namespace lib +} // namespace icing
diff --git a/icing/join/qualified-id-join-index-impl-v3.h b/icing/join/qualified-id-join-index-impl-v3.h new file mode 100644 index 0000000..2f5a8a5 --- /dev/null +++ b/icing/join/qualified-id-join-index-impl-v3.h
@@ -0,0 +1,414 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_JOIN_QUALIFIED_ID_JOIN_INDEX_IMPL_V3_H_ +#define ICING_JOIN_QUALIFIED_ID_JOIN_INDEX_IMPL_V3_H_ + +#include <cstdint> +#include <memory> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/feature-flags.h" +#include "icing/file/file-backed-vector.h" +#include "icing/file/filesystem.h" +#include "icing/file/memory-mapped-file.h" +#include "icing/file/persistent-storage.h" +#include "icing/join/document-join-id-pair.h" +#include "icing/join/qualified-id-join-index.h" +#include "icing/schema/joinable-property.h" +#include "icing/store/document-filter-data.h" +#include "icing/store/document-id.h" +#include "icing/store/namespace-id-fingerprint.h" +#include "icing/store/namespace-id.h" +#include "icing/util/crc32.h" + +namespace icing { +namespace lib { + +// QualifiedIdJoinIndexImplV3: a class to maintain join data. It uses 2 +// FileBackedVectors to maintain the mapping between parent document id and a +// list of its joinable child infos (document id, joinable property id). +class QualifiedIdJoinIndexImplV3 : public QualifiedIdJoinIndex { + public: + struct Info { + static constexpr int32_t kMagic = 0x1529a926; + + int32_t magic; + int32_t num_data; + DocumentId last_added_document_id; + + // Padding exists just to reserve space for additional values and make the + // size of the metadata file 1024 bytes. + static constexpr int kPaddingSize = 1000; + uint8_t padding[kPaddingSize]; + + Crc32 GetChecksum() const { + return Crc32( + std::string_view(reinterpret_cast<const char*>(this), sizeof(Info))); + } + } __attribute__((packed)); + static_assert(sizeof(Info) == 1012, ""); + + // Metadata file layout: <Crcs><Info<data><padding>> + static constexpr int32_t kCrcsMetadataFileOffset = 0; + static constexpr int32_t kInfoMetadataFileOffset = + static_cast<int32_t>(sizeof(Crcs)); + static constexpr int32_t kMetadataFileSize = sizeof(Crcs) + sizeof(Info); + static_assert(kMetadataFileSize == 1024, ""); + + static constexpr WorkingPathType kWorkingPathType = + WorkingPathType::kDirectory; + + // An internal struct to store the start index and length of a + // DocumentJoinIdPair array in child_document_join_id_pair_array_. + struct ArrayInfo { + int32_t index; + int32_t length; + int32_t used_length; + + constexpr ArrayInfo() : index(-1), length(-1), used_length(-1) {} + + explicit ArrayInfo(int32_t index_in, int32_t length_in, + int32_t used_length_in) + : index(index_in), length(length_in), used_length(used_length_in) {} + + bool operator==(const ArrayInfo& other) const { + return index == other.index && length == other.length && + used_length == other.used_length; + } + + bool IsValid() const { + return index >= 0 && length > 0 && used_length >= 0 && + used_length <= length; + } + } __attribute__((packed)); + static_assert(sizeof(ArrayInfo) == 12, "Invalid ArrayInfo size"); + + // Creates a QualifiedIdJoinIndexImplV3 instance to store parent document to a + // list of its joinable children for future joining search. If any of the + // underlying file is missing, then delete the whole working_path and + // (re)initialize with new ones. Otherwise initialize and create the instance + // by existing files. + // + // filesystem: Object to make system level calls + // working_path: Specifies the working path for PersistentStorage. + // QualifiedIdJoinIndexImplV3 uses working path as working + // directory and all related files will be stored under this + // directory. It takes full ownership and of working_path_, + // including creation/deletion. It is the caller's + // responsibility to specify correct working path and avoid + // mixing different persistent storages together under the same + // path. Also the caller has the ownership for the parent + // directory of working_path_, and it is responsible for parent + // directory creation/deletion. See PersistentStorage for more + // details about the concept of working_path. + // feature_flags: a reference to (global) icing search engine feature flags. + // + // Returns: + // - FAILED_PRECONDITION_ERROR if the state of the join index is + // inconsistent (e.g. checksum or the magic + // doesn't match the stored ones, missing some + // files, etc). + // - INTERNAL_ERROR on I/O errors + // - Any MemoryMappedFile, FileBackedVector errors + static libtextclassifier3::StatusOr< + std::unique_ptr<QualifiedIdJoinIndexImplV3>> + Create(const Filesystem& filesystem, std::string working_path, + const FeatureFlags& feature_flags); + + // Delete copy and move constructor/assignment operator. + QualifiedIdJoinIndexImplV3(const QualifiedIdJoinIndexImplV3&) = delete; + QualifiedIdJoinIndexImplV3& operator=(const QualifiedIdJoinIndexImplV3&) = + delete; + + QualifiedIdJoinIndexImplV3(QualifiedIdJoinIndexImplV3&&) = delete; + QualifiedIdJoinIndexImplV3& operator=(QualifiedIdJoinIndexImplV3&&) = delete; + + ~QualifiedIdJoinIndexImplV3() override; + + // Puts new join data into the index: adds a new child document and its + // referenced parent documents into the join index. + // + // Returns: + // - OK on success + // - INVALID_ARGUMENT_ERROR if child_document_join_id_pair is invalid + // - Any FileBackedVector errors + libtextclassifier3::Status Put( + const DocumentJoinIdPair& child_document_join_id_pair, + std::vector<DocumentId>&& parent_document_ids) override; + + // Gets the list of joinable children for the given parent document id. + // + // Returns: + // - A list of children's DocumentJoinIdPair on success + // - Any FileBackedVector errors + libtextclassifier3::StatusOr<std::vector<DocumentJoinIdPair>> Get( + DocumentId parent_document_id) const override; + + // Migrates existing join data for a parent document from old_document_id to + // new_document_id. + // + // Note: when updating a document, we have to migrate the document's join data + // if it is used as a parent document. For its child join data, it will be + // tokenized and indexed separately, so no migration is needed. + // + // Returns: + // - OK on success + // - INVALID_ARGUMENT_ERROR if any document id is invalid + // - Any FileBackedVector errors + libtextclassifier3::Status MigrateParent(DocumentId old_document_id, + DocumentId new_document_id) override; + + // v1 only API. Returns UNIMPLEMENTED_ERROR. + libtextclassifier3::Status Put( + const DocumentJoinIdPair& document_join_id_pair, + std::string_view ref_qualified_id_str) override { + return absl_ports::UnimplementedError("This API is not supported in V3"); + } + + // v1 only API. Returns UNIMPLEMENTED_ERROR. + libtextclassifier3::StatusOr<std::string_view> Get( + const DocumentJoinIdPair& document_join_id_pair) const override { + return absl_ports::UnimplementedError("This API is not supported in V3"); + } + + // v2 only API. Returns UNIMPLEMENTED_ERROR. + libtextclassifier3::Status Put( + SchemaTypeId schema_type_id, JoinablePropertyId joinable_property_id, + DocumentId document_id, + std::vector<NamespaceIdFingerprint>&& ref_namespace_id_uri_fingerprints) + override { + return absl_ports::UnimplementedError("This API is not supported in V3"); + } + + // v2 only API. Returns UNIMPLEMENTED_ERROR. + libtextclassifier3::StatusOr<std::unique_ptr<JoinDataIteratorBase>> + GetIterator(SchemaTypeId schema_type_id, + JoinablePropertyId joinable_property_id) const override { + return absl_ports::UnimplementedError("This API is not supported in V3"); + } + + libtextclassifier3::Status Optimize( + const std::vector<DocumentId>& document_id_old_to_new, + const std::vector<NamespaceId>& namespace_id_old_to_new, + DocumentId new_last_added_document_id) override; + + libtextclassifier3::Status Clear() override; + + QualifiedIdJoinIndex::Version version() const override { + return QualifiedIdJoinIndex::Version::kV3; + } + + int32_t size() const override { return info().num_data; } + + bool empty() const override { return size() == 0; } + + DocumentId last_added_document_id() const override { + return info().last_added_document_id; + } + + void set_last_added_document_id(DocumentId document_id) override { + SetInfoDirty(); + + Info& info_ref = info(); + if (info_ref.last_added_document_id == kInvalidDocumentId || + document_id > info_ref.last_added_document_id) { + info_ref.last_added_document_id = document_id; + } + } + + private: + // Set max # of child DocumentJoinIdPair per parent to 2^16. + // - It is unlikely that a single parent document will have extremely large + // # of children. + // - Also prevent over extending the array. + static constexpr int32_t kMaxNumChildrenPerParent = INT32_C(1) << 16; + static_assert(kMaxNumChildrenPerParent <= (1u << 31), + "Required for math_util::NextPowerOf2"); + + explicit QualifiedIdJoinIndexImplV3( + const Filesystem& filesystem, std::string working_path, + std::unique_ptr<MemoryMappedFile> metadata_mmapped_file, + std::unique_ptr<FileBackedVector<ArrayInfo>> + parent_document_id_to_child_array_info, + std::unique_ptr<FileBackedVector<DocumentJoinIdPair>> + child_document_join_id_pair_array, + const FeatureFlags& feature_flags) + : QualifiedIdJoinIndex(filesystem, std::move(working_path)), + metadata_mmapped_file_(std::move(metadata_mmapped_file)), + parent_document_id_to_child_array_info_( + std::move(parent_document_id_to_child_array_info)), + child_document_join_id_pair_array_( + std::move(child_document_join_id_pair_array)), + feature_flags_(feature_flags), + is_info_dirty_(false), + is_storage_dirty_(false) {} + + static libtextclassifier3::StatusOr< + std::unique_ptr<QualifiedIdJoinIndexImplV3>> + InitializeNewFiles(const Filesystem& filesystem, std::string working_path, + const FeatureFlags& feature_flags); + + static libtextclassifier3::StatusOr< + std::unique_ptr<QualifiedIdJoinIndexImplV3>> + InitializeExistingFiles(const Filesystem& filesystem, + std::string working_path, + const FeatureFlags& feature_flags); + + // Appends a list of new child DocumentJoinIdPair to the parent's + // DocumentJoinIdPair (extensible) array. If the array is invalid or doesn't + // have enough space for new elements, then extend it and set the new array + // info. + // + // Returns: + // - OK on success + // - RESOURCE_EXHAUSTED_ERROR if the new # of elements exceed + // kMaxNumChildrenPerParent + // - Any FileBackedVector errors + libtextclassifier3::Status AppendChildDocumentJoinIdPairsForParent( + DocumentId parent_document_id, + std::vector<DocumentJoinIdPair>&& child_document_join_id_pairs); + + // Extends the parent document id to child array info if necessary, according + // to the new parent document id. + // + // Returns: + // - OK on success + // - Any FileBackedVector errors + libtextclassifier3::Status ExtendParentDocumentIdToChildArrayInfoIfNecessary( + DocumentId parent_document_id); + + // Gets the DocumentJoinIdPair mutable array and extends it if necessary to + // fit num_to_add new elements in the future for the caller. If extended: + // - Allocate a new array with size rounded up to the next 2's power to fit + // all existing elements and num_to_add new elements. + // - Old elements will be migrated to the new array. + // - The old array will be invalidated. + // + // Note: this function only prepares the array to fit "future" num_to_add + // elements for the caller. It potentially does the extension but without + // adding any new elements into the array, so the returned ArrayInfo's + // used_length will remain unchanged. + // + // Returns: + // - GetOrExtendResult on success + // - RESOURCE_EXHAUSTED_ERROR if the new # of elements exceed + // kMaxNumChildrenPerParent + // - Any FileBackedVector errors + struct GetMutableAndExtendResult { + // ArrayInfo of the DocumentJoinIdPair array. + // - If extended (allocated a new array), then the new array's ArrayInfo + // will be returned. + // - Otherwise, the input (original) ArrayInfo will be returned. + ArrayInfo array_info; + + // MutableArrayView object of the DocumentJoinIdPair array corresponding to + // array_info. It will be either the new array after extension or the + // original array, depending on whether it is extended or not. + FileBackedVector<DocumentJoinIdPair>::MutableArrayView mutable_arr; + }; + libtextclassifier3::StatusOr<GetMutableAndExtendResult> + GetMutableAndExtendChildDocumentJoinIdPairArrayIfNecessary( + const ArrayInfo& array_info, int32_t num_to_add); + + // Transfers qualified id join index data from the current to new_index and + // convert to new document id according to document_id_old_to_new. It is a + // helper function for Optimize. + // + // Returns: + // - OK on success + // - INTERNAL_ERROR on I/O error + libtextclassifier3::Status TransferIndex( + const std::vector<DocumentId>& document_id_old_to_new, + QualifiedIdJoinIndexImplV3* new_index) const; + + libtextclassifier3::Status PersistMetadataToDisk() override; + + libtextclassifier3::Status PersistStoragesToDisk() override; + + libtextclassifier3::Status WriteMetadata() override { + // QualifiedIdJoinIndexImplV3::Header is mmapped. Therefore, writes occur + // when the metadata is modified. So just return OK. + return libtextclassifier3::Status::OK; + } + + libtextclassifier3::StatusOr<Crc32> UpdateStoragesChecksum() override; + + libtextclassifier3::StatusOr<Crc32> GetInfoChecksum() const override; + + libtextclassifier3::StatusOr<Crc32> GetStoragesChecksum() const override; + + Crcs& crcs() override { + return *reinterpret_cast<Crcs*>(metadata_mmapped_file_->mutable_region() + + kCrcsMetadataFileOffset); + } + + const Crcs& crcs() const override { + return *reinterpret_cast<const Crcs*>(metadata_mmapped_file_->region() + + kCrcsMetadataFileOffset); + } + + Info& info() { + return *reinterpret_cast<Info*>(metadata_mmapped_file_->mutable_region() + + kInfoMetadataFileOffset); + } + + const Info& info() const { + return *reinterpret_cast<const Info*>(metadata_mmapped_file_->region() + + kInfoMetadataFileOffset); + } + + void SetInfoDirty() { is_info_dirty_ = true; } + // When storage is dirty, we have to set info dirty as well. So just expose + // SetDirty to set both. + void SetDirty() { + is_info_dirty_ = true; + is_storage_dirty_ = true; + } + + bool is_info_dirty() const { return is_info_dirty_; } + bool is_storage_dirty() const { return is_storage_dirty_; } + + std::unique_ptr<MemoryMappedFile> metadata_mmapped_file_; + + // Storage for mapping parent document id to the ArrayInfo, which points to an + // extensible array stored in child_document_join_id_pair_array_, and the + // extensible array contains the parent's joinable children information. + std::unique_ptr<FileBackedVector<ArrayInfo>> + parent_document_id_to_child_array_info_; + + // Storage for DocumentJoinIdPair. + // - It is a collection of multiple extensible arrays for parents. + // - Each extensible array contains a list of child DocumentJoinIdPair. + // - The extensible array information (starting index of the FileBackedVector, + // length) is stored in parent_document_id_to_child_array_info_. + std::unique_ptr<FileBackedVector<DocumentJoinIdPair>> + child_document_join_id_pair_array_; + + const FeatureFlags& feature_flags_; // Does not own. + + bool is_info_dirty_; + bool is_storage_dirty_; +}; + +} // namespace lib +} // namespace icing + +#endif // ICING_JOIN_QUALIFIED_ID_JOIN_INDEX_IMPL_V3_H_
diff --git a/icing/join/qualified-id-join-index-impl-v3_test.cc b/icing/join/qualified-id-join-index-impl-v3_test.cc new file mode 100644 index 0000000..529100b --- /dev/null +++ b/icing/join/qualified-id-join-index-impl-v3_test.cc
@@ -0,0 +1,1388 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/join/qualified-id-join-index-impl-v3.h" + +#include <cstdint> +#include <memory> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/absl_ports/str_cat.h" +#include "icing/feature-flags.h" +#include "icing/file/file-backed-vector.h" +#include "icing/file/filesystem.h" +#include "icing/file/memory-mapped-file.h" +#include "icing/file/persistent-storage.h" +#include "icing/join/document-join-id-pair.h" +#include "icing/store/document-id.h" +#include "icing/testing/common-matchers.h" +#include "icing/testing/test-feature-flags.h" +#include "icing/testing/tmp-directory.h" +#include "icing/util/crc32.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::IsFalse; +using ::testing::IsTrue; +using ::testing::Lt; +using ::testing::Not; +using ::testing::Pointee; +using ::testing::SizeIs; + +using Crcs = PersistentStorage::Crcs; +using Info = QualifiedIdJoinIndexImplV3::Info; +using ArrayInfo = QualifiedIdJoinIndexImplV3::ArrayInfo; + +class QualifiedIdJoinIndexImplV3Test : public ::testing::Test { + protected: + void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + + base_dir_ = GetTestTempDir() + "/icing"; + ASSERT_THAT(filesystem_.CreateDirectoryRecursively(base_dir_.c_str()), + IsTrue()); + + working_path_ = base_dir_ + "/qualified_id_join_index_impl_v3"; + } + + void TearDown() override { + filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); + } + + std::unique_ptr<FeatureFlags> feature_flags_; + Filesystem filesystem_; + std::string base_dir_; + std::string working_path_; +}; + +TEST_F(QualifiedIdJoinIndexImplV3Test, InvalidWorkingPath) { + EXPECT_THAT(QualifiedIdJoinIndexImplV3::Create( + filesystem_, "/dev/null/qualified_id_join_index_impl_v3", + *feature_flags_), + StatusIs(libtextclassifier3::StatusCode::INTERNAL)); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, InitializeNewFiles) { + { + // Create new qualified id join index + ASSERT_FALSE(filesystem_.DirectoryExists(working_path_.c_str())); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + EXPECT_THAT(index, Pointee(IsEmpty())); + + ICING_ASSERT_OK(index->PersistToDisk()); + } + + // Metadata file should be initialized correctly for both info and crcs + // sections. + const std::string metadata_file_path = + absl_ports::StrCat(working_path_, "/metadata"); + auto metadata_buffer = std::make_unique<uint8_t[]>( + QualifiedIdJoinIndexImplV3::kMetadataFileSize); + ASSERT_THAT( + filesystem_.PRead(metadata_file_path.c_str(), metadata_buffer.get(), + QualifiedIdJoinIndexImplV3::kMetadataFileSize, + /*offset=*/0), + IsTrue()); + + // Check info section + const Info* info = reinterpret_cast<const Info*>( + metadata_buffer.get() + + QualifiedIdJoinIndexImplV3::kInfoMetadataFileOffset); + EXPECT_THAT(info->magic, Eq(Info::kMagic)); + EXPECT_THAT(info->num_data, Eq(0)); + EXPECT_THAT(info->last_added_document_id, Eq(kInvalidDocumentId)); + + // Check crcs section + const Crcs* crcs = reinterpret_cast<const Crcs*>( + metadata_buffer.get() + + QualifiedIdJoinIndexImplV3::kCrcsMetadataFileOffset); + // There are no data in FileBackedVectors, so storages_crc should be zero. + EXPECT_THAT(crcs->component_crcs.storages_crc, Eq(0)); + EXPECT_THAT(crcs->component_crcs.info_crc, + Eq(Crc32(std::string_view(reinterpret_cast<const char*>(info), + sizeof(Info))) + .Get())); + EXPECT_THAT(crcs->all_crc, + Eq(Crc32(std::string_view( + reinterpret_cast<const char*>(&crcs->component_crcs), + sizeof(Crcs::ComponentCrcs))) + .Get())); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializationShouldFailIfMissingMetadataFile) { + { + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + + // Insert some data. + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{0})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ASSERT_THAT(index, Pointee(SizeIs(2))); + + ICING_ASSERT_OK(index->PersistToDisk()); + } + + // Manually delete the metadata file. + const std::string metadata_file_path = + absl_ports::StrCat(working_path_, "/metadata"); + ASSERT_THAT(filesystem_.DeleteFile(metadata_file_path.c_str()), IsTrue()); + + // Attempt to create the qualified id join index with missing metadata file. + // This should fail. + EXPECT_THAT( + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_), + StatusIs( + libtextclassifier3::StatusCode::FAILED_PRECONDITION, + HasSubstr("Inconsistent state of qualified id join index (v3)"))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializationShouldFailIfMissingParentDocumentIdToChildArrayInfoFile) { + { + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + + // Insert some data. + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{0})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ASSERT_THAT(index, Pointee(SizeIs(2))); + + ICING_ASSERT_OK(index->PersistToDisk()); + } + + // Manually delete parent_document_id_to_child_array_info file. + const std::string array_working_path = absl_ports::StrCat( + working_path_, "/parent_document_id_to_child_array_info"); + ASSERT_THAT(filesystem_.DeleteFile(array_working_path.c_str()), IsTrue()); + + // Attempt to create the qualified id join index with missing + // parent_document_id_to_child_array_info file. This should fail. + EXPECT_THAT( + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_), + StatusIs( + libtextclassifier3::StatusCode::FAILED_PRECONDITION, + HasSubstr("Inconsistent state of qualified id join index (v3)"))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializationShouldFailIfMissingChildDocumentJoinIdPairArrayFile) { + { + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + + // Insert some data. + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{0})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ASSERT_THAT(index, Pointee(SizeIs(2))); + + ICING_ASSERT_OK(index->PersistToDisk()); + } + + // Manually delete child_document_join_id_pair_array file. + const std::string array_working_path = + absl_ports::StrCat(working_path_, "/child_document_join_id_pair_array"); + ASSERT_THAT(filesystem_.DeleteFile(array_working_path.c_str()), IsTrue()); + + // Attempt to create the qualified id join index with missing + // child_document_join_id_pair_array file. This should fail. + EXPECT_THAT( + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_), + StatusIs( + libtextclassifier3::StatusCode::FAILED_PRECONDITION, + HasSubstr("Inconsistent state of qualified id join index (v3)"))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializationShouldFailWithoutPersistToDiskOrDestruction) { + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Insert some data. + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{0})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ASSERT_THAT(index, Pointee(SizeIs(2))); + + // GetChecksum should succeed without updating the checksum. + EXPECT_THAT(index->GetChecksum(), IsOk()); + + // Without calling PersistToDisk, checksums will not be recomputed or synced + // to disk, so initializing another instance on the same files should fail. + EXPECT_THAT(QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializationShouldSucceedWithUpdateChecksums) { + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index1, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Insert some data. + ICING_ASSERT_OK( + index1->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{0})); + ICING_ASSERT_OK( + index1->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ASSERT_THAT(index1, Pointee(SizeIs(2))); + + // After calling UpdateChecksums, all checksums should be recomputed and + // synced correctly to disk, so initializing another instance on the same + // files should succeed, and we should be able to get the same contents. + ICING_ASSERT_OK_AND_ASSIGN(Crc32 crc, index1->GetChecksum()); + EXPECT_THAT(index1->UpdateChecksums(), IsOkAndHolds(Eq(crc))); + EXPECT_THAT(index1->GetChecksum(), IsOkAndHolds(Eq(crc))); + + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index2, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + EXPECT_THAT(index2, Pointee(SizeIs(2))); + EXPECT_THAT(index2->Get(/*parent_document_id=*/0), + IsOkAndHolds(ElementsAre(child_join_id_pair1))); + EXPECT_THAT(index2->Get(/*parent_document_id=*/1), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializationShouldSucceedWithPersistToDisk) { + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index1, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Insert some data. + ICING_ASSERT_OK( + index1->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{0})); + ICING_ASSERT_OK( + index1->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ASSERT_THAT(index1, Pointee(SizeIs(2))); + + // After calling PersistToDisk, all checksums should be recomputed and synced + // correctly to disk, so initializing another instance on the same files + // should succeed, and we should be able to get the same contents. + ICING_EXPECT_OK(index1->PersistToDisk()); + + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index2, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + EXPECT_THAT(index2, Pointee(SizeIs(2))); + EXPECT_THAT(index2->Get(/*parent_document_id=*/0), + IsOkAndHolds(ElementsAre(child_join_id_pair1))); + EXPECT_THAT(index2->Get(/*parent_document_id=*/1), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializationShouldSucceedAfterDestruction) { + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + + { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + + // Insert some data. + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{0})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ASSERT_THAT(index, Pointee(SizeIs(2))); + } + + { + // The previous instance went out of scope and was destructed. Although we + // didn't call PersistToDisk explicitly, the destructor should invoke it and + // thus initializing another instance on the same files should succeed, and + // we should be able to get the same contents. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + EXPECT_THAT(index, Pointee(SizeIs(2))); + EXPECT_THAT(index->Get(/*parent_document_id=*/0), + IsOkAndHolds(ElementsAre(child_join_id_pair1))); + EXPECT_THAT(index->Get(/*parent_document_id=*/1), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); + } +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializeExistingFilesWithDifferentMagicShouldFail) { + { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/100, /*joinable_property_id=*/20), + /*parent_document_ids=*/std::vector<DocumentId>{0})); + + ICING_ASSERT_OK(index->PersistToDisk()); + } + + { + const std::string metadata_file_path = + absl_ports::StrCat(working_path_, "/metadata"); + ScopedFd metadata_sfd(filesystem_.OpenForWrite(metadata_file_path.c_str())); + ASSERT_THAT(metadata_sfd.is_valid(), IsTrue()); + + auto metadata_buffer = std::make_unique<uint8_t[]>( + QualifiedIdJoinIndexImplV3::kMetadataFileSize); + ASSERT_THAT(filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(), + QualifiedIdJoinIndexImplV3::kMetadataFileSize, + /*offset=*/0), + IsTrue()); + + // Manually change magic and update checksum + Crcs* crcs = reinterpret_cast<Crcs*>( + metadata_buffer.get() + + QualifiedIdJoinIndexImplV3::kCrcsMetadataFileOffset); + Info* info = reinterpret_cast<Info*>( + metadata_buffer.get() + + QualifiedIdJoinIndexImplV3::kInfoMetadataFileOffset); + info->magic += 1; + crcs->component_crcs.info_crc = info->GetChecksum().Get(); + crcs->all_crc = crcs->component_crcs.GetChecksum().Get(); + ASSERT_THAT(filesystem_.PWrite( + metadata_sfd.get(), /*offset=*/0, metadata_buffer.get(), + QualifiedIdJoinIndexImplV3::kMetadataFileSize), + IsTrue()); + } + + // Attempt to create the qualified id join index with different magic. This + // should fail. + EXPECT_THAT(QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, + HasSubstr("Incorrect magic value"))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializeExistingFilesWithWrongAllCrcShouldFail) { + { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/100, /*joinable_property_id=*/20), + /*parent_document_ids=*/std::vector<DocumentId>{0})); + + ICING_ASSERT_OK(index->PersistToDisk()); + } + + { + const std::string metadata_file_path = + absl_ports::StrCat(working_path_, "/metadata"); + ScopedFd metadata_sfd(filesystem_.OpenForWrite(metadata_file_path.c_str())); + ASSERT_THAT(metadata_sfd.is_valid(), IsTrue()); + + auto metadata_buffer = std::make_unique<uint8_t[]>( + QualifiedIdJoinIndexImplV3::kMetadataFileSize); + ASSERT_THAT(filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(), + QualifiedIdJoinIndexImplV3::kMetadataFileSize, + /*offset=*/0), + IsTrue()); + + // Manually corrupt all_crc + Crcs* crcs = reinterpret_cast<Crcs*>( + metadata_buffer.get() + + QualifiedIdJoinIndexImplV3::kCrcsMetadataFileOffset); + crcs->all_crc += 1; + + ASSERT_THAT(filesystem_.PWrite( + metadata_sfd.get(), /*offset=*/0, metadata_buffer.get(), + QualifiedIdJoinIndexImplV3::kMetadataFileSize), + IsTrue()); + } + + // Attempt to create the qualified id join index with metadata containing + // corrupted all_crc. This should fail. + EXPECT_THAT(QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, + HasSubstr("Invalid all crc"))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + InitializeExistingFilesWithCorruptedInfoShouldFail) { + { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/100, /*joinable_property_id=*/20), + /*parent_document_ids=*/std::vector<DocumentId>{0})); + + ICING_ASSERT_OK(index->PersistToDisk()); + } + + { + const std::string metadata_file_path = + absl_ports::StrCat(working_path_, "/metadata"); + ScopedFd metadata_sfd(filesystem_.OpenForWrite(metadata_file_path.c_str())); + ASSERT_THAT(metadata_sfd.is_valid(), IsTrue()); + + auto metadata_buffer = std::make_unique<uint8_t[]>( + QualifiedIdJoinIndexImplV3::kMetadataFileSize); + ASSERT_THAT(filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(), + QualifiedIdJoinIndexImplV3::kMetadataFileSize, + /*offset=*/0), + IsTrue()); + + // Modify info, but don't update the checksum. This would be similar to + // corruption of info. + Info* info = reinterpret_cast<Info*>( + metadata_buffer.get() + + QualifiedIdJoinIndexImplV3::kInfoMetadataFileOffset); + info->last_added_document_id += 1; + + ASSERT_THAT(filesystem_.PWrite( + metadata_sfd.get(), /*offset=*/0, metadata_buffer.get(), + QualifiedIdJoinIndexImplV3::kMetadataFileSize), + IsTrue()); + } + + // Attempt to create the qualified id join index with info that doesn't match + // its checksum. This should fail. + EXPECT_THAT(QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, + HasSubstr("Invalid info crc"))); +} + +TEST_F( + QualifiedIdJoinIndexImplV3Test, + InitializeExistingFilesWithCorruptedParentDocumentIdToChildArrayInfoShouldFail) { + { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/100, /*joinable_property_id=*/20), + /*parent_document_ids=*/std::vector<DocumentId>{0})); + + ICING_ASSERT_OK(index->PersistToDisk()); + } + + // Corrupt parent_document_id_to_child_array_info manually. + { + const std::string array_working_path = absl_ports::StrCat( + working_path_, "/parent_document_id_to_child_array_info"); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<FileBackedVector<ArrayInfo>> fbv, + FileBackedVector<ArrayInfo>::Create( + filesystem_, std::move(array_working_path), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<ArrayInfo>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + ICING_ASSERT_OK_AND_ASSIGN(Crc32 old_crc, fbv->UpdateChecksum()); + ICING_ASSERT_OK(fbv->Append( + ArrayInfo(/*index_in=*/100, /*length_in=*/10, /*used_length_in=*/0))); + ICING_ASSERT_OK(fbv->PersistToDisk()); + ICING_ASSERT_OK_AND_ASSIGN(Crc32 new_crc, fbv->UpdateChecksum()); + ASSERT_THAT(old_crc, Not(Eq(new_crc))); + } + + // Attempt to create the qualified id join index with corrupted + // parent_document_id_to_child_array_info. This should fail. + EXPECT_THAT(QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, + HasSubstr("Invalid storages crc"))); +} + +TEST_F( + QualifiedIdJoinIndexImplV3Test, + InitializeExistingFilesWithCorruptedChildDocumentJoinIdPairArrayShouldFail) { + { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + ICING_ASSERT_OK(index->Put( + DocumentJoinIdPair(/*document_id=*/100, /*joinable_property_id=*/20), + /*parent_document_ids=*/std::vector<DocumentId>{0})); + + ICING_ASSERT_OK(index->PersistToDisk()); + } + + // Corrupt child_document_join_id_pair_array manually. + { + const std::string array_working_path = + absl_ports::StrCat(working_path_, "/child_document_join_id_pair_array"); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<FileBackedVector<DocumentJoinIdPair>> fbv, + FileBackedVector<DocumentJoinIdPair>::Create( + filesystem_, std::move(array_working_path), + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, + FileBackedVector<DocumentJoinIdPair>::kMaxFileSize, + /*pre_mapping_mmap_size=*/0)); + ICING_ASSERT_OK_AND_ASSIGN(Crc32 old_crc, fbv->UpdateChecksum()); + ICING_ASSERT_OK(fbv->Append(DocumentJoinIdPair(/*value=*/12345))); + ICING_ASSERT_OK(fbv->PersistToDisk()); + ICING_ASSERT_OK_AND_ASSIGN(Crc32 new_crc, fbv->UpdateChecksum()); + ASSERT_THAT(old_crc, Not(Eq(new_crc))); + } + + // Attempt to create the qualified id join index with corrupted + // child_document_join_id_pair_array. This should fail. + EXPECT_THAT(QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, + HasSubstr("Invalid storages crc"))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, Put) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Add 6 children with their parents to the index. + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + DocumentJoinIdPair child_join_id_pair3(/*document_id=*/104, + /*joinable_property_id=*/4); + DocumentJoinIdPair child_join_id_pair4(/*document_id=*/105, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair5(/*document_id=*/109, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair6(/*document_id=*/121, + /*joinable_property_id=*/3); + EXPECT_THAT(index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{1}), + IsOk()); + EXPECT_THAT(index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1}), + IsOk()); + EXPECT_THAT(index->Put(child_join_id_pair3, + /*parent_document_ids=*/std::vector<DocumentId>{2}), + IsOk()); + EXPECT_THAT(index->Put(child_join_id_pair4, + /*parent_document_ids=*/std::vector<DocumentId>{0}), + IsOk()); + EXPECT_THAT(index->Put(child_join_id_pair5, + /*parent_document_ids=*/std::vector<DocumentId>{1}), + IsOk()); + EXPECT_THAT(index->Put(child_join_id_pair6, + /*parent_document_ids=*/std::vector<DocumentId>{5}), + IsOk()); + + EXPECT_THAT(index, Pointee(SizeIs(6))); + + // Verify Get API. + EXPECT_THAT(index->Get(/*parent_document_id=*/0), + IsOkAndHolds(ElementsAre(child_join_id_pair4))); + EXPECT_THAT(index->Get(/*parent_document_id=*/1), + IsOkAndHolds(ElementsAre(child_join_id_pair1, child_join_id_pair2, + child_join_id_pair5))); + EXPECT_THAT(index->Get(/*parent_document_id=*/2), + IsOkAndHolds(ElementsAre(child_join_id_pair3))); + EXPECT_THAT(index->Get(/*parent_document_id=*/3), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/4), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/5), + IsOkAndHolds(ElementsAre(child_join_id_pair6))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + Put_multipleParentsInASingleJoinableProperty) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Add 3 children with multiple parents to the index. + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + DocumentJoinIdPair child_join_id_pair3(/*document_id=*/104, + /*joinable_property_id=*/4); + EXPECT_THAT( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{1, 4, 7, 10}), + IsOk()); + EXPECT_THAT( + index->Put( + child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{0, 1, 2, 3, 5, 8}), + IsOk()); + EXPECT_THAT( + index->Put(child_join_id_pair3, + /*parent_document_ids=*/std::vector<DocumentId>{2, 5, 7}), + IsOk()); + + EXPECT_THAT(index, Pointee(SizeIs(13))); + + // Verify Get API. + EXPECT_THAT(index->Get(/*parent_document_id=*/0), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); + EXPECT_THAT( + index->Get(/*parent_document_id=*/1), + IsOkAndHolds(ElementsAre(child_join_id_pair1, child_join_id_pair2))); + EXPECT_THAT( + index->Get(/*parent_document_id=*/2), + IsOkAndHolds(ElementsAre(child_join_id_pair2, child_join_id_pair3))); + EXPECT_THAT(index->Get(/*parent_document_id=*/3), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); + EXPECT_THAT(index->Get(/*parent_document_id=*/4), + IsOkAndHolds(ElementsAre(child_join_id_pair1))); + EXPECT_THAT( + index->Get(/*parent_document_id=*/5), + IsOkAndHolds(ElementsAre(child_join_id_pair2, child_join_id_pair3))); + EXPECT_THAT(index->Get(/*parent_document_id=*/6), IsOkAndHolds(IsEmpty())); + EXPECT_THAT( + index->Get(/*parent_document_id=*/7), + IsOkAndHolds(ElementsAre(child_join_id_pair1, child_join_id_pair3))); + EXPECT_THAT(index->Get(/*parent_document_id=*/8), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); + EXPECT_THAT(index->Get(/*parent_document_id=*/9), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/10), + IsOkAndHolds(ElementsAre(child_join_id_pair1))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + Put_multipleParentsInMultipleJoinableProperties) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Add 1 child document with multiple parents in multiple joinable properties + // to the index. + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/100, + /*joinable_property_id=*/18); + DocumentJoinIdPair child_join_id_pair3(/*document_id=*/100, + /*joinable_property_id=*/5); + EXPECT_THAT( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{1, 4, 7, 10}), + IsOk()); + EXPECT_THAT( + index->Put( + child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{0, 1, 2, 3, 5, 8}), + IsOk()); + EXPECT_THAT( + index->Put(child_join_id_pair3, + /*parent_document_ids=*/std::vector<DocumentId>{2, 5, 7}), + IsOk()); + + EXPECT_THAT(index, Pointee(SizeIs(13))); + + // Verify Get API. + EXPECT_THAT(index->Get(/*parent_document_id=*/0), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); + EXPECT_THAT( + index->Get(/*parent_document_id=*/1), + IsOkAndHolds(ElementsAre(child_join_id_pair1, child_join_id_pair2))); + EXPECT_THAT( + index->Get(/*parent_document_id=*/2), + IsOkAndHolds(ElementsAre(child_join_id_pair2, child_join_id_pair3))); + EXPECT_THAT(index->Get(/*parent_document_id=*/3), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); + EXPECT_THAT(index->Get(/*parent_document_id=*/4), + IsOkAndHolds(ElementsAre(child_join_id_pair1))); + EXPECT_THAT( + index->Get(/*parent_document_id=*/5), + IsOkAndHolds(ElementsAre(child_join_id_pair2, child_join_id_pair3))); + EXPECT_THAT(index->Get(/*parent_document_id=*/6), IsOkAndHolds(IsEmpty())); + EXPECT_THAT( + index->Get(/*parent_document_id=*/7), + IsOkAndHolds(ElementsAre(child_join_id_pair1, child_join_id_pair3))); + EXPECT_THAT(index->Get(/*parent_document_id=*/8), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); + EXPECT_THAT(index->Get(/*parent_document_id=*/9), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/10), + IsOkAndHolds(ElementsAre(child_join_id_pair1))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + PutShouldResizeParentDocumentIdToChildArrayInfo) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + constexpr DocumentId kParentDocumentId = 10; + DocumentJoinIdPair child_join_id_pair(/*document_id=*/100, + /*joinable_property_id=*/20); + + // Even though document 0 to 9 are missing in the index, adding parent + // document id 10 should resize the FileBackedVector and succeed without + // error. + EXPECT_THAT( + index->Put( + child_join_id_pair, + /*parent_document_ids=*/std::vector<DocumentId>{kParentDocumentId}), + IsOk()); + EXPECT_THAT(index, Pointee(SizeIs(1))); + + // Get API should return empty result for document 0 to 9. + for (DocumentId parent_doc_id = 0; parent_doc_id < kParentDocumentId; + ++parent_doc_id) { + EXPECT_THAT(index->Get(parent_doc_id), IsOkAndHolds(IsEmpty())); + } + // Get API should return the child document for document 10. + EXPECT_THAT(index->Get(kParentDocumentId), + IsOkAndHolds(ElementsAre(child_join_id_pair))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + PutShouldExtendChildDocumentJoinIdPairArray) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Put 1 child for parent1. + DocumentId parent1 = 1; + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{parent1})); + EXPECT_THAT(index, Pointee(SizeIs(1))); + + // Put 1 child for parent2. This makes parent2's array locate right after + // parent1's array. + DocumentId parent2 = 2; + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/20); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{parent2})); + EXPECT_THAT(index, Pointee(SizeIs(2))); + + constexpr int kNumAdditionalChildren = 100; + // Put 100 more children for parent1. The array storing the child document + // join id pairs should be extended correctly to fit all the new elements + // without affecting (overwriting) parent2's array. + std::vector<DocumentJoinIdPair> child_join_id_pairs; + child_join_id_pairs.reserve(kNumAdditionalChildren + 1); + child_join_id_pairs.push_back(child_join_id_pair1); + for (int i = 0; i < kNumAdditionalChildren; ++i) { + DocumentJoinIdPair child_join_id_pair(/*document_id=*/200 + i, + /*joinable_property_id=*/5); + EXPECT_THAT( + index->Put(child_join_id_pair, + /*parent_document_ids=*/std::vector<DocumentId>{parent1}), + IsOk()); + child_join_id_pairs.push_back(std::move(child_join_id_pair)); + } + EXPECT_THAT(index, Pointee(SizeIs(102))); + + EXPECT_THAT(index->Get(parent1), IsOkAndHolds(child_join_id_pairs)); + EXPECT_THAT(index->Get(parent2), + IsOkAndHolds(ElementsAre(child_join_id_pair2))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, PutShouldSkipInvalidParentDocumentId) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + DocumentJoinIdPair child_join_id_pair(/*document_id=*/100, + /*joinable_property_id=*/20); + EXPECT_THAT( + index->Put( + child_join_id_pair, + /*parent_document_ids=*/std::vector<DocumentId>{-1, + kInvalidDocumentId, 1, + 3, 2}), + IsOk()); + + // -1, kInvalidDocumentId should be skipped, so only 3 valid join relations + // should be added to the index. + EXPECT_THAT(index, Pointee(SizeIs(3))); + + // Verify Get API. + EXPECT_THAT(index->Get(/*parent_document_id=*/1), + IsOkAndHolds(ElementsAre(child_join_id_pair))); + EXPECT_THAT(index->Get(/*parent_document_id=*/2), + IsOkAndHolds(ElementsAre(child_join_id_pair))); + EXPECT_THAT(index->Get(/*parent_document_id=*/3), + IsOkAndHolds(ElementsAre(child_join_id_pair))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + PutShouldReturnInvalidArgumentErrorForInvalidChild) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + DocumentJoinIdPair invalid_child_join_id; // Default constructor creates an + // invalid DocumentJoinIdPair. + ASSERT_THAT(invalid_child_join_id.is_valid(), IsFalse()); + + EXPECT_THAT(index->Put(invalid_child_join_id, + /*parent_document_ids=*/std::vector<DocumentId>{0}), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT(index, Pointee(IsEmpty())); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, GetEmptyIndex) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + EXPECT_THAT(index, Pointee(IsEmpty())); + + EXPECT_THAT(index->Get(/*parent_document_id=*/0), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/1), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/2), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/3), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/4), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/5), IsOkAndHolds(IsEmpty())); +} + +TEST_F( + QualifiedIdJoinIndexImplV3Test, + GetShouldReturnEmptyResultWithoutAccessingArrayForNonExistingLargeParent) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + DocumentJoinIdPair child_join_id_pair(/*document_id=*/100, + /*joinable_property_id=*/20); + ICING_ASSERT_OK(index->Put( + child_join_id_pair, /*parent_document_ids=*/std::vector<DocumentId>{1})); + EXPECT_THAT(index, Pointee(SizeIs(1))); + + EXPECT_THAT(index->Get(/*parent_document_id=*/0), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/1), + IsOkAndHolds(ElementsAre(child_join_id_pair))); + + // Now, only parent document id 1 is in the index, so the FileBackedVector has + // been resized to fit parent document id 1. + // Get API for parent document id greater than 1 should return empty result + // without accessing the FileBackedVector. + EXPECT_THAT(index->Get(/*parent_document_id=*/2), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/3), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(kMaxDocumentId), IsOkAndHolds(IsEmpty())); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, + GetShouldReturnInvalidArgumentErrorForInvalidParentDocumentId) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + EXPECT_THAT(index->Get(/*parent_document_id=*/-1), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT(index->Get(kInvalidDocumentId), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, MigrateParent) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + DocumentId parent_doc_id1 = 1; + DocumentId parent_doc_id2 = 1024; + + // Add 6 children with their parents to the index. + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/0); + ICING_ASSERT_OK(index->Put( + child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{parent_doc_id1})); + ICING_ASSERT_OK(index->Put( + child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{parent_doc_id1})); + + // Sanity check. + ASSERT_THAT(index, Pointee(SizeIs(2))); + ASSERT_THAT( + index->Get(parent_doc_id1), + IsOkAndHolds(ElementsAre(child_join_id_pair1, child_join_id_pair2))); + ASSERT_THAT(index->Get(parent_doc_id2), IsOkAndHolds(IsEmpty())); + + // Migrate parent document id 1 to 1024. + EXPECT_THAT(index->MigrateParent(parent_doc_id1, parent_doc_id2), IsOk()); + EXPECT_THAT(index->Get(parent_doc_id1), IsOkAndHolds(IsEmpty())); + EXPECT_THAT( + index->Get(parent_doc_id2), + IsOkAndHolds(ElementsAre(child_join_id_pair1, child_join_id_pair2))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, SetLastAddedDocumentId) { + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId)); + + constexpr DocumentId kDocumentId = 100; + index->set_last_added_document_id(kDocumentId); + EXPECT_THAT(index->last_added_document_id(), Eq(kDocumentId)); + + constexpr DocumentId kNextDocumentId = 123; + index->set_last_added_document_id(kNextDocumentId); + EXPECT_THAT(index->last_added_document_id(), Eq(kNextDocumentId)); +} + +TEST_F( + QualifiedIdJoinIndexImplV3Test, + SetLastAddedDocumentIdShouldIgnoreNewDocumentIdNotGreaterThanTheCurrent) { + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + constexpr DocumentId kDocumentId = 123; + index->set_last_added_document_id(kDocumentId); + ASSERT_THAT(index->last_added_document_id(), Eq(kDocumentId)); + + constexpr DocumentId kNextDocumentId = 100; + ASSERT_THAT(kNextDocumentId, Lt(kDocumentId)); + index->set_last_added_document_id(kNextDocumentId); + // last_added_document_id() should remain unchanged. + EXPECT_THAT(index->last_added_document_id(), Eq(kDocumentId)); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, Optimize) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Create 4 parent and 7 child documents (with N to N joins): + // - Document 1: 101, 103, 104, 105, 107 + // - Document 2: 102, 103, 105 + // - Document 3: 101, 106 + // - Document 4: 103 + // Add 6 children with their parents to the index. + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/101, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/102, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair3(/*document_id=*/103, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair4(/*document_id=*/104, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair5(/*document_id=*/105, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair6(/*document_id=*/106, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair7(/*document_id=*/107, + /*joinable_property_id=*/0); + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{1, 3})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{2})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair3, + /*parent_document_ids=*/std::vector<DocumentId>{1, 2, 4})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair4, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair5, + /*parent_document_ids=*/std::vector<DocumentId>{1, 2})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair6, + /*parent_document_ids=*/std::vector<DocumentId>{3})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair7, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + + ASSERT_THAT(index, Pointee(SizeIs(11))); + index->set_last_added_document_id(107); + ASSERT_THAT(index->last_added_document_id(), Eq(107)); + + // Delete parent 3, child 103, 107. Create a new mapping from old document id + // to new document id. + std::vector<DocumentId> document_id_old_to_new(108, kInvalidDocumentId); + document_id_old_to_new[1] = 0; + document_id_old_to_new[2] = 1; + document_id_old_to_new[4] = 2; + document_id_old_to_new[101] = 11; + document_id_old_to_new[102] = 12; + document_id_old_to_new[104] = 13; + document_id_old_to_new[105] = 14; + document_id_old_to_new[106] = 15; + + // Note: namespace_id_old_to_new is not used in + // QualifiedIdJoinIndexImplV3::Optimize. + DocumentId new_last_added_document_id = 15; + EXPECT_THAT( + index->Optimize(document_id_old_to_new, /*namespace_id_old_to_new=*/{}, + new_last_added_document_id), + IsOk()); + EXPECT_THAT(index, Pointee(SizeIs(5))); + EXPECT_THAT(index->last_added_document_id(), Eq(new_last_added_document_id)); + + // Verify document 0 (originally document 1) + // - Child docs 101, 104, 105 become 11, 13, 14. + // - Child docs 103, 107 are deleted. + EXPECT_THAT( + index->Get(/*parent_document_id=*/0), + IsOkAndHolds(ElementsAre( + DocumentJoinIdPair(/*document_id=*/11, /*joinable_property_id=*/0), + DocumentJoinIdPair(/*document_id=*/13, /*joinable_property_id=*/0), + DocumentJoinIdPair(/*document_id=*/14, /*joinable_property_id=*/0)))); + + // Verify document 1 (originally document 2) + // - Child docs 102, 105 become 12, 14. + // - Child doc 103 is deleted. + EXPECT_THAT( + index->Get(/*parent_document_id=*/1), + IsOkAndHolds(ElementsAre( + DocumentJoinIdPair(/*document_id=*/12, /*joinable_property_id=*/0), + DocumentJoinIdPair(/*document_id=*/14, /*joinable_property_id=*/0)))); + + // Verify document 2 (originally document 4) + // - Child doc 103 is deleted. + EXPECT_THAT(index->Get(/*parent_document_id=*/2), IsOkAndHolds(IsEmpty())); + + // Verify document 3 and 4: + // - These 2 doc ids don't exist after optimize. + // - The relations for the original document 3 and 4 should be deleted. + EXPECT_THAT(index->Get(/*parent_document_id=*/3), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/4), IsOkAndHolds(IsEmpty())); + + // Verify Put API should work normally after Optimize(). + DocumentJoinIdPair another_child_join_id_pair(/*document_id=*/16, + /*joinable_property_id=*/0); + EXPECT_THAT( + index->Put(another_child_join_id_pair, + /*parent_document_ids=*/std::vector<DocumentId>{0, 2, 3}), + IsOk()); + index->set_last_added_document_id(16); + + EXPECT_THAT(index, Pointee(SizeIs(8))); + EXPECT_THAT(index->last_added_document_id(), Eq(16)); + EXPECT_THAT( + index->Get(/*parent_document_id=*/0), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair(/*document_id=*/11, + /*joinable_property_id=*/0), + DocumentJoinIdPair(/*document_id=*/13, + /*joinable_property_id=*/0), + DocumentJoinIdPair(/*document_id=*/14, + /*joinable_property_id=*/0), + another_child_join_id_pair))); + EXPECT_THAT(index->Get(/*parent_document_id=*/2), + IsOkAndHolds(ElementsAre(another_child_join_id_pair))); + EXPECT_THAT(index->Get(/*parent_document_id=*/3), + IsOkAndHolds(ElementsAre(another_child_join_id_pair))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, OptimizeOutOfRangeDocumentId) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Create 2 parent and 3 child documents (with N to N joins): + // - Document 1: 101, 106, 108 + // - Document 120: 101 + // Add 3 children with their parents to the index. + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/101, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/106, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair3(/*document_id=*/108, + /*joinable_property_id=*/0); + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{1, 120})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair3, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + + ASSERT_THAT(index, Pointee(SizeIs(4))); + index->set_last_added_document_id(120); + ASSERT_THAT(index->last_added_document_id(), Eq(120)); + + // Create document_id_old_to_new with size = 107 (from index 0 to 106), which + // makes parent document 120 and child document 108 out of range. + // + // Optimize should handle out of range DocumentId properly without crashing. + std::vector<DocumentId> document_id_old_to_new(107, kInvalidDocumentId); + document_id_old_to_new[1] = 0; + document_id_old_to_new[101] = 11; + document_id_old_to_new[106] = 12; + + // Note: namespace_id_old_to_new is not used in + // QualifiedIdJoinIndexImplV3::Optimize. + DocumentId new_last_added_document_id = 12; + EXPECT_THAT( + index->Optimize(document_id_old_to_new, /*namespace_id_old_to_new=*/{}, + new_last_added_document_id), + IsOk()); + EXPECT_THAT(index, Pointee(SizeIs(2))); + EXPECT_THAT(index->last_added_document_id(), Eq(new_last_added_document_id)); + + // Verify document 0 (originally document 1) + // - Child doc 101, 106 become 11, 12. + // - Child doc 108 is out of range, so it should be deleted. + EXPECT_THAT( + index->Get(/*parent_document_id=*/0), + IsOkAndHolds(ElementsAre( + DocumentJoinIdPair(/*document_id=*/11, /*joinable_property_id=*/0), + DocumentJoinIdPair(/*document_id=*/12, /*joinable_property_id=*/0)))); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, OptimizeDeleteAllDocuments) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Create 4 parent and 7 child documents (with N to N joins): + // - Document 1: 101, 103, 104, 105, 107 + // - Document 2: 102, 103, 105 + // - Document 3: 101, 106 + // - Document 4: 103 + // Add 6 children with their parents to the index. + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/101, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/102, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair3(/*document_id=*/103, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair4(/*document_id=*/104, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair5(/*document_id=*/105, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair6(/*document_id=*/106, + /*joinable_property_id=*/0); + DocumentJoinIdPair child_join_id_pair7(/*document_id=*/107, + /*joinable_property_id=*/0); + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{1, 3})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{2})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair3, + /*parent_document_ids=*/std::vector<DocumentId>{1, 2, 4})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair4, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair5, + /*parent_document_ids=*/std::vector<DocumentId>{1, 2})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair6, + /*parent_document_ids=*/std::vector<DocumentId>{3})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair7, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + + ASSERT_THAT(index, Pointee(SizeIs(11))); + index->set_last_added_document_id(107); + ASSERT_THAT(index->last_added_document_id(), Eq(107)); + + // Delete all documents. + std::vector<DocumentId> document_id_old_to_new(108, kInvalidDocumentId); + + // Note: namespace_id_old_to_new is not used in + // QualifiedIdJoinIndexImplV3::Optimize. + DocumentId new_last_added_document_id = kInvalidDocumentId; + EXPECT_THAT( + index->Optimize(document_id_old_to_new, /*namespace_id_old_to_new=*/{}, + new_last_added_document_id), + IsOk()); + EXPECT_THAT(index, Pointee(IsEmpty())); + EXPECT_THAT(index->last_added_document_id(), Eq(new_last_added_document_id)); + EXPECT_THAT(index->Get(/*parent_document_id=*/0), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/1), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/2), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/3), IsOkAndHolds(IsEmpty())); +} + +TEST_F(QualifiedIdJoinIndexImplV3Test, Clear) { + // Create new qualified id join index + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexImplV3> index, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, working_path_, *feature_flags_)); + + // Add 6 children with their parents to the index. + DocumentJoinIdPair child_join_id_pair1(/*document_id=*/100, + /*joinable_property_id=*/20); + DocumentJoinIdPair child_join_id_pair2(/*document_id=*/101, + /*joinable_property_id=*/2); + DocumentJoinIdPair child_join_id_pair3(/*document_id=*/104, + /*joinable_property_id=*/4); + DocumentJoinIdPair child_join_id_pair4(/*document_id=*/105, + /*joinable_property_id=*/0); + ICING_ASSERT_OK( + index->Put(child_join_id_pair1, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair2, + /*parent_document_ids=*/std::vector<DocumentId>{1})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair3, + /*parent_document_ids=*/std::vector<DocumentId>{2})); + ICING_ASSERT_OK( + index->Put(child_join_id_pair4, + /*parent_document_ids=*/std::vector<DocumentId>{0})); + + ASSERT_THAT(index, Pointee(SizeIs(4))); + index->set_last_added_document_id(105); + ASSERT_THAT(index->last_added_document_id(), Eq(105)); + + // After Clear(), last_added_document_id should be set to kInvalidDocumentId, + // and the previous added data should be deleted. + EXPECT_THAT(index->Clear(), IsOk()); + EXPECT_THAT(index, Pointee(IsEmpty())); + EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId)); + EXPECT_THAT(index->Get(/*parent_document_id=*/0), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/1), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(index->Get(/*parent_document_id=*/2), IsOkAndHolds(IsEmpty())); + + // Join index should be able to work normally after Clear(). + EXPECT_THAT(index->Put(child_join_id_pair4, + /*parent_document_ids=*/std::vector<DocumentId>{5}), + IsOk()); + index->set_last_added_document_id(105); + + EXPECT_THAT(index, Pointee(SizeIs(1))); + EXPECT_THAT(index->last_added_document_id(), Eq(105)); + EXPECT_THAT(index->Get(/*parent_document_id=*/5), + IsOkAndHolds(ElementsAre(child_join_id_pair4))); + + ICING_ASSERT_OK(index->PersistToDisk()); + index.reset(); + + // Verify index after reconstructing. + ICING_ASSERT_OK_AND_ASSIGN( + index, QualifiedIdJoinIndexImplV3::Create(filesystem_, working_path_, + *feature_flags_)); + EXPECT_THAT(index, Pointee(SizeIs(1))); + EXPECT_THAT(index->last_added_document_id(), Eq(105)); + EXPECT_THAT(index->Get(/*parent_document_id=*/5), + IsOkAndHolds(ElementsAre(child_join_id_pair4))); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/join/qualified-id-join-index.h b/icing/join/qualified-id-join-index.h index 9b40c4f..4972c5e 100644 --- a/icing/join/qualified-id-join-index.h +++ b/icing/join/qualified-id-join-index.h
@@ -26,12 +26,12 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/file/filesystem.h" #include "icing/file/persistent-storage.h" -#include "icing/join/doc-join-info.h" #include "icing/join/document-id-to-join-info.h" +#include "icing/join/document-join-id-pair.h" #include "icing/schema/joinable-property.h" #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/util/crc32.h" @@ -48,10 +48,12 @@ virtual libtextclassifier3::Status Advance() = 0; - virtual const DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>& - GetCurrent() const = 0; + virtual const DocumentIdToJoinInfo<NamespaceIdFingerprint>& GetCurrent() + const = 0; }; + enum class Version { kV1, kV2, kV3 }; + static constexpr WorkingPathType kWorkingPathType = WorkingPathType::kDirectory; @@ -68,7 +70,7 @@ virtual ~QualifiedIdJoinIndex() override = default; - // (v1 only) Puts a new data into index: DocJoinInfo (DocumentId, + // (v1 only) Puts a new data into index: DocumentJoinIdPair (DocumentId, // JoinablePropertyId) references to ref_qualified_id_str (the identifier of // another document). // @@ -79,11 +81,11 @@ // - INVALID_ARGUMENT_ERROR if doc_join_info is invalid // - Any KeyMapper errors virtual libtextclassifier3::Status Put( - const DocJoinInfo& doc_join_info, + const DocumentJoinIdPair& document_join_id_pair, std::string_view ref_qualified_id_str) = 0; - // (v2 only) Puts a list of referenced NamespaceFingerprintIdentifier into - // index, given the DocumentId, SchemaTypeId and JoinablePropertyId. + // (v2 only) Puts a list of referenced NamespaceIdFingerprint into index, + // given the DocumentId, SchemaTypeId and JoinablePropertyId. // // Returns: // - OK on success @@ -93,20 +95,31 @@ virtual libtextclassifier3::Status Put( SchemaTypeId schema_type_id, JoinablePropertyId joinable_property_id, DocumentId document_id, - std::vector<NamespaceFingerprintIdentifier>&& - ref_namespace_fingerprint_ids) = 0; + std::vector<NamespaceIdFingerprint>&& + ref_namespace_id_uri_fingerprints) = 0; - // (v1 only) Gets the referenced document's qualified id string by - // DocJoinInfo. + // (v3 only) Puts a new child document and its referenced parent documents + // into the join index. // // Returns: - // - A qualified id string referenced by the given DocJoinInfo (DocumentId, - // JoinablePropertyId) on success + // - OK on success + // - INVALID_ARGUMENT_ERROR if child_document_join_id_pair is invalid + // - Any FileBackedVector errors + virtual libtextclassifier3::Status Put( + const DocumentJoinIdPair& child_document_join_id_pair, + std::vector<DocumentId>&& parent_document_ids) = 0; + + // (v1 only) Gets the referenced document's qualified id string by + // DocumentJoinIdPair. + // + // Returns: + // - A qualified id string referenced by the given DocumentJoinIdPair + // (DocumentId, JoinablePropertyId) on success // - INVALID_ARGUMENT_ERROR if doc_join_info is invalid // - NOT_FOUND_ERROR if doc_join_info doesn't exist // - Any KeyMapper errors virtual libtextclassifier3::StatusOr<std::string_view> Get( - const DocJoinInfo& doc_join_info) const = 0; + const DocumentJoinIdPair& document_join_id_pair) const = 0; // (v2 only) Returns a JoinDataIterator for iterating through all join data of // the specified (schema_type_id, joinable_property_id). @@ -120,6 +133,25 @@ GetIterator(SchemaTypeId schema_type_id, JoinablePropertyId joinable_property_id) const = 0; + // (v3 only) Gets the list of joinable children for the given parent document + // id. + // + // Returns: + // - A list of children's DocumentJoinIdPair on success + // - Any FileBackedVector errors + virtual libtextclassifier3::StatusOr<std::vector<DocumentJoinIdPair>> Get( + DocumentId parent_document_id) const = 0; + + // Migrates existing join data for a parent document from old_document_id to + // new_document_id if necessary. + // + // Returns: + // - OK on success + // - INVALID_ARGUMENT_ERROR if any document id is invalid + // - Any errors, depending on the implementation + virtual libtextclassifier3::Status MigrateParent( + DocumentId old_document_id, DocumentId new_document_id) = 0; + // Reduces internal file sizes by reclaiming space and ids of deleted // documents. Qualified id type joinable index will convert all entries to the // new document ids. @@ -149,7 +181,7 @@ // - INTERNAL_ERROR on I/O error virtual libtextclassifier3::Status Clear() = 0; - virtual bool is_v2() const = 0; + virtual Version version() const = 0; virtual int32_t size() const = 0;
diff --git a/icing/join/qualified-id-join-indexing-handler-v1_test.cc b/icing/join/qualified-id-join-indexing-handler-v1_test.cc index c8fdab4..e1e3881 100644 --- a/icing/join/qualified-id-join-indexing-handler-v1_test.cc +++ b/icing/join/qualified-id-join-indexing-handler-v1_test.cc
@@ -21,8 +21,10 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" +#include "icing/join/document-join-id-pair.h" #include "icing/join/qualified-id-join-index-impl-v1.h" #include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id-join-indexing-handler.h" @@ -37,11 +39,12 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" @@ -81,10 +84,11 @@ class QualifiedIdJoinIndexingHandlerV1Test : public ::testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -111,8 +115,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), IsTrue()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType( @@ -152,13 +156,12 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); doc_store_ = std::move(create_result.document_store); } @@ -172,6 +175,7 @@ filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; FakeClock fake_clock_; std::string base_dir_; @@ -232,12 +236,13 @@ qualified_id_join_index_.get())); EXPECT_THAT( handler->Handle(tokenized_document, kDefaultDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId, kQualifiedIdJoinablePropertyId)), IsOkAndHolds("pkg$db/ns#ref_type/1")); } @@ -283,17 +288,18 @@ std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), qualified_id_join_index_.get())); - EXPECT_THAT(handler->Handle(tokenized_document, kDefaultDocumentId, - /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), - IsOk()); + EXPECT_THAT( + handler->Handle(tokenized_document, kDefaultDocumentId, + /*old_document_id=*/kInvalidDocumentId, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId, kNestedQualifiedIdJoinablePropertyId)), IsOkAndHolds("pkg$db/ns#ref_type/2")); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId, kQualifiedId2JoinablePropertyId)), IsOkAndHolds("pkg$db/ns#ref_type/1")); } @@ -328,11 +334,12 @@ qualified_id_join_index_.get())); EXPECT_THAT( handler->Handle(tokenized_document, kDefaultDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId, kQualifiedIdJoinablePropertyId)), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } @@ -359,11 +366,12 @@ qualified_id_join_index_.get())); EXPECT_THAT( handler->Handle(tokenized_document, kDefaultDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId, kQualifiedIdJoinablePropertyId)), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } @@ -402,22 +410,24 @@ // index data and last_added_document_id should remain unchanged. EXPECT_THAT( handler->Handle(tokenized_document, kInvalidDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kInvalidDocumentId, kQualifiedIdJoinablePropertyId)), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); // Recovery mode should get the same result. EXPECT_THAT( handler->Handle(tokenized_document, kInvalidDocumentId, - /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + /*old_document_id=*/kInvalidDocumentId, + /*recovery_mode=*/true, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kInvalidDocumentId, kQualifiedIdJoinablePropertyId)), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } @@ -458,11 +468,12 @@ ASSERT_THAT(IsDocumentIdValid(kDefaultDocumentId - 1), IsTrue()); EXPECT_THAT( handler->Handle(tokenized_document, kDefaultDocumentId - 1, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId, kQualifiedIdJoinablePropertyId)), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); @@ -471,11 +482,12 @@ // unchanged. EXPECT_THAT( handler->Handle(tokenized_document, kDefaultDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId, kQualifiedIdJoinablePropertyId)), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } @@ -516,11 +528,12 @@ ASSERT_THAT(IsDocumentIdValid(kDefaultDocumentId - 1), IsTrue()); EXPECT_THAT( handler->Handle(tokenized_document, kDefaultDocumentId - 1, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId, kQualifiedIdJoinablePropertyId)), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); @@ -530,11 +543,12 @@ // unchanged. EXPECT_THAT( handler->Handle(tokenized_document, kDefaultDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId, kQualifiedIdJoinablePropertyId)), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); @@ -543,11 +557,12 @@ ASSERT_THAT(IsDocumentIdValid(kDefaultDocumentId + 1), IsTrue()); EXPECT_THAT( handler->Handle(tokenized_document, kDefaultDocumentId + 1, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(kDefaultDocumentId + 1)); - EXPECT_THAT(qualified_id_join_index_->Get(DocJoinInfo( + EXPECT_THAT(qualified_id_join_index_->Get(DocumentJoinIdPair( kDefaultDocumentId + 1, kQualifiedIdJoinablePropertyId)), IsOkAndHolds("pkg$db/ns#ref_type/1")); }
diff --git a/icing/join/qualified-id-join-indexing-handler_test.cc b/icing/join/qualified-id-join-indexing-handler-v2_test.cc similarity index 87% rename from icing/join/qualified-id-join-indexing-handler_test.cc rename to icing/join/qualified-id-join-indexing-handler-v2_test.cc index 95c0327..9c20d45 100644 --- a/icing/join/qualified-id-join-indexing-handler_test.cc +++ b/icing/join/qualified-id-join-indexing-handler-v2_test.cc
@@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "icing/join/qualified-id-join-indexing-handler.h" - #include <memory> #include <string> #include <string_view> @@ -26,11 +24,13 @@ #include "gtest/gtest.h" #include "icing/absl_ports/str_cat.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/join/document-id-to-join-info.h" #include "icing/join/qualified-id-join-index-impl-v2.h" #include "icing/join/qualified-id-join-index.h" +#include "icing/join/qualified-id-join-indexing-handler.h" #include "icing/join/qualified-id.h" #include "icing/portable/platform.h" #include "icing/proto/document.pb.h" @@ -41,15 +41,16 @@ #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" #include "icing/store/document-store.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/status-macros.h" #include "icing/util/tokenized-document.h" #include "unicode/uloc.h" @@ -80,13 +81,14 @@ static constexpr std::string_view kPropertyNestedDoc = "nested"; static constexpr std::string_view kPropertyQualifiedId2 = "qualifiedId2"; -class QualifiedIdJoinIndexingHandlerTest : public ::testing::Test { +class QualifiedIdJoinIndexingHandlerV2Test : public ::testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -112,8 +114,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), IsTrue()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType( @@ -153,13 +155,12 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); doc_store_ = std::move(create_result.document_store); @@ -200,6 +201,7 @@ filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; FakeClock fake_clock_; std::string base_dir_; @@ -239,7 +241,8 @@ return result; } -TEST_F(QualifiedIdJoinIndexingHandlerTest, CreationWithNullPointerShouldFail) { +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, + CreationWithNullPointerShouldFail) { EXPECT_THAT( QualifiedIdJoinIndexingHandler::Create( /*clock=*/nullptr, doc_store_.get(), qualified_id_join_index_.get()), @@ -256,7 +259,7 @@ StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } -TEST_F(QualifiedIdJoinIndexingHandlerTest, HandleJoinableProperty) { +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, HandleJoinableProperty) { // Create and put referenced (parent) document. Get its document id and // namespace id. DocumentProto referenced_document = @@ -271,9 +274,9 @@ ICING_ASSERT_OK_AND_ASSIGN( NamespaceId ref_doc_ns_id, doc_store_->GetNamespaceId(referenced_document.namespace_())); - NamespaceFingerprintIdentifier ref_doc_ns_fingerprint_id( + NamespaceIdFingerprint ref_doc_nsid_uri_fingerprint( /*namespace_id=*/ref_doc_ns_id, /*target_str=*/referenced_document.uri()); - ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_ns_fingerprint_id), + ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_nsid_uri_fingerprint), IsOkAndHolds(ref_doc_id)); // Create and put (child) document. Also tokenize it. @@ -299,24 +302,23 @@ QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), qualified_id_join_index_.get())); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); // Verify the state of qualified_id_join_index_ after Handle(). EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(doc_id)); // (kFakeType, kPropertyQualifiedId) should contain - // [(doc_id, ref_doc_ns_fingerprint_id)]. + // [(doc_id, ref_doc_nsid_uri_fingerprint)]. EXPECT_THAT( GetJoinData(*qualified_id_join_index_, /*schema_type_id=*/fake_type_id_, /*joinable_property_id=*/fake_type_joinable_property_id_), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/doc_id, - /*join_info=*/ref_doc_ns_fingerprint_id)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/doc_id, + /*join_info=*/ref_doc_nsid_uri_fingerprint)))); } -TEST_F(QualifiedIdJoinIndexingHandlerTest, HandleNestedJoinableProperty) { +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, HandleNestedJoinableProperty) { // Create and put referenced (parent) document1. Get its document id and // namespace id. DocumentProto referenced_document1 = @@ -331,10 +333,10 @@ ICING_ASSERT_OK_AND_ASSIGN( NamespaceId ref_doc_ns_id1, doc_store_->GetNamespaceId(referenced_document1.namespace_())); - NamespaceFingerprintIdentifier ref_doc_ns_fingerprint_id1( + NamespaceIdFingerprint ref_doc_nsid_uri_fingerprint1( /*namespace_id=*/ref_doc_ns_id1, /*target_str=*/referenced_document1.uri()); - ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_ns_fingerprint_id1), + ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_nsid_uri_fingerprint1), IsOkAndHolds(ref_doc_id1)); // Create and put referenced (parent) document2. Get its document id and @@ -351,10 +353,10 @@ ICING_ASSERT_OK_AND_ASSIGN( NamespaceId ref_doc_ns_id2, doc_store_->GetNamespaceId(referenced_document2.namespace_())); - NamespaceFingerprintIdentifier ref_doc_ns_fingerprint_id2( + NamespaceIdFingerprint ref_doc_nsid_uri_fingerprint2( /*namespace_id=*/ref_doc_ns_id2, /*target_str=*/referenced_document2.uri()); - ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_ns_fingerprint_id2), + ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_nsid_uri_fingerprint2), IsOkAndHolds(ref_doc_id2)); // Create and put (child) document: @@ -393,8 +395,8 @@ QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), qualified_id_join_index_.get())); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); // Verify the state of qualified_id_join_index_ after Handle(). @@ -405,27 +407,25 @@ /*joinable_property_id=*/fake_type_joinable_property_id_), IsOkAndHolds(IsEmpty())); // (kNestedType, kPropertyNestedDoc.kPropertyQualifiedId) should contain - // [(doc_id, ref_doc_ns_fingerprint_id2)]. + // [(doc_id, ref_doc_nsid_uri_fingerprint2)]. EXPECT_THAT( GetJoinData( *qualified_id_join_index_, /*schema_type_id=*/nested_type_id_, /*joinable_property_id=*/nested_type_nested_joinable_property_id_), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/doc_id, - /*join_info=*/ref_doc_ns_fingerprint_id2)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/doc_id, + /*join_info=*/ref_doc_nsid_uri_fingerprint2)))); // (kNestedType, kPropertyQualifiedId2) should contain - // [(doc_id, ref_doc_ns_fingerprint_id1)]. + // [(doc_id, ref_doc_nsid_uri_fingerprint1)]. EXPECT_THAT( GetJoinData(*qualified_id_join_index_, /*schema_type_id=*/nested_type_id_, /*joinable_property_id=*/nested_type_joinable_property_id_), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/doc_id, - /*join_info=*/ref_doc_ns_fingerprint_id1)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/doc_id, + /*join_info=*/ref_doc_nsid_uri_fingerprint1)))); } -TEST_F(QualifiedIdJoinIndexingHandlerTest, +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, HandleShouldSkipInvalidFormatQualifiedId) { static constexpr std::string_view kInvalidFormatQualifiedId = "invalid_format_qualified_id"; @@ -457,8 +457,8 @@ QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), qualified_id_join_index_.get())); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); // Verify the state of qualified_id_join_index_ after Handle(). Index data @@ -472,7 +472,7 @@ IsOkAndHolds(IsEmpty())); } -TEST_F(QualifiedIdJoinIndexingHandlerTest, +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, HandleShouldSkipNonExistingNamespace) { static constexpr std::string_view kUnknownNamespace = "UnknownNamespace"; // Create and put (child) document which references to a parent qualified id @@ -501,8 +501,8 @@ QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), qualified_id_join_index_.get())); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); // Verify the state of qualified_id_join_index_ after Handle(). @@ -515,7 +515,7 @@ IsOkAndHolds(IsEmpty())); } -TEST_F(QualifiedIdJoinIndexingHandlerTest, HandleShouldSkipEmptyQualifiedId) { +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, HandleShouldSkipEmptyQualifiedId) { // Create and put (child) document without any qualified id. Also tokenize it. DocumentProto document = DocumentBuilder() .SetKey("icing", "fake_type/1") @@ -538,8 +538,8 @@ QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), qualified_id_join_index_.get())); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), IsOk()); // Verify the state of qualified_id_join_index_ after Handle(). Index data @@ -553,7 +553,7 @@ IsOkAndHolds(IsEmpty())); } -TEST_F(QualifiedIdJoinIndexingHandlerTest, +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, HandleInvalidDocumentIdShouldReturnInvalidArgumentError) { // Create and put referenced (parent) document. Get its document id and // namespace id. @@ -569,9 +569,9 @@ ICING_ASSERT_OK_AND_ASSIGN( NamespaceId ref_doc_ns_id, doc_store_->GetNamespaceId(referenced_document.namespace_())); - NamespaceFingerprintIdentifier ref_doc_ns_fingerprint_id( + NamespaceIdFingerprint ref_doc_nsid_uri_fingerprint( /*namespace_id=*/ref_doc_ns_id, /*target_str=*/referenced_document.uri()); - ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_ns_fingerprint_id), + ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_nsid_uri_fingerprint), IsOkAndHolds(ref_doc_id)); // Create and put (child) document. Also tokenize it. @@ -600,6 +600,7 @@ // Handling document with kInvalidDocumentId should cause a failure. EXPECT_THAT( handler->Handle(tokenized_document, kInvalidDocumentId, + /*old_document_id=*/kInvalidDocumentId, /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Verify the state of qualified_id_join_index_ after Handle(). Both index @@ -615,7 +616,8 @@ // Recovery mode should get the same result. EXPECT_THAT( handler->Handle(tokenized_document, kInvalidDocumentId, - /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + /*old_document_id=*/kInvalidDocumentId, + /*recovery_mode=*/true, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(ref_doc_id)); @@ -626,7 +628,7 @@ IsOkAndHolds(IsEmpty())); } -TEST_F(QualifiedIdJoinIndexingHandlerTest, +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, HandleOutOfOrderDocumentIdShouldReturnInvalidArgumentError) { // Create and put referenced (parent) document. Get its document id and // namespace id. @@ -642,9 +644,9 @@ ICING_ASSERT_OK_AND_ASSIGN( NamespaceId ref_doc_ns_id, doc_store_->GetNamespaceId(referenced_document.namespace_())); - NamespaceFingerprintIdentifier ref_doc_ns_fingerprint_id( + NamespaceIdFingerprint ref_doc_nsid_uri_fingerprint( /*namespace_id=*/ref_doc_ns_id, /*target_str=*/referenced_document.uri()); - ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_ns_fingerprint_id), + ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_nsid_uri_fingerprint), IsOkAndHolds(ref_doc_id)); // Create and put (child) document. Also tokenize it. @@ -672,8 +674,8 @@ qualified_id_join_index_->set_last_added_document_id(doc_id); ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(doc_id)); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Verify the state of qualified_id_join_index_ after Handle(). Both index // data and last_added_document_id should remain unchanged. @@ -690,8 +692,8 @@ ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(doc_id + 1)); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/false, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Verify the state of qualified_id_join_index_ after Handle(). Both index // data and last_added_document_id should remain unchanged. @@ -704,7 +706,7 @@ IsOkAndHolds(IsEmpty())); } -TEST_F(QualifiedIdJoinIndexingHandlerTest, +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, HandleRecoveryModeShouldIndexDocsGtLastAddedDocId) { // Create and put referenced (parent) document. Get its document id and // namespace id. @@ -720,9 +722,9 @@ ICING_ASSERT_OK_AND_ASSIGN( NamespaceId ref_doc_ns_id, doc_store_->GetNamespaceId(referenced_document.namespace_())); - NamespaceFingerprintIdentifier ref_doc_ns_fingerprint_id( + NamespaceIdFingerprint ref_doc_nsid_uri_fingerprint( /*namespace_id=*/ref_doc_ns_id, /*target_str=*/referenced_document.uri()); - ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_ns_fingerprint_id), + ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_nsid_uri_fingerprint), IsOkAndHolds(ref_doc_id)); // Create and put (child) document. Also tokenize it. @@ -751,20 +753,19 @@ ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(doc_id - 1)); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/true, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(doc_id)); EXPECT_THAT( GetJoinData(*qualified_id_join_index_, /*schema_type_id=*/fake_type_id_, /*joinable_property_id=*/fake_type_joinable_property_id_), - IsOkAndHolds( - ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>( - /*document_id=*/doc_id, - /*join_info=*/ref_doc_ns_fingerprint_id)))); + IsOkAndHolds(ElementsAre(DocumentIdToJoinInfo<NamespaceIdFingerprint>( + /*document_id=*/doc_id, + /*join_info=*/ref_doc_nsid_uri_fingerprint)))); } -TEST_F(QualifiedIdJoinIndexingHandlerTest, +TEST_F(QualifiedIdJoinIndexingHandlerV2Test, HandleRecoveryModeShouldIgnoreDocsLeLastAddedDocId) { // Create and put referenced (parent) document. Get its document id and // namespace id. @@ -780,9 +781,9 @@ ICING_ASSERT_OK_AND_ASSIGN( NamespaceId ref_doc_ns_id, doc_store_->GetNamespaceId(referenced_document.namespace_())); - NamespaceFingerprintIdentifier ref_doc_ns_fingerprint_id( + NamespaceIdFingerprint ref_doc_nsid_uri_fingerprint( /*namespace_id=*/ref_doc_ns_id, /*target_str=*/referenced_document.uri()); - ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_ns_fingerprint_id), + ASSERT_THAT(doc_store_->GetDocumentId(ref_doc_nsid_uri_fingerprint), IsOkAndHolds(ref_doc_id)); // Create and put (child) document. Also tokenize it. @@ -812,8 +813,8 @@ qualified_id_join_index_->set_last_added_document_id(doc_id); ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(doc_id)); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/true, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(doc_id)); // (kFakeType, kPropertyQualifiedId) should contain nothing. @@ -829,8 +830,8 @@ ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(doc_id + 1)); EXPECT_THAT( - handler->Handle(tokenized_document, doc_id, /*recovery_mode=*/true, - /*put_document_stats=*/nullptr), + handler->Handle(tokenized_document, doc_id, put_result.old_document_id, + /*recovery_mode=*/true, /*put_document_stats=*/nullptr), IsOk()); EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), Eq(doc_id + 1));
diff --git a/icing/join/qualified-id-join-indexing-handler-v3_test.cc b/icing/join/qualified-id-join-indexing-handler-v3_test.cc new file mode 100644 index 0000000..5c87e19 --- /dev/null +++ b/icing/join/qualified-id-join-indexing-handler-v3_test.cc
@@ -0,0 +1,808 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <memory> +#include <string> +#include <string_view> +#include <utility> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/absl_ports/str_cat.h" +#include "icing/document-builder.h" +#include "icing/feature-flags.h" +#include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" +#include "icing/join/document-join-id-pair.h" +#include "icing/join/qualified-id-join-index-impl-v3.h" +#include "icing/join/qualified-id-join-indexing-handler.h" +#include "icing/join/qualified-id.h" +#include "icing/portable/platform.h" +#include "icing/proto/document.pb.h" +#include "icing/proto/schema.pb.h" +#include "icing/schema-builder.h" +#include "icing/schema/joinable-property.h" +#include "icing/schema/schema-store.h" +#include "icing/store/document-filter-data.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" +#include "icing/testing/common-matchers.h" +#include "icing/testing/fake-clock.h" +#include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" +#include "icing/testing/tmp-directory.h" +#include "icing/tokenization/language-segmenter-factory.h" +#include "icing/tokenization/language-segmenter.h" +#include "icing/util/icu-data-file-helper.h" +#include "icing/util/tokenized-document.h" +#include "unicode/uloc.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::IsTrue; +using ::testing::NotNull; +using ::testing::Pointee; +using ::testing::SizeIs; + +// Schema type for referenced documents: ReferencedType +static constexpr std::string_view kReferencedType = "ReferencedType"; +static constexpr std::string_view kPropertyName = "name"; + +// Joinable properties and joinable property id. Joinable property id is +// determined by the lexicographical order of joinable property path. +// Schema type with joinable property: FakeType +static constexpr std::string_view kFakeType = "FakeType"; +static constexpr std::string_view kPropertyQualifiedId = "qualifiedId"; + +// Schema type with nested joinable properties: NestedType +static constexpr std::string_view kNestedType = "NestedType"; +static constexpr std::string_view kPropertyNestedDoc = "nested"; +static constexpr std::string_view kPropertyQualifiedId2 = "qualifiedId2"; + +class QualifiedIdJoinIndexingHandlerV3Test : public ::testing::Test { + protected: + void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + + if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { + ICING_ASSERT_OK( + // File generated via icu_data_file rule in //icing/BUILD. + icu_data_file_helper::SetUpIcuDataFile( + GetTestFilePath("icing/icu.dat"))); + } + + base_dir_ = GetTestTempDir() + "/icing_test"; + ASSERT_THAT(filesystem_.CreateDirectoryRecursively(base_dir_.c_str()), + IsTrue()); + + qualified_id_join_index_dir_ = base_dir_ + "/qualified_id_join_index"; + schema_store_dir_ = base_dir_ + "/schema_store"; + doc_store_dir_ = base_dir_ + "/doc_store"; + + ICING_ASSERT_OK_AND_ASSIGN( + qualified_id_join_index_, + QualifiedIdJoinIndexImplV3::Create( + filesystem_, qualified_id_join_index_dir_, *feature_flags_)); + + language_segmenter_factory::SegmenterOptions segmenter_options(ULOC_US); + ICING_ASSERT_OK_AND_ASSIGN( + lang_segmenter_, + language_segmenter_factory::Create(std::move(segmenter_options))); + + ASSERT_THAT( + filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), + IsTrue()); + ICING_ASSERT_OK_AND_ASSIGN( + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType(kReferencedType) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertyName) + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder().SetType(kFakeType).AddProperty( + PropertyConfigBuilder() + .SetName(kPropertyQualifiedId) + .SetDataTypeJoinableString(JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kNestedType) + .AddProperty( + PropertyConfigBuilder() + .SetName(kPropertyNestedDoc) + .SetDataTypeDocument( + kFakeType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertyQualifiedId2) + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + ICING_ASSERT_OK(schema_store_->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + ASSERT_THAT(filesystem_.CreateDirectoryRecursively(doc_store_dir_.c_str()), + IsTrue()); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); + doc_store_ = std::move(create_result.document_store); + + // Get FakeType related ids. + ICING_ASSERT_OK_AND_ASSIGN(fake_type_id_, + schema_store_->GetSchemaTypeId(kFakeType)); + ICING_ASSERT_OK_AND_ASSIGN( + const JoinablePropertyMetadata* metadata1, + schema_store_->GetJoinablePropertyMetadata( + fake_type_id_, std::string(kPropertyQualifiedId))); + ASSERT_THAT(metadata1, NotNull()); + fake_type_joinable_property_id_ = metadata1->id; + + // Get NestedType related ids. + ICING_ASSERT_OK_AND_ASSIGN(nested_type_id_, + schema_store_->GetSchemaTypeId(kNestedType)); + ICING_ASSERT_OK_AND_ASSIGN( + const JoinablePropertyMetadata* metadata2, + schema_store_->GetJoinablePropertyMetadata( + nested_type_id_, + absl_ports::StrCat(kPropertyNestedDoc, ".", kPropertyQualifiedId))); + ASSERT_THAT(metadata2, NotNull()); + nested_type_nested_joinable_property_id_ = metadata2->id; + ICING_ASSERT_OK_AND_ASSIGN( + const JoinablePropertyMetadata* metadata3, + schema_store_->GetJoinablePropertyMetadata( + nested_type_id_, std::string(kPropertyQualifiedId2))); + ASSERT_THAT(metadata3, NotNull()); + nested_type_joinable_property_id_ = metadata3->id; + } + + void TearDown() override { + doc_store_.reset(); + schema_store_.reset(); + lang_segmenter_.reset(); + qualified_id_join_index_.reset(); + + filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); + } + + std::unique_ptr<FeatureFlags> feature_flags_; + Filesystem filesystem_; + FakeClock fake_clock_; + std::string base_dir_; + std::string qualified_id_join_index_dir_; + std::string schema_store_dir_; + std::string doc_store_dir_; + + std::unique_ptr<QualifiedIdJoinIndexImplV3> qualified_id_join_index_; + std::unique_ptr<LanguageSegmenter> lang_segmenter_; + std::unique_ptr<SchemaStore> schema_store_; + std::unique_ptr<DocumentStore> doc_store_; + + // FakeType related ids. + SchemaTypeId fake_type_id_; + JoinablePropertyId fake_type_joinable_property_id_; + + // NestedType related ids. + SchemaTypeId nested_type_id_; + JoinablePropertyId nested_type_nested_joinable_property_id_; + JoinablePropertyId nested_type_joinable_property_id_; +}; + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, + CreationWithNullPointerShouldFail) { + EXPECT_THAT( + QualifiedIdJoinIndexingHandler::Create( + /*clock=*/nullptr, doc_store_.get(), qualified_id_join_index_.get()), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + + EXPECT_THAT( + QualifiedIdJoinIndexingHandler::Create( + &fake_clock_, /*doc_store=*/nullptr, qualified_id_join_index_.get()), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + + EXPECT_THAT( + QualifiedIdJoinIndexingHandler::Create( + &fake_clock_, doc_store_.get(), /*qualified_id_join_index=*/nullptr), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); +} + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, HandleJoinableProperty) { + // Create and put parent document. + DocumentProto parent_document = + DocumentBuilder() + .SetKey("pkg$db/ns", "ref_type/1") + .SetSchema(std::string(kReferencedType)) + .AddStringProperty(std::string(kPropertyName), "one") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult parent_put_result, + doc_store_->Put(parent_document)); + + // Create and put child document. Also tokenize it. + DocumentProto child_document = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyQualifiedId), + "pkg$db/ns#ref_type/1") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result, + doc_store_->Put(child_document)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(child_document))); + + // Handle document. + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(kInvalidDocumentId)); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + EXPECT_THAT( + handler->Handle(tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + + // Verify the state of qualified_id_join_index_ after Handle(). + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id)); + EXPECT_THAT(qualified_id_join_index_, Pointee(SizeIs(1))); + EXPECT_THAT( + qualified_id_join_index_->Get(parent_put_result.new_document_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + child_put_result.new_document_id, fake_type_joinable_property_id_)))); +} + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, HandleNestedJoinableProperty) { + // Create and put parent document1. Get its document id and namespace id. + DocumentProto parent_document1 = + DocumentBuilder() + .SetKey("pkg$db/ns", "ref_type/1") + .SetSchema(std::string(kReferencedType)) + .AddStringProperty(std::string(kPropertyName), "one") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult parent_put_result1, + doc_store_->Put(parent_document1)); + + // Create and put parent document2. + DocumentProto parent_document2 = + DocumentBuilder() + .SetKey("pkg$db/ns", "ref_type/2") + .SetSchema(std::string(kReferencedType)) + .AddStringProperty(std::string(kPropertyName), "two") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult parent_put_result2, + doc_store_->Put(parent_document2)); + + // Create and put child document: + // - kPropertyNestedDoc.kPropertyQualifiedId refers to parent_document2. + // - kPropertyQualifiedId2 refers to parent_document1. + // + // Also tokenize it. + DocumentProto child_document = + DocumentBuilder() + .SetKey("pkg$db/ns", "nested_type/1") + .SetSchema(std::string(kNestedType)) + .AddDocumentProperty( + std::string(kPropertyNestedDoc), + DocumentBuilder() + .SetKey("pkg$db/ns", "nested_fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyQualifiedId), + "pkg$db/ns#ref_type/2") + .Build()) + .AddStringProperty(std::string(kPropertyQualifiedId2), + "pkg$db/ns#ref_type/1") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result, + doc_store_->Put(child_document)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + child_document)); + + // Handle nested_document. + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(kInvalidDocumentId)); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + EXPECT_THAT( + handler->Handle(tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + + // Verify the state of qualified_id_join_index_ after Handle(). + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id)); + EXPECT_THAT(qualified_id_join_index_, Pointee(SizeIs(2))); + EXPECT_THAT(qualified_id_join_index_->Get(parent_put_result1.new_document_id), + IsOkAndHolds(ElementsAre( + DocumentJoinIdPair(child_put_result.new_document_id, + nested_type_joinable_property_id_)))); + EXPECT_THAT(qualified_id_join_index_->Get(parent_put_result2.new_document_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + child_put_result.new_document_id, + nested_type_nested_joinable_property_id_)))); +} + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, + HandleShouldSkipInvalidFormatQualifiedId) { + static constexpr std::string_view kInvalidFormatQualifiedId = + "invalid_format_qualified_id"; + ASSERT_THAT(QualifiedId::Parse(kInvalidFormatQualifiedId), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + + // Create and put child document with an invalid format referenced qualified + // id. Also tokenize it. + DocumentProto document = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyQualifiedId), + std::string(kInvalidFormatQualifiedId)) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result, + doc_store_->Put(document)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + document)); + + // Handle document. Should ignore invalid format qualified id. + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(kInvalidDocumentId)); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + EXPECT_THAT( + handler->Handle(tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + + // Verify the state of qualified_id_join_index_ after Handle(). Index data + // should remain unchanged since there is no valid qualified id, but + // last_added_document_id should be updated. + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id)); + EXPECT_THAT(qualified_id_join_index_, Pointee(IsEmpty())); +} + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, HandleShouldSkipEmptyQualifiedId) { + // Create and put child document without any qualified id. Also tokenize it. + DocumentProto document = DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result, + doc_store_->Put(document)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + document)); + ASSERT_THAT(tokenized_document.qualified_id_join_properties(), IsEmpty()); + + // Handle document. + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(kInvalidDocumentId)); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + EXPECT_THAT( + handler->Handle(tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr), + IsOk()); + + // Verify the state of qualified_id_join_index_ after Handle(). Index data + // should remain unchanged since there is no qualified id, but + // last_added_document_id should be updated. + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id)); + EXPECT_THAT(qualified_id_join_index_, Pointee(IsEmpty())); +} + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, HandleShouldMigrateParent) { + // Create and put parent document. + DocumentProto parent_document = + DocumentBuilder() + .SetKey("pkg$db/ns", "ref_type/1") + .SetSchema(std::string(kReferencedType)) + .AddStringProperty(std::string(kPropertyName), "one") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult parent_put_result, + doc_store_->Put(parent_document)); + + // Create and put child and grandchild document with relations: + // parent_document <- child_document <- grandchild_document + // Also tokenize them. + DocumentProto child_document = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyQualifiedId), + "pkg$db/ns#ref_type/1") + .Build(); + DocumentProto grandchild_document = + DocumentBuilder() + .SetKey("icing", "fake_type/2") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyQualifiedId), + "icing#fake_type/1") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + + // Put and index child document. + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result, + doc_store_->Put(child_document)); + ASSERT_THAT(child_put_result.old_document_id, Eq(kInvalidDocumentId)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument child_tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + child_document)); + ICING_ASSERT_OK(handler->Handle( + child_tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr)); + + // Put and index grandchild document. + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult grandchild_put_result, + doc_store_->Put(grandchild_document)); + ASSERT_THAT(grandchild_put_result.old_document_id, Eq(kInvalidDocumentId)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument grandchild_tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(grandchild_document))); + ICING_ASSERT_OK(handler->Handle( + grandchild_tokenized_document, grandchild_put_result.new_document_id, + grandchild_put_result.old_document_id, + /*recovery_mode=*/false, /*put_document_stats=*/nullptr)); + + // Sanity check: parent contains child join id pair and child contains + // grandchild join id pair. + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(grandchild_put_result.new_document_id)); + ASSERT_THAT(qualified_id_join_index_, Pointee(SizeIs(2))); + ASSERT_THAT( + qualified_id_join_index_->Get(parent_put_result.new_document_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + child_put_result.new_document_id, fake_type_joinable_property_id_)))); + ASSERT_THAT(qualified_id_join_index_->Get(child_put_result.new_document_id), + IsOkAndHolds(ElementsAre( + DocumentJoinIdPair(grandchild_put_result.new_document_id, + fake_type_joinable_property_id_)))); + + // Update the child document and index it again. + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result2, + doc_store_->Put(child_document)); + ASSERT_THAT(child_put_result2.old_document_id, + Eq(child_put_result.new_document_id)); + + // Handle should migrate. + EXPECT_THAT(handler->Handle( + child_tokenized_document, child_put_result2.new_document_id, + child_put_result2.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(qualified_id_join_index_, Pointee(SizeIs(3))); + // Get() with parent document id should return DocumentJoinIdPairs for both + // old and new child document id. + EXPECT_THAT(qualified_id_join_index_->Get(parent_put_result.new_document_id), + IsOkAndHolds(ElementsAre( + DocumentJoinIdPair(child_put_result.new_document_id, + fake_type_joinable_property_id_), + DocumentJoinIdPair(child_put_result2.new_document_id, + fake_type_joinable_property_id_)))); + // Get() with old child document id should return empty list. + EXPECT_THAT(qualified_id_join_index_->Get(child_put_result.new_document_id), + IsOkAndHolds(IsEmpty())); + // Get() with new child document id should return grandchild join id pair. + EXPECT_THAT(qualified_id_join_index_->Get(child_put_result2.new_document_id), + IsOkAndHolds(ElementsAre( + DocumentJoinIdPair(grandchild_put_result.new_document_id, + fake_type_joinable_property_id_)))); +} + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, + HandleInvalidNewDocumentIdShouldReturnInvalidArgumentError) { + // Create and put parent document. + DocumentProto parent_document = + DocumentBuilder() + .SetKey("pkg$db/ns", "ref_type/1") + .SetSchema(std::string(kReferencedType)) + .AddStringProperty(std::string(kPropertyName), "one") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult parent_put_result, + doc_store_->Put(parent_document)); + + // Create and put child document. Also tokenize it. + DocumentProto document = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyQualifiedId), + "pkg$db/ns#ref_type/1") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result, + doc_store_->Put(document)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document))); + + qualified_id_join_index_->set_last_added_document_id( + parent_put_result.new_document_id); + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(parent_put_result.new_document_id)); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + + // Handling document with kInvalidDocumentId should cause a failure. + EXPECT_THAT( + handler->Handle(tokenized_document, kInvalidDocumentId, + child_put_result.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + // Verify the state of qualified_id_join_index_ after Handle(). Both index + // data and last_added_document_id should remain unchanged. + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(parent_put_result.new_document_id)); + EXPECT_THAT(qualified_id_join_index_, Pointee(IsEmpty())); + + // Recovery mode should get the same result. + EXPECT_THAT( + handler->Handle(tokenized_document, kInvalidDocumentId, + child_put_result.old_document_id, /*recovery_mode=*/true, + /*put_document_stats=*/nullptr), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(parent_put_result.new_document_id)); + EXPECT_THAT(qualified_id_join_index_, Pointee(IsEmpty())); + EXPECT_THAT(qualified_id_join_index_->Get(parent_put_result.new_document_id), + IsOkAndHolds(IsEmpty())); +} + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, + HandleOutOfOrderDocumentIdShouldReturnInvalidArgumentError) { + // Create and put parent document. + DocumentProto parent_document = + DocumentBuilder() + .SetKey("pkg$db/ns", "ref_type/1") + .SetSchema(std::string(kReferencedType)) + .AddStringProperty(std::string(kPropertyName), "one") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult parent_put_result, + doc_store_->Put(parent_document)); + + // Create and put child document. Also tokenize it. + DocumentProto document = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyQualifiedId), + "pkg$db/ns#ref_type/1") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result, + doc_store_->Put(document)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document))); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + + // Handling document with document_id == last_added_document_id should cause a + // failure. + qualified_id_join_index_->set_last_added_document_id( + child_put_result.new_document_id); + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id)); + EXPECT_THAT( + handler->Handle(tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + // Verify the state of qualified_id_join_index_ after Handle(). Both index + // data and last_added_document_id should remain unchanged. + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id)); + EXPECT_THAT(qualified_id_join_index_, Pointee(IsEmpty())); + EXPECT_THAT(qualified_id_join_index_->Get(parent_put_result.new_document_id), + IsOkAndHolds(IsEmpty())); + + // Handling document with document_id < last_added_document_id should cause a + // failure. + qualified_id_join_index_->set_last_added_document_id( + child_put_result.new_document_id + 1); + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id + 1)); + EXPECT_THAT( + handler->Handle(tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + // Verify the state of qualified_id_join_index_ after Handle(). Both index + // data and last_added_document_id should remain unchanged. + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id + 1)); + EXPECT_THAT(qualified_id_join_index_, Pointee(IsEmpty())); + EXPECT_THAT(qualified_id_join_index_->Get(parent_put_result.new_document_id), + IsOkAndHolds(IsEmpty())); +} + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, + HandleRecoveryModeShouldIndexDocsGtLastAddedDocId) { + // Create and put parent document. + DocumentProto parent_document = + DocumentBuilder() + .SetKey("pkg$db/ns", "ref_type/1") + .SetSchema(std::string(kReferencedType)) + .AddStringProperty(std::string(kPropertyName), "one") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult parent_put_result, + doc_store_->Put(parent_document)); + + // Create and put child document. Also tokenize it. + DocumentProto document = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyQualifiedId), + "pkg$db/ns#ref_type/1") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result, + doc_store_->Put(document)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document))); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + + // Handle document with document_id > last_added_document_id in recovery mode. + // The handler should index this document and update last_added_document_id. + qualified_id_join_index_->set_last_added_document_id( + child_put_result.new_document_id - 1); + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id - 1)); + EXPECT_THAT( + handler->Handle(tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, /*recovery_mode=*/true, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id)); + EXPECT_THAT(qualified_id_join_index_, Pointee(SizeIs(1))); + EXPECT_THAT( + qualified_id_join_index_->Get(parent_put_result.new_document_id), + IsOkAndHolds(ElementsAre(DocumentJoinIdPair( + child_put_result.new_document_id, fake_type_joinable_property_id_)))); +} + +TEST_F(QualifiedIdJoinIndexingHandlerV3Test, + HandleRecoveryModeShouldIgnoreDocsLeLastAddedDocId) { + // Create and put parent document. + DocumentProto parent_document = + DocumentBuilder() + .SetKey("pkg$db/ns", "ref_type/1") + .SetSchema(std::string(kReferencedType)) + .AddStringProperty(std::string(kPropertyName), "one") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult parent_put_result, + doc_store_->Put(parent_document)); + + // Create and put child document. Also tokenize it. + DocumentProto document = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyQualifiedId), + "pkg$db/ns#ref_type/1") + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult child_put_result, + doc_store_->Put(document)); + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document))); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<QualifiedIdJoinIndexingHandler> handler, + QualifiedIdJoinIndexingHandler::Create(&fake_clock_, doc_store_.get(), + qualified_id_join_index_.get())); + + // Handle document with document_id == last_added_document_id in recovery + // mode. We should not get any error, but the handler should ignore the + // document, so both index data and last_added_document_id should remain + // unchanged. + qualified_id_join_index_->set_last_added_document_id( + child_put_result.new_document_id); + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id)); + EXPECT_THAT( + handler->Handle(tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, /*recovery_mode=*/true, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id)); + EXPECT_THAT(qualified_id_join_index_, Pointee(IsEmpty())); + EXPECT_THAT(qualified_id_join_index_->Get(parent_put_result.new_document_id), + IsOkAndHolds(IsEmpty())); + + // Handle document with document_id < last_added_document_id in recovery mode. + // We should not get any error, but the handler should ignore the document, so + // both index data and last_added_document_id should remain unchanged. + qualified_id_join_index_->set_last_added_document_id( + child_put_result.new_document_id + 1); + ASSERT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id + 1)); + EXPECT_THAT( + handler->Handle(tokenized_document, child_put_result.new_document_id, + child_put_result.old_document_id, /*recovery_mode=*/true, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(qualified_id_join_index_->last_added_document_id(), + Eq(child_put_result.new_document_id + 1)); + EXPECT_THAT(qualified_id_join_index_, Pointee(IsEmpty())); + EXPECT_THAT(qualified_id_join_index_->Get(parent_put_result.new_document_id), + IsOkAndHolds(IsEmpty())); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/join/qualified-id-join-indexing-handler.cc b/icing/join/qualified-id-join-indexing-handler.cc index df86cba..14d8231 100644 --- a/icing/join/qualified-id-join-indexing-handler.cc +++ b/icing/join/qualified-id-join-indexing-handler.cc
@@ -25,7 +25,7 @@ #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" -#include "icing/join/doc-join-info.h" +#include "icing/join/document-join-id-pair.h" #include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id.h" #include "icing/legacy/core/icing-string-util.h" @@ -34,7 +34,7 @@ #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" #include "icing/store/document-store.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/util/clock.h" #include "icing/util/logging.h" @@ -60,7 +60,8 @@ libtextclassifier3::Status QualifiedIdJoinIndexingHandler::Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, - bool recovery_mode, PutDocumentStatsProto* put_document_stats) { + DocumentId old_document_id, bool recovery_mode, + PutDocumentStatsProto* put_document_stats) { std::unique_ptr<Timer> index_timer = clock_.GetNewTimer(); if (!IsDocumentIdValid(document_id)) { @@ -81,90 +82,17 @@ } qualified_id_join_index_.set_last_added_document_id(document_id); - if (qualified_id_join_index_.is_v2()) { - // v2 - std::optional<DocumentFilterData> filter_data = - doc_store_.GetAliveDocumentFilterData( - document_id, - /*current_time_ms=*/std::numeric_limits<int64_t>::min()); - if (!filter_data) { - // This should not happen. - return absl_ports::InternalError( - "Failed to get alive document filter data when indexing"); - } - - for (const JoinableProperty<std::string_view>& qualified_id_property : - tokenized_document.qualified_id_join_properties()) { - // Parse all qualified id strings and convert them to - // NamespaceFingerprintIdentifier. - std::vector<NamespaceFingerprintIdentifier> ref_doc_ns_fingerprint_ids; - for (std::string_view ref_qualified_id_str : - qualified_id_property.values) { - // Attempt to parse qualified id string to make sure the format is - // correct. - auto ref_qualified_id_or = QualifiedId::Parse(ref_qualified_id_str); - if (!ref_qualified_id_or.ok()) { - // Skip incorrect format of qualified id string. - continue; - } - - QualifiedId ref_qualified_id = - std::move(ref_qualified_id_or).ValueOrDie(); - auto ref_namespace_id_or = - doc_store_.GetNamespaceId(ref_qualified_id.name_space()); - if (!ref_namespace_id_or.ok()) { - // Skip invalid namespace id. - continue; - } - NamespaceId ref_namespace_id = - std::move(ref_namespace_id_or).ValueOrDie(); - - ref_doc_ns_fingerprint_ids.push_back(NamespaceFingerprintIdentifier( - ref_namespace_id, ref_qualified_id.uri())); - } - - // Batch add all join data of this (schema_type_id, joinable_property_id) - // into to the index. - libtextclassifier3::Status status = qualified_id_join_index_.Put( - filter_data->schema_type_id(), qualified_id_property.metadata.id, - document_id, std::move(ref_doc_ns_fingerprint_ids)); - if (!status.ok()) { - ICING_LOG(WARNING) - << "Failed to add data into qualified id join index v2 due to: " - << status.error_message(); - return status; - } - } - } else { - // v1 - // TODO(b/275121148): deprecate this part after rollout v2. - for (const JoinableProperty<std::string_view>& qualified_id_property : - tokenized_document.qualified_id_join_properties()) { - if (qualified_id_property.values.empty()) { - continue; - } - - DocJoinInfo info(document_id, qualified_id_property.metadata.id); - // Currently we only support single (non-repeated) joinable value under a - // property. - std::string_view ref_qualified_id_str = qualified_id_property.values[0]; - - // Attempt to parse qualified id string to make sure the format is - // correct. - if (!QualifiedId::Parse(ref_qualified_id_str).ok()) { - // Skip incorrect format of qualified id string to save disk space. - continue; - } - - libtextclassifier3::Status status = - qualified_id_join_index_.Put(info, ref_qualified_id_str); - if (!status.ok()) { - ICING_LOG(WARNING) - << "Failed to add data into qualified id join index due to: " - << status.error_message(); - return status; - } - } + switch (qualified_id_join_index_.version()) { + case QualifiedIdJoinIndex::Version::kV1: + ICING_RETURN_IF_ERROR(HandleV1(tokenized_document, document_id)); + break; + case QualifiedIdJoinIndex::Version::kV2: + ICING_RETURN_IF_ERROR(HandleV2(tokenized_document, document_id)); + break; + case QualifiedIdJoinIndex::Version::kV3: + ICING_RETURN_IF_ERROR( + HandleV3(tokenized_document, document_id, old_document_id)); + break; } if (put_document_stats != nullptr) { @@ -175,5 +103,155 @@ return libtextclassifier3::Status::OK; } +libtextclassifier3::Status QualifiedIdJoinIndexingHandler::HandleV1( + const TokenizedDocument& tokenized_document, DocumentId document_id) { + for (const JoinableProperty<std::string_view>& qualified_id_property : + tokenized_document.qualified_id_join_properties()) { + if (qualified_id_property.values.empty()) { + continue; + } + + DocumentJoinIdPair document_join_id_pair(document_id, + qualified_id_property.metadata.id); + // Currently we only support single (non-repeated) joinable value under a + // property. + std::string_view ref_qualified_id_str = qualified_id_property.values[0]; + + // Attempt to parse qualified id string to make sure the format is + // correct. + if (!QualifiedId::Parse(ref_qualified_id_str).ok()) { + // Skip incorrect format of qualified id string to save disk space. + continue; + } + + libtextclassifier3::Status status = qualified_id_join_index_.Put( + document_join_id_pair, ref_qualified_id_str); + if (!status.ok()) { + ICING_LOG(WARNING) + << "Failed to add data into qualified id join index due to: " + << status.error_message(); + return status; + } + } + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::Status QualifiedIdJoinIndexingHandler::HandleV2( + const TokenizedDocument& tokenized_document, DocumentId document_id) { + std::optional<DocumentFilterData> filter_data = + doc_store_.GetAliveDocumentFilterData( + document_id, + /*current_time_ms=*/std::numeric_limits<int64_t>::min()); + if (!filter_data) { + // This should not happen. + return absl_ports::InternalError( + "Failed to get alive document filter data when indexing"); + } + + for (const JoinableProperty<std::string_view>& qualified_id_property : + tokenized_document.qualified_id_join_properties()) { + // Parse all qualified id strings and convert them to + // NamespaceIdFingerprint. + std::vector<NamespaceIdFingerprint> ref_doc_nsid_uri_fingerprints; + for (std::string_view ref_qualified_id_str : qualified_id_property.values) { + // Attempt to parse qualified id string to make sure the format is + // correct. + auto ref_qualified_id_or = QualifiedId::Parse(ref_qualified_id_str); + if (!ref_qualified_id_or.ok()) { + // Skip incorrect format of qualified id string. + continue; + } + + QualifiedId ref_qualified_id = + std::move(ref_qualified_id_or).ValueOrDie(); + auto ref_namespace_id_or = + doc_store_.GetNamespaceId(ref_qualified_id.name_space()); + if (!ref_namespace_id_or.ok()) { + // Skip invalid namespace id. + continue; + } + NamespaceId ref_namespace_id = + std::move(ref_namespace_id_or).ValueOrDie(); + + ref_doc_nsid_uri_fingerprints.push_back( + NamespaceIdFingerprint(ref_namespace_id, ref_qualified_id.uri())); + } + + // Batch add all join data of this (schema_type_id, joinable_property_id) + // into to the index. + libtextclassifier3::Status status = qualified_id_join_index_.Put( + filter_data->schema_type_id(), qualified_id_property.metadata.id, + document_id, std::move(ref_doc_nsid_uri_fingerprints)); + if (!status.ok()) { + ICING_LOG(WARNING) + << "Failed to add data into qualified id join index v2 due to: " + << status.error_message(); + return status; + } + } + return libtextclassifier3::Status::OK; +} + +libtextclassifier3::Status QualifiedIdJoinIndexingHandler::HandleV3( + const TokenizedDocument& tokenized_document, DocumentId document_id, + DocumentId old_document_id) { + // (Parent perspective) + // When replacement, if there were any existing child documents joining to it, + // then we need to migrate the old document id to the new document id. + if (IsDocumentIdValid(old_document_id)) { + ICING_RETURN_IF_ERROR( + qualified_id_join_index_.MigrateParent(old_document_id, document_id)); + } + + // (Child perspective) + // Add child join data. + for (const JoinableProperty<std::string_view>& qualified_id_property : + tokenized_document.qualified_id_join_properties()) { + if (qualified_id_property.values.empty()) { + continue; + } + + DocumentJoinIdPair child_doc_join_id_pair( + document_id, qualified_id_property.metadata.id); + + // Extract parent qualified ids and lookup their corresponding document ids. + std::vector<DocumentId> parent_doc_ids; + parent_doc_ids.reserve(qualified_id_property.values.size()); + for (std::string_view parent_qualified_id_str : + qualified_id_property.values) { + libtextclassifier3::StatusOr<QualifiedId> parent_qualified_id_or = + QualifiedId::Parse(parent_qualified_id_str); + if (!parent_qualified_id_or.ok()) { + // Skip incorrect format of qualified id string. + continue; + } + QualifiedId parent_qualified_id = + std::move(parent_qualified_id_or).ValueOrDie(); + + // Lookup document store to get the parent document id. + libtextclassifier3::StatusOr<DocumentId> parent_doc_id_or = + doc_store_.GetDocumentId(parent_qualified_id.name_space(), + parent_qualified_id.uri()); + if (!parent_doc_id_or.ok() || + parent_doc_id_or.ValueOrDie() == kInvalidDocumentId) { + // Skip invalid parent document id or parent document does not exist. + continue; + } + parent_doc_ids.push_back(parent_doc_id_or.ValueOrDie()); + } + + // Add all parent document ids to the index. + libtextclassifier3::Status status = qualified_id_join_index_.Put( + child_doc_join_id_pair, std::move(parent_doc_ids)); + if (!status.ok()) { + ICING_LOG(WARNING) + << "Failed to add data into qualified id join index due to: " + << status.error_message(); + return status; + } + } + return libtextclassifier3::Status::OK; +} + } // namespace lib } // namespace icing
diff --git a/icing/join/qualified-id-join-indexing-handler.h b/icing/join/qualified-id-join-indexing-handler.h index 8a11bf9..53c166c 100644 --- a/icing/join/qualified-id-join-indexing-handler.h +++ b/icing/join/qualified-id-join-indexing-handler.h
@@ -49,6 +49,14 @@ // Handles the joinable qualified id data indexing process: add data into the // qualified id join index. // + // old_document_id: + // - It is only used in QualifiedIdJoinIndexImplV3: if we update a parent + // document, then the existing join data for the parent document should be + // migrated from old_document_id to (new) document_id. + // - For other join index versions, since we store (namespace id, + // fingerprint(uri)) or the raw qualified id string for the parent, there is + // no need to migrate. + // /// Returns: // - OK on success. // - INVALID_ARGUMENT_ERROR if document_id is invalid OR document_id is less @@ -58,7 +66,8 @@ // - Any QualifiedIdJoinIndex errors. libtextclassifier3::Status Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, - bool recovery_mode, PutDocumentStatsProto* put_document_stats) override; + DocumentId old_document_id, bool recovery_mode, + PutDocumentStatsProto* put_document_stats) override; private: explicit QualifiedIdJoinIndexingHandler( @@ -68,6 +77,21 @@ doc_store_(*doc_store), qualified_id_join_index_(*qualified_id_join_index) {} + // TODO(b/275121148): deprecate v1, v2 after rollout v3. + + // Helper function to handle indexing for QualfiedIdJoinIndexImplV1. + libtextclassifier3::Status HandleV1( + const TokenizedDocument& tokenized_document, DocumentId document_id); + + // Helper function to handle indexing for QualfiedIdJoinIndexImplV2. + libtextclassifier3::Status HandleV2( + const TokenizedDocument& tokenized_document, DocumentId document_id); + + // Helper function to handle indexing for QualfiedIdJoinIndexImplV3. + libtextclassifier3::Status HandleV3( + const TokenizedDocument& tokenized_document, DocumentId document_id, + DocumentId old_document_id); + const DocumentStore& doc_store_; // Does not own. QualifiedIdJoinIndex& qualified_id_join_index_; // Does not own. };
diff --git a/icing/legacy/index/icing-dynamic-trie.cc b/icing/legacy/index/icing-dynamic-trie.cc index 4514385..e665272 100644 --- a/icing/legacy/index/icing-dynamic-trie.cc +++ b/icing/legacy/index/icing-dynamic-trie.cc
@@ -284,7 +284,11 @@ const Node *GetNode(uint32_t idx) const { return &array_storage_[NODE].array_cast<Node>()[idx]; } + + // REQUIRES: !empty(). Otherwise node 0 could contain invalid data + // (next_index, is_leaf, log2_num_children). const Node *GetRootNode() const { return GetNode(0); } + const Next *GetNext(uint32_t idx, int child) const { return &array_storage_[NEXT].array_cast<Next>()[idx + child]; } @@ -336,6 +340,8 @@ void inc_num_keys() { hdr_.hdr.set_num_keys(hdr_.hdr.num_keys() + 1); } + void dec_num_keys() { hdr_.hdr.set_num_keys(hdr_.hdr.num_keys() - 1); } + private: friend void IcingDynamicTrie::SetHeader( const IcingDynamicTrieHeader &new_hdr); @@ -1072,9 +1078,22 @@ // level - how many levels deep we are in the trie // ret - the stream to pretty print to // keys - the keys encountered are appended to this + // + // REQUIRES: node is valid. + // - Since we only invalidate Next to remove the edge from the trie and Node + // is not invalidated after deletion, the caller should ensure that it + // traverses correctly to a valid node according to the trie structure. + // Calling this function with an invalid node is undefined behavior since + // it could traverse into a deleted subtree, or invalid memory addresses. + // - This also means storage_->empty() should be checked before calling this + // function with the root node. void DumpNodeRecursive(const std::string &prefix, const Node &node, int level, std::ostream *ret, std::vector<std::string> *keys) const { + // This function should be called only if the node is valid. The first call + // is always from the root node, so it means the trie should not be empty at + // this moment. Otherwise (root) node could contain invalid next_index(), + // is_leaf(), and log2_num_children(). if (node.is_leaf()) { // Dump suffix and value. for (int i = 0; i < level; i++) { @@ -1368,8 +1387,19 @@ return storage_->hdr().num_keys(); } +bool IcingDynamicTrie::empty() const { + if (!is_initialized()) { + ICING_LOG(FATAL) << "DynamicTrie not initialized"; + } + return storage_->empty(); +} + void IcingDynamicTrie::CollectStatsRecursive(const Node &node, Stats *stats, uint32_t depth) const { + // This function should be called only if the node is valid. The first call is + // always from the root node, so it means the trie should not be empty at this + // moment. Otherwise (root) node could contain invalid next_index(), + // is_leaf(), and log2_num_children(). if (node.is_leaf()) { stats->num_leaves++; stats->sum_depth += depth; @@ -2577,6 +2607,19 @@ // 2. Remove the suffix and the value. // 3. Reset the nexts that point to the nodes to be removed. // 4. Sort any next array if needed. +// 5. Reset the trie state if the trie is empty after deletion. +// - This is essential for storage_->empty(), which is a critical check for +// all trie APIs before accessing the root node via +// storage_->GetRootNode(). +// - When the trie is empty, it is possible that the root node (i.e. +// Node(0)): +// - Contains an invalid next_index(), and accessing it will cause a crash +// or fetch incorrect data. +// - Points to a valid next array but the next elements in the array +// contain kInvalidNodeIndex. Accessing the next node via the next +// element will cause a crash or fetch incorrect data. +// - So we must reset the trie state to make sure storage_->empty() works +// correctly and prevents trie APIs from accessing the root node. bool IcingDynamicTrie::Delete(std::string_view key) { if (!is_initialized()) { ICING_LOG(ERROR) << "DynamicTrie not initialized"; @@ -2688,6 +2731,19 @@ } } + storage_->dec_num_keys(); + if (storage_->hdr().num_keys() == 0) { + // Reset the trie state to empty by calling Clear() directly. + // + // Note: in this case, last_multichild_node will be nullptr as well. + // - If we never saw a node with multiple children before deletion, then all + // the traversed nodes, including the root node, are single-child nodes + // before deletion. + // - Therefore, after deletion, there should be no valid nodes or nexts in + // the trie. + Clear(); + } + return true; }
diff --git a/icing/legacy/index/icing-dynamic-trie.h b/icing/legacy/index/icing-dynamic-trie.h index 301ee24..53d9bad 100644 --- a/icing/legacy/index/icing-dynamic-trie.h +++ b/icing/legacy/index/icing-dynamic-trie.h
@@ -284,6 +284,8 @@ // Number of keys in trie. uint32_t size() const; + bool empty() const; + // Collecting stats. void CollectStats(Stats *stats) const; @@ -615,6 +617,15 @@ static const uint32_t kInvalidSuffixIndex; // Stats helpers. + // + // REQUIRES: node is valid. + // - Since we only invalidate Next to remove the edge from the trie and Node + // is not invalidated after deletion, the caller should ensure that it + // traverses correctly to a valid node according to the trie structure. + // Calling this function with an invalid node is undefined behavior since + // it could traverse into a deleted subtree, or invalid memory addresses. + // - This also means storage_->empty() should be checked before calling this + // function with the root node. void CollectStatsRecursive(const Node &node, Stats *stats, uint32_t depth = 0) const;
diff --git a/icing/legacy/index/icing-dynamic-trie_test.cc b/icing/legacy/index/icing-dynamic-trie_test.cc index 2b5a8b9..778a1a9 100644 --- a/icing/legacy/index/icing-dynamic-trie_test.cc +++ b/icing/legacy/index/icing-dynamic-trie_test.cc
@@ -40,7 +40,9 @@ using testing::ContainerEq; using testing::ElementsAre; using testing::Eq; +using testing::IsEmpty; using testing::Not; +using testing::SizeIs; constexpr std::string_view kKeys[] = { "", "ab", "ac", "abd", "bac", "bb", "bacd", "abbb", "abcdefg", @@ -946,10 +948,14 @@ uint32_t value = 1; ASSERT_THAT(trie.Insert("foo", &value), IsOk()); ASSERT_TRUE(trie.Find("foo", &value)); + ASSERT_THAT(trie, SizeIs(1)); + ASSERT_THAT(trie, Not(IsEmpty())); // Deletes the key. EXPECT_TRUE(trie.Delete("foo")); EXPECT_FALSE(trie.Find("foo", &value)); + EXPECT_THAT(trie, SizeIs(0)); // Explicitly test size() method. + EXPECT_THAT(trie, IsEmpty()); } TEST_F(IcingDynamicTrieTest, DeletionShouldWorkWhenLastCharIsLeaf) { @@ -972,11 +978,15 @@ ASSERT_THAT(trie.Insert("ba", &value), IsOk()); ASSERT_TRUE(trie.Find("bar", &value)); ASSERT_TRUE(trie.Find("ba", &value)); + ASSERT_THAT(trie, SizeIs(2)); + ASSERT_THAT(trie, Not(IsEmpty())); // Deletes "bar". "r" is a leaf node in the trie. EXPECT_TRUE(trie.Delete("bar")); EXPECT_FALSE(trie.Find("bar", &value)); EXPECT_TRUE(trie.Find("ba", &value)); + EXPECT_THAT(trie, SizeIs(1)); + EXPECT_THAT(trie, Not(IsEmpty())); } TEST_F(IcingDynamicTrieTest, DeletionShouldWorkWithTerminationNode) { @@ -999,11 +1009,15 @@ ASSERT_THAT(trie.Insert("ba", &value), IsOk()); ASSERT_TRUE(trie.Find("bar", &value)); ASSERT_TRUE(trie.Find("ba", &value)); + ASSERT_THAT(trie, SizeIs(2)); + ASSERT_THAT(trie, Not(IsEmpty())); // Deletes "ba" which is a key with termination node in the trie. EXPECT_TRUE(trie.Delete("ba")); EXPECT_FALSE(trie.Find("ba", &value)); EXPECT_TRUE(trie.Find("bar", &value)); + EXPECT_THAT(trie, SizeIs(1)); + EXPECT_THAT(trie, Not(IsEmpty())); } TEST_F(IcingDynamicTrieTest, DeletionShouldWorkWithMultipleNexts) { @@ -1028,6 +1042,8 @@ ASSERT_TRUE(trie.Find("bb", &value)); ASSERT_TRUE(trie.Find("bc", &value)); ASSERT_TRUE(trie.Find("bd", &value)); + ASSERT_THAT(trie, SizeIs(4)); + ASSERT_THAT(trie, Not(IsEmpty())); // Deletes "bc". EXPECT_TRUE(trie.Delete("bc")); @@ -1035,6 +1051,8 @@ EXPECT_TRUE(trie.Find("ba", &value)); EXPECT_TRUE(trie.Find("bb", &value)); EXPECT_TRUE(trie.Find("bd", &value)); + EXPECT_THAT(trie, SizeIs(3)); + EXPECT_THAT(trie, Not(IsEmpty())); } TEST_F(IcingDynamicTrieTest, DeletionShouldWorkWithMultipleTrieBranches) { @@ -1065,12 +1083,49 @@ ASSERT_TRUE(trie.Find("batter", &value)); ASSERT_TRUE(trie.Find("battle", &value)); ASSERT_TRUE(trie.Find("bar", &value)); + ASSERT_THAT(trie, SizeIs(3)); + ASSERT_THAT(trie, Not(IsEmpty())); // Deletes "batter". EXPECT_TRUE(trie.Delete("batter")); EXPECT_FALSE(trie.Find("batter", &value)); EXPECT_TRUE(trie.Find("battle", &value)); EXPECT_TRUE(trie.Find("bar", &value)); + EXPECT_THAT(trie, SizeIs(2)); + EXPECT_THAT(trie, Not(IsEmpty())); +} + +TEST_F(IcingDynamicTrieTest, DeletionShouldResetEmptyStateIfAllKeysAreDeleted) { + IcingFilesystem filesystem; + IcingDynamicTrie trie(trie_files_prefix_, IcingDynamicTrie::RuntimeOptions(), + &filesystem); + ASSERT_TRUE(trie.CreateIfNotExist(IcingDynamicTrie::Options())); + ASSERT_TRUE(trie.Init()); + + uint32_t value1 = 1; + ASSERT_THAT(trie.Insert("foo", &value1), IsOk()); + + uint32_t value2 = 2; + ASSERT_THAT(trie.Insert("bar", &value2), IsOk()); + + uint32_t value3 = 3; + ASSERT_THAT(trie.Insert("baz", &value3), IsOk()); + + ASSERT_THAT(trie, SizeIs(3)); + ASSERT_THAT(trie, Not(IsEmpty())); + + // Delete "foo", "bar", "baz". + EXPECT_TRUE(trie.Delete("foo")); + EXPECT_TRUE(trie.Delete("bar")); + EXPECT_TRUE(trie.Delete("baz")); + + EXPECT_THAT(trie, SizeIs(0)); // Explicitly test size() method. + EXPECT_THAT(trie, IsEmpty()); + + uint32_t value; + EXPECT_FALSE(trie.Find("foo", &value)); + EXPECT_FALSE(trie.Find("bar", &value)); + EXPECT_FALSE(trie.Find("baz", &value)); } TEST_F(IcingDynamicTrieTest, InsertionShouldWorkAfterDeletion) { @@ -1130,6 +1185,43 @@ EXPECT_THAT(results, ElementsAre("bar")); } +TEST_F(IcingDynamicTrieTest, IteratorShouldWorkAfterAllKeysAreDeleted) { + IcingFilesystem filesystem; + IcingDynamicTrie trie(trie_files_prefix_, IcingDynamicTrie::RuntimeOptions(), + &filesystem); + ASSERT_TRUE(trie.CreateIfNotExist(IcingDynamicTrie::Options())); + ASSERT_TRUE(trie.Init()); + + // Inserts some keys. + uint32_t value = 1; + ASSERT_THAT(trie.Insert("bar", &value), IsOk()); + ASSERT_THAT(trie.Insert("bed", &value), IsOk()); + ASSERT_THAT(trie.Insert("foo", &value), IsOk()); + + // Deletes all keys + ASSERT_TRUE(trie.Delete("bar")); + ASSERT_TRUE(trie.Delete("bed")); + ASSERT_TRUE(trie.Delete("foo")); + + EXPECT_THAT(trie, IsEmpty()); + + // Iterates through all keys + IcingDynamicTrie::Iterator iterator_all(trie, ""); + std::vector<std::string> results; + for (; iterator_all.IsValid(); iterator_all.Advance()) { + results.push_back(std::string(iterator_all.GetKey())); + } + EXPECT_THAT(results, IsEmpty()); + + // Iterates through keys that start with "b" + IcingDynamicTrie::Iterator iterator_b(trie, "b"); + results.clear(); + for (; iterator_b.IsValid(); iterator_b.Advance()) { + results.push_back(std::string(iterator_b.GetKey())); + } + EXPECT_THAT(results, IsEmpty()); +} + TEST_F(IcingDynamicTrieTest, DeletingNonExistingKeyShouldReturnTrue) { IcingFilesystem filesystem; IcingDynamicTrie trie(trie_files_prefix_, IcingDynamicTrie::RuntimeOptions(),
diff --git a/icing/monkey_test/icing-monkey-test-runner.cc b/icing/monkey_test/icing-monkey-test-runner.cc index bc04089..14b4e02 100644 --- a/icing/monkey_test/icing-monkey-test-runner.cc +++ b/icing/monkey_test/icing-monkey-test-runner.cc
@@ -551,6 +551,8 @@ // flip this flag to test document store's compatibility. icing_options.set_document_store_namespace_id_fingerprint( GetRandomBoolean(&random_)); + icing_options.set_enable_embedding_index(true); + icing_options.set_enable_embedding_quantization(GetRandomBoolean(&random_)); icing_ = std::make_unique<IcingSearchEngine>(icing_options); ASSERT_THAT(icing_->Initialize().status(), ProtoIsOk()); }
diff --git a/icing/portable/equals-proto.h b/icing/portable/equals-proto.h index 8bb835e..15a0e61 100644 --- a/icing/portable/equals-proto.h +++ b/icing/portable/equals-proto.h
@@ -23,7 +23,6 @@ #include "gmock/gmock.h" // IWYU pragma: export #include <google/protobuf/message_lite.h> // IWYU pragma: export -#if defined(__ANDROID__) || defined(__APPLE__) namespace icing { namespace lib { namespace portable_equals_proto { @@ -31,24 +30,29 @@ // Hence, there is no better way to compare two protos of an arbitrary type. // This matcher enables comparing non-google3 protos on, e.g., Android, with // a known caveat that it is unable to provide detailed difference information. +#if defined(__ANDROID__) || defined(__APPLE__) + MATCHER_P(EqualsProto, other, "Compare MessageLite by serialized string") { return ::testing::ExplainMatchResult(::testing::Eq(other.SerializeAsString()), arg.SerializeAsString(), result_listener); -} // MATCHER_P -} // namespace portable_equals_proto -} // namespace lib -} // namespace icing +} + +MATCHER(EqualsProto, "") { + return ::testing::ExplainMatchResult(EqualsProto(std::get<1>(arg)), + std::get<0>(arg), result_listener); +} + #else -namespace icing { -namespace lib { -namespace portable_equals_proto { + // Leverage the powerful google3 matcher when available, for human readable // differences. using ::testing::EqualsProto; + +#endif // defined(__ANDROID__) || defined(__APPLE__) + } // namespace portable_equals_proto } // namespace lib } // namespace icing -#endif // defined(__ANDROID__) || defined(__APPLE__) #endif // ICING_PORTABLE_EQUALS_PROTO_H_
diff --git a/icing/portable/platform.h b/icing/portable/platform.h index f7d8734..245e41c 100644 --- a/icing/portable/platform.h +++ b/icing/portable/platform.h
@@ -44,8 +44,14 @@ return !IsReverseJniTokenization() && !IsCfStringTokenization(); } +// ICU and Reverse JNI tokenization are enabled. +inline bool IsIcuWithReverseTokenization() { + return IsReverseJniTokenization() && !IsCfStringTokenization(); +} + inline int GetIcuTokenizationVersion() { - return IsIcuTokenization() ? U_ICU_VERSION_MAJOR_NUM : 0; + return (IsIcuTokenization() || IsIcuWithReverseTokenization()) + ? U_ICU_VERSION_MAJOR_NUM : 0; } // Indicates whether stemming is enabled. //
diff --git a/icing/query/advanced_query_parser/query-visitor.cc b/icing/query/advanced_query_parser/query-visitor.cc index 6178525..8a879f9 100644 --- a/icing/query/advanced_query_parser/query-visitor.cc +++ b/icing/query/advanced_query_parser/query-visitor.cc
@@ -23,11 +23,13 @@ #include <string> #include <string_view> #include <unordered_map> +#include <unordered_set> #include <utility> #include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/annotate.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" #include "icing/absl_ports/str_join.h" @@ -36,6 +38,7 @@ #include "icing/index/iterator/doc-hit-info-iterator-all-document-id.h" #include "icing/index/iterator/doc-hit-info-iterator-and.h" #include "icing/index/iterator/doc-hit-info-iterator-filter.h" +#include "icing/index/iterator/doc-hit-info-iterator-match-score-expression.h" #include "icing/index/iterator/doc-hit-info-iterator-none.h" #include "icing/index/iterator/doc-hit-info-iterator-not.h" #include "icing/index/iterator/doc-hit-info-iterator-or.h" @@ -43,7 +46,6 @@ #include "icing/index/iterator/doc-hit-info-iterator-property-in-schema.h" #include "icing/index/iterator/doc-hit-info-iterator-section-restrict.h" #include "icing/index/iterator/doc-hit-info-iterator.h" -#include "icing/index/iterator/section-restrict-data.h" #include "icing/index/property-existence-indexing-handler.h" #include "icing/query/advanced_query_parser/abstract-syntax-tree.h" #include "icing/query/advanced_query_parser/function.h" @@ -55,10 +57,12 @@ #include "icing/query/query-features.h" #include "icing/query/query-results.h" #include "icing/schema/property-util.h" -#include "icing/schema/schema-store.h" #include "icing/schema/section.h" +#include "icing/scoring/advanced_scoring/score-expression-util.h" +#include "icing/scoring/advanced_scoring/score-expression.h" #include "icing/tokenization/token.h" #include "icing/tokenization/tokenizer.h" +#include "icing/transform/normalizer.h" #include "icing/util/embedding-util.h" #include "icing/util/status-macros.h" @@ -295,6 +299,19 @@ registered_functions_.insert( {get_search_string_parameter_function.name(), std::move(get_search_string_parameter_function)}); + + // DocHitInfoIterator matchScoreExpression(std::string, double, double) + auto match_score_expression = [this](std::vector<PendingValue>&& args) { + return this->MatchScoreExpressionFunction(std::move(args)); + }; + Function match_score_expression_function = + Function::Create(DataType::kDocumentIterator, "matchScoreExpression", + {Param(DataType::kString), Param(DataType::kDouble), + Param(DataType::kDouble, Cardinality::kOptional)}, + std::move(match_score_expression)) + .ValueOrDie(); + registered_functions_.insert({match_score_expression_function.name(), + std::move(match_score_expression_function)}); } libtextclassifier3::StatusOr<PendingValue> QueryVisitor::SearchFunction( @@ -333,9 +350,10 @@ } else { QueryVisitor query_visitor( &index_, &numeric_index_, &embedding_index_, &document_store_, - &schema_store_, &normalizer_, &tokenizer_, search_spec_, - filter_options_, needs_term_frequency_info_, - pending_property_restricts_, processing_not_, current_time_ms_); + &schema_store_, &normalizer_, &tokenizer_, join_children_fetcher_, + search_spec_, filter_options_, needs_term_frequency_info_, + &feature_flags_, pending_property_restricts_, processing_not_, + current_time_ms_); tree_root->Accept(&query_visitor); ICING_ASSIGN_OR_RETURN(query_result, std::move(query_visitor).ConsumeResults()); @@ -450,7 +468,8 @@ std::unique_ptr<DocHitInfoIterator> iterator, DocHitInfoIteratorEmbedding::Create( &search_spec_.embedding_query_vectors(vector_index), metric_type, low, - high, score_map, &embedding_index_)); + high, score_map, &embedding_index_, &document_store_, &schema_store_, + current_time_ms_)); return PendingValue(std::move(iterator)); } @@ -471,6 +490,44 @@ return PendingValue(std::move(iterator)); } +libtextclassifier3::StatusOr<PendingValue> +QueryVisitor::MatchScoreExpressionFunction(std::vector<PendingValue>&& args) { + const std::string& scoring_expression_str = + args.at(0).string_val().ValueOrDie()->term; + libtextclassifier3::StatusOr<std::unique_ptr<ScoreExpression>> + scoring_expression = score_expression_util::GetScoreExpression( + scoring_expression_str, + /*default_score=*/-std::numeric_limits<double>::infinity(), + SearchSpecProto::EmbeddingQueryMetricType::UNKNOWN, &document_store_, + &schema_store_, current_time_ms_, join_children_fetcher_, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + &feature_flags_, &scoring_feature_types_enabled_); + if (!scoring_expression.ok()) { + return absl_ports::Annotate( + scoring_expression.status(), + absl_ports::StrCat( + "matchScoreExpression: Failed to handle the score expression ", + scoring_expression_str)); + } + double low = args.at(1).double_val().ValueOrDie(); + double high = std::numeric_limits<double>::infinity(); + if (args.size() == 3) { + high = args.at(2).double_val().ValueOrDie(); + } + if (low > high) { + return absl_ports::InvalidArgumentError( + "The lower bound cannot be greater than the upper bound."); + } + + std::unique_ptr<DocHitInfoIterator> iterator = + std::make_unique<DocHitInfoIteratorMatchScoreExpression>( + document_store_.last_added_document_id(), + std::move(scoring_expression).ValueOrDie(), low, high); + features_.insert(kMatchScoreExpressionFunctionFeature); + return PendingValue(std::move(iterator)); +} + libtextclassifier3::StatusOr<int64_t> QueryVisitor::PopPendingIntValue() { if (pending_values_.empty()) { return absl_ports::InvalidArgumentError("Unable to retrieve int value."); @@ -504,7 +561,7 @@ QueryVisitor::ProduceTextTokenIterators(QueryTerm text_value) { ICING_ASSIGN_OR_RETURN(std::unique_ptr<Tokenizer::Iterator> token_itr, tokenizer_.Tokenize(text_value.term)); - std::string normalized_term; + Normalizer::NormalizedTerm normalized_term; std::vector<std::unique_ptr<DocHitInfoIterator>> iterators; // raw_text is the portion of text_value.raw_term that hasn't yet been // matched to any of the tokens that we've processed. escaped_token will @@ -537,8 +594,9 @@ raw_token, string_util::FindEscapedToken(raw_text, token.text)); } normalized_term = normalizer_.NormalizeTerm(token.text); - QueryTerm term_value{std::move(normalized_term), raw_token, - reached_final_token && text_value.is_prefix_val}; + bool should_prefix_match = reached_final_token && text_value.is_prefix_val; + QueryTerm term_value{std::move(normalized_term.text), raw_token, + should_prefix_match}; ICING_ASSIGN_OR_RETURN(std::unique_ptr<DocHitInfoIterator> iterator, CreateTermIterator(std::move(term_value))); iterators.push_back(std::move(iterator));
diff --git a/icing/query/advanced_query_parser/query-visitor.h b/icing/query/advanced_query_parser/query-visitor.h index 6089ef4..cd9bedd 100644 --- a/icing/query/advanced_query_parser/query-visitor.h +++ b/icing/query/advanced_query_parser/query-visitor.h
@@ -20,7 +20,6 @@ #include <set> #include <stack> #include <string> -#include <string_view> #include <unordered_map> #include <unordered_set> #include <utility> @@ -28,12 +27,14 @@ #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-index.h" #include "icing/index/embed/embedding-query-results.h" #include "icing/index/index.h" #include "icing/index/iterator/doc-hit-info-iterator-filter.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/index/numeric/numeric-index.h" +#include "icing/join/join-children-fetcher.h" #include "icing/query/advanced_query_parser/abstract-syntax-tree.h" #include "icing/query/advanced_query_parser/function.h" #include "icing/query/advanced_query_parser/pending-value.h" @@ -44,7 +45,6 @@ #include "icing/store/document-store.h" #include "icing/tokenization/tokenizer.h" #include "icing/transform/normalizer.h" -#include <google/protobuf/repeated_field.h> namespace icing { namespace lib { @@ -53,21 +53,21 @@ // the parser. class QueryVisitor : public AbstractSyntaxTreeVisitor { public: - explicit QueryVisitor(Index* index, - const NumericIndex<int64_t>* numeric_index, - const EmbeddingIndex* embedding_index, - const DocumentStore* document_store, - const SchemaStore* schema_store, - const Normalizer* normalizer, - const Tokenizer* tokenizer, - const SearchSpecProto& search_spec, - DocHitInfoIteratorFilter::Options filter_options, - bool needs_term_frequency_info, int64_t current_time_ms) + explicit QueryVisitor( + Index* index, const NumericIndex<int64_t>* numeric_index, + const EmbeddingIndex* embedding_index, + const DocumentStore* document_store, const SchemaStore* schema_store, + const Normalizer* normalizer, const Tokenizer* tokenizer, + const JoinChildrenFetcher* join_children_fetcher, + const SearchSpecProto& search_spec, + DocHitInfoIteratorFilter::Options filter_options, + bool needs_term_frequency_info, const FeatureFlags* feature_flags, + int64_t current_time_ms) : QueryVisitor(index, numeric_index, embedding_index, document_store, - schema_store, normalizer, tokenizer, search_spec, - filter_options, needs_term_frequency_info, - PendingPropertyRestricts(), /*processing_not=*/false, - current_time_ms) {} + schema_store, normalizer, tokenizer, join_children_fetcher, + search_spec, filter_options, needs_term_frequency_info, + feature_flags, PendingPropertyRestricts(), + /*processing_not=*/false, current_time_ms) {} void VisitString(const StringNode* node) override; void VisitText(const TextNode* node) override; @@ -111,18 +111,17 @@ std::vector<std::set<std::string>> pending_property_restricts_; }; - explicit QueryVisitor(Index* index, - const NumericIndex<int64_t>* numeric_index, - const EmbeddingIndex* embedding_index, - const DocumentStore* document_store, - const SchemaStore* schema_store, - const Normalizer* normalizer, - const Tokenizer* tokenizer, - const SearchSpecProto& search_spec, - DocHitInfoIteratorFilter::Options filter_options, - bool needs_term_frequency_info, - PendingPropertyRestricts pending_property_restricts, - bool processing_not, int64_t current_time_ms) + explicit QueryVisitor( + Index* index, const NumericIndex<int64_t>* numeric_index, + const EmbeddingIndex* embedding_index, + const DocumentStore* document_store, const SchemaStore* schema_store, + const Normalizer* normalizer, const Tokenizer* tokenizer, + const JoinChildrenFetcher* join_children_fetcher, + const SearchSpecProto& search_spec, + DocHitInfoIteratorFilter::Options filter_options, + bool needs_term_frequency_info, const FeatureFlags* feature_flags, + PendingPropertyRestricts pending_property_restricts, bool processing_not, + int64_t current_time_ms) : index_(*index), numeric_index_(*numeric_index), embedding_index_(*embedding_index), @@ -130,9 +129,11 @@ schema_store_(*schema_store), normalizer_(*normalizer), tokenizer_(*tokenizer), + join_children_fetcher_(join_children_fetcher), search_spec_(search_spec), filter_options_(std::move(filter_options)), needs_term_frequency_info_(needs_term_frequency_info), + feature_flags_(*feature_flags), pending_property_restricts_(std::move(pending_property_restricts)), processing_not_(processing_not), expecting_numeric_arg_(false), @@ -317,6 +318,20 @@ libtextclassifier3::StatusOr<PendingValue> GetSearchStringParameterFunction( std::vector<PendingValue>&& args); + // Implementation of the matchScoreExpression(scoring_expression, low, high) + // custom function. + // + // high is an optional parameter, which defaults to positive infinity. + // + // Returns: + // - a Pending Value of type DocHitIterator that returns all documents with + // a score within [low, high] based on the provided scoring expression. + // Documents that cause an evaluation error during scoring are ignored. + // - INVALID_ARGUMENT if low is greater than high. + // - any errors returned by score_expression_util::GetScoreExpression. + libtextclassifier3::StatusOr<PendingValue> MatchScoreExpressionFunction( + std::vector<PendingValue>&& args); + // Handles a NaryOperatorNode where the operator is HAS (':') and pushes an // iterator with the proper section filter applied. If the current property // restriction represented by pending_property_restricts and the first child @@ -359,6 +374,10 @@ const Normalizer& normalizer_; // Does not own! const Tokenizer& tokenizer_; // Does not own! + // Nullable. A non-null join_children_fetcher_ indicates that this is the + // parent query for a join query, in which case child scores are available. + const JoinChildrenFetcher* join_children_fetcher_; // Does not own. + const SearchSpecProto& search_spec_; DocHitInfoIteratorFilter::Options filter_options_; @@ -368,6 +387,10 @@ // - whether the QueryTermIteratorsMap is populated in the QueryResults. bool needs_term_frequency_info_; + const FeatureFlags& feature_flags_; // Does not own. + // TODO(b/377215223): Pass enabled scoring features from top level. + std::unordered_set<ScoringFeatureType> scoring_feature_types_enabled_; + // The stack of property restricts currently being processed by the visitor. PendingPropertyRestricts pending_property_restricts_; bool processing_not_;
diff --git a/icing/query/advanced_query_parser/query-visitor_test.cc b/icing/query/advanced_query_parser/query-visitor_test.cc index e07e12e..5a60808 100644 --- a/icing/query/advanced_query_parser/query-visitor_test.cc +++ b/icing/query/advanced_query_parser/query-visitor_test.cc
@@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include "icing/absl_ports/str_cat.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-index.h" @@ -59,9 +60,9 @@ #include "icing/store/namespace-id.h" #include "icing/testing/common-matchers.h" #include "icing/testing/embedding-test-utils.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" @@ -70,6 +71,7 @@ #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/status-macros.h" #include "unicode/uloc.h" #include <google/protobuf/repeated_field.h> @@ -168,6 +170,7 @@ class QueryVisitorTest : public ::testing::TestWithParam<QueryType> { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); test_dir_ = GetTestTempDir() + "/icing"; index_dir_ = test_dir_ + "/index"; numeric_index_dir_ = test_dir_ + "/numeric_index"; @@ -189,24 +192,24 @@ // setup doesn't do this. ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, store_dir_, &clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, store_dir_, &clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); Index::Options options(index_dir_.c_str(), @@ -222,7 +225,8 @@ ICING_ASSERT_OK_AND_ASSIGN( embedding_index_, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir_)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir_, &clock_, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN(normalizer_, normalizer_factory::Create( /*max_term_byte_size=*/1000)); @@ -253,8 +257,10 @@ QueryVisitor query_visitor( index_.get(), numeric_index_.get(), embedding_index_.get(), document_store_.get(), schema_store_.get(), normalizer_.get(), - tokenizer_.get(), search_spec, DocHitInfoIteratorFilter::Options(), - /*needs_term_frequency_info=*/true, clock_.GetSystemTimeMilliseconds()); + tokenizer_.get(), /*join_children_fetcher=*/nullptr, search_spec, + DocHitInfoIteratorFilter::Options(), + /*needs_term_frequency_info=*/true, feature_flags_.get(), + clock_.GetSystemTimeMilliseconds()); root_node->Accept(&query_visitor); return std::move(query_visitor).ConsumeResults(); } @@ -303,6 +309,7 @@ } } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; IcingFilesystem icing_filesystem_; std::string test_dir_; @@ -724,8 +731,10 @@ QueryVisitor query_visitor( index_.get(), numeric_index_.get(), embedding_index_.get(), document_store_.get(), schema_store_.get(), normalizer_.get(), - tokenizer_.get(), search_spec, DocHitInfoIteratorFilter::Options(), - /*needs_term_frequency_info_=*/true, clock_.GetSystemTimeMilliseconds()); + tokenizer_.get(), /*join_children_fetcher=*/nullptr, search_spec, + DocHitInfoIteratorFilter::Options(), + /*needs_term_frequency_info=*/true, feature_flags_.get(), + clock_.GetSystemTimeMilliseconds()); EXPECT_THAT(std::move(query_visitor).ConsumeResults(), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -802,12 +811,12 @@ ICING_ASSERT_OK(editor->BufferKey(1)); ICING_ASSERT_OK(editor->BufferKey(2)); ICING_ASSERT_OK(std::move(*editor).IndexAllBufferedKeys()); - Index::Editor term_editor = index_->Edit( - kDocumentId0, kSectionId1, TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(term_editor.BufferTerm("-2")); - ICING_ASSERT_OK(term_editor.BufferTerm("-1")); - ICING_ASSERT_OK(term_editor.BufferTerm("1")); - ICING_ASSERT_OK(term_editor.BufferTerm("2")); + Index::Editor term_editor = + index_->Edit(kDocumentId0, kSectionId1, /*namespace_id=*/0); + ICING_ASSERT_OK(term_editor.BufferTerm("-2", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(term_editor.BufferTerm("-1", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(term_editor.BufferTerm("1", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(term_editor.BufferTerm("2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(term_editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( @@ -821,9 +830,9 @@ editor = numeric_index_->Edit("price", kDocumentId2, kSectionId0); ICING_ASSERT_OK(editor->BufferKey(-1)); ICING_ASSERT_OK(std::move(*editor).IndexAllBufferedKeys()); - term_editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + term_editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(term_editor.BufferTerm("2")); + ICING_ASSERT_OK(term_editor.BufferTerm("2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(term_editor.IndexAllBufferedTerms()); // Translating MINUS chars that are interpreted as NOTs, this query would be @@ -852,18 +861,18 @@ // Setup the index with docs 0, 1 and 2 holding the values "foo", "foo" and // "bar" respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("foo"); @@ -899,18 +908,18 @@ // Setup the index with docs 0, 1 and 2 holding the values "foo", "foo" and // "bar" respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("foo"); @@ -920,8 +929,10 @@ QueryVisitor query_visitor( index_.get(), numeric_index_.get(), embedding_index_.get(), document_store_.get(), schema_store_.get(), normalizer_.get(), - tokenizer_.get(), search_spec, DocHitInfoIteratorFilter::Options(), - /*needs_term_frequency_info=*/false, clock_.GetSystemTimeMilliseconds()); + tokenizer_.get(), /*join_children_fetcher=*/nullptr, search_spec, + DocHitInfoIteratorFilter::Options(), + /*needs_term_frequency_info=*/false, feature_flags_.get(), + clock_.GetSystemTimeMilliseconds()); root_node->Accept(&query_visitor); ICING_ASSERT_OK_AND_ASSIGN(QueryResults query_results, std::move(query_visitor).ConsumeResults()); @@ -951,18 +962,18 @@ // Setup the index with docs 0, 1 and 2 holding the values "foo", "foo" and // "bar" respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // An EXACT query for 'fo' won't match anything. @@ -1012,21 +1023,21 @@ // Setup the index with docs 0, 1 and 2 holding the values ["foo", "ba"], // ["foo", "ba"] and ["bar", "fo"] respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("ba")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("ba", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("ba")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("ba", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); - ICING_ASSERT_OK(editor.BufferTerm("fo")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("fo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // An EXACT query for `ba?fo` will be lexed into a single TEXT token. @@ -1068,18 +1079,18 @@ // Setup the index with docs 0, 1 and 2 holding the values "foo:bar(baz)", // "foo:bar(baz)" and "bar:baz(foo)" respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo:bar(baz)")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo:bar(baz)", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo:bar(baz)")); + ICING_ASSERT_OK(editor.BufferTerm("foo:bar(baz)", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar:baz(foo)")); + ICING_ASSERT_OK(editor.BufferTerm("bar:baz(foo)", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("\"foo:bar(baz)\""); @@ -1105,18 +1116,18 @@ // Setup the index with docs 0, 1 and 2 holding the values "foo:bar(baz)", // "foo:bar(abc)" and "bar:baz(foo)" respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo:bar(baz)")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo:bar(baz)", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo:bar(abc)")); + ICING_ASSERT_OK(editor.BufferTerm("foo:bar(abc)", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar:baz(foo)")); + ICING_ASSERT_OK(editor.BufferTerm("bar:baz(foo)", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Query for `"foo:bar("*`. This should match docs 0 and 1. @@ -1152,18 +1163,18 @@ // Setup the index with docs 0, 1 and 2 holding the values "foobary", // "foobar\" and "foobar"" respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_EXACT, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(R"(foobary)")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobary)", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\)")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\)", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(R"(foobar")")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobar")", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // From the comment above, verbatim_term = `foobar"` and verbatim_query = @@ -1195,19 +1206,19 @@ // Setup the index with docs 0, 1 and 2 holding the values "foobary", // "foobar\" and "foobar"" respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_EXACT, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(R"(foobary)")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobary)", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); // From the comment above, verbatim_term = `foobar\`. - ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\)")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\)", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(R"(foobar")")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobar")", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Issue a query for the verbatim token `foobar\`. @@ -1240,19 +1251,19 @@ // Setup the index with docs 0, 1 and 2 holding the values "foobary", // "foobar\" and "foobar"" respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_EXACT, /*namespace_id=*/0); + /*namespace_id=*/0); // From the comment above, verbatim_term = `foobary`. - ICING_ASSERT_OK(editor.BufferTerm(R"(foobary)")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobary)", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\)")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\)", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\y)")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\y)", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Issue a query for the verbatim token `foobary`. @@ -1304,20 +1315,20 @@ // Setup the index with docs 0, 1 and 2 holding the values "foobar\n", // `foobar\` and `foobar\n` respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_EXACT, /*namespace_id=*/0); + /*namespace_id=*/0); // From the comment above, verbatim_term = `foobar` + '\n'. - ICING_ASSERT_OK(editor.BufferTerm("foobar\n")); + ICING_ASSERT_OK(editor.BufferTerm("foobar\n", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\)")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\)", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); // verbatim_term = `foobar\n`. This is distinct from the term added above. - ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\n)")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foobar\n)", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Issue a query for the verbatim token `foobar` + '\n'. @@ -1363,20 +1374,20 @@ // Setup the index with docs 0, 1 and 2 holding the values `foo\"bar\nbaz"`, // `foo\\\"bar\\nbaz\"` and `foo\\"bar\\nbaz"` respectively. Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_EXACT, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(R"(foo\"bar\nbaz")")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm(R"(foo\"bar\nbaz")", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); // Add the verbatim_term from doc 0 but with all of the escapes left in - ICING_ASSERT_OK(editor.BufferTerm(R"(foo\\\"bar\\nbaz\")")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foo\\\"bar\\nbaz\")", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_EXACT, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); // Add the verbatim_term from doc 0 but with the escapes for '\' chars left in - ICING_ASSERT_OK(editor.BufferTerm(R"(foo\\"bar\\nbaz")")); + ICING_ASSERT_OK(editor.BufferTerm(R"(foo\\"bar\\nbaz")", TERM_MATCH_EXACT)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Issue a query for the verbatim token `foo\"bar\nbaz"`. @@ -1412,22 +1423,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("-foo"); @@ -1457,22 +1468,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("NOT foo"); @@ -1498,26 +1509,26 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("bar")); - ICING_ASSERT_OK(editor.BufferTerm("baz")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("baz", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("baz")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("baz", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); - ICING_ASSERT_OK(editor.BufferTerm("baz")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("baz", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Double negative could be rewritten as `(foo AND NOT bar) baz` @@ -1547,26 +1558,26 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("bar")); - ICING_ASSERT_OK(editor.BufferTerm("baz")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("baz", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("baz")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("baz", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); - ICING_ASSERT_OK(editor.BufferTerm("baz")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("baz", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Simplifying: @@ -1596,19 +1607,19 @@ TEST_P(QueryVisitorTest, ImplicitAndTerms) { Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("foo bar"); @@ -1630,19 +1641,19 @@ TEST_P(QueryVisitorTest, ExplicitAndTerms) { Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("foo AND bar"); @@ -1664,19 +1675,19 @@ TEST_P(QueryVisitorTest, OrTerms) { Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("fo")); - ICING_ASSERT_OK(editor.BufferTerm("ba")); + ICING_ASSERT_OK(editor.BufferTerm("fo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("ba", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("foo OR bar"); @@ -1698,20 +1709,20 @@ TEST_P(QueryVisitorTest, AndOrTermPrecedence) { Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("baz")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("baz", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Should be interpreted like `foo (bar OR baz)` @@ -1780,24 +1791,24 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("baz")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("baz", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Should be interpreted like `foo ((NOT bar) OR baz)` @@ -1851,22 +1862,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop2_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("foo", /*property_restrict=*/"prop1"); @@ -1918,22 +1929,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop2_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop3_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop3_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = R"(search("foo", createList("prop1", "prop2")))"; @@ -2000,22 +2011,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop2_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("foo", /*property_restrict=*/"PROP1"); @@ -2061,22 +2072,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop2_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = @@ -2120,22 +2131,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop2_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("(prop1:foo)", /*property_restrict=*/"prop1"); @@ -2190,22 +2201,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop2_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("(prop2:foo)", /*property_restrict=*/"prop1"); @@ -2256,22 +2267,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop2_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Resulting queries: @@ -2337,22 +2348,22 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop2_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Resulting queries: @@ -2423,35 +2434,35 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("ä¸ç")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("ä¸ç", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId0, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId0, prop2_section_id, /*namespace_id=*/0); if (IsCfStringTokenization()) { - ICING_ASSERT_OK(editor.BufferTerm("ćŻ")); - ICING_ASSERT_OK(editor.BufferTerm("夊")); + ICING_ASSERT_OK(editor.BufferTerm("ćŻ", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("夊", TERM_MATCH_PREFIX)); } else { - ICING_ASSERT_OK(editor.BufferTerm("ćŻĺ¤Š")); + ICING_ASSERT_OK(editor.BufferTerm("ćŻĺ¤Š", TERM_MATCH_PREFIX)); } ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("ä¸ç")); + ICING_ASSERT_OK(editor.BufferTerm("ä¸ç", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop2_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop2_section_id, /*namespace_id=*/0); if (IsCfStringTokenization()) { - ICING_ASSERT_OK(editor.BufferTerm("ćŻ")); - ICING_ASSERT_OK(editor.BufferTerm("夊")); + ICING_ASSERT_OK(editor.BufferTerm("ćŻ", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("夊", TERM_MATCH_PREFIX)); } else { - ICING_ASSERT_OK(editor.BufferTerm("ćŻĺ¤Š")); + ICING_ASSERT_OK(editor.BufferTerm("ćŻĺ¤Š", TERM_MATCH_PREFIX)); } ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); @@ -2509,11 +2520,10 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result0, document_store_->Put(doc)); DocumentId docid0 = put_result0.new_document_id; - Index::Editor editor = - index_->Edit(docid0, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); - ICING_ASSERT_OK(editor.BufferTerm("val1")); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + Index::Editor editor = index_->Edit(docid0, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // - Doc 1: Contains 'val0', 'val1', 'val2' in 'prop1'. Should match. @@ -2521,10 +2531,10 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, document_store_->Put(doc)); DocumentId docid1 = put_result1.new_document_id; - editor = index_->Edit(docid1, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); - ICING_ASSERT_OK(editor.BufferTerm("val1")); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + editor = index_->Edit(docid1, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // - Doc 2: Contains 'val0', 'val1', 'val2' in 'prop2'. Shouldn't match. @@ -2532,10 +2542,10 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, document_store_->Put(doc)); DocumentId docid2 = put_result2.new_document_id; - editor = index_->Edit(docid2, prop2_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); - ICING_ASSERT_OK(editor.BufferTerm("val1")); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + editor = index_->Edit(docid2, prop2_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // - Doc 3: Contains 'val0' in 'prop0', 'val1' in 'prop1' etc. Should match. @@ -2543,14 +2553,14 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result3, document_store_->Put(doc)); DocumentId docid3 = put_result3.new_document_id; - editor = index_->Edit(docid3, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); + editor = index_->Edit(docid3, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid3, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val1")); + editor = index_->Edit(docid3, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid3, prop2_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + editor = index_->Edit(docid3, prop2_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // - Doc 4: Contains 'val1' in 'prop0', 'val2' in 'prop1', 'val0' in 'prop2'. @@ -2559,14 +2569,14 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result4, document_store_->Put(doc)); DocumentId docid4 = put_result4.new_document_id; - editor = index_->Edit(docid4, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val1")); + editor = index_->Edit(docid4, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid4, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + editor = index_->Edit(docid4, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid4, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); + editor = index_->Edit(docid4, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Now issue a query with 'val1' restricted to 'prop1'. This should match only @@ -2625,11 +2635,10 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result0, document_store_->Put(doc)); DocumentId docid0 = put_result0.new_document_id; - Index::Editor editor = - index_->Edit(docid0, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); - ICING_ASSERT_OK(editor.BufferTerm("val1")); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + Index::Editor editor = index_->Edit(docid0, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // - Doc 1: Contains 'val0', 'val1', 'val2' in 'prop1'. Shouldn't match. @@ -2637,10 +2646,10 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, document_store_->Put(doc)); DocumentId docid1 = put_result1.new_document_id; - editor = index_->Edit(docid1, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); - ICING_ASSERT_OK(editor.BufferTerm("val1")); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + editor = index_->Edit(docid1, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // - Doc 2: Contains 'val0', 'val1', 'val2' in 'prop2'. Should match. @@ -2648,10 +2657,10 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, document_store_->Put(doc)); DocumentId docid2 = put_result2.new_document_id; - editor = index_->Edit(docid2, prop2_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); - ICING_ASSERT_OK(editor.BufferTerm("val1")); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + editor = index_->Edit(docid2, prop2_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // - Doc 3: Contains 'val0' in 'prop0', 'val1' in 'prop1' etc. Should match. @@ -2659,14 +2668,14 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result3, document_store_->Put(doc)); DocumentId docid3 = put_result3.new_document_id; - editor = index_->Edit(docid3, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); + editor = index_->Edit(docid3, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid3, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val1")); + editor = index_->Edit(docid3, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid3, prop2_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + editor = index_->Edit(docid3, prop2_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // - Doc 4: Contains 'val1' in 'prop0', 'val2' in 'prop1', 'val0' in 'prop2'. @@ -2675,14 +2684,14 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result4, document_store_->Put(doc)); DocumentId docid4 = put_result4.new_document_id; - editor = index_->Edit(docid4, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val1")); + editor = index_->Edit(docid4, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val1", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid4, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val2")); + editor = index_->Edit(docid4, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val2", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid4, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("val0")); + editor = index_->Edit(docid4, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("val0", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Now issue a query with 'val1' restricted to 'prop1'. This should match only @@ -2783,23 +2792,23 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); Index::Editor editor = index_->Edit(kDocumentId0, prop1_section_id, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId1, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("type").Build())); - editor = index_->Edit(kDocumentId2, prop1_section_id, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, prop1_section_id, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string level_one_query = R"(search("foo", createList("prop1")) bar)"; @@ -2893,65 +2902,64 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result0, document_store_->Put(doc)); DocumentId docid0 = put_result0.new_document_id; - Index::Editor editor = - index_->Edit(kDocumentId0, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + Index::Editor editor = index_->Edit(kDocumentId0, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, document_store_->Put(DocumentBuilder(doc).SetUri("uri1").Build())); DocumentId docid1 = put_result1.new_document_id; - editor = index_->Edit(docid1, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid1, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, document_store_->Put(DocumentBuilder(doc).SetUri("uri2").Build())); DocumentId docid2 = put_result2.new_document_id; - editor = index_->Edit(docid2, prop2_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid2, prop2_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result3, document_store_->Put(DocumentBuilder(doc).SetUri("uri3").Build())); DocumentId docid3 = put_result3.new_document_id; - editor = index_->Edit(docid3, prop3_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid3, prop3_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result4, document_store_->Put(DocumentBuilder(doc).SetUri("uri4").Build())); DocumentId docid4 = put_result4.new_document_id; - editor = index_->Edit(docid4, prop4_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid4, prop4_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result5, document_store_->Put(DocumentBuilder(doc).SetUri("uri5").Build())); DocumentId docid5 = put_result5.new_document_id; - editor = index_->Edit(docid5, prop5_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid5, prop5_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result6, document_store_->Put(DocumentBuilder(doc).SetUri("uri6").Build())); DocumentId docid6 = put_result6.new_document_id; - editor = index_->Edit(docid6, prop6_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid6, prop6_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result7, document_store_->Put(DocumentBuilder(doc).SetUri("uri7").Build())); DocumentId docid7 = put_result7.new_document_id; - editor = index_->Edit(docid7, prop7_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid7, prop7_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // *If* nested function calls were allowed, then this would simplify as: @@ -3055,65 +3063,64 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result0, document_store_->Put(doc)); DocumentId docid0 = put_result0.new_document_id; - Index::Editor editor = - index_->Edit(kDocumentId0, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + Index::Editor editor = index_->Edit(kDocumentId0, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, document_store_->Put(DocumentBuilder(doc).SetUri("uri1").Build())); DocumentId docid1 = put_result1.new_document_id; - editor = index_->Edit(docid1, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid1, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, document_store_->Put(DocumentBuilder(doc).SetUri("uri2").Build())); DocumentId docid2 = put_result2.new_document_id; - editor = index_->Edit(docid2, prop2_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid2, prop2_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result3, document_store_->Put(DocumentBuilder(doc).SetUri("uri3").Build())); DocumentId docid3 = put_result3.new_document_id; - editor = index_->Edit(docid3, prop3_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid3, prop3_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result4, document_store_->Put(DocumentBuilder(doc).SetUri("uri4").Build())); DocumentId docid4 = put_result4.new_document_id; - editor = index_->Edit(docid4, prop4_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid4, prop4_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result5, document_store_->Put(DocumentBuilder(doc).SetUri("uri5").Build())); DocumentId docid5 = put_result5.new_document_id; - editor = index_->Edit(docid5, prop5_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid5, prop5_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result6, document_store_->Put(DocumentBuilder(doc).SetUri("uri6").Build())); DocumentId docid6 = put_result6.new_document_id; - editor = index_->Edit(docid6, prop6_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid6, prop6_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result7, document_store_->Put(DocumentBuilder(doc).SetUri("uri7").Build())); DocumentId docid7 = put_result7.new_document_id; - editor = index_->Edit(docid7, prop7_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid7, prop7_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // *If* nested function calls were allowed, then this would simplify as: @@ -3189,26 +3196,25 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result0, document_store_->Put(doc)); DocumentId docid0 = put_result0.new_document_id; - Index::Editor editor = - index_->Edit(kDocumentId0, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + Index::Editor editor = index_->Edit(kDocumentId0, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, document_store_->Put(DocumentBuilder(doc).SetUri("uri1").Build())); DocumentId docid1 = put_result1.new_document_id; - editor = index_->Edit(docid1, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + editor = index_->Edit(docid1, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, document_store_->Put(DocumentBuilder(doc).SetUri("uri2").Build())); DocumentId docid2 = put_result2.new_document_id; - editor = index_->Edit(docid2, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid2, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = "getSearchStringParameter(0)"; @@ -3310,32 +3316,31 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result0, document_store_->Put(doc)); DocumentId docid0 = put_result0.new_document_id; - Index::Editor editor = - index_->Edit(docid0, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + Index::Editor editor = index_->Edit(docid0, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, document_store_->Put(DocumentBuilder(doc).SetUri("uri1").Build())); DocumentId docid1 = put_result1.new_document_id; - editor = index_->Edit(docid1, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + editor = index_->Edit(docid1, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid1, prop1_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid1, prop1_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, document_store_->Put(DocumentBuilder(doc).SetUri("uri2").Build())); DocumentId docid2 = put_result2.new_document_id; - editor = index_->Edit(docid2, prop0_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + editor = index_->Edit(docid2, prop0_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(docid2, prop2_id, TERM_MATCH_PREFIX, ns_id); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + editor = index_->Edit(docid2, prop2_id, ns_id); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = "prop0:getSearchStringParameter(0)"; @@ -3504,8 +3509,8 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("typeWithUrl").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Document 1 has the term "foo" and its schema DOESN'T have the url property. @@ -3513,17 +3518,17 @@ .SetKey("ns", "uri1") .SetSchema("typeWithoutUrl") .Build())); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Document 2 has the term "bar" and its schema has the url property. ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("typeWithUrl").Build())); - editor = index_->Edit(kDocumentId2, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("foo propertyDefined(\"url\")"); @@ -3554,8 +3559,8 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("typeWithUrl").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Document 1 has the term "foo" and its schema DOESN'T have the url property. @@ -3563,9 +3568,9 @@ .SetKey("ns", "uri1") .SetSchema("typeWithoutUrl") .Build())); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Attempt to query a non-existent property. @@ -3596,8 +3601,8 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("typeWithUrl").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Document 1 has the term "foo" and its schema DOESN'T have the url property. @@ -3605,9 +3610,9 @@ .SetKey("ns", "uri1") .SetSchema("typeWithoutUrl") .Build())); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); std::string query = CreateQuery("foo AND NOT propertyDefined(\"url\")"); @@ -3668,28 +3673,30 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("Simple").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId0, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.BufferTerm( - absl_ports::StrCat(kPropertyExistenceTokenPrefix, "price").c_str())); + absl_ports::StrCat(kPropertyExistenceTokenPrefix, "price").c_str(), + TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Document 1 has the term "foo" and doesn't have the "price" property. ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("Simple").Build())); - editor = index_->Edit(kDocumentId1, kSectionId0, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId0, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Document 2 has the term "bar" and has the "price" property. ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri2").SetSchema("Simple").Build())); - editor = index_->Edit(kDocumentId2, kSectionId0, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId2, kSectionId0, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.BufferTerm( - absl_ports::StrCat(kPropertyExistenceTokenPrefix, "price").c_str())); + absl_ports::StrCat(kPropertyExistenceTokenPrefix, "price").c_str(), + TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Test that `foo hasProperty("price")` matches document 0 only. @@ -3734,18 +3741,19 @@ ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri0").SetSchema("Simple").Build())); Index::Editor editor = index_->Edit(kDocumentId0, kSectionId0, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.BufferTerm( - absl_ports::StrCat(kPropertyExistenceTokenPrefix, "price").c_str())); + absl_ports::StrCat(kPropertyExistenceTokenPrefix, "price").c_str(), + TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Document 1 has the term "foo" and doesn't have the "price" property. ICING_ASSERT_OK(document_store_->Put( DocumentBuilder().SetKey("ns", "uri1").SetSchema("Simple").Build())); - editor = index_->Edit(kDocumentId1, kSectionId0, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId0, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Attempt to query a non-existent property. @@ -3934,15 +3942,38 @@ } TEST_F(QueryVisitorTest, SemanticSearchFunctionSimpleLowerBound) { + // Set up + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("type") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("prop2") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); + // Index two embedding vectors. PropertyProto::VectorProto vector0 = CreateVector("my_model", {0.1, 0.2, 0.3}); PropertyProto::VectorProto vector1 = CreateVector("my_model", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(kSectionId0, kDocumentId0), vector0)); + BasicHit(kSectionId0, kDocumentId0), vector0, QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(kSectionId0, kDocumentId1), vector1)); + BasicHit(kSectionId0, kDocumentId1), vector1, QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); // Create an embedding query that has a semantic score of 1 with vector0 and @@ -4004,15 +4035,38 @@ } TEST_F(QueryVisitorTest, SemanticSearchFunctionSimpleUpperBound) { + // Set up + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("type") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("prop2") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); + // Index two embedding vectors. PropertyProto::VectorProto vector0 = CreateVector("my_model", {0.1, 0.2, 0.3}); PropertyProto::VectorProto vector1 = CreateVector("my_model", {-0.1, -0.2, -0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(kSectionId0, kDocumentId0), vector0)); + BasicHit(kSectionId0, kDocumentId0), vector0, QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(kSectionId0, kDocumentId1), vector1)); + BasicHit(kSectionId0, kDocumentId1), vector1, QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); // Create an embedding query that has a semantic score of 1 with vector0 and @@ -4074,10 +4128,31 @@ } TEST_F(QueryVisitorTest, SemanticSearchFunctionMetricOverride) { + // Set up + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("type") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("prop2") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); + // Index a embedding vector. PropertyProto::VectorProto vector = CreateVector("my_model", {0.1, 0.2, 0.3}); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( - BasicHit(kSectionId0, kDocumentId0), vector)); + BasicHit(kSectionId0, kDocumentId0), vector, QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); // Create an embedding query that has: @@ -4144,23 +4219,51 @@ } TEST_F(QueryVisitorTest, SemanticSearchFunctionMultipleQueries) { + // Set up + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("type") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("prop2") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("prop3") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); + // Index 3 embedding vectors for document 0. ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId0, kDocumentId0), - CreateVector("my_model1", {1, -2, -3}))); + CreateVector("my_model1", {1, -2, -3}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId1, kDocumentId0), - CreateVector("my_model1", {-1, -2, -3}))); + CreateVector("my_model1", {-1, -2, -3}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId2, kDocumentId0), - CreateVector("my_model2", {-1, 2, 3, -4}))); + CreateVector("my_model2", {-1, 2, 3, -4}), QUANTIZATION_TYPE_NONE)); // Index 2 embedding vectors for document 1. ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId0, kDocumentId1), - CreateVector("my_model1", {-1, -2, 3}))); + CreateVector("my_model1", {-1, -2, 3}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId1, kDocumentId1), - CreateVector("my_model2", {1, -2, 3, -4}))); + CreateVector("my_model2", {1, -2, 3, -4}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); // Create two embedding queries. @@ -4265,17 +4368,40 @@ TEST_F(QueryVisitorTest, SemanticSearchFunctionMultipleQueriesScoresMergedRepeat) { + // Set up + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("type") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("prop2") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); + // Index 3 embedding vectors for document 0. ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId0, kDocumentId0), - CreateVector("my_model1", {1, -2, -3}))); + CreateVector("my_model1", {1, -2, -3}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId1, kDocumentId0), - CreateVector("my_model1", {-1, -2, -3}))); + CreateVector("my_model1", {-1, -2, -3}), QUANTIZATION_TYPE_NONE)); // Index 2 embedding vectors for document 1. ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId0, kDocumentId1), - CreateVector("my_model1", {-1, -2, 3}))); + CreateVector("my_model1", {-1, -2, 3}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); // Create two embedding queries. @@ -4359,23 +4485,46 @@ } TEST_F(QueryVisitorTest, SemanticSearchFunctionHybridQueries) { + // Set up + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("type") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("prop2") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri0").SetSchema("type").Build())); + ICING_ASSERT_OK(document_store_->Put( + DocumentBuilder().SetKey("ns", "uri1").SetSchema("type").Build())); + // Index terms Index::Editor editor = index_->Edit(kDocumentId0, kSectionId1, - TERM_MATCH_PREFIX, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("foo")); + /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm("foo", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); - editor = index_->Edit(kDocumentId1, kSectionId1, TERM_MATCH_PREFIX, + editor = index_->Edit(kDocumentId1, kSectionId1, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm("bar")); + ICING_ASSERT_OK(editor.BufferTerm("bar", TERM_MATCH_PREFIX)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); // Index embedding vectors ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId0, kDocumentId0), - CreateVector("my_model1", {1, -2, -3}))); + CreateVector("my_model1", {1, -2, -3}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId0, kDocumentId1), - CreateVector("my_model1", {-1, -2, 3}))); + CreateVector("my_model1", {-1, -2, 3}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); // Create an embedding query with semantic scores: @@ -4480,16 +4629,16 @@ // Add embedding vectors into different sections for the two documents. ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId0, kDocumentId0), - CreateVector("my_model1", {1, -2, -3}))); + CreateVector("my_model1", {1, -2, -3}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->BufferEmbedding( BasicHit(kSectionId1, kDocumentId0), - CreateVector("my_model1", {-1, -2, 3}))); - ICING_ASSERT_OK( - embedding_index_->BufferEmbedding(BasicHit(kSectionId0, kDocumentId1), - CreateVector("my_model1", {-1, 2, 3}))); - ICING_ASSERT_OK( - embedding_index_->BufferEmbedding(BasicHit(kSectionId1, kDocumentId1), - CreateVector("my_model1", {1, 2, -3}))); + CreateVector("my_model1", {-1, -2, 3}), QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionId0, kDocumentId1), + CreateVector("my_model1", {-1, 2, 3}), QUANTIZATION_TYPE_NONE)); + ICING_ASSERT_OK(embedding_index_->BufferEmbedding( + BasicHit(kSectionId1, kDocumentId1), + CreateVector("my_model1", {1, 2, -3}), QUANTIZATION_TYPE_NONE)); ICING_ASSERT_OK(embedding_index_->CommitBufferToIndex()); // Create an embedding query with semantic scores: @@ -4534,6 +4683,203 @@ StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); } +TEST_F(QueryVisitorTest, + MatchScoreExpressionFunctionWithInvalidRangeReturnsInvalidArgument) { + // The expression is invalid, since low > high. + EXPECT_THAT(ProcessQuery("matchScoreExpression(\"1 + 1\", 10, -10)"), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + + // Floating point values are also checked. + EXPECT_THAT(ProcessQuery("matchScoreExpression(\"1 + 1\", 10.2, 10.1)"), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + + // low == high is allowed. + EXPECT_THAT(ProcessQuery("matchScoreExpression(\"1 + 1\", 10.1, 10.1)"), + IsOk()); +} + +TEST_F(QueryVisitorTest, MatchScoreExpressionFunctionSimpleLowerBound) { + // Create two documents with different document scores. + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Simple")) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("ns", "uri0") + .SetSchema("Simple") + .SetScore(4) + .Build())); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("ns", "uri1") + .SetSchema("Simple") + .SetScore(0) + .Build())); + + // Test that `matchScoreExpression("this.documentScore()", 0)` matches + // all documents. + ICING_ASSERT_OK_AND_ASSIGN( + QueryResults query_results, + ProcessQuery("matchScoreExpression(\"this.documentScore()\", 0)")); + EXPECT_THAT(query_results.features_in_use, + UnorderedElementsAre(kMatchScoreExpressionFunctionFeature, + kListFilterQueryLanguageFeature)); + EXPECT_THAT(GetDocumentIds(query_results.root_iterator.get()), + UnorderedElementsAre(kDocumentId0, kDocumentId1)); + + // Test that `matchScoreExpression("this.documentScore()", 2)` matches + // document 0 only. + ICING_ASSERT_OK_AND_ASSIGN( + query_results, + ProcessQuery("matchScoreExpression(\"this.documentScore()\", 2)")); + EXPECT_THAT(query_results.features_in_use, + UnorderedElementsAre(kMatchScoreExpressionFunctionFeature, + kListFilterQueryLanguageFeature)); + EXPECT_THAT(GetDocumentIds(query_results.root_iterator.get()), + UnorderedElementsAre(kDocumentId0)); + + // Test that `matchScoreExpression("this.documentScore()", 5)` matches + // no documents. + ICING_ASSERT_OK_AND_ASSIGN( + query_results, + ProcessQuery("matchScoreExpression(\"this.documentScore()\", 5)")); + EXPECT_THAT(query_results.features_in_use, + UnorderedElementsAre(kMatchScoreExpressionFunctionFeature, + kListFilterQueryLanguageFeature)); + EXPECT_THAT(GetDocumentIds(query_results.root_iterator.get()), IsEmpty()); +} + +TEST_F(QueryVisitorTest, MatchScoreExpressionFunctionSimpleUpperBound) { + // Create two documents with different document scores. + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Simple")) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("ns", "uri0") + .SetSchema("Simple") + .SetScore(4) + .Build())); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("ns", "uri1") + .SetSchema("Simple") + .SetScore(0) + .Build())); + + // Test that `matchScoreExpression("this.documentScore()", -100, 100)` matches + // all documents. + ICING_ASSERT_OK_AND_ASSIGN( + QueryResults query_results, + ProcessQuery( + "matchScoreExpression(\"this.documentScore()\", -100, 100)")); + EXPECT_THAT(query_results.features_in_use, + UnorderedElementsAre(kMatchScoreExpressionFunctionFeature, + kListFilterQueryLanguageFeature)); + EXPECT_THAT(GetDocumentIds(query_results.root_iterator.get()), + UnorderedElementsAre(kDocumentId0, kDocumentId1)); + + // Test that `matchScoreExpression("this.documentScore()", -100, 2)` matches + // document 1 only. + ICING_ASSERT_OK_AND_ASSIGN( + query_results, + ProcessQuery("matchScoreExpression(\"this.documentScore()\", -100, 2)")); + EXPECT_THAT(query_results.features_in_use, + UnorderedElementsAre(kMatchScoreExpressionFunctionFeature, + kListFilterQueryLanguageFeature)); + EXPECT_THAT(GetDocumentIds(query_results.root_iterator.get()), + UnorderedElementsAre(kDocumentId1)); + + // Test that `matchScoreExpression("this.documentScore()", -100, -1)` matches + // no documents. + ICING_ASSERT_OK_AND_ASSIGN( + query_results, + ProcessQuery("matchScoreExpression(\"this.documentScore()\", -100, -1)")); + EXPECT_THAT(query_results.features_in_use, + UnorderedElementsAre(kMatchScoreExpressionFunctionFeature, + kListFilterQueryLanguageFeature)); + EXPECT_THAT(GetDocumentIds(query_results.root_iterator.get()), IsEmpty()); +} + +TEST_F(QueryVisitorTest, MatchScoreExpressionFunctionComplex) { + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Simple")) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("ns", "uri0") + .SetSchema("Simple") + .SetScore(4) + .SetCreationTimestampMs(1) + .Build())); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("ns", "uri1") + .SetSchema("Simple") + .SetScore(2) + .SetCreationTimestampMs(2) + .Build())); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("ns", "uri2") + .SetSchema("Simple") + .SetScore(0) + .SetCreationTimestampMs(3) + .Build())); + + // Query with a complex score expression: + // `pow(this.creationTimestamp(), 2) + this.documentScore() - 1`. + // The score of each document is: + // - document 0: 1 * 1 + 4 - 1 = 4 + // - document 1: 2 * 2 + 2 - 1 = 5 + // - document 2: 3 * 3 + 0 - 1 = 8 + // Therefore, filtering with a range of [4.5, 7] will only match document 1. + ICING_ASSERT_OK_AND_ASSIGN( + QueryResults query_results, + ProcessQuery("matchScoreExpression(\"pow(this.creationTimestamp(), 2) + " + "this.documentScore() - 1\", 4.5, 7)")); + EXPECT_THAT(query_results.features_in_use, + UnorderedElementsAre(kMatchScoreExpressionFunctionFeature, + kListFilterQueryLanguageFeature)); + EXPECT_THAT(GetDocumentIds(query_results.root_iterator.get()), + UnorderedElementsAre(kDocumentId1)); +} + +TEST_F(QueryVisitorTest, MatchScoreExpressionFunctionWithEvaluationErrors) { + // Create two documents with different document scores. + ICING_ASSERT_OK(schema_store_->SetSchema( + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Simple")) + .Build(), + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("ns", "uri0") + .SetSchema("Simple") + .SetScore(4) + .Build())); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("ns", "uri1") + .SetSchema("Simple") + .SetScore(0) + .Build())); + + // Test that documents with evaluation errors will be filtered out. + // Specifically, document1 will be filtered out because its document score is + // 0, and the expression `1 / 0` is an error. + ICING_ASSERT_OK_AND_ASSIGN( + QueryResults query_results, + ProcessQuery( + "matchScoreExpression(\"1 / this.documentScore()\", -10000, 10000)")); + EXPECT_THAT(query_results.features_in_use, + UnorderedElementsAre(kMatchScoreExpressionFunctionFeature, + kListFilterQueryLanguageFeature)); + EXPECT_THAT(GetDocumentIds(query_results.root_iterator.get()), + UnorderedElementsAre(kDocumentId0)); +} + INSTANTIATE_TEST_SUITE_P(QueryVisitorTest, QueryVisitorTest, testing::Values(QueryType::kPlain, QueryType::kSearch));
diff --git a/icing/query/query-features.h b/icing/query/query-features.h index c84dc7d..10649ba 100644 --- a/icing/query/query-features.h +++ b/icing/query/query-features.h
@@ -62,10 +62,20 @@ constexpr Feature kEmbeddingSearchFeatureDeprecated = "EMBEDDING_SEARCH"; // Features#EMBEDDING_SEARCH +// This feature relates to the use of the +// "matchScoreExpression(scoring_expression, low, high)" function. +// +// Features#MATCH_SCORE_EXPRESSION_FUNCTION +constexpr Feature kMatchScoreExpressionFunctionFeature = + "MATCH_SCORE_EXPRESSION_FUNCTION"; + inline std::unordered_set<Feature> GetQueryFeaturesSet() { - return {kNumericSearchFeature, kVerbatimSearchFeature, - kListFilterQueryLanguageFeature, kHasPropertyFunctionFeature, - kEmbeddingSearchFeatureDeprecated}; + return {kNumericSearchFeature, + kVerbatimSearchFeature, + kListFilterQueryLanguageFeature, + kHasPropertyFunctionFeature, + kEmbeddingSearchFeatureDeprecated, + kMatchScoreExpressionFunctionFeature}; } } // namespace lib
diff --git a/icing/query/query-processor.cc b/icing/query/query-processor.cc index fcd4f73..c331bb8 100644 --- a/icing/query/query-processor.cc +++ b/icing/query/query-processor.cc
@@ -23,12 +23,17 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-index.h" #include "icing/index/index.h" #include "icing/index/iterator/doc-hit-info-iterator-all-document-id.h" +#include "icing/index/iterator/doc-hit-info-iterator-and.h" +#include "icing/index/iterator/doc-hit-info-iterator-by-uri.h" #include "icing/index/iterator/doc-hit-info-iterator-filter.h" #include "icing/index/iterator/doc-hit-info-iterator-section-restrict.h" +#include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/index/numeric/numeric-index.h" +#include "icing/join/join-children-fetcher.h" #include "icing/proto/logging.pb.h" #include "icing/proto/search.pb.h" #include "icing/query/advanced_query_parser/abstract-syntax-tree.h" @@ -56,7 +61,9 @@ const LanguageSegmenter* language_segmenter, const Normalizer* normalizer, const DocumentStore* document_store, - const SchemaStore* schema_store, const Clock* clock) { + const SchemaStore* schema_store, + const JoinChildrenFetcher* join_children_fetcher, + const Clock* clock, const FeatureFlags* feature_flags) { ICING_RETURN_ERROR_IF_NULL(index); ICING_RETURN_ERROR_IF_NULL(numeric_index); ICING_RETURN_ERROR_IF_NULL(embedding_index); @@ -65,20 +72,21 @@ ICING_RETURN_ERROR_IF_NULL(document_store); ICING_RETURN_ERROR_IF_NULL(schema_store); ICING_RETURN_ERROR_IF_NULL(clock); + ICING_RETURN_ERROR_IF_NULL(feature_flags); return std::unique_ptr<QueryProcessor>(new QueryProcessor( index, numeric_index, embedding_index, language_segmenter, normalizer, - document_store, schema_store, clock)); + document_store, schema_store, join_children_fetcher, clock, + feature_flags)); } -QueryProcessor::QueryProcessor(Index* index, - const NumericIndex<int64_t>* numeric_index, - const EmbeddingIndex* embedding_index, - const LanguageSegmenter* language_segmenter, - const Normalizer* normalizer, - const DocumentStore* document_store, - const SchemaStore* schema_store, - const Clock* clock) +QueryProcessor::QueryProcessor( + Index* index, const NumericIndex<int64_t>* numeric_index, + const EmbeddingIndex* embedding_index, + const LanguageSegmenter* language_segmenter, const Normalizer* normalizer, + const DocumentStore* document_store, const SchemaStore* schema_store, + const JoinChildrenFetcher* join_children_fetcher, const Clock* clock, + const FeatureFlags* feature_flags) : index_(*index), numeric_index_(*numeric_index), embedding_index_(*embedding_index), @@ -86,7 +94,9 @@ normalizer_(*normalizer), document_store_(*document_store), schema_store_(*schema_store), - clock_(*clock) {} + join_children_fetcher_(join_children_fetcher), + clock_(*clock), + feature_flags_(*feature_flags) {} libtextclassifier3::StatusOr<QueryResults> QueryProcessor::ParseSearch( const SearchSpecProto& search_spec, @@ -110,6 +120,22 @@ } } + std::vector<std::unique_ptr<DocHitInfoIterator>> iterators; + if (search_spec.document_uri_filters_size() > 0) { + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<DocHitInfoIteratorByUri> uri_iterator, + DocHitInfoIteratorByUri::Create(&document_store_, search_spec)); + iterators.push_back(std::move(uri_iterator)); + } + if (results.root_iterator != nullptr) { + iterators.push_back(std::move(results.root_iterator)); + } + if (iterators.empty()) { + iterators.push_back(std::make_unique<DocHitInfoIteratorAllDocumentId>( + document_store_.last_added_document_id())); + } + results.root_iterator = CreateAndIterator(std::move(iterators)); + DocHitInfoIteratorFilter::Options options = GetFilterOptions(search_spec, document_store_, schema_store_); results.root_iterator = std::make_unique<DocHitInfoIteratorFilter>( @@ -147,10 +173,7 @@ } if (tree_root == nullptr) { - QueryResults results; - results.root_iterator = std::make_unique<DocHitInfoIteratorAllDocumentId>( - document_store_.last_added_document_id()); - return results; + return QueryResults{/*root_iterator=*/nullptr}; } ICING_ASSIGN_OR_RETURN( std::unique_ptr<Tokenizer> plain_tokenizer, @@ -164,8 +187,9 @@ std::unique_ptr<Timer> query_visitor_timer = clock_.GetNewTimer(); QueryVisitor query_visitor( &index_, &numeric_index_, &embedding_index_, &document_store_, - &schema_store_, &normalizer_, plain_tokenizer.get(), search_spec, - std::move(options), needs_term_frequency_info, current_time_ms); + &schema_store_, &normalizer_, plain_tokenizer.get(), + join_children_fetcher_, search_spec, std::move(options), + needs_term_frequency_info, &feature_flags_, current_time_ms); tree_root->Accept(&query_visitor); ICING_ASSIGN_OR_RETURN(QueryResults results, std::move(query_visitor).ConsumeResults());
diff --git a/icing/query/query-processor.h b/icing/query/query-processor.h index d90b5f6..7368ef6 100644 --- a/icing/query/query-processor.h +++ b/icing/query/query-processor.h
@@ -19,9 +19,11 @@ #include <memory> #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-index.h" #include "icing/index/index.h" #include "icing/index/numeric/numeric-index.h" +#include "icing/join/join-children-fetcher.h" #include "icing/proto/logging.pb.h" #include "icing/proto/search.pb.h" #include "icing/query/query-results.h" @@ -51,7 +53,8 @@ const EmbeddingIndex* embedding_index, const LanguageSegmenter* language_segmenter, const Normalizer* normalizer, const DocumentStore* document_store, const SchemaStore* schema_store, - const Clock* clock); + const JoinChildrenFetcher* join_children_fetcher, const Clock* clock, + const FeatureFlags* feature_flags); // Parse the search configurations (including the query, any additional // filters, etc.) in the SearchSpecProto into one DocHitInfoIterator. @@ -74,20 +77,22 @@ QueryStatsProto::SearchStats* search_stats = nullptr); private: - explicit QueryProcessor(Index* index, - const NumericIndex<int64_t>* numeric_index, - const EmbeddingIndex* embedding_index, - const LanguageSegmenter* language_segmenter, - const Normalizer* normalizer, - const DocumentStore* document_store, - const SchemaStore* schema_store, const Clock* clock); + explicit QueryProcessor( + Index* index, const NumericIndex<int64_t>* numeric_index, + const EmbeddingIndex* embedding_index, + const LanguageSegmenter* language_segmenter, const Normalizer* normalizer, + const DocumentStore* document_store, const SchemaStore* schema_store, + const JoinChildrenFetcher* join_children_fetcher, const Clock* clock, + const FeatureFlags* feature_flags); - // Parse the query into a one DocHitInfoIterator that represents the root of a - // query tree in our new Advanced Query Language. + // Parse the query into a QueryResults object, which holds a + // DocHitInfoIterator that represents the root of a query tree in our new + // Advanced Query Language. // // Returns: // On success, - // - One iterator that represents the entire query + // - A QueryResults instance. If the query is empty, the + // DocHitInfoIterator that it holds will be nullptr. // INVALID_ARGUMENT if query syntax is incorrect and cannot be tokenized libtextclassifier3::StatusOr<QueryResults> ParseAdvancedQuery( const SearchSpecProto& search_spec, @@ -118,7 +123,11 @@ const Normalizer& normalizer_; // Does not own. const DocumentStore& document_store_; // Does not own. const SchemaStore& schema_store_; // Does not own. - const Clock& clock_; // Does not own. + // Nullable. A non-null join_children_fetcher_ indicates that this is the + // parent query for a join query, in which case child scores are available. + const JoinChildrenFetcher* join_children_fetcher_; // Does not own. + const Clock& clock_; // Does not own. + const FeatureFlags& feature_flags_; // Does not own. }; } // namespace lib
diff --git a/icing/query/query-processor_benchmark.cc b/icing/query/query-processor_benchmark.cc index fd096bf..27120c6 100644 --- a/icing/query/query-processor_benchmark.cc +++ b/icing/query/query-processor_benchmark.cc
@@ -23,6 +23,7 @@ #include "third_party/absl/flags/flag.h" #include "icing/absl_ports/str_cat.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-index.h" @@ -39,14 +40,15 @@ #include "icing/store/document-id.h" #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/logging.h" #include "unicode/uloc.h" @@ -87,8 +89,8 @@ TermMatchType::Code term_match_type, const std::string& token) { Index::Editor editor = - index->Edit(document_id, section_id, term_match_type, /*namespace_id=*/0); - ICING_ASSERT_OK(editor.BufferTerm(token.c_str())); + index->Edit(document_id, section_id, /*namespace_id=*/0); + ICING_ASSERT_OK(editor.BufferTerm(token, term_match_type)); ICING_ASSERT_OK(editor.IndexAllBufferedTerms()); } @@ -110,23 +112,24 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, const SchemaStore* schema_store) { + const Clock* clock, const SchemaStore* schema_store, + const FeatureFlags& feature_flags) { return DocumentStore::Create( - filesystem, base_dir, clock, schema_store, + filesystem, base_dir, clock, schema_store, &feature_flags, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr); } void BM_QueryOneTerm(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); IcingFilesystem icing_filesystem; Filesystem filesystem; const std::string base_dir = GetTestTempDir() + "/query_processor_benchmark"; @@ -149,9 +152,6 @@ ICING_ASSERT_OK_AND_ASSIGN( auto numeric_index, DummyNumericIndex<int64_t>::Create(filesystem, numeric_index_dir)); - ICING_ASSERT_OK_AND_ASSIGN( - auto embedding_index, - EmbeddingIndex::Create(&filesystem, embedding_index_dir)); language_segmenter_factory::SegmenterOptions options(ULOC_US); std::unique_ptr<LanguageSegmenter> language_segmenter = @@ -164,18 +164,23 @@ Clock clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_dir, &clock)); + SchemaStore::Create(&filesystem, schema_dir, &clock, &feature_flags)); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); DocumentStore::CreateResult create_result = CreateDocumentStore(&filesystem, doc_store_dir, &clock, - schema_store.get()) + schema_store.get(), feature_flags) .ValueOrDie(); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); + ICING_ASSERT_OK_AND_ASSIGN( + auto embedding_index, + EmbeddingIndex::Create(&filesystem, embedding_index_dir, &clock, + &feature_flags)); + DocumentId document_id = document_store ->Put(DocumentBuilder() .SetKey("icing", "type1") @@ -190,10 +195,11 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<QueryProcessor> query_processor, - QueryProcessor::Create(index.get(), numeric_index.get(), - embedding_index.get(), language_segmenter.get(), - normalizer.get(), document_store.get(), - schema_store.get(), &clock)); + QueryProcessor::Create( + index.get(), numeric_index.get(), embedding_index.get(), + language_segmenter.get(), normalizer.get(), document_store.get(), + schema_store.get(), /*join_children_fetcher=*/nullptr, &clock, + &feature_flags)); SearchSpecProto search_spec; search_spec.set_query(input_string); @@ -258,10 +264,11 @@ void BM_QueryFiveTerms(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); IcingFilesystem icing_filesystem; Filesystem filesystem; const std::string base_dir = GetTestTempDir() + "/query_processor_benchmark"; @@ -284,9 +291,6 @@ ICING_ASSERT_OK_AND_ASSIGN( auto numeric_index, DummyNumericIndex<int64_t>::Create(filesystem, numeric_index_dir)); - ICING_ASSERT_OK_AND_ASSIGN( - auto embedding_index, - EmbeddingIndex::Create(&filesystem, embedding_index_dir)); language_segmenter_factory::SegmenterOptions options(ULOC_US); std::unique_ptr<LanguageSegmenter> language_segmenter = @@ -299,18 +303,23 @@ Clock clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_dir, &clock)); + SchemaStore::Create(&filesystem, schema_dir, &clock, &feature_flags)); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); DocumentStore::CreateResult create_result = CreateDocumentStore(&filesystem, doc_store_dir, &clock, - schema_store.get()) + schema_store.get(), feature_flags) .ValueOrDie(); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); + ICING_ASSERT_OK_AND_ASSIGN( + auto embedding_index, + EmbeddingIndex::Create(&filesystem, embedding_index_dir, &clock, + &feature_flags)); + DocumentId document_id = document_store ->Put(DocumentBuilder() .SetKey("icing", "type1") @@ -339,10 +348,11 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<QueryProcessor> query_processor, - QueryProcessor::Create(index.get(), numeric_index.get(), - embedding_index.get(), language_segmenter.get(), - normalizer.get(), document_store.get(), - schema_store.get(), &clock)); + QueryProcessor::Create( + index.get(), numeric_index.get(), embedding_index.get(), + language_segmenter.get(), normalizer.get(), document_store.get(), + schema_store.get(), /*join_children_fetcher=*/nullptr, &clock, + &feature_flags)); const std::string query_string = absl_ports::StrCat( input_string_a, " ", input_string_b, " ", input_string_c, " ", @@ -411,10 +421,11 @@ void BM_QueryDiacriticTerm(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); IcingFilesystem icing_filesystem; Filesystem filesystem; const std::string base_dir = GetTestTempDir() + "/query_processor_benchmark"; @@ -437,9 +448,6 @@ ICING_ASSERT_OK_AND_ASSIGN( auto numeric_index, DummyNumericIndex<int64_t>::Create(filesystem, numeric_index_dir)); - ICING_ASSERT_OK_AND_ASSIGN( - auto embedding_index, - EmbeddingIndex::Create(&filesystem, embedding_index_dir)); language_segmenter_factory::SegmenterOptions options(ULOC_US); std::unique_ptr<LanguageSegmenter> language_segmenter = @@ -452,18 +460,23 @@ Clock clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_dir, &clock)); + SchemaStore::Create(&filesystem, schema_dir, &clock, &feature_flags)); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); DocumentStore::CreateResult create_result = CreateDocumentStore(&filesystem, doc_store_dir, &clock, - schema_store.get()) + schema_store.get(), feature_flags) .ValueOrDie(); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); + ICING_ASSERT_OK_AND_ASSIGN( + auto embedding_index, + EmbeddingIndex::Create(&filesystem, embedding_index_dir, &clock, + &feature_flags)); + DocumentId document_id = document_store ->Put(DocumentBuilder() .SetKey("icing", "type1") @@ -481,10 +494,11 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<QueryProcessor> query_processor, - QueryProcessor::Create(index.get(), numeric_index.get(), - embedding_index.get(), language_segmenter.get(), - normalizer.get(), document_store.get(), - schema_store.get(), &clock)); + QueryProcessor::Create( + index.get(), numeric_index.get(), embedding_index.get(), + language_segmenter.get(), normalizer.get(), document_store.get(), + schema_store.get(), /*join_children_fetcher=*/nullptr, &clock, + &feature_flags)); SearchSpecProto search_spec; search_spec.set_query(input_string); @@ -549,10 +563,11 @@ void BM_QueryHiragana(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); IcingFilesystem icing_filesystem; Filesystem filesystem; const std::string base_dir = GetTestTempDir() + "/query_processor_benchmark"; @@ -575,9 +590,6 @@ ICING_ASSERT_OK_AND_ASSIGN( auto numeric_index, DummyNumericIndex<int64_t>::Create(filesystem, numeric_index_dir)); - ICING_ASSERT_OK_AND_ASSIGN( - auto embedding_index, - EmbeddingIndex::Create(&filesystem, embedding_index_dir)); language_segmenter_factory::SegmenterOptions options(ULOC_US); std::unique_ptr<LanguageSegmenter> language_segmenter = @@ -590,18 +602,23 @@ Clock clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_dir, &clock)); + SchemaStore::Create(&filesystem, schema_dir, &clock, &feature_flags)); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); DocumentStore::CreateResult create_result = CreateDocumentStore(&filesystem, doc_store_dir, &clock, - schema_store.get()) + schema_store.get(), feature_flags) .ValueOrDie(); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); + ICING_ASSERT_OK_AND_ASSIGN( + auto embedding_index, + EmbeddingIndex::Create(&filesystem, embedding_index_dir, &clock, + &feature_flags)); + DocumentId document_id = document_store ->Put(DocumentBuilder() .SetKey("icing", "type1") @@ -619,10 +636,11 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<QueryProcessor> query_processor, - QueryProcessor::Create(index.get(), numeric_index.get(), - embedding_index.get(), language_segmenter.get(), - normalizer.get(), document_store.get(), - schema_store.get(), &clock)); + QueryProcessor::Create( + index.get(), numeric_index.get(), embedding_index.get(), + language_segmenter.get(), normalizer.get(), document_store.get(), + schema_store.get(), /*join_children_fetcher=*/nullptr, &clock, + &feature_flags)); SearchSpecProto search_spec; search_spec.set_query(input_string);
diff --git a/icing/query/query-processor_test.cc b/icing/query/query-processor_test.cc index 536ab5f..b5d3c57 100644 --- a/icing/query/query-processor_test.cc +++ b/icing/query/query-processor_test.cc
@@ -27,6 +27,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-index.h" @@ -53,15 +54,16 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/status-macros.h" #include "unicode/uloc.h" @@ -78,13 +80,13 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, const SchemaStore* schema_store) { + const Clock* clock, const SchemaStore* schema_store, + const FeatureFlags& feature_flags) { return DocumentStore::Create( - filesystem, base_dir, clock, schema_store, + filesystem, base_dir, clock, schema_store, &feature_flags, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr); } @@ -99,6 +101,7 @@ embedding_index_dir_(test_dir_ + "/embedding_index") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(index_dir_.c_str()); filesystem_.CreateDirectoryRecursively(store_dir_.c_str()); @@ -112,17 +115,18 @@ // setup doesn't do this. ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, store_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); document_store_ = std::move(create_result.document_store); Index::Options options(index_dir_, @@ -137,7 +141,8 @@ DummyNumericIndex<int64_t>::Create(filesystem_, numeric_index_dir_)); ICING_ASSERT_OK_AND_ASSIGN( embedding_index_, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir_)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir_, &fake_clock_, + feature_flags_.get())); language_segmenter_factory::SegmenterOptions segmenter_options( ULOC_US, jni_cache_.get()); @@ -153,15 +158,16 @@ QueryProcessor::Create( index_.get(), numeric_index_.get(), embedding_index_.get(), language_segmenter_.get(), normalizer_.get(), document_store_.get(), - schema_store_.get(), &fake_clock_)); + schema_store_.get(), /*join_children_fetcher=*/nullptr, + &fake_clock_, feature_flags_.get())); } libtextclassifier3::Status AddTokenToIndex( DocumentId document_id, SectionId section_id, TermMatchType::Code term_match_type, const std::string& token) { Index::Editor editor = index_->Edit(document_id, section_id, - term_match_type, /*namespace_id=*/0); - auto status = editor.BufferTerm(token.c_str()); + /*namespace_id=*/0); + auto status = editor.BufferTerm(token, term_match_type); return status.ok() ? editor.IndexAllBufferedTerms() : status; } @@ -180,6 +186,7 @@ schema_store_.reset(); filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; const std::string test_dir_; const std::string store_dir_; @@ -206,52 +213,63 @@ TEST_F(QueryProcessorTest, CreationWithNullPointerShouldFail) { EXPECT_THAT( - QueryProcessor::Create(/*index=*/nullptr, numeric_index_.get(), - embedding_index_.get(), language_segmenter_.get(), - normalizer_.get(), document_store_.get(), - schema_store_.get(), &fake_clock_), + QueryProcessor::Create( + /*index=*/nullptr, numeric_index_.get(), embedding_index_.get(), + language_segmenter_.get(), normalizer_.get(), document_store_.get(), + schema_store_.get(), /*join_children_fetcher=*/nullptr, &fake_clock_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT( - QueryProcessor::Create(index_.get(), /*numeric_index_=*/nullptr, - embedding_index_.get(), language_segmenter_.get(), - normalizer_.get(), document_store_.get(), - schema_store_.get(), &fake_clock_), + QueryProcessor::Create( + index_.get(), /*numeric_index_=*/nullptr, embedding_index_.get(), + language_segmenter_.get(), normalizer_.get(), document_store_.get(), + schema_store_.get(), /*join_children_fetcher=*/nullptr, &fake_clock_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); - EXPECT_THAT(QueryProcessor::Create(index_.get(), numeric_index_.get(), - /*embedding_index=*/nullptr, - language_segmenter_.get(), - normalizer_.get(), document_store_.get(), - schema_store_.get(), &fake_clock_), + EXPECT_THAT(QueryProcessor::Create( + index_.get(), numeric_index_.get(), + /*embedding_index=*/nullptr, language_segmenter_.get(), + normalizer_.get(), document_store_.get(), schema_store_.get(), + /*join_children_fetcher=*/nullptr, &fake_clock_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT(QueryProcessor::Create( index_.get(), numeric_index_.get(), embedding_index_.get(), /*language_segmenter=*/nullptr, normalizer_.get(), - document_store_.get(), schema_store_.get(), &fake_clock_), + document_store_.get(), schema_store_.get(), + /*join_children_fetcher=*/nullptr, &fake_clock_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT( QueryProcessor::Create(index_.get(), numeric_index_.get(), embedding_index_.get(), language_segmenter_.get(), /*normalizer=*/nullptr, document_store_.get(), - schema_store_.get(), &fake_clock_), + schema_store_.get(), + /*join_children_fetcher=*/nullptr, &fake_clock_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); - EXPECT_THAT( - QueryProcessor::Create( - index_.get(), numeric_index_.get(), embedding_index_.get(), - language_segmenter_.get(), normalizer_.get(), - /*document_store=*/nullptr, schema_store_.get(), &fake_clock_), - StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + EXPECT_THAT(QueryProcessor::Create( + index_.get(), numeric_index_.get(), embedding_index_.get(), + language_segmenter_.get(), normalizer_.get(), + /*document_store=*/nullptr, schema_store_.get(), + /*join_children_fetcher=*/nullptr, &fake_clock_, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT( QueryProcessor::Create(index_.get(), numeric_index_.get(), embedding_index_.get(), language_segmenter_.get(), normalizer_.get(), document_store_.get(), - /*schema_store=*/nullptr, &fake_clock_), + /*schema_store=*/nullptr, + /*join_children_fetcher=*/nullptr, &fake_clock_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); - EXPECT_THAT( - QueryProcessor::Create(index_.get(), numeric_index_.get(), - embedding_index_.get(), language_segmenter_.get(), - normalizer_.get(), document_store_.get(), - schema_store_.get(), /*clock=*/nullptr), - StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + EXPECT_THAT(QueryProcessor::Create( + index_.get(), numeric_index_.get(), embedding_index_.get(), + language_segmenter_.get(), normalizer_.get(), + document_store_.get(), schema_store_.get(), + /*join_children_fetcher=*/nullptr, /*clock=*/nullptr, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } TEST_F(QueryProcessorTest, EmptyGroupMatchAllDocuments) { @@ -2945,7 +2963,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, store_dir_, &fake_clock, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); document_store_ = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN( @@ -2972,7 +2990,9 @@ QueryProcessor::Create(index_.get(), numeric_index_.get(), embedding_index_.get(), language_segmenter_.get(), normalizer_.get(), document_store_.get(), - schema_store_.get(), &fake_clock_)); + schema_store_.get(), + /*join_children_fetcher=*/nullptr, &fake_clock_, + feature_flags_.get())); SearchSpecProto search_spec; search_spec.set_query("hello"); @@ -3008,7 +3028,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, store_dir_, &fake_clock_local, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); document_store_ = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN( @@ -3035,7 +3055,9 @@ QueryProcessor::Create(index_.get(), numeric_index_.get(), embedding_index_.get(), language_segmenter_.get(), normalizer_.get(), document_store_.get(), - schema_store_.get(), &fake_clock_)); + schema_store_.get(), + /*join_children_fetcher=*/nullptr, &fake_clock_, + feature_flags_.get())); SearchSpecProto search_spec; search_spec.set_query("hello"); @@ -3387,6 +3409,40 @@ Eq(kSearchStatsLatencyMs)); } +TEST_F(QueryProcessorTest, UriFiltersIsNotTheRightMostNode) { + SchemaProto schema = SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email")) + .Build(); + ASSERT_THAT(schema_store_->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOk()); + ICING_ASSERT_OK(document_store_->Put(DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("email") + .Build())); + + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query("foo"); + NamespaceDocumentUriGroup* uris = search_spec.add_document_uri_filters(); + uris->set_namespace_("namespace"); + uris->add_document_uris("uri1"); + uris->add_document_uris("uri3"); + + QueryStatsProto::SearchStats search_stats; + ICING_ASSERT_OK_AND_ASSIGN( + QueryResults results, + query_processor_->ParseSearch( + search_spec, ScoringSpecProto::RankingStrategy::NONE, + fake_clock_.GetSystemTimeMilliseconds(), &search_stats)); + + ICING_ASSERT_OK_AND_ASSIGN( + DocHitInfoIterator::TrimmedNode trimmed_node, + std::move(*results.root_iterator).TrimRightMostNode()); + EXPECT_THAT(trimmed_node.iterator_->ToString(), Eq("uri_iterator")); +} + } // namespace } // namespace lib
diff --git a/icing/query/suggestion-processor.cc b/icing/query/suggestion-processor.cc index ba80093..cebd205 100644 --- a/icing/query/suggestion-processor.cc +++ b/icing/query/suggestion-processor.cc
@@ -26,6 +26,7 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-index.h" #include "icing/index/index.h" #include "icing/index/iterator/doc-hit-info-iterator.h" @@ -56,8 +57,8 @@ const LanguageSegmenter* language_segmenter, const Normalizer* normalizer, const DocumentStore* document_store, - const SchemaStore* schema_store, - const Clock* clock) { + const SchemaStore* schema_store, const Clock* clock, + const FeatureFlags* feature_flags) { ICING_RETURN_ERROR_IF_NULL(index); ICING_RETURN_ERROR_IF_NULL(numeric_index); ICING_RETURN_ERROR_IF_NULL(embedding_index); @@ -66,10 +67,11 @@ ICING_RETURN_ERROR_IF_NULL(document_store); ICING_RETURN_ERROR_IF_NULL(schema_store); ICING_RETURN_ERROR_IF_NULL(clock); + ICING_RETURN_ERROR_IF_NULL(feature_flags); return std::unique_ptr<SuggestionProcessor>(new SuggestionProcessor( index, numeric_index, embedding_index, language_segmenter, normalizer, - document_store, schema_store, clock)); + document_store, schema_store, clock, feature_flags)); } libtextclassifier3::StatusOr< @@ -251,9 +253,10 @@ ICING_ASSIGN_OR_RETURN( std::unique_ptr<QueryProcessor> query_processor, - QueryProcessor::Create(&index_, &numeric_index_, &embedding_index_, - &language_segmenter_, &normalizer_, - &document_store_, &schema_store_, &clock_)); + QueryProcessor::Create( + &index_, &numeric_index_, &embedding_index_, &language_segmenter_, + &normalizer_, &document_store_, &schema_store_, + /*join_children_fetcher=*/nullptr, &clock_, &feature_flags_)); SearchSpecProto search_spec; search_spec.set_query(suggestion_spec.prefix()); @@ -343,7 +346,7 @@ const EmbeddingIndex* embedding_index, const LanguageSegmenter* language_segmenter, const Normalizer* normalizer, const DocumentStore* document_store, const SchemaStore* schema_store, - const Clock* clock) + const Clock* clock, const FeatureFlags* feature_flags) : index_(*index), numeric_index_(*numeric_index), embedding_index_(*embedding_index), @@ -351,7 +354,8 @@ normalizer_(*normalizer), document_store_(*document_store), schema_store_(*schema_store), - clock_(*clock) {} + clock_(*clock), + feature_flags_(*feature_flags) {} } // namespace lib } // namespace icing
diff --git a/icing/query/suggestion-processor.h b/icing/query/suggestion-processor.h index cf393b4..5b21611 100644 --- a/icing/query/suggestion-processor.h +++ b/icing/query/suggestion-processor.h
@@ -20,9 +20,11 @@ #include <vector> #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-index.h" #include "icing/index/index.h" #include "icing/index/numeric/numeric-index.h" +#include "icing/index/term-metadata.h" #include "icing/proto/search.pb.h" #include "icing/schema/schema-store.h" #include "icing/store/document-store.h" @@ -50,7 +52,8 @@ const EmbeddingIndex* embedding_index, const LanguageSegmenter* language_segmenter, const Normalizer* normalizer, const DocumentStore* document_store, - const SchemaStore* schema_store, const Clock* clock); + const SchemaStore* schema_store, const Clock* clock, + const FeatureFlags* feature_flags); // Query suggestions based on the given SuggestionSpecProto. // @@ -62,14 +65,12 @@ const SuggestionSpecProto& suggestion_spec, int64_t current_time_ms); private: - explicit SuggestionProcessor(Index* index, - const NumericIndex<int64_t>* numeric_index, - const EmbeddingIndex* embedding_index, - const LanguageSegmenter* language_segmenter, - const Normalizer* normalizer, - const DocumentStore* document_store, - const SchemaStore* schema_store, - const Clock* clock); + explicit SuggestionProcessor( + Index* index, const NumericIndex<int64_t>* numeric_index, + const EmbeddingIndex* embedding_index, + const LanguageSegmenter* language_segmenter, const Normalizer* normalizer, + const DocumentStore* document_store, const SchemaStore* schema_store, + const Clock* clock, const FeatureFlags* feature_flags); // Not const because we could modify/sort the TermMetaData buffer in the lite // index. @@ -81,6 +82,7 @@ const DocumentStore& document_store_; // Does not own. const SchemaStore& schema_store_; // Does not own. const Clock& clock_; // Does not own. + const FeatureFlags& feature_flags_; // Does not own. }; } // namespace lib
diff --git a/icing/query/suggestion-processor_test.cc b/icing/query/suggestion-processor_test.cc index 4347a69..ba6be86 100644 --- a/icing/query/suggestion-processor_test.cc +++ b/icing/query/suggestion-processor_test.cc
@@ -24,6 +24,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-index.h" @@ -42,14 +43,15 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -82,6 +84,7 @@ embedding_index_dir_(test_dir_ + "/embedding_index") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(index_dir_.c_str()); filesystem_.CreateDirectoryRecursively(store_dir_.c_str()); @@ -95,24 +98,24 @@ // setup doesn't do this. ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, store_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, store_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); Index::Options options(index_dir_, @@ -127,7 +130,8 @@ DummyNumericIndex<int64_t>::Create(filesystem_, numeric_index_dir_)); ICING_ASSERT_OK_AND_ASSIGN( embedding_index_, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir_)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir_, &fake_clock_, + feature_flags_.get())); language_segmenter_factory::SegmenterOptions segmenter_options( ULOC_US, jni_cache_.get()); @@ -143,15 +147,15 @@ SuggestionProcessor::Create( index_.get(), numeric_index_.get(), embedding_index_.get(), language_segmenter_.get(), normalizer_.get(), document_store_.get(), - schema_store_.get(), &fake_clock_)); + schema_store_.get(), &fake_clock_, feature_flags_.get())); } libtextclassifier3::Status AddTokenToIndex( DocumentId document_id, SectionId section_id, TermMatchType::Code term_match_type, const std::string& token) { Index::Editor editor = index_->Edit(document_id, section_id, - term_match_type, /*namespace_id=*/0); - auto status = editor.BufferTerm(token.c_str()); + /*namespace_id=*/0); + auto status = editor.BufferTerm(token, term_match_type); return status.ok() ? editor.IndexAllBufferedTerms() : status; } @@ -161,6 +165,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; const std::string test_dir_; const std::string store_dir_;
diff --git a/icing/result/result-adjustment-info_test.cc b/icing/result/result-adjustment-info_test.cc index cbce557..fd0551e 100644 --- a/icing/result/result-adjustment-info_test.cc +++ b/icing/result/result-adjustment-info_test.cc
@@ -14,11 +14,13 @@ #include "icing/result/result-adjustment-info.h" +#include <memory> #include <string> #include <unordered_set> #include <vector> #include "gtest/gtest.h" +#include "icing/feature-flags.h" #include "icing/proto/scoring.pb.h" #include "icing/proto/search.pb.h" #include "icing/proto/term.pb.h" @@ -28,6 +30,7 @@ #include "icing/schema/schema-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" namespace icing { @@ -49,9 +52,10 @@ } void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() @@ -68,6 +72,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; const Filesystem filesystem_; const std::string test_dir_; std::unique_ptr<SchemaStore> schema_store_;
diff --git a/icing/result/result-retriever-v2_group-result-limiter_test.cc b/icing/result/result-retriever-v2_group-result-limiter_test.cc index 215035e..0d59e41 100644 --- a/icing/result/result-retriever-v2_group-result-limiter_test.cc +++ b/icing/result/result-retriever-v2_group-result-limiter_test.cc
@@ -17,6 +17,8 @@ #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/portable/equals-proto.h" #include "icing/portable/platform.h" #include "icing/proto/document.pb.h" @@ -30,14 +32,16 @@ #include "icing/scoring/priority-queue-scored-document-hits-ranker.h" #include "icing/scoring/scored-document-hit.h" #include "icing/store/document-id.h" +#include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -62,10 +66,11 @@ } void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } language_segmenter_factory::SegmenterOptions options(ULOC_US); @@ -74,8 +79,8 @@ language_segmenter_factory::Create(std::move(options))); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN(normalizer_, normalizer_factory::Create( /*max_term_byte_size=*/10000)); @@ -89,14 +94,14 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } @@ -104,6 +109,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; const Filesystem filesystem_; const std::string test_dir_; std::unique_ptr<LanguageSegmenter> language_segmenter_;
diff --git a/icing/result/result-retriever-v2_projection_test.cc b/icing/result/result-retriever-v2_projection_test.cc index f846ef6..1ac0681 100644 --- a/icing/result/result-retriever-v2_projection_test.cc +++ b/icing/result/result-retriever-v2_projection_test.cc
@@ -18,6 +18,8 @@ #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/portable/equals-proto.h" #include "icing/portable/platform.h" #include "icing/proto/document.pb.h" @@ -35,14 +37,16 @@ #include "icing/scoring/priority-queue-scored-document-hits-ranker.h" #include "icing/scoring/scored-document-hit.h" #include "icing/store/document-id.h" +#include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -60,10 +64,11 @@ } void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } language_segmenter_factory::SegmenterOptions options(ULOC_US); @@ -72,8 +77,8 @@ language_segmenter_factory::Create(std::move(options))); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN(normalizer_, normalizer_factory::Create( /*max_term_byte_size=*/10000)); @@ -184,14 +189,14 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } @@ -218,6 +223,7 @@ return kInvalidSectionId; } + std::unique_ptr<FeatureFlags> feature_flags_; const Filesystem filesystem_; const std::string test_dir_; std::unique_ptr<LanguageSegmenter> language_segmenter_;
diff --git a/icing/result/result-retriever-v2_snippet_test.cc b/icing/result/result-retriever-v2_snippet_test.cc index 7bad0d8..114637e 100644 --- a/icing/result/result-retriever-v2_snippet_test.cc +++ b/icing/result/result-retriever-v2_snippet_test.cc
@@ -19,6 +19,8 @@ #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/portable/equals-proto.h" #include "icing/portable/platform.h" #include "icing/proto/document.pb.h" @@ -35,14 +37,16 @@ #include "icing/scoring/priority-queue-scored-document-hits-ranker.h" #include "icing/scoring/scored-document-hit.h" #include "icing/store/document-id.h" +#include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/snippet-helpers.h" #include "unicode/uloc.h" @@ -64,10 +68,11 @@ } void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } language_segmenter_factory::SegmenterOptions options(ULOC_US); @@ -76,8 +81,8 @@ language_segmenter_factory::Create(std::move(options))); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN(normalizer_, normalizer_factory::Create( /*max_term_byte_size=*/10000)); @@ -109,14 +114,14 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } @@ -143,6 +148,7 @@ return kInvalidSectionId; } + std::unique_ptr<FeatureFlags> feature_flags_; const Filesystem filesystem_; const std::string test_dir_; std::unique_ptr<LanguageSegmenter> language_segmenter_;
diff --git a/icing/result/result-retriever-v2_test.cc b/icing/result/result-retriever-v2_test.cc index 4ae4e4c..77e6fa2 100644 --- a/icing/result/result-retriever-v2_test.cc +++ b/icing/result/result-retriever-v2_test.cc
@@ -29,6 +29,7 @@ #include "gtest/gtest.h" #include "icing/absl_ports/mutex.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/mock-filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" @@ -50,14 +51,15 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -97,10 +99,11 @@ } void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } language_segmenter_factory::SegmenterOptions options(ULOC_US); @@ -109,8 +112,8 @@ language_segmenter_factory::Create(std::move(options))); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN(normalizer_, normalizer_factory::Create( /*max_term_byte_size=*/10000)); @@ -179,6 +182,7 @@ return kInvalidSectionId; } + std::unique_ptr<FeatureFlags> feature_flags_; const Filesystem filesystem_; const std::string test_dir_; std::unique_ptr<LanguageSegmenter> language_segmenter_; @@ -216,13 +220,13 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, const SchemaStore* schema_store) { + const Clock* clock, const SchemaStore* schema_store, + const FeatureFlags& feature_flags) { return DocumentStore::Create( - filesystem, base_dir, clock, schema_store, + filesystem, base_dir, clock, schema_store, &feature_flags, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr); } @@ -235,7 +239,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -257,7 +261,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -355,7 +359,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -430,7 +434,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -592,7 +596,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&mock_filesystem, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -642,7 +646,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -741,7 +745,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -854,7 +858,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -916,7 +920,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -980,7 +984,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get())); + schema_store_.get(), *feature_flags_)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store);
diff --git a/icing/result/result-state-manager_test.cc b/icing/result/result-state-manager_test.cc index b06eb94..e998c99 100644 --- a/icing/result/result-state-manager_test.cc +++ b/icing/result/result-state-manager_test.cc
@@ -14,10 +14,14 @@ #include "icing/result/result-state-manager.h" +#include <memory> + #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/portable/equals-proto.h" #include "icing/result/page-result.h" #include "icing/result/result-adjustment-info.h" @@ -28,13 +32,14 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -79,10 +84,11 @@ } void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -94,8 +100,8 @@ language_segmenter_factory::Create(std::move(options))); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, clock_.get())); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + clock_.get(), feature_flags_.get())); SchemaProto schema; schema.add_types()->set_schema_type("Document"); ICING_ASSERT_OK(schema_store_->SetSchema( @@ -107,14 +113,14 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult result, - DocumentStore::Create( - &filesystem_, test_dir_, clock_.get(), schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, test_dir_, clock_.get(), + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(result.document_store); ICING_ASSERT_OK_AND_ASSIGN( @@ -173,6 +179,7 @@ } private: + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; const std::string test_dir_; std::unique_ptr<FakeClock> clock_;
diff --git a/icing/result/result-state-manager_thread-safety_test.cc b/icing/result/result-state-manager_thread-safety_test.cc index dd0f6f1..33cde3f 100644 --- a/icing/result/result-state-manager_thread-safety_test.cc +++ b/icing/result/result-state-manager_thread-safety_test.cc
@@ -13,13 +13,16 @@ // limitations under the License. #include <algorithm> +#include <memory> #include <optional> #include <thread> // NOLINT #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/portable/equals-proto.h" #include "icing/result/page-result.h" #include "icing/result/result-retriever-v2.h" @@ -29,13 +32,14 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -72,10 +76,11 @@ } void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -87,8 +92,8 @@ language_segmenter_factory::Create(std::move(options))); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, clock_.get())); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + clock_.get(), feature_flags_.get())); SchemaProto schema; schema.add_types()->set_schema_type("Document"); ICING_ASSERT_OK(schema_store_->SetSchema( @@ -100,14 +105,14 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult result, - DocumentStore::Create( - &filesystem_, test_dir_, clock_.get(), schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, test_dir_, clock_.get(), + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(result.document_store); ICING_ASSERT_OK_AND_ASSIGN( @@ -121,6 +126,7 @@ clock_.reset(); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; const std::string test_dir_; std::unique_ptr<FakeClock> clock_;
diff --git a/icing/result/result-state-v2_test.cc b/icing/result/result-state-v2_test.cc index 9915dbf..bb2031c 100644 --- a/icing/result/result-state-v2_test.cc +++ b/icing/result/result-state-v2_test.cc
@@ -25,6 +25,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/absl_ports/mutex.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/proto/document.pb.h" @@ -38,6 +39,7 @@ #include "icing/store/document-id.h" #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/util/clock.h" @@ -61,11 +63,13 @@ class ResultStateV2Test : public ::testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); schema_store_base_dir_ = GetTestTempDir() + "/schema_store"; filesystem_.CreateDirectoryRecursively(schema_store_base_dir_.c_str()); + ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_base_dir_, &clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_base_dir_, + &clock_, feature_flags_.get())); SchemaProto schema; schema.add_types()->set_schema_type("Document"); ICING_ASSERT_OK(schema_store_->SetSchema( @@ -76,14 +80,14 @@ filesystem_.CreateDirectoryRecursively(doc_store_base_dir_.c_str()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult result, - DocumentStore::Create( - &filesystem_, doc_store_base_dir_, &clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, doc_store_base_dir_, &clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(result.document_store); num_total_hits_ = 0; @@ -110,6 +114,7 @@ const std::atomic<int>& num_total_hits() const { return num_total_hits_; } private: + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; std::string doc_store_base_dir_; std::string schema_store_base_dir_;
diff --git a/icing/result/snippet-retriever.cc b/icing/result/snippet-retriever.cc index 9b10c33..53bff43 100644 --- a/icing/result/snippet-retriever.cc +++ b/icing/result/snippet-retriever.cc
@@ -61,7 +61,8 @@ // Returns a string of the normalized text of the input Token. Normalization // is applied based on the Token's type. -std::string NormalizeToken(const Normalizer& normalizer, const Token& token) { +Normalizer::NormalizedTerm NormalizeToken(const Normalizer& normalizer, + const Token& token) { switch (token.type) { case Token::Type::RFC822_NAME: [[fallthrough]]; @@ -106,7 +107,7 @@ case Token::Type::REGULAR: return normalizer.NormalizeTerm(token.text); case Token::Type::VERBATIM: - return std::string(token.text); + return {std::string(token.text)}; case Token::Type::QUERY_EXCLUSION: [[fallthrough]]; case Token::Type::QUERY_LEFT_PARENTHESES: @@ -120,7 +121,7 @@ case Token::Type::INVALID: ICING_LOG(WARNING) << "Unable to normalize token of type: " << static_cast<int>(token.type); - return std::string(token.text); + return {std::string(token.text)}; } } @@ -227,10 +228,11 @@ normalizer_(normalizer) {} CharacterIterator Matches(Token token) const override { - std::string s = NormalizeToken(normalizer_, token); - auto itr = unrestricted_query_terms_.find(s); + Normalizer::NormalizedTerm normalized_term = + NormalizeToken(normalizer_, token); + auto itr = unrestricted_query_terms_.find(normalized_term.text); if (itr == unrestricted_query_terms_.end()) { - itr = restricted_query_terms_.find(s); + itr = restricted_query_terms_.find(normalized_term.text); } if (itr != unrestricted_query_terms_.end() && itr != restricted_query_terms_.end()) { @@ -257,16 +259,19 @@ normalizer_(normalizer) {} CharacterIterator Matches(Token token) const override { - std::string s = NormalizeToken(normalizer_, token); + Normalizer::NormalizedTerm normalized_term = + NormalizeToken(normalizer_, token); for (const std::string& query_term : unrestricted_query_terms_) { - if (query_term.length() <= s.length() && - s.compare(0, query_term.length(), query_term) == 0) { + if (query_term.length() <= normalized_term.text.length() && + normalized_term.text.compare(0, query_term.length(), query_term) == + 0) { return FindMatchEnd(normalizer_, token, query_term); } } for (const std::string& query_term : restricted_query_terms_) { - if (query_term.length() <= s.length() && - s.compare(0, query_term.length(), query_term) == 0) { + if (query_term.length() <= normalized_term.text.length() && + normalized_term.text.compare(0, query_term.length(), query_term) == + 0) { return FindMatchEnd(normalizer_, token, query_term); } }
diff --git a/icing/result/snippet-retriever_benchmark.cc b/icing/result/snippet-retriever_benchmark.cc index e574325..b4d10da 100644 --- a/icing/result/snippet-retriever_benchmark.cc +++ b/icing/result/snippet-retriever_benchmark.cc
@@ -16,6 +16,7 @@ #include "gmock/gmock.h" #include "third_party/absl/flags/flag.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/proto/schema.pb.h" #include "icing/proto/search.pb.h" @@ -24,13 +25,14 @@ #include "icing/schema/schema-store.h" #include "icing/schema/section.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/random-string.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/transform/normalizer-factory.h" #include "icing/util/clock.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/logging.h" #include "unicode/uloc.h" @@ -72,10 +74,11 @@ void BM_SnippetOneProperty(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); const std::string base_dir = GetTestTempDir() + "/query_processor_benchmark"; const std::string schema_dir = base_dir + "/schema"; Filesystem filesystem; @@ -103,7 +106,7 @@ Clock clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_dir, &clock)); + SchemaStore::Create(&filesystem, schema_dir, &clock, &feature_flags)); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -201,10 +204,11 @@ void BM_SnippetRfcOneProperty(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } + FeatureFlags feature_flags = GetTestFeatureFlags(); const std::string base_dir = GetTestTempDir() + "/query_processor_benchmark"; const std::string schema_dir = base_dir + "/schema"; Filesystem filesystem; @@ -232,7 +236,7 @@ Clock clock; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_dir, &clock)); + SchemaStore::Create(&filesystem, schema_dir, &clock, &feature_flags)); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false));
diff --git a/icing/result/snippet-retriever_test.cc b/icing/result/snippet-retriever_test.cc index 8d81b43..8c07e7a 100644 --- a/icing/result/snippet-retriever_test.cc +++ b/icing/result/snippet-retriever_test.cc
@@ -21,6 +21,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/mock-filesystem.h" #include "icing/portable/equals-proto.h" #include "icing/portable/platform.h" @@ -36,15 +37,16 @@ #include "icing/store/key-mapper.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/map/map-normalizer.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/snippet-helpers.h" #include "unicode/uloc.h" @@ -76,13 +78,14 @@ class SnippetRetrieverTest : public testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); test_dir_ = GetTestTempDir() + "/icing"; filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -95,8 +98,8 @@ // Setup the schema ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType( @@ -136,6 +139,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; FakeClock fake_clock_; std::unique_ptr<SchemaStore> schema_store_;
diff --git a/icing/schema-builder.h b/icing/schema-builder.h index 9387440..ab14d7e 100644 --- a/icing/schema-builder.h +++ b/icing/schema-builder.h
@@ -63,6 +63,11 @@ constexpr EmbeddingIndexingConfig::EmbeddingIndexingType::Code EMBEDDING_INDEXING_LINEAR_SEARCH = EmbeddingIndexingConfig::EmbeddingIndexingType::LINEAR_SEARCH; +constexpr EmbeddingIndexingConfig::QuantizationType::Code + QUANTIZATION_TYPE_NONE = EmbeddingIndexingConfig::QuantizationType::NONE; +constexpr EmbeddingIndexingConfig::QuantizationType::Code + QUANTIZATION_TYPE_QUANTIZE_8_BIT = + EmbeddingIndexingConfig::QuantizationType::QUANTIZE_8_BIT; constexpr PropertyConfigProto::DataType::Code TYPE_UNKNOWN = PropertyConfigProto::DataType::UNKNOWN; @@ -88,6 +93,19 @@ constexpr JoinableConfig::ValueType::Code JOINABLE_VALUE_TYPE_QUALIFIED_ID = JoinableConfig::ValueType::QUALIFIED_ID; +constexpr JoinableConfig::DeletePropagationType::Code + DELETE_PROPAGATION_TYPE_NONE = JoinableConfig::DeletePropagationType::NONE; +constexpr JoinableConfig::DeletePropagationType::Code + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM = + JoinableConfig::DeletePropagationType::PROPAGATE_FROM; + +constexpr PropertyConfigProto::ScorableType::Code SCORABLE_TYPE_ENABLED = + PropertyConfigProto::ScorableType::ENABLED; +constexpr PropertyConfigProto::ScorableType::Code SCORABLE_TYPE_DISABLED = + PropertyConfigProto::ScorableType::DISABLED; +constexpr PropertyConfigProto::ScorableType::Code SCORABLE_TYPE_UNKNOWN = + PropertyConfigProto::ScorableType::UNKNOWN; + class PropertyConfigBuilder { public: PropertyConfigBuilder() = default; @@ -116,12 +134,12 @@ PropertyConfigBuilder& SetDataTypeJoinableString( JoinableConfig::ValueType::Code join_value_type, - TermMatchType::Code match_type = TERM_MATCH_UNKNOWN, - StringIndexingConfig::TokenizerType::Code tokenizer = TOKENIZER_NONE) { + JoinableConfig::DeletePropagationType::Code delete_propagation_type = + DELETE_PROPAGATION_TYPE_NONE) { property_.set_data_type(PropertyConfigProto::DataType::STRING); property_.mutable_joinable_config()->set_value_type(join_value_type); - property_.mutable_string_indexing_config()->set_term_match_type(match_type); - property_.mutable_string_indexing_config()->set_tokenizer_type(tokenizer); + property_.mutable_joinable_config()->set_delete_propagation_type( + delete_propagation_type); return *this; } @@ -160,17 +178,24 @@ PropertyConfigBuilder& SetDataTypeVector( EmbeddingIndexingConfig::EmbeddingIndexingType::Code - embedding_indexing_type) { + embedding_indexing_type, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type = + EmbeddingIndexingConfig::QuantizationType::NONE) { property_.set_data_type(PropertyConfigProto::DataType::VECTOR); - property_.mutable_embedding_indexing_config()->set_embedding_indexing_type( + EmbeddingIndexingConfig* embedding_indexing_config = + property_.mutable_embedding_indexing_config(); + embedding_indexing_config->set_embedding_indexing_type( embedding_indexing_type); + embedding_indexing_config->set_quantization_type(quantization_type); return *this; } PropertyConfigBuilder& SetJoinable( - JoinableConfig::ValueType::Code join_value_type, bool propagate_delete) { + JoinableConfig::ValueType::Code join_value_type, + JoinableConfig::DeletePropagationType::Code delete_propagation_type) { property_.mutable_joinable_config()->set_value_type(join_value_type); - property_.mutable_joinable_config()->set_propagate_delete(propagate_delete); + property_.mutable_joinable_config()->set_delete_propagation_type( + delete_propagation_type); return *this; } @@ -185,6 +210,12 @@ return *this; } + PropertyConfigBuilder& SetScorableType( + PropertyConfigProto::ScorableType::Code scorable_type) { + property_.set_scorable_type(scorable_type); + return *this; + } + PropertyConfigProto Build() const { return std::move(property_); } private: @@ -217,6 +248,11 @@ return *this; } + SchemaTypeConfigBuilder& SetDatabase(std::string database) { + type_config_.set_database(std::move(database)); + return *this; + } + SchemaTypeConfigBuilder& AddProperty(PropertyConfigProto property) { *type_config_.add_properties() = std::move(property); return *this;
diff --git a/icing/schema/joinable-property-manager-builder_test.cc b/icing/schema/joinable-property-manager-builder_test.cc index ac48faa..f573ac7 100644 --- a/icing/schema/joinable-property-manager-builder_test.cc +++ b/icing/schema/joinable-property-manager-builder_test.cc
@@ -62,21 +62,21 @@ PropertyConfigBuilder() .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_OPTIONAL) .Build(); PropertyConfigProto prop_bar = PropertyConfigBuilder() .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL) .Build(); PropertyConfigProto prop_baz = PropertyConfigBuilder() .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(); @@ -123,7 +123,7 @@ PropertyConfigBuilder() .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(); ICING_ASSERT_OK(builder.ProcessSchemaTypePropertyConfig( @@ -136,7 +136,7 @@ PropertyConfigBuilder() .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(); EXPECT_THAT(builder.ProcessSchemaTypePropertyConfig( @@ -159,7 +159,7 @@ PropertyConfigBuilder() .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(); @@ -189,7 +189,7 @@ PropertyConfigBuilder() .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(); @@ -216,56 +216,56 @@ .SetName("int1") .SetDataType(TYPE_INT64) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(), PropertyConfigBuilder() .SetName("int2") .SetDataType(TYPE_INT64) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED) .Build(), PropertyConfigBuilder() .SetName("double1") .SetDataType(TYPE_DOUBLE) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(), PropertyConfigBuilder() .SetName("double2") .SetDataType(TYPE_DOUBLE) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED) .Build(), PropertyConfigBuilder() .SetName("boolean1") .SetDataType(TYPE_BOOLEAN) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(), PropertyConfigBuilder() .SetName("boolean2") .SetDataType(TYPE_BOOLEAN) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED) .Build(), PropertyConfigBuilder() .SetName("bytes1") .SetDataType(TYPE_BYTES) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(), PropertyConfigBuilder() .SetName("bytes2") .SetDataType(TYPE_BYTES) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED) .Build(), PropertyConfigBuilder() @@ -273,7 +273,7 @@ .SetDataTypeDocument(/*schema_type=*/"SchemaTypeTwo", /*index_nested_properties=*/true) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(), PropertyConfigBuilder() @@ -281,7 +281,7 @@ .SetDataTypeDocument(/*schema_type=*/"SchemaTypeTwo", /*index_nested_properties=*/true) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED) .Build()}; @@ -335,14 +335,14 @@ .SetName("property") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_OPTIONAL) .Build(), PropertyConfigBuilder() .SetName("property") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL) .Build(), // Indexable string can be configured joinable as well. For @@ -351,14 +351,14 @@ .SetName("property") .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_OPTIONAL) .Build(), PropertyConfigBuilder() .SetName("property") .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL) .Build()));
diff --git a/icing/schema/joinable-property-manager.cc b/icing/schema/joinable-property-manager.cc index 1606abb..e51e143 100644 --- a/icing/schema/joinable-property-manager.cc +++ b/icing/schema/joinable-property-manager.cc
@@ -14,7 +14,7 @@ #include "icing/schema/joinable-property-manager.h" -#include <memory> +#include <cstdint> #include <string> #include <string_view> #include <utility> @@ -42,7 +42,8 @@ metadata_list_wrapper, std::string&& concatenated_path, PropertyConfigProto::DataType::Code data_type, - JoinableConfig::ValueType::Code value_type) { + JoinableConfig::ValueType::Code value_type, + JoinableConfig::DeletePropagationType::Code delete_propagation_type) { // Validates next joinable property id, makes sure that joinable property id // is the same as the list index so that we could find any joinable property // metadata by id in O(1) later. @@ -58,7 +59,8 @@ // Creates joinable property metadata metadata_list_wrapper->metadata_list.push_back(JoinablePropertyMetadata( - new_id, data_type, value_type, std::move(concatenated_path))); + new_id, data_type, value_type, delete_propagation_type, + std::move(concatenated_path))); metadata_list_wrapper->property_path_to_id_map.insert( {metadata_list_wrapper->metadata_list.back().path, new_id}); return libtextclassifier3::Status::OK; @@ -101,7 +103,8 @@ ICING_RETURN_IF_ERROR(AppendNewJoinablePropertyMetadata( &joinable_property_metadata_cache_[schema_type_id], std::move(property_path), PropertyConfigProto::DataType::STRING, - JoinableConfig::ValueType::QUALIFIED_ID)); + JoinableConfig::ValueType::QUALIFIED_ID, + property_config.joinable_config().delete_propagation_type())); } break; }
diff --git a/icing/schema/joinable-property-manager_test.cc b/icing/schema/joinable-property-manager_test.cc index ceaaa18..f3a6a2d 100644 --- a/icing/schema/joinable-property-manager_test.cc +++ b/icing/schema/joinable-property-manager_test.cc
@@ -72,7 +72,8 @@ return PropertyConfigBuilder() .SetName(kPropertySenderQualifiedId) .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) - .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, /*propagate_delete=*/true) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_OPTIONAL) .Build(); } @@ -81,7 +82,8 @@ return PropertyConfigBuilder() .SetName(kPropertyReceiverQualifiedId) .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) - .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, /*propagate_delete=*/true) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_OPTIONAL) .Build(); } @@ -90,7 +92,8 @@ return PropertyConfigBuilder() .SetName(kPropertyGroupQualifiedId) .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) - .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, /*propagate_delete=*/false) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL) .Build(); }
diff --git a/icing/schema/joinable-property.h b/icing/schema/joinable-property.h index 057bb74..416d870 100644 --- a/icing/schema/joinable-property.h +++ b/icing/schema/joinable-property.h
@@ -70,14 +70,30 @@ // non-string DataType. JoinableConfig::ValueType::Code value_type; + // How to propagate the deletion between the document and the referenced + // joinable document. + // + // JoinableConfig::DeletePropagationType::NONE: + // No propagation. + // + // JoinableConfig::DeletePropagationType::PROPAGATE_FROM: + // Delete the document when the referenced (parent) document is deleted. + // + // This is only applicable to joinable properties with value type + // JoinableConfig::ValueType::QUALIFIED_ID. + JoinableConfig::DeletePropagationType::Code delete_propagation_type; + explicit JoinablePropertyMetadata( JoinablePropertyId id_in, PropertyConfigProto::DataType::Code data_type_in, - JoinableConfig::ValueType::Code value_type_in, std::string&& path_in) + JoinableConfig::ValueType::Code value_type_in, + JoinableConfig::DeletePropagationType::Code delete_propagation_type_in, + std::string&& path_in) : path(std::move(path_in)), id(id_in), data_type(data_type_in), - value_type(value_type_in) {} + value_type(value_type_in), + delete_propagation_type(delete_propagation_type_in) {} JoinablePropertyMetadata(const JoinablePropertyMetadata& other) = default; JoinablePropertyMetadata& operator=(const JoinablePropertyMetadata& other) = @@ -89,7 +105,8 @@ bool operator==(const JoinablePropertyMetadata& rhs) const { return path == rhs.path && id == rhs.id && data_type == rhs.data_type && - value_type == rhs.value_type; + value_type == rhs.value_type && + delete_propagation_type == rhs.delete_propagation_type; } }; @@ -115,6 +132,10 @@ JoinableConfig::ValueType::Code value_type() const { return metadata.value_type; } + + JoinableConfig::DeletePropagationType::Code delete_propagation_type() const { + return metadata.delete_propagation_type; + } }; // Groups of different type joinable properties. Callers can access joinable
diff --git a/icing/schema/property-util.cc b/icing/schema/property-util.cc index 4c07306..e8eeec7 100644 --- a/icing/schema/property-util.cc +++ b/icing/schema/property-util.cc
@@ -134,6 +134,20 @@ } template <> +libtextclassifier3::StatusOr<std::vector<double>> ExtractPropertyValues<double>( + const PropertyProto& property) { + return std::vector<double>(property.double_values().begin(), + property.double_values().end()); +} + +template <> +libtextclassifier3::StatusOr<std::vector<bool>> ExtractPropertyValues<bool>( + const PropertyProto& property) { + return std::vector<bool>(property.boolean_values().begin(), + property.boolean_values().end()); +} + +template <> libtextclassifier3::StatusOr<std::vector<PropertyProto::VectorProto>> ExtractPropertyValues<PropertyProto::VectorProto>( const PropertyProto& property) {
diff --git a/icing/schema/property-util.h b/icing/schema/property-util.h index 55e4f20..d07489e 100644 --- a/icing/schema/property-util.h +++ b/icing/schema/property-util.h
@@ -167,6 +167,14 @@ ExtractPropertyValues<int64_t>(const PropertyProto& property); template <> +libtextclassifier3::StatusOr<std::vector<double>> ExtractPropertyValues<double>( + const PropertyProto& property); + +template <> +libtextclassifier3::StatusOr<std::vector<bool>> ExtractPropertyValues<bool>( + const PropertyProto& property); + +template <> libtextclassifier3::StatusOr<std::vector<PropertyProto::VectorProto>> ExtractPropertyValues<PropertyProto::VectorProto>( const PropertyProto& property);
diff --git a/icing/schema/property-util_test.cc b/icing/schema/property-util_test.cc index 1bfd5cd..c401bd3 100644 --- a/icing/schema/property-util_test.cc +++ b/icing/schema/property-util_test.cc
@@ -33,9 +33,12 @@ namespace { using ::icing::lib::portable_equals_proto::EqualsProto; +using ::testing::DoubleNear; using ::testing::ElementsAre; using ::testing::IsEmpty; +constexpr double kEps = 0.0000001; // 1e-7 + static constexpr std::string_view kTypeTest = "Test"; static constexpr std::string_view kPropertySingleString = "singleString"; static constexpr std::string_view kPropertyRepeatedString = "repeatedString"; @@ -88,6 +91,25 @@ IsOkAndHolds(ElementsAre(123, -456, 0))); } +TEST(PropertyUtilTest, ExtractPropertyValuesTypeDouble) { + PropertyProto property; + property.mutable_double_values()->Add(3.5); + property.mutable_double_values()->Add(-2.0); + + EXPECT_THAT( + property_util::ExtractPropertyValues<double>(property), + IsOkAndHolds(ElementsAre(DoubleNear(3.5, kEps), DoubleNear(-2.0, kEps)))); +} + +TEST(PropertyUtilTest, ExtractPropertyValuesTypeBoolean) { + PropertyProto property; + property.mutable_boolean_values()->Add(true); + property.mutable_boolean_values()->Add(false); + + EXPECT_THAT(property_util::ExtractPropertyValues<bool>(property), + IsOkAndHolds(ElementsAre(true, false))); +} + TEST(PropertyUtilTest, ExtractPropertyValuesTypeVector) { PropertyProto::VectorProto vector1; vector1.set_model_signature("my_model1"); @@ -112,12 +134,12 @@ TEST(PropertyUtilTest, ExtractPropertyValuesTypeBlobHandle) { PropertyProto::BlobHandleProto blob_handle1; - blob_handle1.set_label("label1"); + blob_handle1.set_namespace_("namespace1"); std::vector<unsigned char> bytes1(32); std::string digestString1 = std::string(bytes1.begin(), bytes1.end()); blob_handle1.set_digest(digestString1); PropertyProto::BlobHandleProto blob_handle2; - blob_handle2.set_label("label2"); + blob_handle2.set_namespace_("namespace2"); std::vector<unsigned char> bytes2(32); std::string digestString2 = std::string(bytes2.begin(), bytes2.end()); blob_handle2.set_digest(digestString2);
diff --git a/icing/schema/schema-store.cc b/icing/schema/schema-store.cc index 2b04adb..aaa9f84 100644 --- a/icing/schema/schema-store.cc +++ b/icing/schema/schema-store.cc
@@ -15,9 +15,11 @@ #include "icing/schema/schema-store.h" #include <cinttypes> +#include <cstddef> #include <cstdint> #include <limits> #include <memory> +#include <optional> #include <string> #include <string_view> #include <unordered_map> @@ -29,6 +31,7 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" +#include "icing/feature-flags.h" #include "icing/file/destructible-directory.h" #include "icing/file/file-backed-proto.h" #include "icing/file/filesystem.h" @@ -46,6 +49,7 @@ #include "icing/schema/schema-property-iterator.h" #include "icing/schema/schema-type-manager.h" #include "icing/schema/schema-util.h" +#include "icing/schema/scorable_property_manager.h" #include "icing/schema/section.h" #include "icing/store/document-filter-data.h" #include "icing/store/dynamic-trie-key-mapper.h" @@ -64,6 +68,12 @@ constexpr char kOverlaySchemaFilename[] = "overlay_schema.pb"; constexpr char kSchemaTypeMapperFilename[] = "schema_type_mapper"; +// This should be kept consistent with the delimiter used in AppSearch. +// See: +// https://cs.android.com/androidx/platform/frameworks/support/+/androidx-main:appsearch/appsearch-local-storage/src/main/java/androidx/appsearch/localstorage/util/PrefixUtil.java;l=42;drc=ffaf979c6f0cbd26caafd7a9d07a6bad12fe3a2a + +constexpr char kAppSearchDatabaseDelimiter = '/'; + // A DynamicTrieKeyMapper stores its data across 3 arrays internally. Giving // each array 128KiB for storage means the entire DynamicTrieKeyMapper requires // 384KiB. @@ -93,12 +103,12 @@ std::unordered_set<SchemaTypeId> old_schema_type_ids_changed; std::unordered_map<std::string, int> old_types_and_index; - for (int i = 0; i < old_schema.types_size(); ++i) { + for (int i = 0; i < old_schema.types().size(); ++i) { old_types_and_index.emplace(old_schema.types(i).schema_type(), i); } std::unordered_map<std::string, int> new_types_and_index; - for (int i = 0; i < new_schema.types_size(); ++i) { + for (int i = 0; i < new_schema.types().size(); ++i) { new_types_and_index.emplace(new_schema.types(i).schema_type(), i); } @@ -120,6 +130,46 @@ return old_schema_type_ids_changed; } +// Returns the database from the schema type name if it exists. +// +// The schema type is expected to be in the format of +// <database><delimiter><actual_type_name>. +// +// Returns an empty string if the schema type name is not in the database +// format. +std::string GetDatabaseFromSchemaType(const std::string& schema_type, + char database_delimeter) { + size_t db_index = schema_type.find(database_delimeter); + std::string database; + if (db_index != std::string::npos) { + database = schema_type.substr(0, db_index); + } + return database; +} + +// For each schema type in the schema proto, parses out the database from the +// type name, and sets it as the database field in the input proto in +// place. The schema_type name field itself is not modified. +// +// If the schema type name does not contain an AppSearch database, then +// SchemaTypeConfigProto is not modified. +// +// Returns: +// - True if any SchemaTypeConfigProto in the schema proto is rewritten. +// - False otherwise. +bool ParseAndPopulateAppSearchDatabaseField(SchemaProto& schema_proto) { + bool populated_database_field = false; + for (auto& type : *schema_proto.mutable_types()) { + std::string database = GetDatabaseFromSchemaType( + type.schema_type(), kAppSearchDatabaseDelimiter); + if (type.database() != database) { + type.set_database(std::move(database)); + populated_database_field = true; + } + } + return populated_database_field; +} + } // namespace /* static */ libtextclassifier3::StatusOr<SchemaStore::Header> @@ -202,36 +252,94 @@ libtextclassifier3::StatusOr<std::unique_ptr<SchemaStore>> SchemaStore::Create( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, InitializeStatsProto* initialize_stats) { + const Clock* clock, const FeatureFlags* feature_flags, + bool enable_schema_database, InitializeStatsProto* initialize_stats) { ICING_RETURN_ERROR_IF_NULL(filesystem); ICING_RETURN_ERROR_IF_NULL(clock); + ICING_RETURN_ERROR_IF_NULL(feature_flags); if (!filesystem->DirectoryExists(base_dir.c_str())) { return absl_ports::FailedPreconditionError( "Schema store base directory does not exist!"); } - std::unique_ptr<SchemaStore> schema_store = std::unique_ptr<SchemaStore>( - new SchemaStore(filesystem, base_dir, clock)); + std::unique_ptr<SchemaStore> schema_store = + std::unique_ptr<SchemaStore>(new SchemaStore( + filesystem, base_dir, clock, feature_flags, enable_schema_database)); ICING_RETURN_IF_ERROR(schema_store->Initialize(initialize_stats)); return schema_store; } libtextclassifier3::StatusOr<std::unique_ptr<SchemaStore>> SchemaStore::Create( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, SchemaProto schema) { + const Clock* clock, const FeatureFlags* feature_flags, SchemaProto schema, + bool enable_schema_database) { ICING_RETURN_ERROR_IF_NULL(filesystem); ICING_RETURN_ERROR_IF_NULL(clock); + ICING_RETURN_ERROR_IF_NULL(feature_flags); if (!filesystem->DirectoryExists(base_dir.c_str())) { return absl_ports::FailedPreconditionError( "Schema store base directory does not exist!"); } - std::unique_ptr<SchemaStore> schema_store = std::unique_ptr<SchemaStore>( - new SchemaStore(filesystem, base_dir, clock)); + std::unique_ptr<SchemaStore> schema_store = + std::unique_ptr<SchemaStore>(new SchemaStore( + filesystem, base_dir, clock, feature_flags, enable_schema_database)); ICING_RETURN_IF_ERROR(schema_store->Initialize(std::move(schema))); return schema_store; } +/* static */ libtextclassifier3::Status +SchemaStore::PopulateSchemaDatabaseFieldForSchemaFile( + const Filesystem* filesystem, const std::string& schema_filename) { + FileBackedProto<SchemaProto> schema_file(*filesystem, schema_filename); + auto schema_proto_or = schema_file.Read(); + if (absl_ports::IsNotFound(schema_proto_or.status())) { + // Don't have an existing schema proto, that's fine + return libtextclassifier3::Status::OK; + } else if (!schema_proto_or.ok()) { + // Real error when trying to read the existing schema + return schema_proto_or.status(); + } + + SchemaProto schema_proto_copy = *schema_proto_or.ValueOrDie(); + bool schema_changed = + ParseAndPopulateAppSearchDatabaseField(schema_proto_copy); + if (!schema_changed) { + // Nothing to do if the schema is not changed. + return libtextclassifier3::Status::OK; + } + + // Create a temporary schema file and schema proto copy to update the + // schema. + std::string temp_schema_filename = schema_filename + ".tmp"; + if (!filesystem->DeleteFile(temp_schema_filename.c_str())) { + return absl_ports::InternalError( + "Unable to delete temp schema file to prepare for schema database " + "migration."); + } + + { + FileBackedProto<SchemaProto> temp_schema_file(*filesystem, + temp_schema_filename); + ICING_RETURN_IF_ERROR(temp_schema_file.Write( + std::make_unique<SchemaProto>(schema_proto_copy))); + } + + // Swap the temp schema file with the original schema file. + if (!filesystem->SwapFiles(temp_schema_filename.c_str(), + schema_filename.c_str())) { + return absl_ports::InternalError( + "Unable to apply migrated schema with database due to failed swap!"); + } + // Clean up the temp schema file. + if (!filesystem->DeleteFile(temp_schema_filename.c_str())) { + return absl_ports::InternalError( + "Unable to delete temp schema file after schema database migration."); + } + + return libtextclassifier3::Status::OK; +} + /* static */ libtextclassifier3::Status SchemaStore::DiscardOverlaySchema( const Filesystem* filesystem, const std::string& base_dir, Header& header) { std::string header_filename = MakeHeaderFilename(base_dir); @@ -252,7 +360,8 @@ /* static */ libtextclassifier3::Status SchemaStore::MigrateSchema( const Filesystem* filesystem, const std::string& base_dir, - version_util::StateChange version_state_change, int32_t new_version) { + version_util::StateChange version_state_change, int32_t new_version, + bool perform_schema_database_migration) { if (!filesystem->DirectoryExists(base_dir.c_str())) { // Situations when schema store directory doesn't exist: // - Initializing new Icing instance: don't have to do anything now. The @@ -264,6 +373,33 @@ return libtextclassifier3::Status::OK; } + ICING_RETURN_IF_ERROR(HandleOverlaySchemaForVersionChange( + filesystem, base_dir, version_state_change, new_version)); + + // Perform schema database migration if needed. + // - This populates the the database field in the schema proto and writes it + // to the schema file. + // - If the overlay schema file exists at this point, does the same for the + // overlay schema. + if (perform_schema_database_migration) { + std::string base_schema_filename = MakeSchemaFilename(base_dir); + ICING_RETURN_IF_ERROR(PopulateSchemaDatabaseFieldForSchemaFile( + filesystem, base_schema_filename)); + + std::string overlay_schema_filename = MakeOverlaySchemaFilename(base_dir); + if (filesystem->FileExists(overlay_schema_filename.c_str())) { + ICING_RETURN_IF_ERROR(PopulateSchemaDatabaseFieldForSchemaFile( + filesystem, overlay_schema_filename)); + } + } + + return libtextclassifier3::Status::OK; +} + +/* static */ libtextclassifier3::Status +SchemaStore::HandleOverlaySchemaForVersionChange( + const Filesystem* filesystem, const std::string& base_dir, + version_util::StateChange version_state_change, int32_t new_version) { std::string overlay_schema_filename = MakeOverlaySchemaFilename(base_dir); if (!filesystem->FileExists(overlay_schema_filename.c_str())) { // The overlay doesn't exist. So there should be nothing particularly @@ -329,12 +465,15 @@ } SchemaStore::SchemaStore(const Filesystem* filesystem, std::string base_dir, - const Clock* clock) + const Clock* clock, const FeatureFlags* feature_flags, + bool enable_schema_database) : filesystem_(filesystem), base_dir_(std::move(base_dir)), clock_(clock), + feature_flags_(feature_flags), schema_file_(std::make_unique<FileBackedProto<SchemaProto>>( - *filesystem, MakeSchemaFilename(base_dir_))) {} + *filesystem, MakeSchemaFilename(base_dir_))), + enable_schema_database_(enable_schema_database) {} SchemaStore::~SchemaStore() { if (has_schema_successfully_set_ && schema_file_ != nullptr && @@ -509,7 +648,7 @@ ICING_RETURN_IF_ERROR(schema_file_->Write(std::move(base_schema_ptr))); // LINT.IfChange(min_overlay_version_compatibility) - // Although the current version is 4, the schema is compatible with + // Although the current version is 5, the schema is compatible with // version 1, so min_overlay_version_compatibility should be 1. int32_t min_overlay_version_compatibility = version_util::kVersionOne; // LINT.ThenChange(//depot/google3/icing/file/version-util.h:kVersion) @@ -533,18 +672,23 @@ SchemaUtil::BuildTransitiveInheritanceGraph(*schema_proto)); reverse_schema_type_mapper_.clear(); + database_type_map_.clear(); type_config_map_.clear(); schema_subtype_id_map_.clear(); for (const SchemaTypeConfigProto& type_config : schema_proto->types()) { - std::string_view type_name = type_config.schema_type(); + const std::string& database = type_config.database(); + const std::string& type_name = type_config.schema_type(); ICING_ASSIGN_OR_RETURN(SchemaTypeId type_id, schema_type_mapper_->Get(type_name)); // Build reverse_schema_type_mapper_ - reverse_schema_type_mapper_.insert({type_id, std::string(type_name)}); + reverse_schema_type_mapper_.insert({type_id, type_name}); + + // Build database_type_map_ + database_type_map_[database].push_back(type_name); // Build type_config_map_ - type_config_map_.insert({std::string(type_name), type_config}); + type_config_map_.insert({type_name, type_config}); // Build schema_subtype_id_map_ std::unordered_set<SchemaTypeId>& subtype_id_set = @@ -568,6 +712,9 @@ ICING_ASSIGN_OR_RETURN( schema_type_manager_, SchemaTypeManager::Create(type_config_map_, schema_type_mapper_.get())); + + scorable_property_manager_ = std::make_unique<ScorablePropertyManager>(); + return libtextclassifier3::Status::OK; } @@ -661,11 +808,32 @@ return schema_file_->Read(); } +libtextclassifier3::StatusOr<SchemaProto> SchemaStore::GetSchema( + const std::string& database) const { + if (!has_schema_successfully_set_) { + return absl_ports::NotFoundError("No schema found."); + } + + const auto database_type_map_itr_ = database_type_map_.find(database); + if (database_type_map_itr_ == database_type_map_.end()) { + return absl_ports::NotFoundError( + absl_ports::StrCat("No schema found for database '", database, "'.")); + } + + SchemaProto schema_proto; + for (const std::string& type_name : database_type_map_itr_->second) { + ICING_ASSIGN_OR_RETURN(const SchemaTypeConfigProto* type_config, + GetSchemaTypeConfig(type_name)); + *schema_proto.add_types() = *type_config; + } + return schema_proto; +} + // TODO(cassiewang): Consider removing this definition of SetSchema if it's not // needed by production code. It's currently being used by our tests, but maybe // it's trivial to change our test code to also use the // SetSchema(SchemaProto&& new_schema) -libtextclassifier3::StatusOr<const SchemaStore::SetSchemaResult> +libtextclassifier3::StatusOr<SchemaStore::SetSchemaResult> SchemaStore::SetSchema(const SchemaProto& new_schema, bool ignore_errors_and_delete_documents, bool allow_circular_schema_definitions) { @@ -673,92 +841,194 @@ allow_circular_schema_definitions); } -libtextclassifier3::StatusOr<const SchemaStore::SetSchemaResult> +libtextclassifier3::StatusOr<SchemaStore::SetSchemaResult> SchemaStore::SetSchema(SchemaProto&& new_schema, bool ignore_errors_and_delete_documents, bool allow_circular_schema_definitions) { + if (enable_schema_database_) { + // Step 1: (Only required if schema database is enabled) + // Do some preliminary checks on the new schema before formal validation and + // delta computation. This checks that: + // - The new schema only contains types from a single database. + // - The new schema's type names are not already in use from other + // databases. + ICING_ASSIGN_OR_RETURN(std::string database, + ValidateAndGetDatabase(new_schema)); + + // Step 2: Schema validation and delta computation -- try to get the + // existing schema for the database to compare to the new schema. + libtextclassifier3::StatusOr<SchemaProto> schema_proto = + GetSchema(database); + if (absl_ports::IsNotFound(schema_proto.status())) { + // Case 1: No preexisting schema for this database. + return SetInitialSchemaForDatabase(std::move(new_schema), + ignore_errors_and_delete_documents, + allow_circular_schema_definitions); + } + + if (!schema_proto.ok()) { + // Case 2: Real error + return schema_proto.status(); + } + + // Case 3: At this point, we're guaranteed that we have an existing schema + // for this database. + const SchemaProto& old_schema = schema_proto.ValueOrDie(); + return SetSchemaWithDatabaseOverride(std::move(new_schema), old_schema, + ignore_errors_and_delete_documents, + allow_circular_schema_definitions); + } + + // Get the full schema if schema database is disabled. + libtextclassifier3::StatusOr<const SchemaProto*> schema_proto = GetSchema(); + if (absl_ports::IsNotFound(schema_proto.status())) { + // Case 1: No preexisting schema + return SetInitialSchemaForDatabase(std::move(new_schema), + ignore_errors_and_delete_documents, + allow_circular_schema_definitions); + } + + if (!schema_proto.ok()) { + // Case 2: Real error + return schema_proto.status(); + } + + // Case 3: At this point, we're guaranteed that we have an existing schema + const SchemaProto& old_schema = *schema_proto.ValueOrDie(); + return SetSchemaWithDatabaseOverride(std::move(new_schema), old_schema, + ignore_errors_and_delete_documents, + allow_circular_schema_definitions); +} + +libtextclassifier3::StatusOr<SchemaStore::SetSchemaResult> +SchemaStore::SetInitialSchemaForDatabase( + SchemaProto new_schema, bool ignore_errors_and_delete_documents, + bool allow_circular_schema_definitions) { SetSchemaResult result; - auto schema_proto_or = GetSchema(); - if (absl_ports::IsNotFound(schema_proto_or.status())) { - // We don't have a pre-existing schema, so anything is valid as long as the - // new schema is valid. - ICING_RETURN_IF_ERROR( - SchemaUtil::Validate(new_schema, allow_circular_schema_definitions)); + ICING_RETURN_IF_ERROR(SchemaUtil::Validate( + new_schema, *feature_flags_, allow_circular_schema_definitions)); - result.success = true; - for (const SchemaTypeConfigProto& type_config : new_schema.types()) { - result.schema_types_new_by_name.insert(type_config.schema_type()); - } - } else if (!schema_proto_or.ok()) { - // Real error - return schema_proto_or.status(); - } else { - // At this point, we're guaranteed that we have a schema. - const SchemaProto old_schema = *schema_proto_or.ValueOrDie(); - - // Assume we can set the schema unless proven otherwise. - result.success = true; - - if (new_schema.SerializeAsString() == old_schema.SerializeAsString()) { - // Same schema as before. No need to update anything - return result; - } - - // Different schema -- validate the new schema, track the differences and - // see if we can still write it - ICING_ASSIGN_OR_RETURN( - SchemaUtil::DependentMap new_dependent_map, - SchemaUtil::Validate(new_schema, allow_circular_schema_definitions)); - SchemaUtil::SchemaDelta schema_delta = - SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - new_dependent_map); - - result.schema_types_new_by_name = std::move(schema_delta.schema_types_new); - result.schema_types_changed_fully_compatible_by_name = - std::move(schema_delta.schema_types_changed_fully_compatible); - result.schema_types_index_incompatible_by_name = - std::move(schema_delta.schema_types_index_incompatible); - result.schema_types_join_incompatible_by_name = - std::move(schema_delta.schema_types_join_incompatible); - - for (const auto& schema_type : schema_delta.schema_types_deleted) { - // We currently don't support deletions, so mark this as not possible. - // This will change once we allow force-set schemas. - result.success = false; - - result.schema_types_deleted_by_name.emplace(schema_type); - - ICING_ASSIGN_OR_RETURN(SchemaTypeId schema_type_id, - GetSchemaTypeId(schema_type)); - result.schema_types_deleted_by_id.emplace(schema_type_id); - } - - for (const auto& schema_type : schema_delta.schema_types_incompatible) { - // We currently don't support incompatible schemas, so mark this as - // not possible. This will change once we allow force-set schemas. - result.success = false; - - result.schema_types_incompatible_by_name.emplace(schema_type); - - ICING_ASSIGN_OR_RETURN(SchemaTypeId schema_type_id, - GetSchemaTypeId(schema_type)); - result.schema_types_incompatible_by_id.emplace(schema_type_id); - } - - // SchemaTypeIds changing is fine, we can update the DocumentStore - result.old_schema_type_ids_changed = - SchemaTypeIdsChanged(old_schema, new_schema); + result.success = true; + for (const SchemaTypeConfigProto& type_config : new_schema.types()) { + result.schema_types_new_by_name.insert(type_config.schema_type()); } + // Get the full new SchemaProto that is a combination of the existing schema + // and new_schema. This is needed as we can only write the full proto to the + // schema file. + ICING_ASSIGN_OR_RETURN( + SchemaProto full_new_schema, + GetFullSchemaProtoWithUpdatedDb(std::move(new_schema))); + ICING_RETURN_IF_ERROR(ApplySchemaChange(std::move(full_new_schema))); + has_schema_successfully_set_ = true; + + return result; +} + +libtextclassifier3::StatusOr<SchemaStore::SetSchemaResult> +SchemaStore::SetSchemaWithDatabaseOverride( + SchemaProto new_schema, const SchemaProto& old_schema, + bool ignore_errors_and_delete_documents, + bool allow_circular_schema_definitions) { + // Assume we can set the schema unless proven otherwise. + SetSchemaResult result; + result.success = true; + + if (new_schema.SerializeAsString() == old_schema.SerializeAsString()) { + // Same schema as before. No need to update anything + return result; + } + + // Different schema -- we need to validate the schema and track the + // differences to see if we can still write it. + // + // Validate the new schema and compute the delta between the old and new + // schema. + ICING_ASSIGN_OR_RETURN( + SchemaUtil::DependentMap new_dependent_map, + SchemaUtil::Validate(new_schema, *feature_flags_, + allow_circular_schema_definitions)); + SchemaUtil::SchemaDelta schema_delta = SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, new_dependent_map, *feature_flags_); + + result.schema_types_new_by_name = std::move(schema_delta.schema_types_new); + result.schema_types_changed_fully_compatible_by_name = + std::move(schema_delta.schema_types_changed_fully_compatible); + result.schema_types_index_incompatible_by_name = + std::move(schema_delta.schema_types_index_incompatible); + result.schema_types_join_incompatible_by_name = + std::move(schema_delta.schema_types_join_incompatible); + result.schema_types_scorable_property_inconsistent_by_name = + std::move(schema_delta.schema_types_scorable_property_inconsistent); + + for (const std::string& schema_type : schema_delta.schema_types_deleted) { + // We currently don't support deletions, so mark this as not possible. + // This will change once we allow force-set schemas. + result.success = false; + + result.schema_types_deleted_by_name.emplace(schema_type); + + ICING_ASSIGN_OR_RETURN(SchemaTypeId schema_type_id, + GetSchemaTypeId(schema_type)); + result.schema_types_deleted_by_id.emplace(schema_type_id); + } + + for (const std::string& schema_type : + schema_delta.schema_types_incompatible) { + // We currently don't support incompatible schemas, so mark this as + // not possible. This will change once we allow force-set schemas. + result.success = false; + + result.schema_types_incompatible_by_name.emplace(schema_type); + + ICING_ASSIGN_OR_RETURN(SchemaTypeId schema_type_id, + GetSchemaTypeId(schema_type)); + result.schema_types_incompatible_by_id.emplace(schema_type_id); + } + + // Get the full new SchemaProto that is a combination of the existing schema + // and new_schema. This is needed to calculate the updated SchemaTypeIds, and + // for writing the full proto to the schema file. + ICING_ASSIGN_OR_RETURN( + SchemaProto full_new_schema, + GetFullSchemaProtoWithUpdatedDb(std::move(new_schema))); + + // We still need to update old_schema_type_ids_changed. We need to retrieve + // the entire old schema for this, as type ids are assigned for the entire + // schema, and not on a per-database level. + // + // SchemaTypeIds changing is fine, we can update the DocumentStore. + ICING_ASSIGN_OR_RETURN(const SchemaProto* full_old_schema, GetSchema()); + result.old_schema_type_ids_changed = + SchemaTypeIdsChanged(*full_old_schema, full_new_schema); // We can force set the schema if the caller has told us to ignore any errors result.success = result.success || ignore_errors_and_delete_documents; + // Step 3: Apply the schema change if success. This updates persisted files + // and derived data structures. if (result.success) { - ICING_RETURN_IF_ERROR(ApplySchemaChange(std::move(new_schema))); + ICING_RETURN_IF_ERROR(ApplySchemaChange(std::move(full_new_schema))); has_schema_successfully_set_ = true; } + // Convert schema types to SchemaTypeIds after the new schema is applied. + if (feature_flags_->enable_scorable_properties()) { + for (const std::string& schema_type : + result.schema_types_scorable_property_inconsistent_by_name) { + libtextclassifier3::StatusOr<SchemaTypeId> schema_type_id_or = + GetSchemaTypeId(schema_type); + if (!schema_type_id_or.ok()) { + if (absl_ports::IsNotFound(schema_type_id_or.status())) { + continue; + } + return schema_type_id_or.status(); + } + result.schema_types_scorable_property_inconsistent_by_id.insert( + schema_type_id_or.ValueOrDie()); + } + } + return result; } @@ -790,7 +1060,8 @@ ICING_ASSIGN_OR_RETURN( std::unique_ptr<SchemaStore> new_schema_store, SchemaStore::Create(filesystem_, temp_schema_store_dir.dir(), clock_, - std::move(new_schema))); + feature_flags_, std::move(new_schema), + enable_schema_database_)); // Then we swap the new schema file + new derived files with the old files. if (!filesystem_->SwapFiles(base_dir_.c_str(), @@ -880,6 +1151,15 @@ .GetJoinablePropertyMetadata(schema_type_id, property_path); } +libtextclassifier3::StatusOr<const JoinablePropertyMetadata*> +SchemaStore::GetJoinablePropertyMetadata( + SchemaTypeId schema_type_id, + JoinablePropertyId joinable_property_id) const { + ICING_RETURN_IF_ERROR(CheckSchemaSet()); + return schema_type_manager_->joinable_property_manager() + .GetJoinablePropertyMetadata(schema_type_id, joinable_property_id); +} + libtextclassifier3::StatusOr<JoinablePropertyGroup> SchemaStore::ExtractJoinableProperties(const DocumentProto& document) const { ICING_RETURN_IF_ERROR(CheckSchemaSet()); @@ -887,6 +1167,29 @@ .ExtractJoinableProperties(document); } +libtextclassifier3::StatusOr<std::optional<int>> +SchemaStore::GetScorablePropertyIndex(SchemaTypeId schema_type_id, + std::string_view property_path) const { + ICING_RETURN_IF_ERROR(CheckSchemaSet()); + if (!feature_flags_->enable_scorable_properties()) { + return std::nullopt; + } + return scorable_property_manager_->GetScorablePropertyIndex( + schema_type_id, property_path, type_config_map_, + reverse_schema_type_mapper_); +} + +libtextclassifier3::StatusOr< + const std::vector<ScorablePropertyManager::ScorablePropertyInfo>*> +SchemaStore::GetOrderedScorablePropertyInfo(SchemaTypeId schema_type_id) const { + ICING_RETURN_IF_ERROR(CheckSchemaSet()); + if (!feature_flags_->enable_scorable_properties()) { + return nullptr; + } + return scorable_property_manager_->GetOrderedScorablePropertyInfo( + schema_type_id, type_config_map_, reverse_schema_type_mapper_); +} + libtextclassifier3::Status SchemaStore::PersistToDisk() { if (!has_schema_successfully_set_) { return libtextclassifier3::Status::OK; @@ -903,7 +1206,7 @@ storage_info.set_schema_store_size( Filesystem::SanitizeFileSize(directory_size)); ICING_ASSIGN_OR_RETURN(const SchemaProto* schema, GetSchema(), storage_info); - storage_info.set_num_schema_types(schema->types_size()); + storage_info.set_num_schema_types(schema->types().size()); int total_sections = 0; int num_types_sections_exhausted = 0; for (const SchemaTypeConfigProto& type : schema->types()) { @@ -1064,5 +1367,152 @@ return blob_property_map; } +libtextclassifier3::StatusOr<std::string> SchemaStore::ValidateAndGetDatabase( + const SchemaProto& new_schema) const { + std::string database; + + if (!enable_schema_database_ || new_schema.types().empty()) { + return database; + } + + database = new_schema.types(0).database(); + // Loop through new_schema's types and validate it. The input SchemaProto + // contains a list of SchemaTypeConfigProtos without deduplication. We need to + // check that: + // 1. All SchemaTypeConfigProtos have the same database value. + // 2. The SchemaTypeConfigProtos's schema_type field is unique within both + // new_schema, as well as the existing schema (recorded in + // type_config_map_). + for (const SchemaTypeConfigProto& type_config : new_schema.types()) { + // Check database consistency. + if (database != type_config.database()) { + return absl_ports::InvalidArgumentError( + "SetSchema only accepts a SchemaProto with types from a single " + "database at a time. Please make separate calls for each database if " + "you need to set the schema for multiple databases."); + } + + // Check type name uniqueness. This is only necessary if there is a + // pre-existing schema. + if (has_schema_successfully_set_) { + auto iter = type_config_map_.find(type_config.schema_type()); + if (iter != type_config_map_.end() && + database != iter->second.database()) { + return absl_ports::AlreadyExistsError( + absl_ports::StrCat("schema_type name: '", type_config.schema_type(), + "' is already in use by a different database.")); + } + } + } + return database; +} + +libtextclassifier3::StatusOr<SchemaProto> +SchemaStore::GetFullSchemaProtoWithUpdatedDb( + SchemaProto input_database_schema) const { + if (!enable_schema_database_) { + // If the schema database is not enabled, the input schema is already the + // full schema, so we don't need to do any merges. + return input_database_schema; + } + + libtextclassifier3::StatusOr<const SchemaProto*> schema_proto = GetSchema(); + if (absl_ports::IsNotFound(schema_proto.status())) { + // We don't have a pre-existing schema -- we can return the input database + // schema as it's already the full schema. + return input_database_schema; + } + + if (!schema_proto.ok()) { + // Real error. + return schema_proto.status(); + } + + if (!has_schema_successfully_set_) { + return absl_ports::InternalError( + "Schema store was not initialized properly."); + } + + // At this point, we have a pre-existing schema -- we need to merge the + // updated database with the existing schema. + if (input_database_schema.types().empty()) { + return *schema_proto.ValueOrDie(); + } + + std::string input_database = input_database_schema.types(0).database(); + if (database_type_map_.size() == 1 && + database_type_map_.find(input_database) != database_type_map_.end()) { + // No other databases in the schema -- we can return the input database + // schema. + return input_database_schema; + } + + const SchemaProto* existing_schema = schema_proto.ValueOrDie(); + SchemaProto full_schema; + + // 1. Add types from the existing schema, replacing existing types with the + // input types if the database is the one being updated by the input schema. + // - For the input_database, we replace the existing types with the input + // types. An exisiting type is deleted if it's not included in + // input_database. + // - If there are more input types than existing types for the input_database, + // the rest of the input types are appended to the end of the full_schema. + // - If there are fewer input types than existing types for the + // input_database, we shift all existing that come after input_database + // forward. + // - For existing types from other databases, we add the types in their + // original order to full_schema. Note that the type-ids of existing types + // might still change if some types deleted in input_database as this will + // cause all subsequent types ids to shift forward. + int input_schema_index = 0, existing_schema_index = 0; + while (input_schema_index < input_database_schema.types().size() && + existing_schema_index < existing_schema->types().size()) { + const SchemaTypeConfigProto& existing_type_config = + existing_schema->types(existing_schema_index); + SchemaTypeConfigProto& input_type_config = + *input_database_schema.mutable_types(input_schema_index); + + if (input_type_config.database() != input_database) { + return absl_ports::InvalidArgumentError( + "Can only update a single database at a time."); + } + + if (existing_type_config.database() == input_database) { + // If the database is the one being updated by the input schema, replace + // the existing type with a type from the input schema. + *full_schema.add_types() = std::move(input_type_config); + ++input_schema_index; + } else { + *full_schema.add_types() = existing_type_config; + } + ++existing_schema_index; + } + + // 2. Append remaining types to the end of the SchemaProto. + for (; input_schema_index < input_database_schema.types().size(); + ++input_schema_index) { + // Case 1: Append all remaining types from the input schema. This happens + // when more types are added in input_database_schema than what's in the + // existing schema. In this case, we've used up the space for the database + // in the existing schema, so we can just append the rest of the types to + // the end. + SchemaTypeConfigProto& input_type_config = + *input_database_schema.mutable_types(input_schema_index); + *full_schema.add_types() = std::move(input_type_config); + } + for (; existing_schema_index < existing_schema->types().size(); + ++existing_schema_index) { + // Case 2: Add remaining types from the existing schema, but skip the ones + // that are from input_database, since existing types from input_database + // are replaced with input_database_schema. + if (existing_schema->types(existing_schema_index).database() != + input_database) { + *full_schema.add_types() = existing_schema->types(existing_schema_index); + } + } + + return full_schema; +} + } // namespace lib } // namespace icing
diff --git a/icing/schema/schema-store.h b/icing/schema/schema-store.h index 77cd87f..e7b1e36 100644 --- a/icing/schema/schema-store.h +++ b/icing/schema/schema-store.h
@@ -19,15 +19,18 @@ #include <cstring> #include <limits> #include <memory> +#include <optional> #include <string> #include <string_view> #include <unordered_map> #include <unordered_set> +#include <utility> #include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" +#include "icing/feature-flags.h" #include "icing/file/file-backed-proto.h" #include "icing/file/filesystem.h" #include "icing/file/version-util.h" @@ -40,11 +43,13 @@ #include "icing/schema/joinable-property.h" #include "icing/schema/schema-type-manager.h" #include "icing/schema/schema-util.h" +#include "icing/schema/scorable_property_manager.h" #include "icing/schema/section.h" #include "icing/store/document-filter-data.h" #include "icing/store/key-mapper.h" #include "icing/util/clock.h" #include "icing/util/crc32.h" +#include "icing/util/status-macros.h" namespace icing { namespace lib { @@ -226,6 +231,18 @@ // but invalidated the joinable cache. Represented by the `schema_type` // field in the SchemaTypeConfigProto. std::unordered_set<std::string> schema_types_join_incompatible_by_name; + + // Schema types that were changed in a way that was backwards compatible, + // but inconsistent with the old schema so that the scorable property cache + // needs to be re-generated. + std::unordered_set<SchemaTypeId> + schema_types_scorable_property_inconsistent_by_id; + + // Schema types that were changed in a way that was backwards compatible, + // but inconsistent with the old schema so that the scorable property cache + // needs to be re-generated. + std::unordered_set<std::string> + schema_types_scorable_property_inconsistent_by_name; }; struct ExpandedTypePropertyMask { @@ -249,16 +266,20 @@ // INTERNAL_ERROR on any IO errors static libtextclassifier3::StatusOr<std::unique_ptr<SchemaStore>> Create( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, InitializeStatsProto* initialize_stats = nullptr); + const Clock* clock, const FeatureFlags* feature_flags, + bool enable_schema_database = false, + InitializeStatsProto* initialize_stats = nullptr); // Migrates schema files (backup v.s. new schema) according to version state - // change. + // change. Also performs schema database migration and populates the database + // fields in the persisted schema file if necessary. // // Returns: // OK on success or nothing to migrate static libtextclassifier3::Status MigrateSchema( const Filesystem* filesystem, const std::string& base_dir, - version_util::StateChange version_state_change, int32_t new_version); + version_util::StateChange version_state_change, int32_t new_version, + bool perform_schema_database_migration); // Discards all derived data in the schema store. // @@ -280,30 +301,49 @@ // Retrieve the current schema if it exists. // // Returns: - // SchemaProto* if exists - // INTERNAL_ERROR on any IO errors - // NOT_FOUND_ERROR if a schema hasn't been set before + // - SchemaProto* if exists + // - INTERNAL_ERROR on any IO errors + // - NOT_FOUND_ERROR if a schema hasn't been set before libtextclassifier3::StatusOr<const SchemaProto*> GetSchema() const; + // Retrieve the current schema for a given database if it exists. + // + // This is an expensive operation. Use GetSchema() when retrieving the entire + // schema, or if there is only a single database in the schema store. + // + // Returns: + // - SchemaProto* containing only schema types from the database, if exists + // - INTERNAL_ERROR on any IO errors + // - NOT_FOUND_ERROR if the database doesn't exist in the schema, or if a + // schema hasn't been set before + libtextclassifier3::StatusOr<SchemaProto> GetSchema( + const std::string& database) const; + // Update our current schema if it's compatible. Does not accept incompatible - // schema. Compatibility rules defined by - // SchemaUtil::ComputeCompatibilityDelta. + // schema or schema with types from multiple databases. Compatibility rules + // defined by SchemaUtil::ComputeCompatibilityDelta. + // + // The schema types in the new schema proto must all be from a single + // database. Does not support setting schema types across multiple databases + // at once. // // If ignore_errors_and_delete_documents is set to true, then incompatible // schema are allowed and we'll force set the schema, meaning // SetSchemaResult.success will always be true. // // Returns: - // SetSchemaResult that encapsulates the differences between the old and new - // schema, as well as if the new schema can be set. - // INTERNAL_ERROR on any IO errors - libtextclassifier3::StatusOr<const SetSchemaResult> SetSchema( - const SchemaProto& new_schema, - bool ignore_errors_and_delete_documents, + // - SetSchemaResult that encapsulates the differences between the old and + // new schema, as well as if the new schema can be set. + // - INTERNAL_ERROR on any IO errors + // - ALREADY_EXISTS_ERROR if type names in the new schema are already in use + // by a different database. + // - INVALID_ARGUMENT_ERROR if the schema is invalid, or if the schema types + // are from multiple databases (once schema database is enabled). + libtextclassifier3::StatusOr<SetSchemaResult> SetSchema( + const SchemaProto& new_schema, bool ignore_errors_and_delete_documents, bool allow_circular_schema_definitions); - libtextclassifier3::StatusOr<const SetSchemaResult> SetSchema( - SchemaProto&& new_schema, - bool ignore_errors_and_delete_documents, + libtextclassifier3::StatusOr<SetSchemaResult> SetSchema( + SchemaProto&& new_schema, bool ignore_errors_and_delete_documents, bool allow_circular_schema_definitions); // Get the SchemaTypeConfigProto of schema_type name. @@ -401,6 +441,17 @@ GetJoinablePropertyMetadata(SchemaTypeId schema_type_id, const std::string& property_path) const; + // Returns the JoinablePropertyMetadata associated with joinable_property_id + // that's in the SchemaTypeId. + // + // Returns: + // Valid pointer to JoinablePropertyMetadata on success + // FAILED_PRECONDITION if schema hasn't been set yet + // INVALID_ARGUMENT if schema type id or joinable property id is invalid + libtextclassifier3::StatusOr<const JoinablePropertyMetadata*> + GetJoinablePropertyMetadata(SchemaTypeId schema_type_id, + JoinablePropertyId joinable_property_id) const; + // Extracts all joinable property contents of different types from the given // document and group them by joinable value type. // - Joinable properties are sorted by joinable property id in ascending @@ -416,6 +467,19 @@ libtextclassifier3::StatusOr<JoinablePropertyGroup> ExtractJoinableProperties( const DocumentProto& document) const; + // Returns the quantization type for the given schema_type_id and section_id. + // + // Returns: + // - The quantization type on success. + // - INVALID_ARGUMENT_ERROR if schema_type_id or section_id is invalid. + // - Any error from schema store. + libtextclassifier3::StatusOr<EmbeddingIndexingConfig::QuantizationType::Code> + GetQuantizationType(SchemaTypeId schema_type_id, SectionId section_id) const { + ICING_ASSIGN_OR_RETURN(const SectionMetadata* section_metadata, + GetSectionMetadata(schema_type_id, section_id)); + return section_metadata->quantization_type; + } + // Syncs all the data changes to disk. // // Returns: @@ -445,6 +509,32 @@ libtextclassifier3::StatusOr<const std::vector<SectionMetadata>*> GetSectionMetadata(const std::string& schema_type) const; + // Gets the index of the given |property_path|, where the index N means that + // it is the Nth scorable property path in the schema config of the given + // |schema_type_id|, in lexicographical order. + // + // Returns: + // - Index on success + // - std::nullopt if the |property_path| doesn't point to a scorable + // property under the |schema_type_id| + // - FAILED_PRECONDITION if the schema hasn't been set yet + // - INVALID_ARGUMENT if |schema_type_id| is invalid + libtextclassifier3::StatusOr<std::optional<int>> GetScorablePropertyIndex( + SchemaTypeId schema_type_id, std::string_view property_path) const; + + // Returns the list of ScorablePropertyInfo for the given |schema_type_id|, + // in lexicographical order of its property path. + // + // Returns: + // - Vector of scorable property info on success. The vector can be empty + // if no scorable property is found under the schema config of + // |schema_type_id|. + // - FAILED_PRECONDITION if the schema hasn't been set yet + // - INVALID_ARGUMENT if |schema_type_id| is invalid + libtextclassifier3::StatusOr< + const std::vector<ScorablePropertyManager::ScorablePropertyInfo>*> + GetOrderedScorablePropertyInfo(SchemaTypeId schema_type_id) const; + // Calculates the StorageInfo for the Schema Store. // // If an IO error occurs while trying to calculate the value for a field, then @@ -489,11 +579,13 @@ // INTERNAL_ERROR on any IO errors static libtextclassifier3::StatusOr<std::unique_ptr<SchemaStore>> Create( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, SchemaProto schema); + const Clock* clock, const FeatureFlags* feature_flags, SchemaProto schema, + bool enable_schema_database); // Use SchemaStore::Create instead. explicit SchemaStore(const Filesystem* filesystem, std::string base_dir, - const Clock* clock); + const Clock* clock, const FeatureFlags* feature_flags, + bool enable_schema_database); // Deletes the overlay schema and ensures that the Header is correctly set. // @@ -504,6 +596,27 @@ const Filesystem* filesystem, const std::string& base_dir, Header& header); + // Handles the overlay schema after a version change by deleting it if it is + // no longer compatible with the new version. + // + // Requires: base_dir exists. + // + // Returns: + // OK on success + // INTERNAL_ERROR on any IO errors + static libtextclassifier3::Status HandleOverlaySchemaForVersionChange( + const Filesystem* filesystem, const std::string& base_dir, + version_util::StateChange version_state_change, int32_t new_version); + + // Populates the schema database field in the schema proto that is stored in + // the input schema file. + // + // Returns: + // OK on success or nothing to migrate + // INTERNAL_ERROR on IO error + static libtextclassifier3::Status PopulateSchemaDatabaseFieldForSchemaFile( + const Filesystem* filesystem, const std::string& schema_filename); + // Verifies that there is no error retrieving a previously set schema. Then // initializes like normal. // @@ -593,9 +706,101 @@ // Or an invalid schema configuration is present. libtextclassifier3::Status LoadSchema(); + // Sets the schema for a database for the first time. + // + // Note that when schema database is disabled, this function sets the entire + // schema, with all under the default empty database. + // + // Requires: + // - All types in new_schema are from the same database. + // - new_schema does not contain type names that are already in use by a + // different database. + // + // Returns: + // - SetSchemaResult that indicates if the new schema can be set. + // - INTERNAL_ERROR on any IO errors. + // - INVALID_ARGUMENT_ERROR if the schema is invalid. + libtextclassifier3::StatusOr<SchemaStore::SetSchemaResult> + SetInitialSchemaForDatabase(SchemaProto new_schema, + bool ignore_errors_and_delete_documents, + bool allow_circular_schema_definitions); + + // Sets the schema for a database, overriding any existing schema for that + // database. + // + // Note that when schema database is disabled, this function sets and + // overrides the entire schema. + // + // Requires: + // - All types in new_schema are from the same database. + // - new_schema does not contain type names that are already in use by a + // different database. + // + // Returns: + // - SetSchemaResult that encapsulates the differences between the old and + // new schema, as well as if the new schema can be set. + // - INTERNAL_ERROR on any IO errors. + // - INVALID_ARGUMENT_ERROR if the schema is invalid. + libtextclassifier3::StatusOr<SchemaStore::SetSchemaResult> + SetSchemaWithDatabaseOverride(SchemaProto new_schema, + const SchemaProto& old_schema, + bool ignore_errors_and_delete_documents, + bool allow_circular_schema_definitions); + + // Initial validation on the SchemaProto for SetSchema. This is intended as a + // preliminary check before any expensive operations are performed during + // `SetSchema::Validate`. Returns the schema's database if it's valid. + // + // Note that when schema database is disabled, any schema input is valid and + // an empty string is returned as the database. + // + // Checks that: + // - The new schema only contains types from a single database. + // - The schema's type names are not already in use in other databases. This + // is done outside of `SchemaUtil::Validate` because we need to know all + // existing type names, which is stored in the SchemaStore and not known to + // SchemaUtil. + // + // Returns: + // - new_schema's database on success + // - INVALID_ARGUMENT_ERROR if new_schema contains types from multiple + // databases + // - ALREADY_EXISTS_ERROR if new_schema's types names are not unique + libtextclassifier3::StatusOr<std::string> ValidateAndGetDatabase( + const SchemaProto& new_schema) const; + + // Returns a SchemaProto representing the full schema, which is a combination + // of the existing schema and the input database schema. + // + // For the database being updated by the input database schema: + // - If the existing schema does not contain the database, the input types + // are appended to the end of the SchemaProto, without changing the order + // of the existing schema types. + // - Otherwise, the existing schema types are replaced with types from the + // input database schema in their original position in the existing + // SchemaProto. + // - Types from input_database_schema are added in the order in which they + // appear. + // - If more types are added to the database, the additional types are + // appended at the end of the SchemaProto, without changing the order of + // existing types from unaffected databases. + // + // Requires: + // - input_database_schema must not contain types from multiple databases. + // + // Returns: + // - SchemaProto on success + // - INTERNAL_ERROR on any IO errors, or if the schema store was not + // previously initialized properly. + // - INVALID_ARGUMENT_ERROR if the input schema contains types from multiple + // databases. + libtextclassifier3::StatusOr<SchemaProto> GetFullSchemaProtoWithUpdatedDb( + SchemaProto input_database_schema) const; + const Filesystem* filesystem_; std::string base_dir_; const Clock* clock_; + const FeatureFlags* feature_flags_; // Does not own. // Used internally to indicate whether the class has been successfully // initialized with a valid schema. Will be false if Initialize failed or no @@ -616,6 +821,17 @@ // map of schema_type_mapper_. std::unordered_map<SchemaTypeId, std::string> reverse_schema_type_mapper_; + // A hash map of (database -> vector of type config names in the database). + // + // We use a vector instead of a set because we need to preserve the order of + // the types (i.e. the order in which they appear in the input SchemaProto + // during SetSchema), so that we can return the correct SchemaProto for + // GetSchema. + // + // This keeps track of the type configs defined in each database, which allows + // schema operations to be performed on a per-database basis. + std::unordered_map<std::string, std::vector<std::string>> database_type_map_; + // A hash map of (type config name -> type config), allows faster lookup of // type config in schema. The O(1) type config access makes schema-related and // section-related operations faster. @@ -635,7 +851,18 @@ // metadata for all Schemas. std::unique_ptr<const SchemaTypeManager> schema_type_manager_; + // Used to cache and manage the schema's scorable properties. + std::unique_ptr<ScorablePropertyManager> scorable_property_manager_; + std::unique_ptr<Header> header_; + + // Whether to use the database field for the schema. + // + // This is a temporary flag to control the rollout of the schema database. It + // affects the `SetSchema` and `GetSchema(std::string database)` methods. + // TODO - b/337913932: Remove this flag once the schema database is fully + // rolled out. + bool enable_schema_database_ = false; }; } // namespace lib
diff --git a/icing/schema/schema-store_test.cc b/icing/schema/schema-store_test.cc index b2da7fa..271ab4a 100644 --- a/icing/schema/schema-store_test.cc +++ b/icing/schema/schema-store_test.cc
@@ -16,6 +16,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include <utility> #include <vector> @@ -25,6 +26,7 @@ #include "gtest/gtest.h" #include "icing/absl_ports/str_cat.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/file-backed-proto.h" #include "icing/file/filesystem.h" #include "icing/file/mock-filesystem.h" @@ -40,6 +42,7 @@ #include "icing/store/document-filter-data.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/util/crc32.h" @@ -66,6 +69,7 @@ class SchemaStoreTest : public ::testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); test_dir_ = GetTestTempDir() + "/icing"; schema_store_dir_ = test_dir_ + "/schema_store"; filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()); @@ -85,7 +89,8 @@ PropertyConfigBuilder() .SetName("timestamp") .SetDataTypeInt64(NUMERIC_MATCH_RANGE) - .SetCardinality(CARDINALITY_OPTIONAL))) + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED))) .Build(); } @@ -101,6 +106,7 @@ ASSERT_TRUE(filesystem_.DeleteDirectoryRecursively(test_dir_.c_str())); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; std::string test_dir_; std::string schema_store_dir_; @@ -108,9 +114,21 @@ FakeClock fake_clock_; }; -TEST_F(SchemaStoreTest, CreationWithNullPointerShouldFail) { +TEST_F(SchemaStoreTest, CreationWithFilesystemNullPointerShouldFail) { EXPECT_THAT(SchemaStore::Create(/*filesystem=*/nullptr, schema_store_dir_, - &fake_clock_), + &fake_clock_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); +} + +TEST_F(SchemaStoreTest, CreationWithClockNullPointerShouldFail) { + EXPECT_THAT(SchemaStore::Create(&filesystem_, schema_store_dir_, + /*clock=*/nullptr, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); +} + +TEST_F(SchemaStoreTest, CreationWithFeatureFlagsNullPointerShouldFail) { + EXPECT_THAT(SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + /*feature_flags=*/nullptr), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } @@ -127,7 +145,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, @@ -143,7 +162,8 @@ IsOkAndHolds(Eq(expected_checksum))); SectionMetadata expected_metadata(/*id_in=*/0, TYPE_STRING, TOKENIZER_PLAIN, TERM_MATCH_EXACT, NUMERIC_MATCH_UNKNOWN, - EMBEDDING_INDEXING_UNKNOWN, "prop1"); + EMBEDDING_INDEXING_UNKNOWN, + QUANTIZATION_TYPE_NONE, "prop1"); EXPECT_THAT(move_constructed_schema_store.GetSectionMetadata("type_a"), IsOkAndHolds(Pointee(ElementsAre(expected_metadata)))); } @@ -161,7 +181,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema1, /*ignore_errors_and_delete_documents=*/false, @@ -181,7 +202,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> move_assigned_schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema2, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -194,7 +216,8 @@ IsOkAndHolds(Eq(expected_checksum))); SectionMetadata expected_metadata(/*id_in=*/0, TYPE_STRING, TOKENIZER_PLAIN, TERM_MATCH_EXACT, NUMERIC_MATCH_UNKNOWN, - EMBEDDING_INDEXING_UNKNOWN, "prop1"); + EMBEDDING_INDEXING_UNKNOWN, + QUANTIZATION_TYPE_NONE, "prop1"); EXPECT_THAT(move_assigned_schema_store->GetSectionMetadata("type_a"), IsOkAndHolds(Pointee(ElementsAre(expected_metadata)))); } @@ -203,7 +226,8 @@ { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Set it for the first time SchemaStore::SetSchemaResult result; @@ -234,16 +258,17 @@ serialized_schema.size()); // If ground truth was corrupted, we won't know what to do - EXPECT_THAT( - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_), - StatusIs(libtextclassifier3::StatusCode::INTERNAL)); + EXPECT_THAT(SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } TEST_F(SchemaStoreTest, RecoverCorruptDerivedFileOk) { { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Set it for the first time SchemaStore::SetSchemaResult result; @@ -258,6 +283,16 @@ EXPECT_THAT(*actual_schema, EqualsProto(schema_)); EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); + + // Scorable property manager working as expected. + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); } // "Corrupt" the derived SchemaTypeIds by deleting the entire directory. This @@ -273,7 +308,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, - &initialize_stats)); + feature_flags_.get(), + /*enable_schema_database=*/false, &initialize_stats)); EXPECT_THAT(initialize_stats.schema_store_recovery_cause(), Eq(InitializeStatsProto::IO_ERROR)); EXPECT_THAT(initialize_stats.schema_store_recovery_latency_ms(), Eq(123)); @@ -283,13 +319,24 @@ schema_store->GetSchema()); EXPECT_THAT(*actual_schema, EqualsProto(schema_)); EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); + + // Scorable property manager working as expected. + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); } TEST_F(SchemaStoreTest, RecoverDiscardDerivedFilesOk) { { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Set it for the first time SchemaStore::SetSchemaResult result; @@ -304,6 +351,16 @@ EXPECT_THAT(*actual_schema, EqualsProto(schema_)); EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); + + // Scorable property manager working as expected. + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); } ICING_ASSERT_OK( @@ -314,7 +371,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, - &initialize_stats)); + feature_flags_.get(), + /*enable_schema_database=*/false, &initialize_stats)); EXPECT_THAT(initialize_stats.schema_store_recovery_cause(), Eq(InitializeStatsProto::IO_ERROR)); EXPECT_THAT(initialize_stats.schema_store_recovery_latency_ms(), Eq(123)); @@ -324,13 +382,24 @@ schema_store->GetSchema()); EXPECT_THAT(*actual_schema, EqualsProto(schema_)); EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); + + // Scorable property manager working as expected. + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); } TEST_F(SchemaStoreTest, RecoverBadChecksumOk) { { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Set it for the first time SchemaStore::SetSchemaResult result; @@ -345,6 +414,16 @@ EXPECT_THAT(*actual_schema, EqualsProto(schema_)); EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); + + // Scorable property manager working as expected. + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); } // Change the SchemaStore's header combined checksum so that it won't match @@ -360,19 +439,31 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Everything looks fine, ground truth and derived data ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_schema, schema_store->GetSchema()); EXPECT_THAT(*actual_schema, EqualsProto(schema_)); EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); + + // Scorable property manager working as expected. + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); } TEST_F(SchemaStoreTest, CreateNoPreviousSchemaOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // The apis to retrieve information about the schema should fail gracefully. EXPECT_THAT(store->GetSchema(), @@ -406,7 +497,8 @@ TEST_F(SchemaStoreTest, CreateWithPreviousSchemaOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaStore::SetSchemaResult result; result.success = true; @@ -417,9 +509,9 @@ IsOkAndHolds(EqualsSetSchemaResult(result))); schema_store.reset(); - EXPECT_THAT( - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_), - IsOk()); + EXPECT_THAT(SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get()), + IsOk()); } TEST_F(SchemaStoreTest, MultipleCreateOk) { @@ -434,7 +526,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaStore::SetSchemaResult result; result.success = true; @@ -454,13 +547,23 @@ EXPECT_THAT(section_group.integer_sections[0].content, ElementsAre(kDefaultTimestamp)); + // Scorable property manager working as expected. + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); + // Verify that our persisted data is ok EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); schema_store.reset(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); // Verify that our in-memory structures are ok EXPECT_THAT(schema_store->GetSchemaTypeConfig("email"), @@ -473,6 +576,16 @@ EXPECT_THAT(section_group.integer_sections[0].content, ElementsAre(kDefaultTimestamp)); + // Scorable property manager working as expected. + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); + // Verify that our persisted data is ok EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); } @@ -480,7 +593,8 @@ TEST_F(SchemaStoreTest, SetNewSchemaOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Set it for the first time SchemaStore::SetSchemaResult result; @@ -495,10 +609,81 @@ EXPECT_THAT(*actual_schema, EqualsProto(schema_)); } +TEST_F(SchemaStoreTest, SetNewSchemaInDifferentDatabaseOk) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + SchemaProto db1_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + SchemaStore::SetSchemaResult result; + result.success = true; + result.schema_types_new_by_name.insert("db1_email"); + result.schema_types_new_by_name.insert("db1_message"); + EXPECT_THAT(schema_store->SetSchema( + db1_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(db1_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + + // Set a schema in a different database + SchemaProto db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db2_email"); + result.schema_types_new_by_name.insert("db2_message"); + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + + // Check the full schema. Databases that are updated last are appended to the + // schema proto + SchemaProto expected_full_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(expected_full_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + EXPECT_THAT(schema_store->GetSchema("db2"), + IsOkAndHolds(EqualsProto(db2_schema))); +} + TEST_F(SchemaStoreTest, SetSameSchemaOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Set it for the first time SchemaStore::SetSchemaResult result; @@ -523,10 +708,577 @@ EXPECT_THAT(*actual_schema, EqualsProto(schema_)); } +TEST_F(SchemaStoreTest, SetSameDatabaseSchemaOk) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + // Set schema for the first time + SchemaProto db1_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + SchemaProto db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + SchemaProto expected_full_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + SchemaStore::SetSchemaResult result; + result.success = true; + result.schema_types_new_by_name.insert("db1_email"); + result.schema_types_new_by_name.insert("db1_message"); + EXPECT_THAT(schema_store->SetSchema( + db1_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db2_email"); + result.schema_types_new_by_name.insert("db2_message"); + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_full_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_full_schema, EqualsProto(expected_full_schema)); + + // Reset db1 with the same SchemaProto. The schema should be exactly the same. + result = SchemaStore::SetSchemaResult(); + result.success = true; + EXPECT_THAT( + schema_store->SetSchema(db1_schema, + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + + // Check the schema, this should not have changed + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(expected_full_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + EXPECT_THAT(schema_store->GetSchema("db2"), + IsOkAndHolds(EqualsProto(db2_schema))); +} + +TEST_F(SchemaStoreTest, SetDatabaseReorderedTypesPreservesSchemaTypeIds) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + // Set schema for the first time + SchemaProto db1_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + SchemaProto db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + SchemaProto db3_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db3_email").SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_message") + .SetDatabase("db3")) + .Build(); + SchemaProto expected_full_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .AddType( + SchemaTypeConfigBuilder().SetType("db3_email").SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_message") + .SetDatabase("db3")) + .Build(); + + // Set schema for db1 + SchemaStore::SetSchemaResult result; + result.success = true; + result.schema_types_new_by_name.insert("db1_email"); + result.schema_types_new_by_name.insert("db1_message"); + EXPECT_THAT(schema_store->SetSchema( + db1_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + // Set schema for db2 + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db2_email"); + result.schema_types_new_by_name.insert("db2_message"); + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + // Set schema for db3 + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db3_email"); + result.schema_types_new_by_name.insert("db3_message"); + EXPECT_THAT(schema_store->SetSchema( + db3_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + // Verify schema. + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_full_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_full_schema, EqualsProto(expected_full_schema)); + + // Reset db2 with the types reordered. The expected full schema will be + // different, but the SchemaTypeIds for db1 and db3 should not change. + db2_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .Build(); + expected_full_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType( + SchemaTypeConfigBuilder().SetType("db3_email").SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_message") + .SetDatabase("db3")) + .Build(); + result = SchemaStore::SetSchemaResult(); + result.success = true; + // Only db2's schema type ids should change. + result.old_schema_type_ids_changed.insert(2); + result.old_schema_type_ids_changed.insert(3); + EXPECT_THAT( + schema_store->SetSchema(db2_schema, + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + + // Check the schema + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(expected_full_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + EXPECT_THAT(schema_store->GetSchema("db2"), + IsOkAndHolds(EqualsProto(db2_schema))); + EXPECT_THAT(schema_store->GetSchema("db3"), + IsOkAndHolds(EqualsProto(db3_schema))); +} + +TEST_F(SchemaStoreTest, SetDatabaseAddedTypesPreservesSchemaTypeIds) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + // Set schema for the first time + SchemaProto db1_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + SchemaProto db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + SchemaProto db3_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db3_email").SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_message") + .SetDatabase("db3")) + .Build(); + SchemaProto expected_full_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .AddType( + SchemaTypeConfigBuilder().SetType("db3_email").SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_message") + .SetDatabase("db3")) + .Build(); + + // Set schema for db1 + SchemaStore::SetSchemaResult result; + result.success = true; + result.schema_types_new_by_name.insert("db1_email"); + result.schema_types_new_by_name.insert("db1_message"); + EXPECT_THAT(schema_store->SetSchema( + db1_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + // Set schema for db2 + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db2_email"); + result.schema_types_new_by_name.insert("db2_message"); + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + // Set schema for db3 + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db3_email"); + result.schema_types_new_by_name.insert("db3_message"); + EXPECT_THAT(schema_store->SetSchema( + db3_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + // Verify schema. + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_full_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_full_schema, EqualsProto(expected_full_schema)); + + // Reset db2 and add a type. The added type should be appended to the end of + // the SchemaProto, and SchemaTypeIds for db1 and db3 should not change. + // + // Whether or not the SchemaTypeIds for db2 change depends on the order in the + // new db2 SchemaProto (in this case, existing type's order and ids do not + // change) + db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_recipient") + .SetDatabase("db2")) + .Build(); + expected_full_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() // db1 types + .SetType("db1_email") + .SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() // db2 types + .SetType("db2_email") + .SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() // db3 types + .SetType("db3_email") + .SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_message") + .SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() // Additional db2 type is appended + // at the end + .SetType("db2_recipient") + .SetDatabase("db2")) + .Build(); + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db2_recipient"); + EXPECT_THAT( + schema_store->SetSchema(db2_schema, + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + + // Check the schema + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(expected_full_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + EXPECT_THAT(schema_store->GetSchema("db2"), + IsOkAndHolds(EqualsProto(db2_schema))); + EXPECT_THAT(schema_store->GetSchema("db3"), + IsOkAndHolds(EqualsProto(db3_schema))); +} + +TEST_F(SchemaStoreTest, SetDatabaseDeletedTypesOk) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + // Set schema for the first time + SchemaProto db1_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + SchemaProto db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + SchemaProto db3_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db3_email").SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_message") + .SetDatabase("db3")) + .Build(); + + // Set schema for db1 + SchemaStore::SetSchemaResult result; + result.success = true; + result.schema_types_new_by_name.insert("db1_email"); + result.schema_types_new_by_name.insert("db1_message"); + EXPECT_THAT(schema_store->SetSchema( + db1_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + // Set schema for db2 + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db2_email"); + result.schema_types_new_by_name.insert("db2_message"); + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + // Set schema for db3 + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db3_email"); + result.schema_types_new_by_name.insert("db3_message"); + EXPECT_THAT(schema_store->SetSchema( + db3_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + // Set schema again for db2 and add a type. The added type should be appended + // to the end of the SchemaProto. + db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_recipient") + .SetDatabase("db2")) + .Build(); + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db2_recipient"); + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + SchemaProto expected_full_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_email") // SchemaTypeId 0 + .SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") // SchemaTypeId 1 + .SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_email") // SchemaTypeId 2 + .SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") // SchemaTypeId 3 + .SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_email") // SchemaTypeId 4 + .SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_message") // SchemaTypeId 5 + .SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_recipient") // SchemaTypeId 6 + .SetDatabase("db2")) + .Build(); + // Verify schema. + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_full_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_full_schema, EqualsProto(expected_full_schema)); + + // Reset db2 and delete some types. All types that were originally added after + // db2 should have their type ids changed. + db2_schema = SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + expected_full_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_email") // SchemaTypeId 0 + .SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") // SchemaTypeId 1 + .SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") // SchemaTypeId 2 + .SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_email") // SchemaTypeId 3 + .SetDatabase("db3")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db3_message") // SchemaTypeId 4 + .SetDatabase("db3")) + .Build(); + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_deleted_by_name.insert("db2_email"); + result.schema_types_deleted_by_name.insert("db2_recipient"); + result.schema_types_deleted_by_id.insert(2); // db2_email + result.schema_types_deleted_by_id.insert(6); // db2_recipient + result.old_schema_type_ids_changed.insert(3); // db2_message + result.old_schema_type_ids_changed.insert(4); // db3_email + result.old_schema_type_ids_changed.insert(5); // db3_message + EXPECT_THAT( + schema_store->SetSchema(db2_schema, + /*ignore_errors_and_delete_documents=*/true, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + + // Check the schema + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(expected_full_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + EXPECT_THAT(schema_store->GetSchema("db2"), + IsOkAndHolds(EqualsProto(db2_schema))); + EXPECT_THAT(schema_store->GetSchema("db3"), + IsOkAndHolds(EqualsProto(db3_schema))); +} + +TEST_F(SchemaStoreTest, SetEmptySchemaInDifferentDatabaseOk) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + SchemaProto db1_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + SchemaStore::SetSchemaResult result; + result.success = true; + result.schema_types_new_by_name.insert("db1_email"); + result.schema_types_new_by_name.insert("db1_message"); + EXPECT_THAT(schema_store->SetSchema( + db1_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(db1_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + + // Set an empty schema in a different database + SchemaProto db2_schema; + result = SchemaStore::SetSchemaResult(); + result.success = true; + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + + // Check the schema, this should not have changed + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(db1_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + + // GetSchema for an empty database should return NotFoundError + EXPECT_THAT(schema_store->GetSchema("db2"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); +} + TEST_F(SchemaStoreTest, SetIncompatibleSchemaOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Set it for the first time SchemaStore::SetSchemaResult result; @@ -554,10 +1306,264 @@ IsOkAndHolds(EqualsSetSchemaResult(result))); } +TEST_F(SchemaStoreTest, SetIncompatibleInDifferentDatabaseOk) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + // Set schema for the first time + SchemaProto db1_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + SchemaProto db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + SchemaProto expected_full_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + SchemaStore::SetSchemaResult result; + result.success = true; + result.schema_types_new_by_name.insert("db1_email"); + result.schema_types_new_by_name.insert("db1_message"); + EXPECT_THAT(schema_store->SetSchema( + db1_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db2_email"); + result.schema_types_new_by_name.insert("db2_message"); + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_full_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_full_schema, EqualsProto(expected_full_schema)); + + // Make db2 incompatible by changing a type name + SchemaProto db2_schema_incompatible = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_recipient") + .SetDatabase("db2")) + .Build(); + result = SchemaStore::SetSchemaResult(); + result.success = false; + result.schema_types_deleted_by_name.insert("db2_message"); + result.schema_types_new_by_name.insert("db2_recipient"); + result.schema_types_deleted_by_id.insert(3); // db2_message + EXPECT_THAT( + schema_store->SetSchema(db2_schema_incompatible, + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + + // Check the schema, this should not have changed + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(expected_full_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + EXPECT_THAT(schema_store->GetSchema("db2"), + IsOkAndHolds(EqualsProto(db2_schema))); +} + +TEST_F(SchemaStoreTest, SetInvalidInDifferentDatabaseFails) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + // Set schema for the first time + SchemaProto db1_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + SchemaProto db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + SchemaProto expected_full_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + SchemaStore::SetSchemaResult result; + result.success = true; + result.schema_types_new_by_name.insert("db1_email"); + result.schema_types_new_by_name.insert("db1_message"); + EXPECT_THAT(schema_store->SetSchema( + db1_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + result = SchemaStore::SetSchemaResult(); + result.success = true; + result.schema_types_new_by_name.insert("db2_email"); + result.schema_types_new_by_name.insert("db2_message"); + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_full_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_full_schema, EqualsProto(expected_full_schema)); + + // Make db2 invalid by duplicating a property name + PropertyConfigProto prop = + PropertyConfigBuilder() + .SetName("prop0") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL) + .Build(); + SchemaProto db2_schema_incompatible = SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_email") + .SetDatabase("db2") + .AddProperty(prop) + .AddProperty(prop)) + .Build(); + EXPECT_THAT( + schema_store->SetSchema(db2_schema_incompatible, + /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + StatusIs(libtextclassifier3::StatusCode::ALREADY_EXISTS)); + + // Check the schema, this should not have changed + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(expected_full_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + EXPECT_THAT(schema_store->GetSchema("db2"), + IsOkAndHolds(EqualsProto(db2_schema))); +} + +TEST_F(SchemaStoreTest, SetSchemaWithMultipleDbFails) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + // Set schema for the first time + SchemaProto combined_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("db2_email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .AddType( + SchemaTypeConfigBuilder().SetType("db1_email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + EXPECT_THAT(schema_store->SetSchema( + combined_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(SchemaStoreTest, SetSchemaWithDuplicateTypeNameAcrossDifferentDbFails) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/true, + /*initialize_stats=*/nullptr)); + + // Set schema for the first time + SchemaProto db1_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("email").SetDatabase("db1")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db1_message") + .SetDatabase("db1")) + .Build(); + SchemaStore::SetSchemaResult result; + result.success = true; + result.schema_types_new_by_name.insert("email"); + result.schema_types_new_by_name.insert("db1_message"); + EXPECT_THAT(schema_store->SetSchema( + db1_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(result))); + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(db1_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); + + // Set schema in db2 with the same type name + SchemaProto db2_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder().SetType("email").SetDatabase("db2")) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2_message") + .SetDatabase("db2")) + .Build(); + EXPECT_THAT(schema_store->SetSchema( + db2_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + StatusIs(libtextclassifier3::StatusCode::ALREADY_EXISTS)); + + // Check schema, this should not have changed + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(db1_schema)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema))); +} + TEST_F(SchemaStoreTest, SetSchemaWithAddedTypeOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("email")) @@ -595,7 +1601,8 @@ TEST_F(SchemaStoreTest, SetSchemaWithDeletedTypeOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder() @@ -658,7 +1665,8 @@ TEST_F(SchemaStoreTest, SetSchemaWithReorderedTypesOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder() @@ -705,7 +1713,8 @@ TEST_F(SchemaStoreTest, IndexedPropertyChangeRequiresReindexingOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder() @@ -753,7 +1762,8 @@ TEST_F(SchemaStoreTest, IndexNestedDocumentsChangeRequiresReindexingOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Make two schemas. One that sets index_nested_properties to false and one // that sets it to true. @@ -831,7 +1841,8 @@ TEST_F(SchemaStoreTest, SetSchemaWithIncompatibleTypesOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder() @@ -898,7 +1909,8 @@ TEST_F(SchemaStoreTest, SetSchemaWithIncompatibleNestedTypesOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // 1. Create a ContactPoint type with a repeated property and set that schema SchemaTypeConfigBuilder contact_point_repeated_label = @@ -971,7 +1983,8 @@ TEST_F(SchemaStoreTest, SetSchemaWithIndexIncompatibleNestedTypesOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // 1. Create a ContactPoint type with label that matches prefix and set that // schema @@ -1030,7 +2043,8 @@ TEST_F(SchemaStoreTest, SetSchemaWithCompatibleNestedTypesOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // 1. Create a ContactPoint type with a optional property and set that schema SchemaTypeConfigBuilder contact_point_optional_label = @@ -1088,7 +2102,8 @@ TEST_F(SchemaStoreTest, SetSchemaWithAddedIndexableNestedTypeOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // 1. Create a ContactPoint type with a optional property, and a type that // references the ContactPoint type. @@ -1154,7 +2169,8 @@ TEST_F(SchemaStoreTest, SetSchemaWithAddedJoinableNestedTypeOk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // 1. Create a ContactPoint type with a optional property, and a type that // references the ContactPoint type. @@ -1166,7 +2182,7 @@ .SetName("label") .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED)); SchemaTypeConfigBuilder person = SchemaTypeConfigBuilder().SetType("Person").AddProperty( @@ -1216,10 +2232,140 @@ EXPECT_THAT(*actual_schema, EqualsProto(new_schema)); } +TEST_F(SchemaStoreTest, SetSchemaByUpdatingScorablePropertyOk) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email").AddProperty( + PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + SchemaProto new_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("score") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // Set old schema + SchemaStore::SetSchemaResult expected_result; + expected_result.success = true; + expected_result.schema_types_new_by_name.insert("email"); + EXPECT_THAT(schema_store->SetSchema( + old_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(expected_result))); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_schema, EqualsProto(old_schema)); + + // Set new schema. + // The new schema adds "score" as scorable_type ENABLED from type "email". + SchemaStore::SetSchemaResult new_expected_result; + new_expected_result.success = true; + new_expected_result.schema_types_scorable_property_inconsistent_by_id.insert( + 0); + new_expected_result.schema_types_changed_fully_compatible_by_name.insert( + "email"); + EXPECT_THAT(schema_store->SetSchema( + new_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(new_expected_result))); + ICING_ASSERT_OK_AND_ASSIGN(actual_schema, schema_store->GetSchema()); + EXPECT_THAT(*actual_schema, EqualsProto(new_schema)); +} + +TEST_F(SchemaStoreTest, + SetSchemaWithReorderedSchemeTypesAndUpdatedScorablePropertyOk) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("message")) + .AddType(SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("score") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_DISABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + // The new schema updates "score" as scorable_type ENABLED from type "email", + // and it also reorders the schema types of "email" and "message". + SchemaProto new_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("score") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder().SetType("message")) + .Build(); + + // Set old schema + SchemaStore::SetSchemaResult expected_result; + expected_result.success = true; + expected_result.schema_types_new_by_name.insert("email"); + expected_result.schema_types_new_by_name.insert("message"); + EXPECT_THAT(schema_store->SetSchema( + old_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(expected_result))); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_schema, EqualsProto(old_schema)); + + // Set new schema. + SchemaStore::SetSchemaResult new_expected_result; + new_expected_result.success = true; + // Schema type id of "email" is updated to 0. + SchemaTypeId email_schema_type_id = 0; + new_expected_result.schema_types_scorable_property_inconsistent_by_id.insert( + email_schema_type_id); + new_expected_result.schema_types_changed_fully_compatible_by_name.insert( + "email"); + new_expected_result.old_schema_type_ids_changed.insert(0); + new_expected_result.old_schema_type_ids_changed.insert(1); + EXPECT_THAT(schema_store->SetSchema( + new_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(new_expected_result))); + ICING_ASSERT_OK_AND_ASSIGN(actual_schema, schema_store->GetSchema()); + EXPECT_THAT(*actual_schema, EqualsProto(new_schema)); +} + TEST_F(SchemaStoreTest, GetSchemaTypeId) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); schema_.clear_types(); @@ -1249,7 +2395,8 @@ TEST_F(SchemaStoreTest, UpdateChecksumDefaultOnEmptySchemaStore) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetChecksum(), IsOkAndHolds(Crc32())); EXPECT_THAT(schema_store->UpdateChecksum(), IsOkAndHolds(Crc32())); @@ -1259,7 +2406,8 @@ TEST_F(SchemaStoreTest, UpdateChecksumSameBetweenCalls) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto foo_schema = SchemaBuilder().AddType(SchemaTypeConfigBuilder().SetType("foo")).Build(); @@ -1280,7 +2428,8 @@ TEST_F(SchemaStoreTest, UpdateChecksumSameAcrossInstances) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto foo_schema = SchemaBuilder().AddType(SchemaTypeConfigBuilder().SetType("foo")).Build(); @@ -1297,8 +2446,8 @@ schema_store.reset(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); EXPECT_THAT(schema_store->GetChecksum(), IsOkAndHolds(checksum)); EXPECT_THAT(schema_store->UpdateChecksum(), IsOkAndHolds(checksum)); EXPECT_THAT(schema_store->GetChecksum(), IsOkAndHolds(checksum)); @@ -1307,7 +2456,8 @@ TEST_F(SchemaStoreTest, UpdateChecksumChangesOnModification) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto foo_schema = SchemaBuilder().AddType(SchemaTypeConfigBuilder().SetType("foo")).Build(); @@ -1341,7 +2491,8 @@ TEST_F(SchemaStoreTest, PersistToDiskFineForEmptySchemaStore) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Persisting is fine and shouldn't affect anything ICING_EXPECT_OK(schema_store->PersistToDisk()); @@ -1350,7 +2501,8 @@ TEST_F(SchemaStoreTest, UpdateChecksumAvoidsRecovery) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder().AddType(SchemaTypeConfigBuilder().SetType("foo")).Build(); @@ -1374,7 +2526,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store_two, SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, - &initialize_stats)); + feature_flags_.get(), + /*enable_schema_database=*/false, &initialize_stats)); EXPECT_THAT(initialize_stats.schema_store_recovery_cause(), Eq(InitializeStatsProto::NONE)); ICING_ASSERT_OK_AND_ASSIGN(actual_schema, schema_store_two->GetSchema()); @@ -1389,7 +2542,8 @@ TEST_F(SchemaStoreTest, PersistToDiskPreservesAcrossInstances) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder().AddType(SchemaTypeConfigBuilder().SetType("foo")).Build(); @@ -1419,8 +2573,10 @@ // And we get the same schema back on reinitialization InitializeStatsProto initialize_stats; ICING_ASSERT_OK_AND_ASSIGN( - schema_store, SchemaStore::Create(&filesystem_, schema_store_dir_, - &fake_clock_, &initialize_stats)); + schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get(), + /*enable_schema_database=*/false, &initialize_stats)); EXPECT_THAT(initialize_stats.schema_store_recovery_cause(), Eq(InitializeStatsProto::NONE)); ICING_ASSERT_OK_AND_ASSIGN(actual_schema, schema_store->GetSchema()); @@ -1430,7 +2586,8 @@ TEST_F(SchemaStoreTest, SchemaStoreStorageInfoProto) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Create a schema with two types: one simple type and one type that uses all // 64 sections. @@ -1472,7 +2629,8 @@ TEST_F(SchemaStoreTest, GetDebugInfo) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Set schema ASSERT_THAT( @@ -1493,7 +2651,8 @@ TEST_F(SchemaStoreTest, GetDebugInfoForEmptySchemaStore) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Check debug info before setting a schema ICING_ASSERT_OK_AND_ASSIGN(SchemaDebugInfoProto out, @@ -1510,7 +2669,8 @@ { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("Type")) .Build(); @@ -1525,7 +2685,7 @@ .WillByDefault(Return(false)); { EXPECT_THAT(SchemaStore::Create(mock_filesystem.get(), schema_store_dir_, - &fake_clock_), + &fake_clock_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } } @@ -1549,7 +2709,8 @@ { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder().AddType(type).Build(); ICING_ASSERT_OK(schema_store->SetSchema( std::move(schema), /*ignore_errors_and_delete_documents=*/false, @@ -1561,7 +2722,7 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, SchemaStore::Create(mock_filesystem.get(), schema_store_dir_, - &fake_clock_)); + &fake_clock_, feature_flags_.get())); ON_CALL(*mock_filesystem, CreateDirectoryRecursively(HasSubstr("key_mapper_dir"))) @@ -1584,10 +2745,12 @@ .Build(); SectionMetadata expected_int_prop1_metadata( /*id_in=*/0, TYPE_INT64, TOKENIZER_NONE, TERM_MATCH_UNKNOWN, - NUMERIC_MATCH_RANGE, EMBEDDING_INDEXING_UNKNOWN, "intProp1"); + NUMERIC_MATCH_RANGE, EMBEDDING_INDEXING_UNKNOWN, QUANTIZATION_TYPE_NONE, + "intProp1"); SectionMetadata expected_string_prop1_metadata( /*id_in=*/1, TYPE_STRING, TOKENIZER_PLAIN, TERM_MATCH_EXACT, - NUMERIC_MATCH_UNKNOWN, EMBEDDING_INDEXING_UNKNOWN, "stringProp1"); + NUMERIC_MATCH_UNKNOWN, EMBEDDING_INDEXING_UNKNOWN, + QUANTIZATION_TYPE_NONE, "stringProp1"); ICING_ASSERT_OK_AND_ASSIGN(SectionGroup section_group, schema_store->ExtractSections(document)); ASSERT_THAT(section_group.string_sections, SizeIs(1)); @@ -1606,7 +2769,8 @@ TEST_F(SchemaStoreTest, CanCheckForPropertiesDefinedInSchema) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Set it for the first time SchemaStore::SetSchemaResult result; @@ -1648,7 +2812,8 @@ TEST_F(SchemaStoreTest, GetSchemaTypeIdsWithChildren) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Create a schema with the following inheritance relation: // A @@ -1717,7 +2882,8 @@ TEST_F(SchemaStoreTest, DiamondGetSchemaTypeIdsWithChildren) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); // Create a schema with the following inheritance relation: // A @@ -1805,7 +2971,7 @@ .SetName("senderQualifiedId") .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED)) .AddProperty(PropertyConfigBuilder() .SetName("recipients") @@ -1824,7 +2990,8 @@ SchemaProto schema = SchemaBuilder().AddType(email_type).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -1851,21 +3018,22 @@ .SetName("tagQualifiedId") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED)) .AddProperty( PropertyConfigBuilder() .SetName("senderQualifiedId") .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED)) .Build(); SchemaProto schema = SchemaBuilder().AddType(email_type).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -1900,7 +3068,8 @@ SchemaProto schema = SchemaBuilder().AddType(email_type).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -1929,7 +3098,7 @@ .SetName("senderQualifiedId") .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED)) .AddProperty(PropertyConfigBuilder() .SetName("timestamp") @@ -1944,7 +3113,8 @@ SchemaProto schema = SchemaBuilder().AddType(email_type).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -1967,7 +3137,7 @@ .SetName("tagQualifiedId") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED)) .AddProperty( PropertyConfigBuilder() @@ -2004,7 +3174,8 @@ SchemaBuilder().AddType(email_type).AddType(conversation_type).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -2025,7 +3196,7 @@ .SetName("tagQualifiedId") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED)) .AddProperty( PropertyConfigBuilder() @@ -2062,7 +3233,8 @@ SchemaBuilder().AddType(email_type).AddType(conversation_type).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -2083,7 +3255,7 @@ .SetName("tagQualifiedId") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED)) .AddProperty( PropertyConfigBuilder() @@ -2120,7 +3292,8 @@ SchemaBuilder().AddType(email_type).AddType(conversation_type).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -2145,7 +3318,7 @@ .SetName("tagQualifiedId") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED)) .AddProperty( PropertyConfigBuilder() @@ -2182,7 +3355,8 @@ SchemaBuilder().AddType(email_type).AddType(conversation_type).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -2207,7 +3381,7 @@ .SetName("tagQualifiedId") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED)) .AddProperty( PropertyConfigBuilder() @@ -2244,7 +3418,8 @@ SchemaBuilder().AddType(email_type).AddType(conversation_type).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -2290,7 +3465,8 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -2360,7 +3536,8 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -2433,7 +3610,8 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/true)); @@ -2489,7 +3667,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2503,7 +3682,8 @@ // present. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(schema)))); @@ -2559,7 +3739,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2573,7 +3754,8 @@ // is present. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(schema)))); @@ -2611,7 +3793,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2627,9 +3810,9 @@ { // Create a new instance of the schema store and check that it fails because // the backup schema is not available. - EXPECT_THAT( - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_), - StatusIs(libtextclassifier3::StatusCode::INTERNAL)); + EXPECT_THAT(SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } } @@ -2661,7 +3844,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2677,9 +3861,9 @@ { // Create a new instance of the schema store and check that it fails because // the overlay schema is not available when we expected it to be. - EXPECT_THAT( - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_), - StatusIs(libtextclassifier3::StatusCode::INTERNAL)); + EXPECT_THAT(SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } } @@ -2711,7 +3895,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2727,9 +3912,9 @@ { // Create a new of the schema store and check that the same schema is // present. - EXPECT_THAT( - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_), - StatusIs(libtextclassifier3::StatusCode::INTERNAL)); + EXPECT_THAT(SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } } @@ -2760,7 +3945,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2777,9 +3963,9 @@ // Create a new instance of the schema store and check that it fails because // the schema header (which is now a part of the ground truth) is not // available. - EXPECT_THAT( - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_), - StatusIs(libtextclassifier3::StatusCode::INTERNAL)); + EXPECT_THAT(SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } } @@ -2801,7 +3987,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2812,14 +3999,15 @@ ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kCompatible, - version_util::kVersion)); + version_util::kVersion, /*perform_schema_database_migration=*/false)); { // Create a new of the schema store and check that the same schema is // present. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(schema)))); } @@ -2843,7 +4031,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2854,14 +4043,15 @@ ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kUpgrade, - version_util::kVersion + 1)); + version_util::kVersion + 1, /*perform_schema_database_migration=*/false)); { // Create a new of the schema store and check that the same schema is // present. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(schema)))); } @@ -2885,7 +4075,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2894,17 +4085,18 @@ IsOkAndHolds(Pointee(EqualsProto(schema)))); } - ICING_EXPECT_OK( - SchemaStore::MigrateSchema(&filesystem_, schema_store_dir_, - version_util::StateChange::kVersionZeroUpgrade, - version_util::kVersion + 1)); + ICING_EXPECT_OK(SchemaStore::MigrateSchema( + &filesystem_, schema_store_dir_, + version_util::StateChange::kVersionZeroUpgrade, + version_util::kVersion + 1, /*perform_schema_database_migration=*/false)); { // Create a new of the schema store and check that the same schema is // present. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(schema)))); } @@ -2929,7 +4121,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2943,14 +4136,16 @@ // So kVersionOne - 1 is incompatible and will throw out the schema. ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kRollBack, - version_util::kVersionOne - 1)); + version_util::kVersionOne - 1, + /*perform_schema_database_migration=*/false)); { // Create a new of the schema store and check that we fell back to the // base schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); SchemaTypeConfigProto other_type_a = SchemaTypeConfigBuilder() @@ -2984,7 +4179,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2998,14 +4194,15 @@ // compatible and retain the overlay schema. ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kRollBack, - version_util::kVersion)); + version_util::kVersion, /*perform_schema_database_migration=*/false)); { // Create a new of the schema store and check that the same schema is // present. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(schema)))); } @@ -3026,7 +4223,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3040,7 +4238,8 @@ // So kVersionOne - 1 is incompatible and will throw out the schema. ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kRollBack, - version_util::kVersionOne - 1)); + version_util::kVersionOne - 1, + /*perform_schema_database_migration=*/false)); SchemaTypeConfigProto other_type_a = SchemaTypeConfigBuilder() @@ -3057,7 +4256,8 @@ // base schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(base_schema)))); @@ -3067,13 +4267,14 @@ // present (currently base schema) ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kRollForward, - version_util::kVersion)); + version_util::kVersion, /*perform_schema_database_migration=*/false)); { // Create a new of the schema store and check that we fell back to the // base schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(base_schema)))); @@ -3095,7 +4296,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3109,14 +4311,15 @@ // compatible and retain the overlay schema. ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kRollBack, - version_util::kVersion)); + version_util::kVersion, /*perform_schema_database_migration=*/false)); { // Create a new of the schema store and check that the same schema is // present. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(schema)))); @@ -3126,13 +4329,14 @@ // present (currently overlay schema) ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kRollForward, - version_util::kVersion)); + version_util::kVersion, /*perform_schema_database_migration=*/false)); { // Create a new of the schema store and check that the same schema is // present. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(schema)))); @@ -3155,7 +4359,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3169,7 +4374,7 @@ ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kVersionZeroRollForward, - version_util::kVersion)); + version_util::kVersion, /*perform_schema_database_migration=*/false)); SchemaTypeConfigProto other_type_a = SchemaTypeConfigBuilder() @@ -3186,7 +4391,8 @@ // base schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(base_schema)))); @@ -3208,7 +4414,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3222,7 +4429,7 @@ // it should always be valid. ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kUndetermined, - version_util::kVersion)); + version_util::kVersion, /*perform_schema_database_migration=*/false)); SchemaTypeConfigProto other_type_a = SchemaTypeConfigBuilder() @@ -3239,7 +4446,8 @@ // base schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); EXPECT_THAT(schema_store->GetSchema(), IsOkAndHolds(Pointee(EqualsProto(base_schema)))); @@ -3296,7 +4504,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3367,7 +4576,8 @@ // Create an instance of the schema store and set the schema. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3381,6 +4591,501 @@ } } +TEST_F(SchemaStoreTest, GetScorablePropertyIndex_SchemaNotSet) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); +} + +TEST_F(SchemaStoreTest, GetScorablePropertyIndex_InvalidSchemaTypeId) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + // Set schema + ICING_ASSERT_OK(schema_store->SetSchema( + schema_, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + // non-existing schema type id + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/100, + /*property_path=*/"timestamp"), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(SchemaStoreTest, GetScorablePropertyIndex_InvalidPropertyName) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("scoreDouble") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED)) + .AddProperty(PropertyConfigBuilder() + .SetName("timestamp") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED))) + .Build(); + + // Set schema + ICING_ASSERT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + // non-scorable property + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"subject"), + IsOkAndHolds(Eq(std::nullopt))); + // non-existing property + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"non_existing"), + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(SchemaStoreTest, GetScorablePropertyIndex_Ok) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + // Set schema + ICING_ASSERT_OK(schema_store->SetSchema( + schema_, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); +} + +TEST_F(SchemaStoreTest, GetOrderedScorablePropertyPaths_SchemaNotSet) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); +} + +TEST_F(SchemaStoreTest, GetOrderedScorablePropertyPaths_InvalidSchemaTypeId) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + // Set schema + ICING_ASSERT_OK(schema_store->SetSchema( + schema_, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/100), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(SchemaStoreTest, GetOrderedScorablePropertyPaths_Ok) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("scoreDouble") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED)) + .AddProperty(PropertyConfigBuilder() + .SetName("timestamp") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED))) + .AddType(SchemaTypeConfigBuilder().SetType("message").AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // Set schema + ICING_ASSERT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); + EXPECT_THAT(schema_store->GetSchemaTypeId("message"), IsOkAndHolds(1)); + + // no scorable properties under the schema, 'message'. + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/1), + IsOkAndHolds(Pointee(ElementsAre()))); + + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("scoreDouble", TYPE_DOUBLE), + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); +} + +TEST_F(SchemaStoreTest, ScorablePropertyManagerUpdatesUponSchemaChange) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + // Sets the initial schema + ICING_ASSERT_OK(schema_store->SetSchema( + schema_, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(0)); + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("timestamp", TYPE_INT64))))); + + // The new schema drops the type 'email', and adds a new type 'message'. + SchemaProto new_schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("message") + .AddProperty( + // Add an indexed property so we generate + // section metadata on it + PropertyConfigBuilder() + .SetName("content") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("scoreInt") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("scoreDouble") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED))) + .Build(); + + // Force updates the schema. + ICING_ASSERT_OK(schema_store->SetSchema( + new_schema, /*ignore_errors_and_delete_documents=*/true, + /*allow_circular_schema_definitions=*/false)); + + // "timestamp" is no longer a valid property name. + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"timestamp"), + IsOkAndHolds(Eq(std::nullopt))); + + // ok cases for the new schema. + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"scoreInt"), + IsOkAndHolds(1)); + EXPECT_THAT(schema_store->GetScorablePropertyIndex( + /*schema_type_id=*/0, + /*property_path=*/"scoreDouble"), + IsOkAndHolds(0)); + EXPECT_THAT(schema_store->GetOrderedScorablePropertyInfo( + /*schema_type_id=*/0), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("scoreDouble", TYPE_DOUBLE), + EqualsScorablePropertyInfo("scoreInt", TYPE_INT64))))); +} + +class SchemaStoreTestWithParam + : public SchemaStoreTest, + public testing::WithParamInterface<version_util::StateChange> {}; + +TEST_P(SchemaStoreTestWithParam, MigrateSchemaWithDatabaseMigration) { + SchemaProto schema_no_database = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("db1/email") + .AddProperty(PropertyConfigBuilder() + .SetName("db1Subject") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_RFC822) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("db2/email") + .AddProperty(PropertyConfigBuilder() + .SetName("db2Subject") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + { + // Create an instance of the schema store and set the schema. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + ICING_ASSERT_OK(schema_store->SetSchema( + schema_no_database, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(schema_no_database)))); + } + + SchemaTypeConfigProto db1_email_rfc = + SchemaTypeConfigBuilder() + .SetType("db1/email") + .SetDatabase("db1") + .AddProperty( + PropertyConfigBuilder() + .SetName("db1Subject") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_RFC822)) + .Build(); + SchemaTypeConfigProto db1_email_no_rfc = + SchemaTypeConfigBuilder() + .SetType("db1/email") + .SetDatabase("db1") + .AddProperty(PropertyConfigBuilder() + .SetName("db1Subject") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetDataType(TYPE_STRING)) + .Build(); + SchemaTypeConfigProto db2_email = + SchemaTypeConfigBuilder() + .SetType("db2/email") + .SetDatabase("db2") + .AddProperty(PropertyConfigBuilder() + .SetName("db2Subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + SchemaProto full_schema_with_database_rfc = + SchemaBuilder().AddType(db1_email_rfc).AddType(db2_email).Build(); + SchemaProto full_schema_with_database_no_rfc = + SchemaBuilder().AddType(db1_email_no_rfc).AddType(db2_email).Build(); + SchemaProto db1_schema_rfc = SchemaBuilder().AddType(db1_email_rfc).Build(); + SchemaProto db1_schema_no_rfc = + SchemaBuilder().AddType(db1_email_no_rfc).Build(); + SchemaProto db2_schema = SchemaBuilder().AddType(db2_email).Build(); + + ICING_EXPECT_OK(SchemaStore::MigrateSchema( + &filesystem_, schema_store_dir_, /*version_state_change=*/GetParam(), + version_util::kVersion, /*perform_schema_database_migration=*/true)); + { + // Create a new instance of the schema store and check that the database + // field is populated. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + if (GetParam() == version_util::StateChange::kVersionZeroRollForward || + GetParam() == version_util::StateChange::kUndetermined) { + // For these cases, the overlay schema is discarded and we fall back to + // the backup schema (no rfc version). + EXPECT_THAT( + schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(full_schema_with_database_no_rfc)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema_no_rfc))); + EXPECT_THAT(schema_store->GetSchema("db2"), + IsOkAndHolds(EqualsProto(db2_schema))); + } else { + EXPECT_THAT( + schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(full_schema_with_database_rfc)))); + EXPECT_THAT(schema_store->GetSchema("db1"), + IsOkAndHolds(EqualsProto(db1_schema_rfc))); + EXPECT_THAT(schema_store->GetSchema("db2"), + IsOkAndHolds(EqualsProto(db2_schema))); + + DocumentProto db1_email_doc = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("db1/email") + .AddStringProperty("db1Subject", "db1_subject") + .Build(); + DocumentProto db2_email_doc = + DocumentBuilder() + .SetKey("namespace", "uri3") + .SetSchema("db2/email") + .AddStringProperty("db2Subject", "db2_subject") + .Build(); + + // Verify that our in-memory structures are ok + ICING_ASSERT_OK_AND_ASSIGN(SectionGroup section_group, + schema_store->ExtractSections(db1_email_doc)); + EXPECT_THAT(section_group.string_sections[0].content, + ElementsAre("db1_subject")); + ICING_ASSERT_OK_AND_ASSIGN(section_group, + schema_store->ExtractSections(db2_email_doc)); + EXPECT_THAT(section_group.string_sections[0].content, + ElementsAre("db2_subject")); + + // Verify that our persisted data are ok + EXPECT_THAT(schema_store->GetSchemaTypeId("db1/email"), IsOkAndHolds(0)); + EXPECT_THAT(schema_store->GetSchemaTypeId("db2/email"), IsOkAndHolds(1)); + } + } +} + +TEST_P(SchemaStoreTestWithParam, + MigrateSchemaWithDatabaseMigration_noDbPrefix) { + SchemaTypeConfigProto email_rfc = + SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_RFC822)) + .Build(); + SchemaTypeConfigProto email_no_rfc = + SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetDataType(TYPE_STRING)) + .Build(); + SchemaTypeConfigProto message = + SchemaTypeConfigBuilder() + .SetType("message") + .AddProperty(PropertyConfigBuilder() + .SetName("content") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + + SchemaProto original_schema = + SchemaBuilder().AddType(email_rfc).AddType(message).Build(); + + { + // Create an instance of the schema store and set the schema. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + ICING_ASSERT_OK(schema_store->SetSchema( + original_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(original_schema)))); + } + + SchemaProto backup_schema = + SchemaBuilder().AddType(email_no_rfc).AddType(message).Build(); + + ICING_EXPECT_OK(SchemaStore::MigrateSchema( + &filesystem_, schema_store_dir_, /*version_state_change=*/GetParam(), + version_util::kVersion, /*perform_schema_database_migration=*/true)); + + { + // Create a new instance of the schema store and check that the database + // field is populated. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); + + if (GetParam() == version_util::StateChange::kVersionZeroRollForward || + GetParam() == version_util::StateChange::kUndetermined) { + // For these cases, the overlay schema is discarded and we fall back to + // the backup schema (no rfc version). + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(backup_schema)))); + EXPECT_THAT(schema_store->GetSchema("db"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); + } else { + EXPECT_THAT(schema_store->GetSchema(), + IsOkAndHolds(Pointee(EqualsProto(original_schema)))); + EXPECT_THAT(schema_store->GetSchema("db"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); + + DocumentProto email_doc = DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("email") + .AddStringProperty("subject", "subject") + .Build(); + DocumentProto message_doc = DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("message") + .AddStringProperty("content", "content") + .Build(); + + // Verify that our in-memory structures are ok + ICING_ASSERT_OK_AND_ASSIGN(SectionGroup section_group, + schema_store->ExtractSections(email_doc)); + EXPECT_THAT(section_group.string_sections[0].content, + ElementsAre("subject")); + ICING_ASSERT_OK_AND_ASSIGN(section_group, + schema_store->ExtractSections(message_doc)); + EXPECT_THAT(section_group.string_sections[0].content, + ElementsAre("content")); + + // Verify that our persisted data are ok + EXPECT_THAT(schema_store->GetSchemaTypeId("email"), IsOkAndHolds(0)); + EXPECT_THAT(schema_store->GetSchemaTypeId("message"), IsOkAndHolds(1)); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + SchemaStoreTestWithParam, SchemaStoreTestWithParam, + testing::Values( + /*version_state_change=*/ + version_util::StateChange::kUndetermined, + version_util::StateChange::kCompatible, + version_util::StateChange::kRollForward, + version_util::StateChange::kRollBack, + version_util::StateChange::kUpgrade, + version_util::StateChange::kVersionZeroUpgrade, + version_util::StateChange::kVersionZeroRollForward)); + } // namespace } // namespace lib
diff --git a/icing/schema/schema-type-manager_test.cc b/icing/schema/schema-type-manager_test.cc index eafc612..df1c493 100644 --- a/icing/schema/schema-type-manager_test.cc +++ b/icing/schema/schema-type-manager_test.cc
@@ -89,7 +89,8 @@ return PropertyConfigBuilder() .SetName(kPropertySenderQualifiedId) .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) - .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, /*propagate_delete=*/true) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(); } @@ -114,7 +115,8 @@ return PropertyConfigBuilder() .SetName(kPropertyTagQualifiedId) .SetDataType(TYPE_STRING) - .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, /*propagate_delete=*/true) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(); } @@ -123,7 +125,8 @@ return PropertyConfigBuilder() .SetName(kPropertyGroupQualifiedId) .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) - .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, /*propagate_delete=*/true) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(); } @@ -132,7 +135,8 @@ return PropertyConfigBuilder() .SetName(kPropertySuperTagQualifiedId) .SetDataType(TYPE_STRING) - .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, /*propagate_delete=*/true) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED) .Build(); }
diff --git a/icing/schema/schema-util.cc b/icing/schema/schema-util.cc index cd591bc..22ce523 100644 --- a/icing/schema/schema-util.cc +++ b/icing/schema/schema-util.cc
@@ -15,7 +15,7 @@ #include "icing/schema/schema-util.h" #include <algorithm> -#include <cstdint> +#include <cctype> #include <queue> #include <string> #include <string_view> @@ -25,10 +25,12 @@ #include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/annotate.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" #include "icing/absl_ports/str_join.h" +#include "icing/feature-flags.h" #include "icing/proto/schema.pb.h" #include "icing/proto/term.pb.h" #include "icing/util/logging.h" @@ -39,18 +41,58 @@ namespace { +bool AreStringIndexingConfigsEqual(const StringIndexingConfig& old_config, + const StringIndexingConfig& new_config) { + return old_config.term_match_type() == new_config.term_match_type() && + old_config.tokenizer_type() == new_config.tokenizer_type(); +} + +bool AreDocumentIndexingConfigsEqual(const DocumentIndexingConfig& old_config, + const DocumentIndexingConfig& new_config) { + return old_config.index_nested_properties() == + new_config.index_nested_properties(); +} + +bool AreIntegerIndexingConfigsEqual(const IntegerIndexingConfig& old_config, + const IntegerIndexingConfig& new_config) { + return old_config.numeric_match_type() == new_config.numeric_match_type(); +} + +bool AreJoinableConfigsEqual(const JoinableConfig& old_config, + const JoinableConfig& new_config) { + return old_config.value_type() == new_config.value_type() && + old_config.delete_propagation_type() == + new_config.delete_propagation_type(); +} + +bool AreEmbeddingIndexingConfigsEqual( + const EmbeddingIndexingConfig& old_config, + const EmbeddingIndexingConfig& new_config) { + return old_config.embedding_indexing_type() == + new_config.embedding_indexing_type() && + old_config.quantization_type() == new_config.quantization_type(); +} + bool ArePropertiesEqual(const PropertyConfigProto& old_property, const PropertyConfigProto& new_property) { return old_property.property_name() == new_property.property_name() && old_property.data_type() == new_property.data_type() && old_property.schema_type() == new_property.schema_type() && old_property.cardinality() == new_property.cardinality() && - old_property.string_indexing_config().term_match_type() == - new_property.string_indexing_config().term_match_type() && - old_property.string_indexing_config().tokenizer_type() == - new_property.string_indexing_config().tokenizer_type() && - old_property.document_indexing_config().index_nested_properties() == - new_property.document_indexing_config().index_nested_properties(); + old_property.scorable_type() == new_property.scorable_type() && + AreStringIndexingConfigsEqual(old_property.string_indexing_config(), + new_property.string_indexing_config()) && + AreDocumentIndexingConfigsEqual( + old_property.document_indexing_config(), + new_property.document_indexing_config()) && + AreIntegerIndexingConfigsEqual( + old_property.integer_indexing_config(), + new_property.integer_indexing_config()) && + AreJoinableConfigsEqual(old_property.joinable_config(), + new_property.joinable_config()) && + AreEmbeddingIndexingConfigsEqual( + old_property.embedding_indexing_config(), + new_property.embedding_indexing_config()); } bool IsCardinalityCompatible(const PropertyConfigProto& old_property, @@ -118,7 +160,8 @@ bool IsEmbeddingIndexingCompatible(const EmbeddingIndexingConfig& old_indexed, const EmbeddingIndexingConfig& new_indexed) { return old_indexed.embedding_indexing_type() == - new_indexed.embedding_indexing_type(); + new_indexed.embedding_indexing_type() && + old_indexed.quantization_type() == new_indexed.quantization_type(); } bool IsDocumentIndexingCompatible(const DocumentIndexingConfig& old_indexed, @@ -207,6 +250,90 @@ return true; } +// Builds a map of {schema_type -> set of scorable property names} +std::unordered_map<std::string_view, std::unordered_set<std::string_view>> +BuildTypeToScorablePropertyNamesMap( + const SchemaUtil::TypeConfigMap& type_config_map) { + std::unordered_map<std::string_view, std::unordered_set<std::string_view>> + type_to_scorable_property_names_map; + for (const auto& [schema_type, schema_type_config] : type_config_map) { + for (const PropertyConfigProto& property_config : + schema_type_config.properties()) { + if (property_config.scorable_type() == + PropertyConfigProto::ScorableType::ENABLED) { + type_to_scorable_property_names_map[schema_type].insert( + property_config.property_name()); + } + } + } + return type_to_scorable_property_names_map; +} + +// Finds the schema types that have inconsistent scorable properties, which will +// be added in place in the `schema_delta`. +void FindScorablePropertyInconsistentTypes( + const SchemaUtil::TypeConfigMap& old_type_config_map, + const SchemaUtil::TypeConfigMap& new_type_config_map, + const SchemaUtil::DependentMap& new_schema_dependent_map, + SchemaUtil::SchemaDelta* schema_delta) { + std::unordered_map<std::string_view, std::unordered_set<std::string_view>> + new_type_to_scorable_property_names_map = + BuildTypeToScorablePropertyNamesMap(new_type_config_map); + std::unordered_map<std::string_view, std::unordered_set<std::string_view>> + old_type_to_scorable_property_names_map = + BuildTypeToScorablePropertyNamesMap(old_type_config_map); + for (const auto& [schema_type, _] : old_type_config_map) { + if (new_type_config_map.find(schema_type) == new_type_config_map.end()) { + // The type has been deleted in the new schema. + continue; + } + auto old_schema_type_property_names_iter = + old_type_to_scorable_property_names_map.find(schema_type); + auto new_schema_type_property_names_iter = + new_type_to_scorable_property_names_map.find(schema_type); + bool has_scorable_properties_in_old_schema = + old_schema_type_property_names_iter != + old_type_to_scorable_property_names_map.end(); + bool has_scorable_properties_in_new_schema = + new_schema_type_property_names_iter != + new_type_to_scorable_property_names_map.end(); + if (has_scorable_properties_in_old_schema && + !has_scorable_properties_in_new_schema) { + schema_delta->schema_types_scorable_property_inconsistent.insert( + schema_type); + } else if (!has_scorable_properties_in_old_schema && + has_scorable_properties_in_new_schema) { + schema_delta->schema_types_scorable_property_inconsistent.insert( + schema_type); + } else if (has_scorable_properties_in_old_schema && + has_scorable_properties_in_new_schema) { + // The sets of scorable properties from the old and new schema are + // different. + if (old_schema_type_property_names_iter->second != + new_schema_type_property_names_iter->second) { + schema_delta->schema_types_scorable_property_inconsistent.insert( + schema_type); + } + } + } + + // Now, look up the DependentMap of the new schema config and find the parent + // types that depend on the currently discovered inconsistent types. + std::vector<std::string_view> parent_types; + for (const std::string& schema_type : + schema_delta->schema_types_scorable_property_inconsistent) { + auto parent_type_maps_iter = new_schema_dependent_map.find(schema_type); + if (parent_type_maps_iter == new_schema_dependent_map.end()) { + continue; + } + for (const auto& [parent_type, _] : parent_type_maps_iter->second) { + parent_types.push_back(parent_type); + } + } + schema_delta->schema_types_scorable_property_inconsistent.insert( + parent_types.begin(), parent_types.end()); +} + } // namespace libtextclassifier3::Status CalculateTransitiveNestedTypeRelations( @@ -562,7 +689,8 @@ } libtextclassifier3::StatusOr<SchemaUtil::DependentMap> SchemaUtil::Validate( - const SchemaProto& schema, bool allow_circular_schema_definitions) { + const SchemaProto& schema, const FeatureFlags& feature_flags, + bool allow_circular_schema_definitions) { // 1. Build the dependent map. This will detect any cycles, non-existent or // duplicate types in the schema. ICING_ASSIGN_OR_RETURN( @@ -620,6 +748,10 @@ ICING_RETURN_IF_ERROR(ValidateCardinality(property_config.cardinality(), schema_type, property_name)); + if (feature_flags.enable_scorable_properties()) { + ICING_RETURN_IF_ERROR( + ValidateScorableType(schema_type, property_config)); + } if (data_type == PropertyConfigProto::DataType::STRING) { ICING_RETURN_IF_ERROR(ValidateStringIndexingConfig( @@ -627,9 +759,10 @@ property_name)); } - ICING_RETURN_IF_ERROR(ValidateJoinableConfig( - property_config.joinable_config(), data_type, - property_config.cardinality(), schema_type, property_name)); + ICING_RETURN_IF_ERROR( + ValidateJoinableConfig(property_config.joinable_config(), data_type, + property_config.cardinality(), schema_type, + property_name, feature_flags)); if (property_config.joinable_config().value_type() != JoinableConfig::ValueType::NONE) { schema_types_with_joinable_property.insert(schema_type); @@ -744,6 +877,42 @@ return libtextclassifier3::Status::OK; } +libtextclassifier3::Status SchemaUtil::ValidateScorableType( + std::string_view schema_type, + const PropertyConfigProto& property_config_proto) { + if (property_config_proto.data_type() == + PropertyConfigProto::DataType::DOCUMENT) { + if (property_config_proto.scorable_type() != + PropertyConfigProto::ScorableType::UNKNOWN) { + return absl_ports::InvalidArgumentError(absl_ports::StrCat( + "Field 'scorable_type' shouldn't be explicitly set for data type " + "DOCUMENT. It is considered scorable if any of its or its " + "dependency's property is scorable.")); + } + } + + if (property_config_proto.scorable_type() == + PropertyConfigProto::ScorableType::DISABLED || + property_config_proto.scorable_type() == + PropertyConfigProto::ScorableType::UNKNOWN) { + return libtextclassifier3::Status::OK; + } + + switch (property_config_proto.data_type()) { + case PropertyConfigProto::DataType::INT64: + case PropertyConfigProto::DataType::DOUBLE: + case PropertyConfigProto::DataType::BOOLEAN: + return libtextclassifier3::Status::OK; + default: + return absl_ports::InvalidArgumentError(absl_ports::StrCat( + "Field 'scorable_type' cannot be enabled for data type '", + PropertyConfigProto::DataType::Code_Name( + property_config_proto.data_type()), + "' for schema property '", schema_type, ".", + property_config_proto.property_name(), "'")); + } +} + libtextclassifier3::Status SchemaUtil::ValidateStringIndexingConfig( const StringIndexingConfig& config, PropertyConfigProto::DataType::Code data_type, std::string_view schema_type, @@ -770,7 +939,8 @@ libtextclassifier3::Status SchemaUtil::ValidateJoinableConfig( const JoinableConfig& config, PropertyConfigProto::DataType::Code data_type, PropertyConfigProto::Cardinality::Code cardinality, - std::string_view schema_type, std::string_view property_name) { + std::string_view schema_type, std::string_view property_name, + const FeatureFlags& feature_flags) { if (config.value_type() == JoinableConfig::ValueType::QUALIFIED_ID) { if (data_type != PropertyConfigProto::DataType::STRING) { return absl_ports::InvalidArgumentError( @@ -778,14 +948,16 @@ "' is required to have STRING data type")); } - if (cardinality == PropertyConfigProto::Cardinality::REPEATED) { + if (!feature_flags.enable_repeated_field_joins() && + cardinality == PropertyConfigProto::Cardinality::REPEATED) { return absl_ports::InvalidArgumentError( absl_ports::StrCat("Qualified id joinable property '", property_name, "' cannot have REPEATED cardinality")); } } - if (config.propagate_delete() && + if (config.delete_propagation_type() != + JoinableConfig::DeletePropagationType::NONE && config.value_type() != JoinableConfig::ValueType::QUALIFIED_ID) { return absl_ports::InvalidArgumentError( absl_ports::StrCat("Field 'property_name' '", property_name, @@ -986,13 +1158,20 @@ const SchemaUtil::SchemaDelta SchemaUtil::ComputeCompatibilityDelta( const SchemaProto& old_schema, const SchemaProto& new_schema, - const DependentMap& new_schema_dependent_map) { + const DependentMap& new_schema_dependent_map, + const FeatureFlags& feature_flags) { SchemaDelta schema_delta; TypeConfigMap old_type_config_map, new_type_config_map; BuildTypeConfigMap(old_schema, &old_type_config_map); BuildTypeConfigMap(new_schema, &new_type_config_map); + if (feature_flags.enable_scorable_properties()) { + FindScorablePropertyInconsistentTypes( + old_type_config_map, new_type_config_map, new_schema_dependent_map, + &schema_delta); + } + // Iterate through and check each field of the old schema for (const auto& old_type_config : old_schema.types()) { auto new_schema_type_and_config =
diff --git a/icing/schema/schema-util.h b/icing/schema/schema-util.h index 4f09915..27ad3e4 100644 --- a/icing/schema/schema-util.h +++ b/icing/schema/schema-util.h
@@ -23,6 +23,7 @@ #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/feature-flags.h" #include "icing/proto/schema.pb.h" namespace icing { @@ -95,6 +96,11 @@ // field in the SchemaTypeConfigProto. std::unordered_set<std::string> schema_types_join_incompatible; + // Schema types that were changed in a way that was backwards compatible, + // but inconsistent with the old schema so that the scorable property cache + // needs to be re-generated. + std::unordered_set<std::string> schema_types_scorable_property_inconsistent; + bool operator==(const SchemaDelta& other) const { return schema_types_deleted == other.schema_types_deleted && schema_types_incompatible == other.schema_types_incompatible && @@ -104,7 +110,9 @@ schema_types_index_incompatible == other.schema_types_index_incompatible && schema_types_join_incompatible == - other.schema_types_join_incompatible; + other.schema_types_join_incompatible && + schema_types_scorable_property_inconsistent == + other.schema_types_scorable_property_inconsistent; } }; @@ -163,6 +171,14 @@ // 15. For DOCUMENT data types, if // DocumentIndexingConfig.indexable_nested_properties_list is non-empty, // DocumentIndexingConfig.index_nested_properties must be false. + // 16. Validate the PropertyConfigProtos.scorable_type: + // - It can only be set to ENABLED for the following data types: + // a. Int64 + // b. Double + // c. Boolean + // - Documment type can't be explicitly set to DISABLED OR + // ENABLED. It is implicitly considered scorable if any of its or its + // dependency's property is scorable. // // Returns: // On success, a dependent map from each types to their dependent types @@ -170,7 +186,8 @@ // ALREADY_EXISTS for case 1 and 2 // INVALID_ARGUMENT for 3-15 static libtextclassifier3::StatusOr<DependentMap> Validate( - const SchemaProto& schema, bool allow_circular_schema_definitions); + const SchemaProto& schema, const FeatureFlags& feature_flags, + bool allow_circular_schema_definitions); // Builds a transitive inheritance map. // @@ -215,6 +232,8 @@ // definition is now incompatible. // 4. The derived join index would be incompatible. This is held in // `SchemaDelta.join_incompatible`. + // 5. The scorable properties of two schema are inconsistent. This is held + // in `SchemaDelta.schema_types_scorable_property_inconsistent`. // // For case 1, the two schemas would result in an incompatible index if: // 1.1. The new SchemaProto has a different set of indexed properties than @@ -242,13 +261,28 @@ // different set of joinable properties than it did in the old // SchemaProto. // + // For case 5, a schema type is considered to have inconsistent scorable + // properties if it is present in both the old and new schemas, and that: + // 5.1. The schema type contains different sets of scorable properties in + // the old and new schemas. It could be that: + // a. The type contains scorable properties in the new schema, but + // not in the old schema. + // b. The type contains scorable properties in the old schema, but + // not in the new schema. + // c. The type contains scorable properties in both the old and new + // schemas, but the set of properties are different. + // 5.2. The type has dependency on the types that are considered to have + // inconsistent scorable properties, based on the new schema's + // dependent map. + // // A property is defined by the combination of the // SchemaTypeConfig.schema_type and the PropertyConfigProto.property_name. // // Returns a SchemaDelta that captures the aforementioned differences. static const SchemaDelta ComputeCompatibilityDelta( const SchemaProto& old_schema, const SchemaProto& new_schema, - const DependentMap& new_schema_dependent_map); + const DependentMap& new_schema_dependent_map, + const FeatureFlags& feature_flags); // Validates the 'property_name' field. // 1. Can't be an empty string @@ -293,6 +327,15 @@ PropertyConfigProto::Cardinality::Code cardinality, std::string_view schema_type, std::string_view property_name); + // Validates the scorable_type of the given |property_config_proto|. + // + // Returns: + // INVALID_ARGUMENT if any scorable_type is found to be set incorrectly. + // OK on success + static libtextclassifier3::Status ValidateScorableType( + std::string_view schema_type, + const PropertyConfigProto& property_config_proto); + // Checks that the 'string_indexing_config' satisfies the following rules: // 1. Only STRING data types can be indexed // 2. An indexed property must have a valid tokenizer type @@ -319,7 +362,8 @@ const JoinableConfig& config, PropertyConfigProto::DataType::Code data_type, PropertyConfigProto::Cardinality::Code cardinality, - std::string_view schema_type, std::string_view property_name); + std::string_view schema_type, std::string_view property_name, + const FeatureFlags& feature_flags); // Checks that the 'document_indexing_config' satisfies the following rule: // 1. If indexable_nested_properties is non-empty, index_nested_properties
diff --git a/icing/schema/schema-util_test.cc b/icing/schema/schema-util_test.cc index e77ffab..414525e 100644 --- a/icing/schema/schema-util_test.cc +++ b/icing/schema/schema-util_test.cc
@@ -15,15 +15,20 @@ #include "icing/schema/schema-util.h" #include <initializer_list> +#include <memory> #include <string> #include <string_view> #include <unordered_set> +#include <utility> +#include "icing/text_classifier/lib3/utils/base/status.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "icing/feature-flags.h" #include "icing/proto/schema.pb.h" #include "icing/schema-builder.h" #include "icing/testing/common-matchers.h" +#include "icing/testing/test-feature-flags.h" namespace icing { namespace lib { @@ -45,7 +50,14 @@ constexpr char kMessageType[] = "Text"; constexpr char kPersonType[] = "Person"; -class SchemaUtilTest : public ::testing::TestWithParam<bool> {}; +class SchemaUtilTest : public ::testing::TestWithParam<bool> { + protected: + void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + } + + std::unique_ptr<FeatureFlags> feature_flags_; +}; TEST_P(SchemaUtilTest, DependentGraphAlphabeticalOrder) { // Create a schema with the following dependent relation: @@ -123,8 +135,9 @@ .AddType(type_e) .AddType(type_f) .Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, testing::SizeIs(5)); EXPECT_THAT( d_map["F"], @@ -231,8 +244,9 @@ .AddType(type_b) .AddType(type_a) .Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, testing::SizeIs(5)); EXPECT_THAT( d_map["F"], @@ -338,8 +352,9 @@ .AddType(type_b) .AddType(type_d) .Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, testing::SizeIs(5)); EXPECT_THAT( d_map["F"], @@ -392,7 +407,7 @@ .Build(); SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -428,8 +443,9 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); // Assert Validate status is OK and check dependent map - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(1)); EXPECT_THAT(d_map["B"], UnorderedElementsAre( @@ -473,7 +489,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs((libtextclassifier3::StatusCode::INVALID_ARGUMENT), HasSubstr("Invalid cycle"))); } @@ -517,7 +533,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::OK)); } @@ -563,8 +579,9 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); // Assert Validate status is OK and check dependent map - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(3)); EXPECT_THAT( d_map["A"], @@ -636,7 +653,7 @@ .AddType(type_c) .AddType(type_d) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -704,8 +721,9 @@ .AddType(type_d) .Build(); // Assert Validate status is OK and check dependent map - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(3)); EXPECT_THAT(d_map["B"], UnorderedElementsAre( @@ -780,7 +798,7 @@ .AddType(type_c) .AddType(type_b) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -850,7 +868,7 @@ .AddType(type_b) .AddType(type_a) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -891,8 +909,9 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); // Assert Validate status is OK and check dependent map - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(2)); EXPECT_THAT( d_map["A"], @@ -938,7 +957,7 @@ .Build(); SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -986,7 +1005,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -1038,8 +1057,9 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); // Assert Validate status is OK and check dependent map - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(2)); EXPECT_THAT(d_map["B"], UnorderedElementsAre( @@ -1114,7 +1134,7 @@ .AddType(type_c) .AddType(type_d) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -1187,7 +1207,7 @@ .AddType(type_d) .AddType(type_e) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -1261,7 +1281,7 @@ .AddType(type_d) .AddType(type_e) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -1357,7 +1377,7 @@ .AddType(type_e) .AddType(type_f) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -1449,7 +1469,7 @@ .AddType(type_e) .AddType(type_f) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -1523,8 +1543,9 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(3)); // Both A-B and A-C are inheritance relations. EXPECT_THAT(d_map["A"], @@ -1617,7 +1638,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -1683,7 +1704,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("inherits from itself"))); } @@ -1721,7 +1742,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -1761,8 +1782,9 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(2)); EXPECT_THAT(d_map["A"], UnorderedElementsAre( @@ -1788,8 +1810,9 @@ SchemaTypeConfigBuilder().SetType("B").AddParentType("A").Build(); SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(1)); EXPECT_THAT(d_map["A"], UnorderedElementsAre(Pair("B", IsEmpty()))); @@ -1819,8 +1842,9 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(2)); EXPECT_THAT(d_map["A"], UnorderedElementsAre(Pair("B", IsEmpty()), Pair("C", IsEmpty()))); @@ -1864,8 +1888,9 @@ .AddType(type_e) .AddType(type_f) .Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(3)); EXPECT_THAT(d_map["A"], UnorderedElementsAre(Pair("B", IsEmpty()), Pair("C", IsEmpty()), @@ -1902,7 +1927,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -1911,7 +1936,7 @@ SchemaTypeConfigBuilder().SetType("A").AddParentType("A").Build(); SchemaProto schema = SchemaBuilder().AddType(type_a).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -1927,7 +1952,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -1950,8 +1975,9 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).AddType(type_c).Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(2)); // Nested-type dependency and inheritance dependencies are not transitive. EXPECT_THAT(d_map["A"], UnorderedElementsAre(Pair("B", IsEmpty()))); @@ -2022,8 +2048,9 @@ .AddType(type_e) .AddType(type_f) .Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(3)); EXPECT_THAT( d_map["A"], @@ -2065,13 +2092,13 @@ SchemaTypeConfigBuilder().SetType("B").AddParentType("A").Build(); SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } TEST_P(SchemaUtilTest, EmptySchemaProtoIsValid) { SchemaProto schema; - ICING_ASSERT_OK(SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK(SchemaUtil::Validate(schema, *feature_flags_, GetParam())); } TEST_P(SchemaUtilTest, Valid_Nested) { @@ -2097,7 +2124,7 @@ .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - ICING_ASSERT_OK(SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK(SchemaUtil::Validate(schema, *feature_flags_, GetParam())); } TEST_P(SchemaUtilTest, ClearedPropertyConfigsIsValid) { @@ -2106,13 +2133,13 @@ SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType(kEmailType)) .Build(); - ICING_ASSERT_OK(SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK(SchemaUtil::Validate(schema, *feature_flags_, GetParam())); } TEST_P(SchemaUtilTest, ClearedSchemaTypeIsInvalid) { SchemaProto schema = SchemaBuilder().AddType(SchemaTypeConfigBuilder()).Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2120,7 +2147,7 @@ SchemaProto schema = SchemaBuilder().AddType(SchemaTypeConfigBuilder().SetType("")).Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2130,7 +2157,7 @@ "abc123!@#$%^&*()_-+=[{]}|\\;:'\",<.>?ä˝ ĺĽ˝")) .Build(); - ICING_ASSERT_OK(SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK(SchemaUtil::Validate(schema, *feature_flags_, GetParam())); } TEST_P(SchemaUtilTest, ClearedPropertyNameIsInvalid) { @@ -2144,7 +2171,7 @@ .SetCardinality(CARDINALITY_REQUIRED))) .Build(); schema.mutable_types(0)->mutable_properties(0)->clear_property_name(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2159,7 +2186,7 @@ .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2174,7 +2201,7 @@ .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2189,7 +2216,7 @@ .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - ICING_ASSERT_OK(SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK(SchemaUtil::Validate(schema, *feature_flags_, GetParam())); } TEST_P(SchemaUtilTest, DuplicatePropertyNameIsInvalid) { @@ -2206,7 +2233,7 @@ .SetDataType(TYPE_STRING) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::ALREADY_EXISTS)); } @@ -2221,7 +2248,7 @@ .SetCardinality(CARDINALITY_REQUIRED))) .Build(); schema.mutable_types(0)->mutable_properties(0)->clear_data_type(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2237,7 +2264,7 @@ .SetDataType(PropertyConfigProto::DataType::UNKNOWN) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2252,7 +2279,7 @@ .SetCardinality(CARDINALITY_REQUIRED))) .Build(); schema.mutable_types(0)->mutable_properties(0)->clear_cardinality(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2266,7 +2293,7 @@ .SetDataType(TYPE_STRING) .SetCardinality(CARDINALITY_UNKNOWN))) .Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2280,7 +2307,7 @@ .SetDataType(TYPE_DOCUMENT) .SetCardinality(CARDINALITY_REPEATED))) .Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2297,7 +2324,7 @@ .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -2314,7 +2341,7 @@ .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - ASSERT_THAT(SchemaUtil::Validate(schema, GetParam()), + ASSERT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Undefined 'schema_type'"))); } @@ -2351,7 +2378,8 @@ schema_delta.schema_types_changed_fully_compatible.insert(kEmailType); SchemaUtil::DependentMap no_dependents_map; EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( - old_schema, new_schema_with_optional, no_dependents_map), + old_schema, new_schema_with_optional, no_dependents_map, + *feature_flags_), Eq(schema_delta)); } @@ -2387,7 +2415,8 @@ schema_delta.schema_types_incompatible.emplace(kEmailType); SchemaUtil::DependentMap no_dependents_map; EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( - old_schema, new_schema_with_required, no_dependents_map), + old_schema, new_schema_with_required, no_dependents_map, + *feature_flags_), Eq(schema_delta)); } @@ -2422,8 +2451,8 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_incompatible.emplace(kEmailType); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map), + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_), Eq(schema_delta)); } @@ -2456,7 +2485,8 @@ SchemaUtil::DependentMap no_dependents_map; EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( /*old_schema=*/less_restrictive_schema, - /*new_schema=*/more_restrictive_schema, no_dependents_map), + /*new_schema=*/more_restrictive_schema, no_dependents_map, + *feature_flags_), Eq(incompatible_schema_delta)); // We can have the new schema be less restrictive, OPTIONAL->REPEATED; @@ -2465,7 +2495,8 @@ kEmailType); EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( /*old_schema=*/more_restrictive_schema, - /*new_schema=*/less_restrictive_schema, no_dependents_map), + /*new_schema=*/less_restrictive_schema, no_dependents_map, + *feature_flags_), Eq(compatible_schema_delta)); } @@ -2495,8 +2526,8 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_incompatible.emplace(kEmailType); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map), + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_), Eq(schema_delta)); } @@ -2557,7 +2588,7 @@ SchemaUtil::DependentMap dependents_map = { {kMessageType, {{kEmailType, {}}}}}; SchemaUtil::SchemaDelta actual = SchemaUtil::ComputeCompatibilityDelta( - old_schema, new_schema, dependents_map); + old_schema, new_schema, dependents_map, *feature_flags_); EXPECT_THAT(actual, Eq(schema_delta)); EXPECT_THAT(actual.schema_types_incompatible, testing::ElementsAre(kEmailType)); @@ -2594,7 +2625,7 @@ .Build(); SchemaUtil::SchemaDelta delta = SchemaUtil::ComputeCompatibilityDelta( - old_schema, new_schema, /*new_schema_dependent_map=*/{}); + old_schema, new_schema, /*new_schema_dependent_map=*/{}, *feature_flags_); EXPECT_THAT(delta.schema_types_incompatible, testing::ElementsAre(kEmailType)); EXPECT_THAT(delta.schema_types_index_incompatible, testing::IsEmpty()); @@ -2629,7 +2660,7 @@ .Build(); SchemaUtil::SchemaDelta delta = SchemaUtil::ComputeCompatibilityDelta( - old_schema, new_schema, /*new_schema_dependent_map=*/{}); + old_schema, new_schema, /*new_schema_dependent_map=*/{}, *feature_flags_); EXPECT_THAT(delta.schema_types_incompatible, testing::IsEmpty()); EXPECT_THAT(delta.schema_types_index_incompatible, testing::ElementsAre(kEmailType)); @@ -2664,7 +2695,7 @@ .Build(); SchemaUtil::SchemaDelta delta = SchemaUtil::ComputeCompatibilityDelta( - old_schema, new_schema, /*new_schema_dependent_map=*/{}); + old_schema, new_schema, /*new_schema_dependent_map=*/{}, *feature_flags_); EXPECT_THAT(delta.schema_types_incompatible, testing::IsEmpty()); EXPECT_THAT(delta.schema_types_index_incompatible, testing::IsEmpty()); EXPECT_THAT(delta.schema_types_deleted, testing::IsEmpty()); @@ -2704,13 +2735,13 @@ SchemaUtil::DependentMap no_dependents_map; EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( schema_with_unindexed_property, schema_with_indexed_property, - no_dependents_map), + no_dependents_map, *feature_flags_), Eq(schema_delta)); // New schema lost an indexed string property. EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( schema_with_indexed_property, schema_with_unindexed_property, - no_dependents_map), + no_dependents_map, *feature_flags_), Eq(schema_delta)); } @@ -2747,8 +2778,8 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_index_incompatible.insert(kPersonType); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map), + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_), Eq(schema_delta)); } @@ -2784,8 +2815,8 @@ .Build(); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map) + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_) .schema_types_index_incompatible, IsEmpty()); } @@ -2820,13 +2851,13 @@ SchemaUtil::DependentMap no_dependents_map; EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( schema_with_unindexed_property, schema_with_indexed_property, - no_dependents_map), + no_dependents_map, *feature_flags_), Eq(schema_delta)); // New schema lost an indexed integer property. EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( schema_with_indexed_property, schema_with_unindexed_property, - no_dependents_map), + no_dependents_map, *feature_flags_), Eq(schema_delta)); } @@ -2860,8 +2891,8 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_index_incompatible.insert(kPersonType); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map), + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_), Eq(schema_delta)); } @@ -2894,8 +2925,8 @@ .Build(); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map) + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_) .schema_types_index_incompatible, IsEmpty()); } @@ -2930,16 +2961,58 @@ SchemaUtil::DependentMap no_dependents_map; EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( schema_with_unindexed_property, schema_with_indexed_property, - no_dependents_map), + no_dependents_map, *feature_flags_), Eq(schema_delta)); // New schema lost an indexed vector property. EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( schema_with_indexed_property, schema_with_unindexed_property, - no_dependents_map), + no_dependents_map, *feature_flags_), Eq(schema_delta)); } +TEST_P(SchemaUtilTest, ChangingQuantizationTypeMakesIndexIncompatible) { + SchemaProto schema_with_unquantized_property = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH, + QUANTIZATION_TYPE_NONE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SchemaProto schema_with_quantized_property = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeVector( + EMBEDDING_INDEXING_LINEAR_SEARCH, + QUANTIZATION_TYPE_QUANTIZE_8_BIT) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_index_incompatible.insert(kPersonType); + + SchemaUtil::DependentMap no_dependents_map; + EXPECT_THAT( + SchemaUtil::ComputeCompatibilityDelta(schema_with_quantized_property, + schema_with_unquantized_property, + no_dependents_map, *feature_flags_), + Eq(schema_delta)); + + EXPECT_THAT( + SchemaUtil::ComputeCompatibilityDelta(schema_with_unquantized_property, + schema_with_quantized_property, + no_dependents_map, *feature_flags_), + Eq(schema_delta)); +} + TEST_P(SchemaUtilTest, AddingNewIndexedVectorPropertyMakesIndexIncompatible) { // Configure old schema SchemaProto old_schema = @@ -2971,8 +3044,8 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_index_incompatible.insert(kPersonType); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map), + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_), Eq(schema_delta)); } @@ -3006,8 +3079,8 @@ .Build(); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map) + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_) .schema_types_index_incompatible, IsEmpty()); } @@ -3060,7 +3133,7 @@ SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; SchemaUtil::SchemaDelta result_schema_delta = SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - dependents_map); + dependents_map, *feature_flags_); EXPECT_THAT(result_schema_delta, Eq(schema_delta)); } @@ -3117,7 +3190,7 @@ SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; SchemaUtil::SchemaDelta result_schema_delta = SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - dependents_map); + dependents_map, *feature_flags_); EXPECT_THAT(result_schema_delta, Eq(schema_delta)); } @@ -3169,7 +3242,7 @@ SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; SchemaUtil::SchemaDelta result_schema_delta = SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - dependents_map); + dependents_map, *feature_flags_); EXPECT_THAT(result_schema_delta, Eq(schema_delta)); } @@ -3233,7 +3306,7 @@ SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; SchemaUtil::SchemaDelta result_schema_delta = SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - dependents_map); + dependents_map, *feature_flags_); EXPECT_THAT(result_schema_delta, Eq(schema_delta)); } @@ -3296,7 +3369,7 @@ SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; SchemaUtil::SchemaDelta result_schema_delta = SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - dependents_map); + dependents_map, *feature_flags_); EXPECT_THAT(result_schema_delta, Eq(schema_delta)); } @@ -3368,7 +3441,7 @@ SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; SchemaUtil::SchemaDelta result_schema_delta = SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - dependents_map); + dependents_map, *feature_flags_); EXPECT_THAT(result_schema_delta, Eq(schema_delta)); } @@ -3439,7 +3512,7 @@ SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; SchemaUtil::SchemaDelta result_schema_delta = SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - dependents_map); + dependents_map, *feature_flags_); EXPECT_THAT(result_schema_delta, Eq(schema_delta)); } @@ -3473,16 +3546,18 @@ // New schema gained a new joinable property. SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( - schema_with_non_joinable_property, - schema_with_joinable_property, no_dependents_map), - Eq(expected_schema_delta)); + EXPECT_THAT( + SchemaUtil::ComputeCompatibilityDelta(schema_with_non_joinable_property, + schema_with_joinable_property, + no_dependents_map, *feature_flags_), + Eq(expected_schema_delta)); // New schema lost a joinable property. - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( - schema_with_joinable_property, - schema_with_non_joinable_property, no_dependents_map), - Eq(expected_schema_delta)); + EXPECT_THAT( + SchemaUtil::ComputeCompatibilityDelta(schema_with_joinable_property, + schema_with_non_joinable_property, + no_dependents_map, *feature_flags_), + Eq(expected_schema_delta)); } TEST_P(SchemaUtilTest, AddingNewJoinablePropertyMakesJoinIncompatible) { @@ -3518,8 +3593,8 @@ SchemaUtil::SchemaDelta expected_schema_delta; expected_schema_delta.schema_types_join_incompatible.insert(kPersonType); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map), + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_), Eq(expected_schema_delta)); } @@ -3554,12 +3629,46 @@ .Build(); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map) + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_) .schema_types_join_incompatible, IsEmpty()); } +TEST_P(SchemaUtilTest, ChangingJoinablePropertiesPropagateDeleteIsCompatible) { + // Configure old schema + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("MyType").AddProperty( + PropertyConfigBuilder() + .SetName("Foo") + .SetDataType(TYPE_STRING) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_NONE) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + + // Configure new schema with delete propagation type PROPAGATE_FROM + SchemaProto new_schema_with_optional = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("MyType").AddProperty( + PropertyConfigBuilder() + .SetName("Foo") + .SetDataType(TYPE_STRING) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_changed_fully_compatible.insert("MyType"); + SchemaUtil::DependentMap no_dependents_map; + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema_with_optional, no_dependents_map, + *feature_flags_), + Eq(schema_delta)); +} + TEST_P(SchemaUtilTest, AddingTypeIsCompatible) { // Can add a new type, existing data isn't incompatible, since none of them // are of this new schema type @@ -3595,8 +3704,8 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_new.insert(kEmailType); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map), + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_), Eq(schema_delta)); } @@ -3636,8 +3745,8 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_deleted.emplace(kPersonType); SchemaUtil::DependentMap no_dependents_map; - EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, - no_dependents_map), + EXPECT_THAT(SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, no_dependents_map, *feature_flags_), Eq(schema_delta)); } @@ -3675,7 +3784,7 @@ schema_delta.schema_types_index_incompatible.emplace(kEmailType); SchemaUtil::DependentMap no_dependents_map; SchemaUtil::SchemaDelta actual = SchemaUtil::ComputeCompatibilityDelta( - old_schema, new_schema, no_dependents_map); + old_schema, new_schema, no_dependents_map, *feature_flags_); EXPECT_THAT(actual, Eq(schema_delta)); } @@ -3723,14 +3832,16 @@ schema_delta.schema_types_index_incompatible.emplace(kPersonType); SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; SchemaUtil::SchemaDelta actual = SchemaUtil::ComputeCompatibilityDelta( - no_nested_index_schema, nested_index_schema, dependents_map); + no_nested_index_schema, nested_index_schema, dependents_map, + *feature_flags_); EXPECT_THAT(actual, Eq(schema_delta)); // Going from index_nested_properties=true to index_nested_properties=false // should also make kPersonType index_incompatible. kEmailType should be // unaffected. actual = SchemaUtil::ComputeCompatibilityDelta( - nested_index_schema, no_nested_index_schema, dependents_map); + nested_index_schema, no_nested_index_schema, dependents_map, + *feature_flags_); EXPECT_THAT(actual, Eq(schema_delta)); } @@ -3785,14 +3896,14 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_index_incompatible.emplace(kPersonType); SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; - SchemaUtil::SchemaDelta actual = - SchemaUtil::ComputeCompatibilityDelta(schema_1, schema_2, dependents_map); + SchemaUtil::SchemaDelta actual = SchemaUtil::ComputeCompatibilityDelta( + schema_1, schema_2, dependents_map, *feature_flags_); EXPECT_THAT(actual, Eq(schema_delta)); // Adding some indexable_nested_properties should also make kPersonType // index_incompatible. kEmailType should be unaffected. - actual = - SchemaUtil::ComputeCompatibilityDelta(schema_2, schema_1, dependents_map); + actual = SchemaUtil::ComputeCompatibilityDelta( + schema_2, schema_1, dependents_map, *feature_flags_); EXPECT_THAT(actual, Eq(schema_delta)); } @@ -3848,8 +3959,8 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_index_incompatible.emplace(kPersonType); SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; - SchemaUtil::SchemaDelta actual = - SchemaUtil::ComputeCompatibilityDelta(schema_1, schema_2, dependents_map); + SchemaUtil::SchemaDelta actual = SchemaUtil::ComputeCompatibilityDelta( + schema_1, schema_2, dependents_map, *feature_flags_); EXPECT_THAT(actual, Eq(schema_delta)); } @@ -3904,8 +4015,8 @@ SchemaUtil::SchemaDelta schema_delta; schema_delta.schema_types_index_incompatible.emplace(kPersonType); SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; - SchemaUtil::SchemaDelta actual = - SchemaUtil::ComputeCompatibilityDelta(schema_1, schema_2, dependents_map); + SchemaUtil::SchemaDelta actual = SchemaUtil::ComputeCompatibilityDelta( + schema_1, schema_2, dependents_map, *feature_flags_); EXPECT_THAT(actual, Eq(schema_delta)); } @@ -3961,12 +4072,305 @@ // no effect on schema compatibility. SchemaUtil::SchemaDelta schema_delta; SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; - SchemaUtil::SchemaDelta actual = - SchemaUtil::ComputeCompatibilityDelta(schema_1, schema_2, dependents_map); + SchemaUtil::SchemaDelta actual = SchemaUtil::ComputeCompatibilityDelta( + schema_1, schema_2, dependents_map, *feature_flags_); EXPECT_THAT(actual, Eq(schema_delta)); EXPECT_THAT(actual.schema_types_index_incompatible, IsEmpty()); } +TEST_P(SchemaUtilTest, SchemasWithConsistentScorableProperties) { + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("score") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + SchemaProto new_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email").AddProperty( + PropertyConfigBuilder() + .SetName("score") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap new_schema_dependent_map, + SchemaUtil::Validate(new_schema, *feature_flags_, GetParam())); + + SchemaUtil::SchemaDelta schema_delta = SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, new_schema_dependent_map, *feature_flags_); + EXPECT_THAT(schema_delta.schema_types_scorable_property_inconsistent, + UnorderedElementsAre()); +} + +TEST_P(SchemaUtilTest, + SchemaDeltaScorablePropertyInconsistent_SchemaTypeRemovedInNewSchema) { + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email").AddProperty( + PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED))) + .AddType(SchemaTypeConfigBuilder().SetType("person").AddProperty( + PropertyConfigBuilder() + .SetName("frequencyScore") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + // New schema removes "person" type. + SchemaProto new_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email").AddProperty( + PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap new_schema_dependent_map, + SchemaUtil::Validate(new_schema, *feature_flags_, GetParam())); + + SchemaUtil::SchemaDelta schema_delta = SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, new_schema_dependent_map, *feature_flags_); + EXPECT_THAT(schema_delta.schema_types_scorable_property_inconsistent, + UnorderedElementsAre()); +} + +TEST_P(SchemaUtilTest, + SchemaDeltaScorablePropertyInconsistent_SchemaTypeAddedInNewSchema) { + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email").AddProperty( + PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED))) + .AddType(SchemaTypeConfigBuilder().SetType("message").AddProperty( + PropertyConfigBuilder() + .SetName("frequencyScore") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + // New schema adds "person" type. + SchemaProto new_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email").AddProperty( + PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED))) + .AddType(SchemaTypeConfigBuilder().SetType("message").AddProperty( + PropertyConfigBuilder() + .SetName("frequencyScore") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder().SetType("person").AddProperty( + PropertyConfigBuilder() + .SetName("frequencyScore") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap new_schema_dependent_map, + SchemaUtil::Validate(new_schema, *feature_flags_, GetParam())); + + SchemaUtil::SchemaDelta schema_delta = SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, new_schema_dependent_map, *feature_flags_); + EXPECT_THAT(schema_delta.schema_types_scorable_property_inconsistent, + UnorderedElementsAre()); +} + +TEST_P(SchemaUtilTest, + SchemaDeltaScorablePropertyInconsistent_AddScorableProperties) { + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email").AddProperty( + PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + // New schema adds "score" as scorable_type ENABLED from type "email". + SchemaProto new_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("score") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap new_schema_dependent_map, + SchemaUtil::Validate(new_schema, *feature_flags_, GetParam())); + + SchemaUtil::SchemaDelta schema_delta = SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, new_schema_dependent_map, *feature_flags_); + EXPECT_THAT(schema_delta.schema_types_scorable_property_inconsistent, + UnorderedElementsAre("email")); +} + +TEST_P(SchemaUtilTest, + SchemaDeltaScorablePropertyInconsistent_RemovesAllScorableProperties) { + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("score") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + // New schema marks "score" as scorable_type DISABLED from type "email". + SchemaProto new_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("title") + .SetDataType(TYPE_STRING) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("score") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_DISABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap new_schema_dependent_map, + SchemaUtil::Validate(new_schema, *feature_flags_, GetParam())); + + SchemaUtil::SchemaDelta schema_delta = SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, new_schema_dependent_map, *feature_flags_); + EXPECT_THAT(schema_delta.schema_types_scorable_property_inconsistent, + UnorderedElementsAre("email")); +} + +TEST_P(SchemaUtilTest, + SchemaDeltaScorablePropertyInconsistent_DifferentScorablePropertyNames) { + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("score1") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("score2") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + SchemaProto new_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("score2") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("score3") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap new_schema_dependent_map, + SchemaUtil::Validate(new_schema, *feature_flags_, GetParam())); + + SchemaUtil::SchemaDelta schema_delta = SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, new_schema_dependent_map, *feature_flags_); + EXPECT_THAT(schema_delta.schema_types_scorable_property_inconsistent, + UnorderedElementsAre("email")); +} + +TEST_P(SchemaUtilTest, + SchemaDeltaScorablePropertyInconsistent_ReportParentTypes) { + SchemaProto old_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("person").AddProperty( + PropertyConfigBuilder() + .SetName("scoreA") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // The new schema's type 'person' has inconsistent scorable property with + // the old schema. + // + // The dependence map of the new schema: + // personalContext -> gmail -> person + // message -> person + SchemaProto new_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("person").AddProperty( + PropertyConfigBuilder() + .SetName("scoreB") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder().SetType("message").AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeDocument("person", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder().SetType("gmail").AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeDocument("person", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("personalContext") + .AddProperty(PropertyConfigBuilder() + .SetName("mailData") + .SetDataTypeDocument( + "gmail", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap new_schema_dependent_map, + SchemaUtil::Validate(new_schema, *feature_flags_, GetParam())); + + SchemaUtil::SchemaDelta schema_delta = SchemaUtil::ComputeCompatibilityDelta( + old_schema, new_schema, new_schema_dependent_map, *feature_flags_); + EXPECT_THAT( + schema_delta.schema_types_scorable_property_inconsistent, + UnorderedElementsAre("person", "gmail", "personalContext", "message")); +} + TEST_P(SchemaUtilTest, ValidateStringIndexingConfigShouldHaveTermMatchType) { SchemaProto schema = SchemaBuilder() @@ -3978,7 +4382,7 @@ .Build(); // Error if we don't set a term match type - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Passes once we set a term match type @@ -3989,7 +4393,8 @@ .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, ValidateStringIndexingConfigShouldHaveTokenizer) { @@ -4003,7 +4408,7 @@ .Build(); // Error if we don't set a tokenizer type - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Passes once we set a tokenizer type @@ -4014,7 +4419,8 @@ .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, @@ -4026,12 +4432,12 @@ .SetName("Foo") .SetDataType(TYPE_INT64) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); // Error if data type is not STRING for qualified id joinable value type. - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Passes once we set STRING as the data type. @@ -4041,14 +4447,20 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, ValidateJoinablePropertyShouldNotHaveRepeatedCardinality) { + // We need to explicitly override enable_repeated_field_joins to false. + feature_flags_ = + std::make_unique<FeatureFlags>(/*enable_scorable_properties=*/true, + /*enable_embedding_quantization=*/true, + /*enable_repeated_field_joins=*/false); SchemaProto schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("MyType").AddProperty( @@ -4056,12 +4468,12 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REPEATED))) .Build(); // Error if using REPEATED cardinality for joinable property. - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Passes once we use OPTIONAL cardinality with joinable property. @@ -4071,10 +4483,11 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); // Passes once we use REQUIRED cardinality with joinable property. schema = SchemaBuilder() @@ -4083,10 +4496,11 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); // Passes once we use REPEATED cardinality with non-joinable property. schema = SchemaBuilder() @@ -4095,10 +4509,34 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_NONE, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REPEATED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); +} + +TEST_P(SchemaUtilTest, ValidateJoinablePropertyCanHaveRepeatedCardinality) { + // We need to explicitly override enable_repeated_field_joins to true. + feature_flags_ = + std::make_unique<FeatureFlags>(/*enable_scorable_properties=*/true, + /*enable_embedding_quantization=*/true, + /*enable_repeated_field_joins=*/true); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("MyType").AddProperty( + PropertyConfigBuilder() + .SetName("Foo") + .SetDataType(TYPE_STRING) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_NONE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + // Error if using REPEATED cardinality for joinable property. + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, @@ -4110,13 +4548,13 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_NONE, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); // Error if enabling delete propagation with non qualified id joinable value // type. - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Passes once we set qualified id joinable value type with delete propagation @@ -4127,10 +4565,11 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/true) + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); // Passes once we disable delete propagation. schema = SchemaBuilder() @@ -4139,10 +4578,11 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_NONE, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_REQUIRED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, @@ -4157,7 +4597,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder().SetType("B").AddProperty( PropertyConfigBuilder() @@ -4172,7 +4612,7 @@ /*index_nested_properties=*/false) .SetCardinality(CARDINALITY_REPEATED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Passes once we use non-REPEATED cardinality for "C.b", i.e. the dependency @@ -4184,7 +4624,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder().SetType("B").AddProperty( PropertyConfigBuilder() @@ -4199,7 +4639,8 @@ /*index_nested_properties=*/false) .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P( @@ -4215,7 +4656,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_NONE, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder() .SetType("B") @@ -4230,7 +4671,7 @@ .SetName("Bar") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder().SetType("C").AddProperty( PropertyConfigBuilder() @@ -4242,7 +4683,8 @@ // Passes since nested schema type with REPEATED cardinality doesn't have // joinable property. - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, @@ -4261,7 +4703,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder() .SetType("B") @@ -4278,7 +4720,7 @@ /*index_nested_properties=*/false) .SetCardinality(CARDINALITY_REPEATED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Passes once we use non-REPEATED cardinality for "B.a2", i.e. the dependency @@ -4295,7 +4737,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder() .SetType("B") @@ -4312,7 +4754,8 @@ /*index_nested_properties=*/false) .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, ValidateNestedJoinablePropertyDiamondRelationship) { @@ -4334,7 +4777,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder().SetType("B").AddProperty( PropertyConfigBuilder() @@ -4363,7 +4806,8 @@ /*index_nested_properties=*/false) .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); // Fails once we change any of edge to REPEATED cardinality. // B @@ -4382,7 +4826,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder().SetType("B").AddProperty( PropertyConfigBuilder() @@ -4411,7 +4855,7 @@ /*index_nested_properties=*/false) .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // B @@ -4430,7 +4874,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder().SetType("B").AddProperty( PropertyConfigBuilder() @@ -4459,7 +4903,7 @@ /*index_nested_properties=*/false) .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // B @@ -4478,7 +4922,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder().SetType("B").AddProperty( PropertyConfigBuilder() @@ -4507,7 +4951,7 @@ /*index_nested_properties=*/false) .SetCardinality(CARDINALITY_REPEATED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // B @@ -4526,7 +4970,7 @@ .SetName("Foo") .SetDataType(TYPE_STRING) .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, - /*propagate_delete=*/false) + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .AddType(SchemaTypeConfigBuilder().SetType("B").AddProperty( PropertyConfigBuilder() @@ -4555,7 +4999,7 @@ /*index_nested_properties=*/false) .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -4590,7 +5034,8 @@ ->mutable_document_indexing_config() ->clear_indexable_nested_properties_list(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, @@ -4624,7 +5069,8 @@ ->mutable_document_indexing_config() ->clear_indexable_nested_properties_list(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, @@ -4654,7 +5100,8 @@ outerSchemaType->mutable_properties(0) ->mutable_document_indexing_config() ->set_index_nested_properties(false); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, InvalidDocumentIndexingConfigFields) { @@ -4686,7 +5133,7 @@ ->mutable_document_indexing_config() ->add_indexable_nested_properties_list("prop"); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -4710,7 +5157,8 @@ .SetCardinality(CARDINALITY_REPEATED))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), + IsOk()); } TEST_P(SchemaUtilTest, InvalidSelfReference) { @@ -4727,7 +5175,7 @@ .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -4751,7 +5199,7 @@ .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -4783,7 +5231,7 @@ .Build(); // Two degrees of referencing: A -> B -> A - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -4824,7 +5272,7 @@ .Build(); // Three degrees of referencing: A -> B -> C -> A - EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + EXPECT_THAT(SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Invalid cycle"))); } @@ -4844,7 +5292,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); EXPECT_THAT( - SchemaUtil::Validate(schema, GetParam()), + SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Property text is not present in child type"))); } @@ -4864,7 +5312,7 @@ SchemaProto schema = SchemaBuilder().AddType(type_a).AddType(type_b).Build(); EXPECT_THAT( - SchemaUtil::Validate(schema, GetParam()), + SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Property text is not present in child type"))); } @@ -4928,8 +5376,9 @@ .AddType(person_type) .AddType(artist_type) .Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(3)); EXPECT_THAT(d_map["Message"], UnorderedElementsAre(Pair("ArtistMessage", IsEmpty()))); @@ -4995,7 +5444,7 @@ .AddType(artist_type) .Build(); EXPECT_THAT( - SchemaUtil::Validate(schema, GetParam()), + SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Property person from child type ArtistMessage is not " "compatible to the parent type Message."))); @@ -5051,7 +5500,7 @@ .AddType(artist_type) .Build(); EXPECT_THAT( - SchemaUtil::Validate(schema, GetParam()), + SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Property text from child type ArtistMessage is not " "compatible to the parent type Message."))); @@ -5108,7 +5557,7 @@ .AddType(artist_type) .Build(); EXPECT_THAT( - SchemaUtil::Validate(schema, GetParam()), + SchemaUtil::Validate(schema, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("Property person from child type ArtistMessage is not " "compatible to the parent type Message."))); @@ -5165,8 +5614,9 @@ .AddType(message_type) .AddType(email_message_type) .Build(); - ICING_ASSERT_OK_AND_ASSIGN(SchemaUtil::DependentMap d_map, - SchemaUtil::Validate(schema, GetParam())); + ICING_ASSERT_OK_AND_ASSIGN( + SchemaUtil::DependentMap d_map, + SchemaUtil::Validate(schema, *feature_flags_, GetParam())); EXPECT_THAT(d_map, SizeIs(2)); EXPECT_THAT(d_map["Email"], UnorderedElementsAre(Pair("EmailMessage", IsEmpty()))); @@ -5222,7 +5672,7 @@ .AddType(email_message_type1) .Build(); EXPECT_THAT( - SchemaUtil::Validate(schema1, GetParam()), + SchemaUtil::Validate(schema1, *feature_flags_, GetParam()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr( "Property sender is not present in child type EmailMessage, " @@ -5251,7 +5701,7 @@ .AddType(email_message_type2) .Build(); EXPECT_THAT( - SchemaUtil::Validate(schema2, GetParam()), + SchemaUtil::Validate(schema2, *feature_flags_, GetParam()), StatusIs( libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr( @@ -5259,6 +5709,149 @@ "but it is defined in the parent type Message."))); } +TEST_P(SchemaUtilTest, ValidateScorableType_EnabledForSupportedDataTypes) { + SchemaTypeConfigProto type_int64 = + SchemaTypeConfigBuilder() + .SetType("A") + .AddProperty(PropertyConfigBuilder() + .SetName("c") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE)) + .Build(); + EXPECT_THAT(SchemaUtil::Validate(SchemaBuilder().AddType(type_int64).Build(), + *feature_flags_, GetParam()), + StatusIs(libtextclassifier3::StatusCode::OK)); + + SchemaTypeConfigProto type_double = + SchemaTypeConfigBuilder() + .SetType("B") + .AddProperty(PropertyConfigBuilder() + .SetName("c") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataType(TYPE_DOUBLE)) + .Build(); + + EXPECT_THAT(SchemaUtil::Validate(SchemaBuilder().AddType(type_double).Build(), + *feature_flags_, GetParam()), + StatusIs(libtextclassifier3::StatusCode::OK)); + + SchemaTypeConfigProto type_boolean = + SchemaTypeConfigBuilder() + .SetType("B") + .AddProperty(PropertyConfigBuilder() + .SetName("c") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataType(TYPE_BOOLEAN)) + .Build(); + + EXPECT_THAT( + SchemaUtil::Validate(SchemaBuilder().AddType(type_boolean).Build(), + *feature_flags_, GetParam()), + StatusIs(libtextclassifier3::StatusCode::OK)); +} + +TEST_P(SchemaUtilTest, ValidateScorableType_EnabledForUnsupportedDataTypes) { + SchemaTypeConfigProto type_a = + SchemaTypeConfigBuilder() + .SetType("A") + .AddProperty(PropertyConfigBuilder() + .SetName("c") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataType(TYPE_STRING)) + .Build(); + EXPECT_THAT( + SchemaUtil::Validate(SchemaBuilder().AddType(type_a).Build(), + *feature_flags_, GetParam()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("Field 'scorable_type' cannot be enabled for data " + "type 'STRING' for schema property 'A.c'"))); +} + +TEST_P(SchemaUtilTest, + ValidateScorableType_CannotBeExplicitlyEnabledForDocumentType) { + SchemaTypeConfigProto message_type = + SchemaTypeConfigBuilder() + .SetType(kMessageType) + .AddProperty(PropertyConfigBuilder() + .SetName("prop") + .SetDataType(TYPE_INT64) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .Build(); + + SchemaTypeConfigProto type_document = + SchemaTypeConfigBuilder() + .SetType("B") + .AddProperty( + PropertyConfigBuilder() + .SetName("c") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeDocument(kMessageType, + /*index_nested_properties=*/true)) + .Build(); + + EXPECT_THAT( + SchemaUtil::Validate( + SchemaBuilder().AddType(type_document).AddType(message_type).Build(), + *feature_flags_, GetParam()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("Field 'scorable_type' shouldn't be explicitly " + "set for data type DOCUMENT"))); +} + +TEST_P(SchemaUtilTest, + ValidateScorableType_CannotBeExplicitlyDisabledForDocumentType) { + SchemaTypeConfigProto message_type = + SchemaTypeConfigBuilder() + .SetType(kMessageType) + .AddProperty(PropertyConfigBuilder() + .SetName("prop") + .SetDataType(TYPE_INT64) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .Build(); + + SchemaTypeConfigProto type_document = + SchemaTypeConfigBuilder() + .SetType("B") + .AddProperty( + PropertyConfigBuilder() + .SetName("c") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_DISABLED) + .SetDataTypeDocument(kMessageType, + /*index_nested_properties=*/true)) + .Build(); + + EXPECT_THAT( + SchemaUtil::Validate( + SchemaBuilder().AddType(type_document).AddType(message_type).Build(), + *feature_flags_, GetParam()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("Field 'scorable_type' shouldn't be explicitly " + "set for data type DOCUMENT"))); +} + +TEST_P(SchemaUtilTest, ValidateScorableType_DisabledForUnsupportedDataTypes) { + SchemaTypeConfigProto type_b = + SchemaTypeConfigBuilder() + .SetType("A") + .AddProperty(PropertyConfigBuilder() + .SetName("c") + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_DISABLED) + .SetDataType(TYPE_STRING)) + .Build(); + EXPECT_THAT(SchemaUtil::Validate(SchemaBuilder().AddType(type_b).Build(), + *feature_flags_, GetParam()), + StatusIs(libtextclassifier3::StatusCode::OK)); +} + INSTANTIATE_TEST_SUITE_P( SchemaUtilTest, SchemaUtilTest, testing::Values(/*allow_circular_schema_definitions=*/true, false));
diff --git a/icing/schema/scorable_property_manager.cc b/icing/schema/scorable_property_manager.cc new file mode 100644 index 0000000..440d1bd --- /dev/null +++ b/icing/schema/scorable_property_manager.cc
@@ -0,0 +1,135 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/schema/scorable_property_manager.h" + +#include <optional> +#include <string> +#include <string_view> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/legacy/core/icing-string-util.h" +#include "icing/schema/schema-property-iterator.h" +#include "icing/schema/schema-util.h" +#include "icing/store/document-filter-data.h" +#include "icing/util/status-macros.h" + +namespace icing { +namespace lib { + +libtextclassifier3::StatusOr<std::optional<int>> +ScorablePropertyManager::GetScorablePropertyIndex( + SchemaTypeId schema_type_id, std::string_view property_path, + const SchemaUtil::TypeConfigMap& type_config_map, + const std::unordered_map<SchemaTypeId, std::string>& + schema_id_to_type_map) { + ICING_ASSIGN_OR_RETURN(auto cache_iter, LookupAndMaybeUpdateCache( + schema_type_id, type_config_map, + schema_id_to_type_map)); + auto iter = + cache_iter->second.property_path_to_index_map.find(property_path.data()); + if (iter == cache_iter->second.property_path_to_index_map.end()) { + return std::nullopt; + } + return iter->second; +} + +libtextclassifier3::StatusOr< + const std::vector<ScorablePropertyManager::ScorablePropertyInfo>*> +ScorablePropertyManager::GetOrderedScorablePropertyInfo( + SchemaTypeId schema_type_id, + const SchemaUtil::TypeConfigMap& type_config_map, + const std::unordered_map<SchemaTypeId, std::string>& + schema_id_to_type_map) { + ICING_ASSIGN_OR_RETURN(auto cache_iter, LookupAndMaybeUpdateCache( + schema_type_id, type_config_map, + schema_id_to_type_map)); + return &cache_iter->second.ordered_scorable_property_info; +} + +libtextclassifier3::StatusOr<std::unordered_map< + SchemaTypeId, + ScorablePropertyManager::DerivedScorablePropertySchema>::iterator> +ScorablePropertyManager::LookupAndMaybeUpdateCache( + SchemaTypeId schema_type_id, + const SchemaUtil::TypeConfigMap& type_config_map, + const std::unordered_map<SchemaTypeId, std::string>& + schema_id_to_type_map) { + auto cache_iter = scorable_property_schema_cache_.find(schema_type_id); + if (cache_iter == scorable_property_schema_cache_.end()) { + if (UpdateCache(schema_type_id, type_config_map, schema_id_to_type_map)) { + cache_iter = scorable_property_schema_cache_.find(schema_type_id); + } else { + // schema type id not found neither in the cache nor in the schema + // config. + return absl_ports::InvalidArgumentError(IcingStringUtil::StringPrintf( + "LookupAndMaybeUpdateCache failed: Schema type id %d not found", + schema_type_id)); + } + } + return cache_iter; +} + +bool ScorablePropertyManager::UpdateCache( + SchemaTypeId schema_type_id, + const SchemaUtil::TypeConfigMap& type_config_map, + const std::unordered_map<SchemaTypeId, std::string>& + schema_id_to_type_map) { + auto schema_id_iter = schema_id_to_type_map.find(schema_type_id); + if (schema_id_iter == schema_id_to_type_map.end()) { + return false; + } + auto schema_config_iter = type_config_map.find(schema_id_iter->second); + if (schema_config_iter == type_config_map.end()) { + return false; + } + std::vector<ScorablePropertyInfo> property_info_vector; + std::unordered_map<std::string, int> index_map; + + SchemaPropertyIterator iterator(schema_config_iter->second, type_config_map); + while (true) { + libtextclassifier3::Status status = iterator.Advance(); + if (!status.ok()) { + if (absl_ports::IsOutOfRange(status)) { + break; + } + // Swallow other type of errors which should not happen if the schema has + // been validated already. + return false; + } + if (iterator.GetCurrentPropertyConfig().scorable_type() == + PropertyConfigProto::ScorableType::ENABLED) { + index_map[iterator.GetCurrentPropertyPath()] = + property_info_vector.size(); + ScorablePropertyInfo scorable_property_info; + scorable_property_info.property_path = iterator.GetCurrentPropertyPath(); + scorable_property_info.data_type = + iterator.GetCurrentPropertyConfig().data_type(); + property_info_vector.push_back(std::move(scorable_property_info)); + } + } + scorable_property_schema_cache_.insert( + {schema_type_id, + DerivedScorablePropertySchema(std::move(property_info_vector), + std::move(index_map))}); + return true; +} + +} // namespace lib +} // namespace icing
diff --git a/icing/schema/scorable_property_manager.h b/icing/schema/scorable_property_manager.h new file mode 100644 index 0000000..0ea1057 --- /dev/null +++ b/icing/schema/scorable_property_manager.h
@@ -0,0 +1,126 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_SCHEMA_SCORABLE_PROPERTY_MANAGER_H_ +#define ICING_SCHEMA_SCORABLE_PROPERTY_MANAGER_H_ + +#include <optional> +#include <string> +#include <string_view> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/schema/schema-util.h" +#include "icing/store/document-filter-data.h" + +namespace icing { +namespace lib { + +// This class serves as a cache layer to access schema configs for scorable +// properties, with the schema-store being the source of truth. +class ScorablePropertyManager { + public: + struct ScorablePropertyInfo { + // Scorable property's property_path in the schema config, eg: + // "property_name", "parent_schema.nested_child_schema.property_name". + std::string property_path; + // Data type of the property. + PropertyConfigProto::DataType::Code data_type; + }; + + ScorablePropertyManager() = default; + + // Delete copy constructor and assignment operator. + ScorablePropertyManager(const ScorablePropertyManager&) = delete; + ScorablePropertyManager& operator=(const ScorablePropertyManager&) = delete; + + // Gets the index of the given |property_path|, where the index N means that + // it is the Nth scorable property path in the schema config of the given + // |schema_type_id|, in lexicographical order. + // + // Returns: + // - Index on success + // - std::nullopt if the |property_path| doesnn't point to a scorable + // property under the |schema_type_id| + // - INVALID_ARGUMENT if |schema_type_id| is invalid + libtextclassifier3::StatusOr<std::optional<int>> GetScorablePropertyIndex( + SchemaTypeId schema_type_id, std::string_view property_path, + const SchemaUtil::TypeConfigMap& type_config_map, + const std::unordered_map<SchemaTypeId, std::string>& + schema_id_to_type_map); + + // Returns the list of ScorablePropertyInfo for the given |schema_type_id|, + // in lexicographical order of its property path. + // + // Returns: + // - Vector of scorable property info on success. The vector can be empty + // if no scorable property is found under the schema config of + // |schema_type_id|. + // - INVALID_ARGUMENT if |schema_type_id| is invalid. + libtextclassifier3::StatusOr<const std::vector<ScorablePropertyInfo>*> + GetOrderedScorablePropertyInfo( + SchemaTypeId schema_type_id, + const SchemaUtil::TypeConfigMap& type_config_map, + const std::unordered_map<SchemaTypeId, std::string>& + schema_id_to_type_map); + + private: + struct DerivedScorablePropertySchema { + // vector of ScorablePropertyInfo, in lexicographical order of its property + // path. + std::vector<ScorablePropertyInfo> ordered_scorable_property_info; + // Map of scorable property path to its index, where index N means that the + // property is the Nth scorable property in the schema config, in + // lexicographical order. + std::unordered_map<std::string, int> property_path_to_index_map; + + explicit DerivedScorablePropertySchema( + std::vector<ScorablePropertyInfo>&& ordered_properties, + std::unordered_map<std::string, int>&& index_map) + : ordered_scorable_property_info(std::move(ordered_properties)), + property_path_to_index_map(std::move(index_map)) {} + }; + + // Looks up the entry in |scorable_property_schema_cache_| for the key + // |schema_type_id|. If the entry is not found, try to update the cache and + // return the iterator of the updated entry. + // + // Returns: + // - INVALID_ARGUMENT if |schema_type_id| is not found from the schema. + libtextclassifier3::StatusOr< + std::unordered_map<SchemaTypeId, DerivedScorablePropertySchema>::iterator> + LookupAndMaybeUpdateCache(SchemaTypeId schema_type_id, + const SchemaUtil::TypeConfigMap& type_config_map, + const std::unordered_map<SchemaTypeId, std::string>& + schema_id_to_type_map); + + // Updates the entry in |scorable_property_schema_cache_| for the key + // |schema_type_id|. + // + // Returns true if an entry is successfully inserted to the cache. + bool UpdateCache(SchemaTypeId schema_type_id, + const SchemaUtil::TypeConfigMap& type_config_map, + const std::unordered_map<SchemaTypeId, std::string>& + schema_id_to_type_map); + + std::unordered_map<SchemaTypeId, DerivedScorablePropertySchema> + scorable_property_schema_cache_; +}; + +} // namespace lib +} // namespace icing + +#endif // ICING_SCHEMA_SCORABLE_PROPERTY_MANAGER_H_
diff --git a/icing/schema/scorable_property_manager_test.cc b/icing/schema/scorable_property_manager_test.cc new file mode 100644 index 0000000..0cbc6d8 --- /dev/null +++ b/icing/schema/scorable_property_manager_test.cc
@@ -0,0 +1,404 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/schema/scorable_property_manager.h" + +#include <optional> +#include <string> +#include <string_view> +#include <unordered_map> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/proto/schema.pb.h" +#include "icing/schema-builder.h" +#include "icing/schema/schema-util.h" +#include "icing/store/document-filter-data.h" +#include "icing/testing/common-matchers.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Pointee; + +constexpr int kEmailSchemaTypeId = 1; +constexpr int kMessageSchemaTypeId = 2; +constexpr int kPersonSchemaTypeId = 3; + +SchemaTypeConfigProto CreateEmailTypeConfig() { + return SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("scorableInt64") + .SetDataType(TYPE_INT64) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("scorableDouble") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .Build(); +} + +SchemaTypeConfigProto CreateMessageTypeConfig() { + return SchemaTypeConfigBuilder() + .SetType("message") + .AddProperty(PropertyConfigBuilder() + .SetName("message") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("scorableBoolean") + .SetDataType(TYPE_BOOLEAN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("scorableDouble") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .Build(); +} + +SchemaTypeConfigProto CreatePersonTypeConfig() { + return SchemaTypeConfigBuilder() + .SetType("person") + .AddProperty(PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); +} + +class ScorablePropertyManagerTest : public ::testing::Test { + protected: + void SetUp() override { + type_config_map_.emplace("email", CreateEmailTypeConfig()); + type_config_map_.emplace("message", CreateMessageTypeConfig()); + type_config_map_.emplace("person", CreatePersonTypeConfig()); + schema_id_to_type_map_.emplace(kEmailSchemaTypeId, "email"); + schema_id_to_type_map_.emplace(kMessageSchemaTypeId, "message"); + schema_id_to_type_map_.emplace(kPersonSchemaTypeId, "person"); + } + + SchemaUtil::TypeConfigMap type_config_map_; + std::unordered_map<SchemaTypeId, std::string> schema_id_to_type_map_; +}; + +TEST_F(ScorablePropertyManagerTest, + GetScorablePropertyIndex_InvalidSchemaTypeId) { + ScorablePropertyManager scorable_property_manager; + + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + /*schema_type_id=*/100, "subject", type_config_map_, + schema_id_to_type_map_), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(ScorablePropertyManagerTest, + GetScorablePropertyIndex_InvalidPropertyName) { + ScorablePropertyManager scorable_property_manager; + + // non-existing property + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kEmailSchemaTypeId, "non_existing", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(Eq(std::nullopt))); + + // non-scorable property + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kEmailSchemaTypeId, "subject", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(Eq(std::nullopt))); +} + +TEST_F(ScorablePropertyManagerTest, GetScorablePropertyIndex_Ok) { + ScorablePropertyManager scorable_property_manager; + + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kEmailSchemaTypeId, "scorableDouble", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(0)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kEmailSchemaTypeId, "scorableInt64", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(1)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kMessageSchemaTypeId, "scorableBoolean", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(0)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kMessageSchemaTypeId, "scorableDouble", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(1)); + + // Repeat those calls to test the cache hits scenarios. + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kEmailSchemaTypeId, "scorableDouble", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(0)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kEmailSchemaTypeId, "scorableInt64", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(1)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kMessageSchemaTypeId, "scorableBoolean", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(0)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + kMessageSchemaTypeId, "scorableDouble", type_config_map_, + schema_id_to_type_map_), + IsOkAndHolds(1)); +} + +TEST_F(ScorablePropertyManagerTest, + GetOrderedScorablePropertyInfo_InvalidSchemaTypeId) { + ScorablePropertyManager scorable_property_manager; + + EXPECT_THAT( + scorable_property_manager.GetOrderedScorablePropertyInfo( + /*schema_type_id=*/100, type_config_map_, schema_id_to_type_map_), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(ScorablePropertyManagerTest, + GetOrderedScorablePropertyInfo_NoScorablePropertyInSchema) { + ScorablePropertyManager scorable_property_manager; + + EXPECT_THAT( + scorable_property_manager.GetOrderedScorablePropertyInfo( + kPersonSchemaTypeId, type_config_map_, schema_id_to_type_map_), + IsOkAndHolds(Pointee(ElementsAre()))); + + // Repeat the call to test the cache hits scenarios. + EXPECT_THAT( + scorable_property_manager.GetOrderedScorablePropertyInfo( + kPersonSchemaTypeId, type_config_map_, schema_id_to_type_map_), + IsOkAndHolds(Pointee(ElementsAre()))); +} + +TEST_F(ScorablePropertyManagerTest, GetOrderedScorablePropertyInfo_Ok) { + ScorablePropertyManager scorable_property_manager; + + EXPECT_THAT(scorable_property_manager.GetOrderedScorablePropertyInfo( + kEmailSchemaTypeId, type_config_map_, schema_id_to_type_map_), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("scorableDouble", TYPE_DOUBLE), + EqualsScorablePropertyInfo("scorableInt64", TYPE_INT64))))); + EXPECT_THAT( + scorable_property_manager.GetOrderedScorablePropertyInfo( + kMessageSchemaTypeId, type_config_map_, schema_id_to_type_map_), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("scorableBoolean", TYPE_BOOLEAN), + EqualsScorablePropertyInfo("scorableDouble", TYPE_DOUBLE))))); + + // Repeat those calls to test the cache hits scenarios. + EXPECT_THAT(scorable_property_manager.GetOrderedScorablePropertyInfo( + kEmailSchemaTypeId, type_config_map_, schema_id_to_type_map_), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("scorableDouble", TYPE_DOUBLE), + EqualsScorablePropertyInfo("scorableInt64", TYPE_INT64))))); + EXPECT_THAT( + scorable_property_manager.GetOrderedScorablePropertyInfo( + kMessageSchemaTypeId, type_config_map_, schema_id_to_type_map_), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("scorableBoolean", TYPE_BOOLEAN), + EqualsScorablePropertyInfo("scorableDouble", TYPE_DOUBLE))))); +} + +TEST_F(ScorablePropertyManagerTest, + GetOrderedScorablePropertyInfo_WithNestedSchemas) { + std::string person_schema_name = "Person"; + std::string gmail_schema_name = "Gmail"; + std::string interaction_log_schema_name = "InteractionLog"; + + SchemaTypeConfigProto person_schema_config = + SchemaTypeConfigBuilder() + .SetType(person_schema_name) + .AddProperty(PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("networth") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + SchemaTypeConfigProto gmail_schema_config = + SchemaTypeConfigBuilder() + .SetType(gmail_schema_name) + .AddProperty( + PropertyConfigBuilder().SetName("subject").SetDataTypeString( + TERM_MATCH_UNKNOWN, TOKENIZER_NONE)) + .AddProperty( + PropertyConfigBuilder().SetName("sender").SetDataTypeDocument( + person_schema_name, + /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("recipient") + .SetDataTypeDocument(person_schema_name, + /*index_nested_properties=*/true)) + .Build(); + SchemaTypeConfigProto interaction_log_schema_config = + SchemaTypeConfigBuilder() + .SetType(interaction_log_schema_name) + .AddProperty( + PropertyConfigBuilder().SetName("summary").SetDataTypeString( + TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("contactGmail") + .SetDataTypeDocument(gmail_schema_name, + /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("mostContactedPerson") + .SetDataTypeDocument(person_schema_name, + /*index_nested_properties=*/true)) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {person_schema_name, person_schema_config}, + {gmail_schema_name, gmail_schema_config}, + {interaction_log_schema_name, interaction_log_schema_config}}; + std::unordered_map<SchemaTypeId, std::string> schema_id_to_type_map = { + {1, person_schema_name}, + {2, gmail_schema_name}, + {3, interaction_log_schema_name}}; + ScorablePropertyManager scorable_property_manager; + + EXPECT_THAT(scorable_property_manager.GetOrderedScorablePropertyInfo( + /*schema_type_id=*/1, type_config_map, schema_id_to_type_map), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("networth", TYPE_DOUBLE))))); + EXPECT_THAT( + scorable_property_manager.GetOrderedScorablePropertyInfo( + /*schema_type_id=*/2, type_config_map, schema_id_to_type_map), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("recipient.networth", TYPE_DOUBLE), + EqualsScorablePropertyInfo("sender.networth", TYPE_DOUBLE))))); + EXPECT_THAT(scorable_property_manager.GetOrderedScorablePropertyInfo( + /*schema_type_id=*/3, type_config_map, schema_id_to_type_map), + IsOkAndHolds(Pointee(ElementsAre( + EqualsScorablePropertyInfo("contactGmail.recipient.networth", + TYPE_DOUBLE), + EqualsScorablePropertyInfo("contactGmail.sender.networth", + TYPE_DOUBLE), + EqualsScorablePropertyInfo("mostContactedPerson.networth", + TYPE_DOUBLE))))); +} + +TEST_F(ScorablePropertyManagerTest, + GetScorablePropertyIndex_WithNestedSchemas) { + std::string person_schema_name = "Person"; + std::string gmail_schema_name = "Gmail"; + std::string interaction_log_schema_name = "InteractionLog"; + + SchemaTypeConfigProto person_schema_config = + SchemaTypeConfigBuilder() + .SetType(person_schema_name) + .AddProperty(PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("networth") + .SetDataType(TYPE_DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + SchemaTypeConfigProto gmail_schema_config = + SchemaTypeConfigBuilder() + .SetType(gmail_schema_name) + .AddProperty( + PropertyConfigBuilder().SetName("subject").SetDataTypeString( + TERM_MATCH_UNKNOWN, TOKENIZER_NONE)) + .AddProperty( + PropertyConfigBuilder().SetName("sender").SetDataTypeDocument( + person_schema_name, + /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("recipient") + .SetDataTypeDocument(person_schema_name, + /*index_nested_properties=*/true)) + .Build(); + SchemaTypeConfigProto interaction_log_schema_config = + SchemaTypeConfigBuilder() + .SetType(interaction_log_schema_name) + .AddProperty( + PropertyConfigBuilder().SetName("summary").SetDataTypeString( + TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("contactGmail") + .SetDataTypeDocument(gmail_schema_name, + /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("mostContactedPerson") + .SetDataTypeDocument(person_schema_name, + /*index_nested_properties=*/true)) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {person_schema_name, person_schema_config}, + {gmail_schema_name, gmail_schema_config}, + {interaction_log_schema_name, interaction_log_schema_config}}; + std::unordered_map<SchemaTypeId, std::string> schema_id_to_type_map = { + {1, person_schema_name}, + {2, gmail_schema_name}, + {3, interaction_log_schema_name}}; + ScorablePropertyManager scorable_property_manager; + + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + /*schema_type_id=*/1, "networth", type_config_map, + schema_id_to_type_map), + IsOkAndHolds(0)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + /*schema_type_id=*/2, "recipient.networth", type_config_map, + schema_id_to_type_map), + IsOkAndHolds(0)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + /*schema_type_id=*/2, "sender.networth", type_config_map, + schema_id_to_type_map), + IsOkAndHolds(1)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + /*schema_type_id=*/3, "contactGmail.recipient.networth", + type_config_map, schema_id_to_type_map), + IsOkAndHolds(0)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + /*schema_type_id=*/3, "contactGmail.sender.networth", + type_config_map, schema_id_to_type_map), + IsOkAndHolds(1)); + EXPECT_THAT(scorable_property_manager.GetScorablePropertyIndex( + /*schema_type_id=*/3, "mostContactedPerson.networth", + type_config_map, schema_id_to_type_map), + IsOkAndHolds(2)); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/schema/section-manager.cc b/icing/schema/section-manager.cc index 8689bf2..7097a17 100644 --- a/icing/schema/section-manager.cc +++ b/icing/schema/section-manager.cc
@@ -62,6 +62,7 @@ property_config.string_indexing_config().term_match_type(), property_config.integer_indexing_config().numeric_match_type(), property_config.embedding_indexing_config().embedding_indexing_type(), + property_config.embedding_indexing_config().quantization_type(), std::move(concatenated_path))); return libtextclassifier3::Status::OK; }
diff --git a/icing/schema/section-manager_test.cc b/icing/schema/section-manager_test.cc index b735fb1..62c2528 100644 --- a/icing/schema/section-manager_test.cc +++ b/icing/schema/section-manager_test.cc
@@ -55,6 +55,8 @@ static constexpr std::string_view kPropertySubject = "subject"; static constexpr std::string_view kPropertySubjectEmbedding = "subjectEmbedding"; +static constexpr std::string_view kPropertySubjectEmbeddingQuantized = + "subjectEmbeddingQuantized"; static constexpr std::string_view kPropertyTimestamp = "timestamp"; // non-indexable static constexpr std::string_view kPropertyAttachment = "attachment"; @@ -124,6 +126,15 @@ .Build(); } +PropertyConfigProto CreateSubjectEmbeddingQuantizedPropertyConfig() { + return PropertyConfigBuilder() + .SetName(kPropertySubjectEmbeddingQuantized) + .SetDataTypeVector(EMBEDDING_INDEXING_LINEAR_SEARCH, + QUANTIZATION_TYPE_QUANTIZE_8_BIT) + .SetCardinality(CARDINALITY_OPTIONAL) + .Build(); +} + PropertyConfigProto CreateTimestampPropertyConfig() { return PropertyConfigBuilder() .SetName(kPropertyTimestamp) @@ -161,6 +172,7 @@ .SetType(kTypeEmail) .AddProperty(CreateSubjectPropertyConfig()) .AddProperty(CreateSubjectEmbeddingPropertyConfig()) + .AddProperty(CreateSubjectEmbeddingQuantizedPropertyConfig()) .AddProperty(PropertyConfigBuilder() .SetName(kPropertyText) .SetDataTypeString(TERM_MATCH_UNKNOWN, TOKENIZER_NONE) @@ -249,6 +261,8 @@ .AddStringProperty(std::string(kPropertySubject), "the subject") .AddVectorProperty(std::string(kPropertySubjectEmbedding), email_subject_embedding_) + .AddVectorProperty(std::string(kPropertySubjectEmbeddingQuantized), + email_subject_embedding_) .AddStringProperty(std::string(kPropertyText), "the text") .AddBytesProperty(std::string(kPropertyAttachment), "attachment bytes") @@ -333,14 +347,14 @@ EXPECT_THAT(section_group.integer_sections[0].content, ElementsAre(1, 2, 3)); EXPECT_THAT(section_group.integer_sections[1].metadata, - EqualsSectionMetadata(/*expected_id=*/4, + EqualsSectionMetadata(/*expected_id=*/5, /*expected_property_path=*/"timestamp", CreateTimestampPropertyConfig())); EXPECT_THAT(section_group.integer_sections[1].content, ElementsAre(kDefaultTimestamp)); // Vector sections - EXPECT_THAT(section_group.vector_sections, SizeIs(1)); + EXPECT_THAT(section_group.vector_sections, SizeIs(2)); EXPECT_THAT( section_group.vector_sections[0].metadata, EqualsSectionMetadata(/*expected_id=*/3, @@ -348,6 +362,13 @@ CreateSubjectEmbeddingPropertyConfig())); EXPECT_THAT(section_group.vector_sections[0].content, ElementsAre(EqualsProto(email_subject_embedding_))); + EXPECT_THAT(section_group.vector_sections[1].metadata, + EqualsSectionMetadata( + /*expected_id=*/4, + /*expected_property_path=*/"subjectEmbeddingQuantized", + CreateSubjectEmbeddingQuantizedPropertyConfig())); + EXPECT_THAT(section_group.vector_sections[1].content, + ElementsAre(EqualsProto(email_subject_embedding_))); } TEST_F(SectionManagerTest, ExtractSectionsNested) { @@ -394,14 +415,14 @@ EXPECT_THAT( section_group.integer_sections[1].metadata, - EqualsSectionMetadata(/*expected_id=*/4, + EqualsSectionMetadata(/*expected_id=*/5, /*expected_property_path=*/"emails.timestamp", CreateTimestampPropertyConfig())); EXPECT_THAT(section_group.integer_sections[1].content, ElementsAre(kDefaultTimestamp, kDefaultTimestamp)); // Vector sections - EXPECT_THAT(section_group.vector_sections, SizeIs(1)); + EXPECT_THAT(section_group.vector_sections, SizeIs(2)); EXPECT_THAT(section_group.vector_sections[0].metadata, EqualsSectionMetadata( /*expected_id=*/3, @@ -410,6 +431,14 @@ EXPECT_THAT(section_group.vector_sections[0].content, ElementsAre(EqualsProto(email_subject_embedding_), EqualsProto(email_subject_embedding_))); + EXPECT_THAT(section_group.vector_sections[1].metadata, + EqualsSectionMetadata( + /*expected_id=*/4, + /*expected_property_path=*/"emails.subjectEmbeddingQuantized", + CreateSubjectEmbeddingQuantizedPropertyConfig())); + EXPECT_THAT(section_group.vector_sections[1].content, + ElementsAre(EqualsProto(email_subject_embedding_), + EqualsProto(email_subject_embedding_))); } TEST_F(SectionManagerTest, ExtractSectionsIndexableNestedPropertiesList) { @@ -508,7 +537,8 @@ // 1 -> recipients // 2 -> subject // 3 -> subjectEmbedding - // 4 -> timestamp + // 4 -> subjectEmbeddingQuantized + // 5 -> timestamp EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( /*schema_type_id=*/0, /*section_id=*/0), IsOkAndHolds(Pointee(EqualsSectionMetadata( @@ -533,7 +563,13 @@ EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( /*schema_type_id=*/0, /*section_id=*/4), IsOkAndHolds(Pointee(EqualsSectionMetadata( - /*expected_id=*/4, /*expected_property_path=*/"timestamp", + /*expected_id=*/4, + /*expected_property_path=*/"subjectEmbeddingQuantized", + CreateSubjectEmbeddingQuantizedPropertyConfig())))); + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/0, /*section_id=*/5), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/5, /*expected_property_path=*/"timestamp", CreateTimestampPropertyConfig())))); // Conversation (section id -> section property path): @@ -541,8 +577,9 @@ // 1 -> emails.recipients // 2 -> emails.subject // 3 -> emails.subjectEmbedding - // 4 -> emails.timestamp - // 5 -> name + // 4 -> emails.subjectEmbeddingQuantized + // 5 -> emails.timestamp + // 6 -> name EXPECT_THAT( schema_type_manager->section_manager().GetSectionMetadata( /*schema_type_id=*/1, /*section_id=*/0), @@ -567,16 +604,22 @@ /*expected_id=*/3, /*expected_property_path=*/"emails.subjectEmbedding", CreateSubjectEmbeddingPropertyConfig())))); + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/1, /*section_id=*/4), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/4, + /*expected_property_path=*/"emails.subjectEmbeddingQuantized", + CreateSubjectEmbeddingQuantizedPropertyConfig())))); EXPECT_THAT( schema_type_manager->section_manager().GetSectionMetadata( - /*schema_type_id=*/1, /*section_id=*/4), + /*schema_type_id=*/1, /*section_id=*/5), IsOkAndHolds(Pointee(EqualsSectionMetadata( - /*expected_id=*/4, /*expected_property_path=*/"emails.timestamp", + /*expected_id=*/5, /*expected_property_path=*/"emails.timestamp", CreateTimestampPropertyConfig())))); EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( - /*schema_type_id=*/1, /*section_id=*/5), + /*schema_type_id=*/1, /*section_id=*/6), IsOkAndHolds(Pointee(EqualsSectionMetadata( - /*expected_id=*/5, /*expected_property_path=*/"name", + /*expected_id=*/6, /*expected_property_path=*/"name", CreateNamePropertyConfig())))); // Group (section id -> section property path): @@ -686,12 +729,13 @@ // 1 -> recipients // 2 -> subject // 3 -> subjectEmbedding - // 4 -> timestamp + // 4 -> subjectEmbeddingQuantized + // 5 -> timestamp EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( /*schema_type_id=*/0, /*section_id=*/-1), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( - /*schema_type_id=*/0, /*section_id=*/5), + /*schema_type_id=*/0, /*section_id=*/6), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Conversation (section id -> section property path): @@ -699,13 +743,14 @@ // 1 -> emails.recipients // 2 -> emails.subject // 3 -> emails.subjectEmbedding - // 4 -> emails.timestamp - // 5 -> name + // 4 -> subjectEmbeddingQuantized + // 5 -> emails.timestamp + // 6 -> name EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( /*schema_type_id=*/1, /*section_id=*/-1), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( - /*schema_type_id=*/1, /*section_id=*/6), + /*schema_type_id=*/1, /*section_id=*/7), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); }
diff --git a/icing/schema/section.h b/icing/schema/section.h index 76e9e57..c866351 100644 --- a/icing/schema/section.h +++ b/icing/schema/section.h
@@ -100,6 +100,15 @@ // Contents will be indexed for linear search. EmbeddingIndexingConfig::EmbeddingIndexingType::Code embedding_indexing_type; + // How vectors in a vector section should be quantized. + // + // QuantizationType::NONE: + // Contents will not be quantized. + // + // QuantizationType::QUANTIZE_8_BIT: + // Contents will be quantized to 8 bits. + EmbeddingIndexingConfig::QuantizationType::Code quantization_type; + explicit SectionMetadata( SectionId id_in, PropertyConfigProto::DataType::Code data_type_in, StringIndexingConfig::TokenizerType::Code tokenizer, @@ -107,6 +116,7 @@ IntegerIndexingConfig::NumericMatchType::Code numeric_match_type_in, EmbeddingIndexingConfig::EmbeddingIndexingType::Code embedding_indexing_type_in, + EmbeddingIndexingConfig::QuantizationType::Code quantization_type_in, std::string&& path_in) : path(std::move(path_in)), id(id_in), @@ -114,7 +124,8 @@ tokenizer(tokenizer), term_match_type(term_match_type_in), numeric_match_type(numeric_match_type_in), - embedding_indexing_type(embedding_indexing_type_in) {} + embedding_indexing_type(embedding_indexing_type_in), + quantization_type(quantization_type_in) {} SectionMetadata(const SectionMetadata& other) = default; SectionMetadata& operator=(const SectionMetadata& other) = default; @@ -126,7 +137,9 @@ return path == rhs.path && id == rhs.id && data_type == rhs.data_type && tokenizer == rhs.tokenizer && term_match_type == rhs.term_match_type && - numeric_match_type == rhs.numeric_match_type; + numeric_match_type == rhs.numeric_match_type && + embedding_indexing_type == rhs.embedding_indexing_type && + quantization_type == rhs.quantization_type; } };
diff --git a/icing/scoring/advanced_scoring/advanced-scorer.cc b/icing/scoring/advanced_scoring/advanced-scorer.cc index 68b6971..d2029c8 100644 --- a/icing/scoring/advanced_scoring/advanced-scorer.cc +++ b/icing/scoring/advanced_scoring/advanced-scorer.cc
@@ -16,20 +16,19 @@ #include <cstdint> #include <memory> -#include <string_view> +#include <string> +#include <unordered_set> #include <utility> #include <vector> #include "icing/text_classifier/lib3/utils/base/statusor.h" -#include "icing/absl_ports/canonical_errors.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-query-results.h" #include "icing/join/join-children-fetcher.h" -#include "icing/query/advanced_query_parser/abstract-syntax-tree.h" -#include "icing/query/advanced_query_parser/lexer.h" -#include "icing/query/advanced_query_parser/parser.h" +#include "icing/proto/scoring.pb.h" #include "icing/schema/schema-store.h" +#include "icing/scoring/advanced_scoring/score-expression-util.h" #include "icing/scoring/advanced_scoring/score-expression.h" -#include "icing/scoring/advanced_scoring/scoring-visitor.h" #include "icing/scoring/bm25f-calculator.h" #include "icing/scoring/section-weights.h" #include "icing/store/document-store.h" @@ -40,35 +39,17 @@ namespace { -libtextclassifier3::StatusOr<std::unique_ptr<ScoreExpression>> -GetScoreExpression(std::string_view scoring_expression, double default_score, - SearchSpecProto::EmbeddingQueryMetricType::Code - default_semantic_metric_type, - const DocumentStore* document_store, - const SchemaStore* schema_store, int64_t current_time_ms, - const JoinChildrenFetcher* join_children_fetcher, - const EmbeddingQueryResults* embedding_query_results, - SectionWeights* section_weights, - Bm25fCalculator* bm25f_calculator) { - Lexer lexer(scoring_expression, Lexer::Language::SCORING); - ICING_ASSIGN_OR_RETURN(std::vector<Lexer::LexerToken> lexer_tokens, - std::move(lexer).ExtractTokens()); - Parser parser = Parser::Create(std::move(lexer_tokens)); - ICING_ASSIGN_OR_RETURN(std::unique_ptr<Node> tree_root, - parser.ConsumeScoring()); - ScoringVisitor visitor(default_score, default_semantic_metric_type, - document_store, schema_store, section_weights, - bm25f_calculator, join_children_fetcher, - embedding_query_results, current_time_ms); - tree_root->Accept(&visitor); - - ICING_ASSIGN_OR_RETURN(std::unique_ptr<ScoreExpression> expression, - std::move(visitor).Expression()); - if (expression->type() != ScoreExpressionType::kDouble) { - return absl_ports::InvalidArgumentError( - "The root scoring expression is not of double type."); +std::unique_ptr<SchemaTypeAliasMap> BuildSchemaTypeAliasMap( + const ScoringSpecProto& scoring_spec) { + std::unique_ptr<SchemaTypeAliasMap> schema_type_alias_map = + std::make_unique<SchemaTypeAliasMap>(); + for (const SchemaTypeAliasMapProto& alias_map_proto : + scoring_spec.schema_type_alias_map_protos()) { + (*schema_type_alias_map)[alias_map_proto.alias_schema_type()] = + std::unordered_set<std::string>(alias_map_proto.schema_types().begin(), + alias_map_proto.schema_types().end()); } - return expression; + return schema_type_alias_map; } } // namespace @@ -81,24 +62,33 @@ const DocumentStore* document_store, const SchemaStore* schema_store, int64_t current_time_ms, const JoinChildrenFetcher* join_children_fetcher, - const EmbeddingQueryResults* embedding_query_results) { + const EmbeddingQueryResults* embedding_query_results, + const FeatureFlags* feature_flags) { ICING_RETURN_ERROR_IF_NULL(document_store); ICING_RETURN_ERROR_IF_NULL(schema_store); ICING_RETURN_ERROR_IF_NULL(embedding_query_results); + ICING_RETURN_ERROR_IF_NULL(feature_flags); ICING_ASSIGN_OR_RETURN(std::unique_ptr<SectionWeights> section_weights, SectionWeights::Create(schema_store, scoring_spec)); std::unique_ptr<Bm25fCalculator> bm25f_calculator = std::make_unique<Bm25fCalculator>(document_store, section_weights.get(), current_time_ms); + std::unique_ptr<SchemaTypeAliasMap> schema_type_alias_map = + BuildSchemaTypeAliasMap(scoring_spec); + std::unique_ptr<std::unordered_set<ScoringFeatureType>> + scoring_feature_types_enabled = + score_expression_util::GetEnabledScoringFeatureTypes(scoring_spec); ICING_ASSIGN_OR_RETURN( std::unique_ptr<ScoreExpression> score_expression, - GetScoreExpression(scoring_spec.advanced_scoring_expression(), - default_score, default_semantic_metric_type, - document_store, schema_store, current_time_ms, - join_children_fetcher, embedding_query_results, - section_weights.get(), bm25f_calculator.get())); + score_expression_util::GetScoreExpression( + scoring_spec.advanced_scoring_expression(), default_score, + default_semantic_metric_type, document_store, schema_store, + current_time_ms, join_children_fetcher, embedding_query_results, + section_weights.get(), bm25f_calculator.get(), + schema_type_alias_map.get(), feature_flags, + scoring_feature_types_enabled.get())); std::vector<std::unique_ptr<ScoreExpression>> additional_score_expressions; additional_score_expressions.reserve( @@ -107,17 +97,22 @@ scoring_spec.additional_advanced_scoring_expressions()) { ICING_ASSIGN_OR_RETURN( std::unique_ptr<ScoreExpression> additional_score_expression, - GetScoreExpression(additional_expression, default_score, - default_semantic_metric_type, document_store, - schema_store, current_time_ms, join_children_fetcher, - embedding_query_results, section_weights.get(), - bm25f_calculator.get())); + score_expression_util::GetScoreExpression( + additional_expression, default_score, default_semantic_metric_type, + document_store, schema_store, current_time_ms, + join_children_fetcher, embedding_query_results, + section_weights.get(), bm25f_calculator.get(), + schema_type_alias_map.get(), feature_flags, + scoring_feature_types_enabled.get())); + additional_score_expressions.push_back( std::move(additional_score_expression)); } return std::unique_ptr<AdvancedScorer>(new AdvancedScorer( std::move(score_expression), std::move(additional_score_expressions), - std::move(section_weights), std::move(bm25f_calculator), default_score)); + std::move(section_weights), std::move(bm25f_calculator), + std::move(schema_type_alias_map), + std::move(scoring_feature_types_enabled), default_score)); } } // namespace lib
diff --git a/icing/scoring/advanced_scoring/advanced-scorer.h b/icing/scoring/advanced_scoring/advanced-scorer.h index 5fc5c48..eeb39c4 100644 --- a/icing/scoring/advanced_scoring/advanced-scorer.h +++ b/icing/scoring/advanced_scoring/advanced-scorer.h
@@ -19,15 +19,18 @@ #include <memory> #include <string> #include <unordered_map> +#include <unordered_set> #include <utility> #include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-query-results.h" #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/join/join-children-fetcher.h" +#include "icing/proto/scoring.pb.h" #include "icing/schema/schema-store.h" #include "icing/scoring/advanced_scoring/score-expression.h" #include "icing/scoring/bm25f-calculator.h" @@ -51,7 +54,8 @@ default_semantic_metric_type, const DocumentStore* document_store, const SchemaStore* schema_store, int64_t current_time_ms, const JoinChildrenFetcher* join_children_fetcher, - const EmbeddingQueryResults* embedding_query_results); + const EmbeddingQueryResults* embedding_query_results, + const FeatureFlags* feature_flags); double GetScore(const DocHitInfo& hit_info, const DocHitInfoIterator* query_it) override { @@ -82,16 +86,23 @@ bool is_constant() const { return score_expression_->is_constant(); } private: - explicit AdvancedScorer(std::unique_ptr<ScoreExpression> score_expression, - std::vector<std::unique_ptr<ScoreExpression>> - additional_score_expressions, - std::unique_ptr<SectionWeights> section_weights, - std::unique_ptr<Bm25fCalculator> bm25f_calculator, - double default_score) + explicit AdvancedScorer( + std::unique_ptr<ScoreExpression> score_expression, + std::vector<std::unique_ptr<ScoreExpression>> + additional_score_expressions, + std::unique_ptr<SectionWeights> section_weights, + std::unique_ptr<Bm25fCalculator> bm25f_calculator, + std::unique_ptr<SchemaTypeAliasMap> alias_schema_type_map, + std::unique_ptr<std::unordered_set<ScoringFeatureType>> + scoring_feature_types_enabled, + double default_score) : score_expression_(std::move(score_expression)), additional_score_expressions_(std::move(additional_score_expressions)), section_weights_(std::move(section_weights)), bm25f_calculator_(std::move(bm25f_calculator)), + alias_schema_type_map_(std::move(alias_schema_type_map)), + scoring_feature_types_enabled_( + std::move(scoring_feature_types_enabled)), default_score_(default_score) { if (is_constant()) { ICING_LOG(WARNING) @@ -118,6 +129,9 @@ std::vector<std::unique_ptr<ScoreExpression>> additional_score_expressions_; std::unique_ptr<SectionWeights> section_weights_; std::unique_ptr<Bm25fCalculator> bm25f_calculator_; + std::unique_ptr<SchemaTypeAliasMap> alias_schema_type_map_; + std::unique_ptr<std::unordered_set<ScoringFeatureType>> + scoring_feature_types_enabled_; double default_score_; };
diff --git a/icing/scoring/advanced_scoring/advanced-scorer_fuzz_test.cc b/icing/scoring/advanced_scoring/advanced-scorer_fuzz_test.cc index 9de9e8a..765ccdb 100644 --- a/icing/scoring/advanced_scoring/advanced-scorer_fuzz_test.cc +++ b/icing/scoring/advanced_scoring/advanced-scorer_fuzz_test.cc
@@ -18,6 +18,7 @@ #include <string> #include <string_view> +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-query-results.h" @@ -25,12 +26,14 @@ #include "icing/scoring/advanced_scoring/advanced-scorer.h" #include "icing/store/document-store.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" namespace icing { namespace lib { extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + FeatureFlags feature_flags = GetTestFeatureFlags(); FakeClock fake_clock; Filesystem filesystem; const std::string test_dir = GetTestTempDir() + "/icing"; @@ -42,15 +45,16 @@ filesystem.CreateDirectoryRecursively(schema_store_dir.c_str()); std::unique_ptr<SchemaStore> schema_store = - SchemaStore::Create(&filesystem, schema_store_dir, &fake_clock) + SchemaStore::Create(&filesystem, schema_store_dir, &fake_clock, + &feature_flags) .ValueOrDie(); std::unique_ptr<DocumentStore> document_store = DocumentStore::Create( &filesystem, doc_store_dir, &fake_clock, schema_store.get(), + &feature_flags, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr) .ValueOrDie() .document_store; @@ -67,7 +71,7 @@ document_store.get(), schema_store.get(), fake_clock.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_); + &empty_embedding_query_results_, &feature_flags); // Not able to test the GetScore method of AdvancedScorer, since it will only // be available after AdvancedScorer is successfully created. However, the
diff --git a/icing/scoring/advanced_scoring/advanced-scorer_test.cc b/icing/scoring/advanced_scoring/advanced-scorer_test.cc index 8841d2c..ccd10c6 100644 --- a/icing/scoring/advanced_scoring/advanced-scorer_test.cc +++ b/icing/scoring/advanced_scoring/advanced-scorer_test.cc
@@ -14,6 +14,7 @@ #include "icing/scoring/advanced_scoring/advanced-scorer.h" +#include <algorithm> #include <cmath> #include <cstdint> #include <memory> @@ -28,11 +29,12 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-query-results.h" #include "icing/index/hit/doc-hit-info.h" -#include "icing/join/join-children-fetcher.h" +#include "icing/join/join-children-fetcher-impl-deprecated.h" #include "icing/proto/document.pb.h" #include "icing/proto/schema.pb.h" #include "icing/proto/scoring.pb.h" @@ -47,6 +49,7 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" namespace icing { @@ -66,28 +69,29 @@ schema_store_dir_(test_dir_ + "/schema_store") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(doc_store_dir_.c_str()); filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, doc_store_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); - // Creates a simple email schema - SchemaProto test_email_schema = + // Creates the schema + SchemaProto test_schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("email").AddProperty( PropertyConfigBuilder() @@ -96,34 +100,74 @@ TermMatchType::PREFIX, StringIndexingConfig::TokenizerType::PLAIN) .SetCardinality(CARDINALITY_OPTIONAL))) - .AddType(SchemaTypeConfigBuilder() - .SetType("person") - .AddProperty( - PropertyConfigBuilder() - .SetName("emailAddress") - .SetDataTypeString( - TermMatchType::PREFIX, - StringIndexingConfig::TokenizerType::PLAIN) - .SetCardinality(CARDINALITY_OPTIONAL)) - .AddProperty( - PropertyConfigBuilder() - .SetName("name") - .SetDataTypeString( - TermMatchType::PREFIX, - StringIndexingConfig::TokenizerType::PLAIN) - .SetCardinality(CARDINALITY_OPTIONAL)) - - .AddProperty( - PropertyConfigBuilder() - .SetName("phoneNumber") - .SetDataTypeString( - TermMatchType::PREFIX, - StringIndexingConfig::TokenizerType::PLAIN) - .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("message") + .AddProperty( + PropertyConfigBuilder() + .SetName("frequencyScore") + .SetDataType(PropertyConfigProto::DataType::INT64) + .SetCardinality(CARDINALITY_REPEATED) + .SetScorableType(SCORABLE_TYPE_ENABLED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("content") + .SetDataTypeString( + TermMatchType::PREFIX, + StringIndexingConfig::TokenizerType::PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("person") + .AddProperty( + PropertyConfigBuilder() + .SetName("emailAddress") + .SetDataTypeString( + TermMatchType::PREFIX, + StringIndexingConfig::TokenizerType::PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString( + TermMatchType::PREFIX, + StringIndexingConfig::TokenizerType::PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("frequencyScore") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_REPEATED) + .SetScorableType(SCORABLE_TYPE_ENABLED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("customizedScore") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("contactTimes") + .SetDataType(PropertyConfigProto::DataType::INT64) + .SetCardinality(CARDINALITY_REPEATED) + .SetScorableType(SCORABLE_TYPE_ENABLED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("isStarred") + .SetDataType(PropertyConfigProto::DataType::BOOLEAN) + .SetCardinality(CARDINALITY_OPTIONAL) + .SetScorableType(SCORABLE_TYPE_ENABLED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("phoneNumber") + .SetDataTypeString( + TermMatchType::PREFIX, + StringIndexingConfig::TokenizerType::PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); ICING_ASSERT_OK(schema_store_->SetSchema( - test_email_schema, /*ignore_errors_and_delete_documents=*/false, + test_schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); } @@ -133,6 +177,7 @@ filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); } + std::unique_ptr<FeatureFlags> feature_flags_; const std::string test_dir_; const std::string doc_store_dir_; const std::string schema_store_dir_; @@ -182,6 +227,16 @@ return scoring_spec; } +void AddSchemaTypeAliasMap(ScoringSpecProto* scoring_spec, + const std::string& alias_schema_type, + const std::vector<std::string>& schema_types) { + SchemaTypeAliasMapProto* alias_map_proto = + scoring_spec->add_schema_type_alias_map_protos(); + alias_map_proto->set_alias_schema_type(alias_schema_type); + alias_map_proto->mutable_schema_types()->Add(schema_types.begin(), + schema_types.end()); +} + PropertyWeight CreatePropertyWeight(std::string path, double weight) { PropertyWeight property_weight; property_weight.set_path(std::move(path)); @@ -208,24 +263,24 @@ ScoringSpecProto scoring_spec; scoring_spec.set_rank_by( ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION); - EXPECT_THAT(scorer_factory::Create(scoring_spec, /*default_score=*/10, - kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(scorer_factory::Create( + scoring_spec, /*default_score=*/10, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // Non-empty scoring expression for normal scoring scoring_spec = ScoringSpecProto::default_instance(); scoring_spec.set_rank_by(ScoringSpecProto::RankingStrategy::DOCUMENT_SCORE); scoring_spec.set_advanced_scoring_expression("1"); - EXPECT_THAT(scorer_factory::Create(scoring_spec, /*default_score=*/10, - kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(scorer_factory::Create( + scoring_spec, /*default_score=*/10, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -242,7 +297,8 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); @@ -263,57 +319,58 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(3)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(CreateAdvancedScoringSpec("-1 + 2"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("-1 + 2"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(1)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(CreateAdvancedScoringSpec("1 + -2"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("1 + -2"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(-1)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(CreateAdvancedScoringSpec("1 - 2"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("1 - 2"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(-1)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(CreateAdvancedScoringSpec("1 * 2"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("1 * 2"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(2)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(CreateAdvancedScoringSpec("1 / 2"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("1 / 2"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(0.5)); } @@ -331,127 +388,128 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(3, kEps)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("log(2.718281828459045)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("log(2.718281828459045)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(1, kEps)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(CreateAdvancedScoringSpec("pow(2, 10)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("pow(2, 10)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(1024)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("max(10, 11, 12, 13, 14)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("max(10, 11, 12, 13, 14)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(14)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("min(10, 11, 12, 13, 14)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("min(10, 11, 12, 13, 14)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(10)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("len(10, 11, 12, 13, 14)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("len(10, 11, 12, 13, 14)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(5)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("sum(10, 11, 12, 13, 14)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("sum(10, 11, 12, 13, 14)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(10 + 11 + 12 + 13 + 14)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("avg(10, 11, 12, 13, 14)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("avg(10, 11, 12, 13, 14)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq((10 + 11 + 12 + 13 + 14) / 5.)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(CreateAdvancedScoringSpec("sqrt(2)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("sqrt(2)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(sqrt(2), kEps)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(CreateAdvancedScoringSpec("abs(-2) + abs(2)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("abs(-2) + abs(2)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(4)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("sin(3.141592653589793)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("sin(3.141592653589793)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(0, kEps)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("cos(3.141592653589793)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("cos(3.141592653589793)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(-1, kEps)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("tan(3.141592653589793 / 4)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("tan(3.141592653589793 / 4)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(1, kEps)); } @@ -471,28 +529,29 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(123)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("this.creationTimestamp()"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("this.creationTimestamp()"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(kDefaultCreationTimestampMs)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec( - "this.documentScore() + this.creationTimestamp()"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec( + "this.documentScore() + this.creationTimestamp()"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(123 + kDefaultCreationTimestampMs)); } @@ -512,7 +571,8 @@ /*default_score=*/10, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(0)); ICING_ASSERT_OK(document_store_->ReportUsage( CreateUsageReport("namespace", "uri", 100000, UsageReport::USAGE_TYPE1))); @@ -525,31 +585,31 @@ EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(300002)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("this.usageLastUsedTimestamp(1)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("this.usageLastUsedTimestamp(1)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(100000)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("this.usageLastUsedTimestamp(2)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("this.usageLastUsedTimestamp(2)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(200000)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec("this.usageLastUsedTimestamp(3)"), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec("this.usageLastUsedTimestamp(3)"), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(300000)); } @@ -571,7 +631,8 @@ CreateAdvancedScoringSpec("this.usageCount(4)"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(default_score)); ICING_ASSERT_OK_AND_ASSIGN( @@ -580,7 +641,8 @@ CreateAdvancedScoringSpec("this.usageCount(0)"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(default_score)); ICING_ASSERT_OK_AND_ASSIGN( @@ -589,7 +651,8 @@ CreateAdvancedScoringSpec("this.usageCount(1.5)"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(default_score)); } @@ -615,7 +678,8 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); scorer->PrepareToScore(/*query_term_iterators=*/{}); // Should get the default score. @@ -660,7 +724,10 @@ map_joinable_qualified_id[document_id_1].push_back(fake_child1); map_joinable_qualified_id[document_id_1].push_back(fake_child2); map_joinable_qualified_id[document_id_2].push_back(fake_child3); - JoinChildrenFetcher fetcher(join_spec, std::move(map_joinable_qualified_id)); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplDeprecated> fetcher, + JoinChildrenFetcherImplDeprecated::Create( + join_spec, std::move(map_joinable_qualified_id))); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<AdvancedScorer> scorer, @@ -668,7 +735,8 @@ CreateAdvancedScoringSpec("len(this.childrenRankingSignals())"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - &fetcher, &empty_embedding_query_results_)); + fetcher.get(), &empty_embedding_query_results_, + feature_flags_.get())); // document_id_1 has two children. EXPECT_THAT(scorer->GetScore(docHitInfo1, /*query_it=*/nullptr), Eq(2)); // document_id_2 has one child. @@ -682,7 +750,8 @@ CreateAdvancedScoringSpec("sum(this.childrenRankingSignals())"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - &fetcher, &empty_embedding_query_results_)); + fetcher.get(), &empty_embedding_query_results_, + feature_flags_.get())); // document_id_1 has two children with scores 1 and 2. EXPECT_THAT(scorer->GetScore(docHitInfo1, /*query_it=*/nullptr), Eq(3)); // document_id_2 has one child with score 4. @@ -696,7 +765,8 @@ CreateAdvancedScoringSpec("avg(this.childrenRankingSignals())"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - &fetcher, &empty_embedding_query_results_)); + fetcher.get(), &empty_embedding_query_results_, + feature_flags_.get())); // document_id_1 has two children with scores 1 and 2. EXPECT_THAT(scorer->GetScore(docHitInfo1, /*query_it=*/nullptr), Eq(3 / 2.)); // document_id_2 has one child with score 4. @@ -707,15 +777,15 @@ Eq(default_score)); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create( - CreateAdvancedScoringSpec( - // Equivalent to "avg(this.childrenRankingSignals())" - "sum(this.childrenRankingSignals()) / " - "len(this.childrenRankingSignals())"), - default_score, kDefaultSemanticMetricType, document_store_.get(), - schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - &fetcher, &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + CreateAdvancedScoringSpec( + // Equivalent to "avg(this.childrenRankingSignals())" + "sum(this.childrenRankingSignals()) / " + "len(this.childrenRankingSignals())"), + default_score, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), fetcher.get(), + &empty_embedding_query_results_, feature_flags_.get())); // document_id_1 has two children with scores 1 and 2. EXPECT_THAT(scorer->GetScore(docHitInfo1, /*query_it=*/nullptr), Eq(3 / 2.)); // document_id_2 has one child with score 4. @@ -782,7 +852,8 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); // min([1]) = 1 EXPECT_THAT(scorer->GetScore(doc_hit_info_1, /*query_it=*/nullptr), Eq(1)); // min([0.5, 0.8]) = 0.5 @@ -792,13 +863,13 @@ spec_proto.set_advanced_scoring_expression("max(this.propertyWeights())"); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(spec_proto, - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + spec_proto, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); // max([1]) = 1 EXPECT_THAT(scorer->GetScore(doc_hit_info_1, /*query_it=*/nullptr), Eq(1)); // max([0.5, 0.8]) = 0.8 @@ -808,13 +879,13 @@ spec_proto.set_advanced_scoring_expression("sum(this.propertyWeights())"); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(spec_proto, - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + spec_proto, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); // sum([1]) = 1 EXPECT_THAT(scorer->GetScore(doc_hit_info_1, /*query_it=*/nullptr), Eq(1)); // sum([0.5, 0.8]) = 1.3 @@ -869,7 +940,8 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); // min([1]) = 1 EXPECT_THAT(scorer->GetScore(doc_hit_info_1, /*query_it=*/nullptr), Eq(1)); // min([0.5, 1, 0.5]) = 0.5 @@ -877,13 +949,13 @@ spec_proto.set_advanced_scoring_expression("max(this.propertyWeights())"); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(spec_proto, - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + spec_proto, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); // max([1]) = 1 EXPECT_THAT(scorer->GetScore(doc_hit_info_1, /*query_it=*/nullptr), Eq(1)); // max([0.5, 1, 0.5]) = 1 @@ -891,13 +963,13 @@ spec_proto.set_advanced_scoring_expression("sum(this.propertyWeights())"); ICING_ASSERT_OK_AND_ASSIGN( - scorer, - AdvancedScorer::Create(spec_proto, - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + scorer, AdvancedScorer::Create( + spec_proto, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); // sum([1]) = 1 EXPECT_THAT(scorer->GetScore(doc_hit_info_1, /*query_it=*/nullptr), Eq(1)); // sum([0.5, 1, 0.5]) = 2 @@ -914,19 +986,26 @@ CreateAdvancedScoringSpec("len(this.childrenRankingSignals())"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); // The root expression can only be of double type, but here it is of list // type. - JoinChildrenFetcher fake_fetcher(JoinSpecProto::default_instance(), - /*map_joinable_qualified_id=*/{}); + JoinSpecProto join_spec; + join_spec.set_parent_property_expression("this.qualifiedId()"); + join_spec.set_child_property_expression("sender"); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<JoinChildrenFetcherImplDeprecated> fetcher, + JoinChildrenFetcherImplDeprecated::Create( + join_spec, + /*map_joinable_qualified_id=*/{})); EXPECT_THAT( AdvancedScorer::Create( CreateAdvancedScoringSpec("this.childrenRankingSignals()"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - &fake_fetcher, &empty_embedding_query_results_), + fetcher.get(), &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -954,7 +1033,8 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_FALSE(scorer->is_constant()); scorer->PrepareToScore(/*query_term_iterators=*/{}); @@ -982,29 +1062,32 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_TRUE(scorer->is_constant()); } // Should be a parsing Error TEST_F(AdvancedScorerTest, EmptyExpression) { - EXPECT_THAT( - AdvancedScorer::Create(CreateAdvancedScoringSpec(""), - /*default_score=*/10, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec(""), + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } TEST_F(AdvancedScorerTest, ConstantEvaluationErrorShouldReturnAnError) { libtextclassifier3::StatusOr<std::unique_ptr<AdvancedScorer>> scorer_or = - AdvancedScorer::Create( - CreateAdvancedScoringSpec("log(0)"), /*default_score=*/0, - kDefaultSemanticMetricType, document_store_.get(), - schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_); + AdvancedScorer::Create(CreateAdvancedScoringSpec("log(0)"), + /*default_score=*/0, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1014,7 +1097,8 @@ CreateAdvancedScoringSpec("1 / 0"), /*default_score=*/0, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1024,7 +1108,8 @@ CreateAdvancedScoringSpec("sqrt(-1)"), /*default_score=*/0, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1034,7 +1119,8 @@ CreateAdvancedScoringSpec("pow(-1, 0.5)"), /*default_score=*/0, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1058,7 +1144,8 @@ CreateAdvancedScoringSpec("log(this.documentScore())"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(default_score, kEps)); ICING_ASSERT_OK_AND_ASSIGN( @@ -1067,7 +1154,8 @@ CreateAdvancedScoringSpec("1 / this.documentScore()"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(default_score, kEps)); ICING_ASSERT_OK_AND_ASSIGN( @@ -1076,7 +1164,8 @@ CreateAdvancedScoringSpec("sqrt(this.documentScore() - 1)"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(default_score, kEps)); ICING_ASSERT_OK_AND_ASSIGN( @@ -1085,7 +1174,8 @@ CreateAdvancedScoringSpec("pow(this.documentScore() - 1, 0.5)"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(docHitInfo), DoubleNear(default_score, kEps)); } @@ -1094,176 +1184,181 @@ TEST_F(AdvancedScorerTest, MathTypeError) { const double default_score = 0; - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("test"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("test"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("log()"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("log()"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("log(1, 2, 3)"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("log(1, 2, 3)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("log(1, this)"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("log(1, this)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("pow(1)"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("pow(1)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("sqrt(1, 2)"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("sqrt(1, 2)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("abs(1, 2)"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("abs(1, 2)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("sin(1, 2)"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("sin(1, 2)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("cos(1, 2)"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("cos(1, 2)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("tan(1, 2)"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("tan(1, 2)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("this"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("this"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("-this"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("-this"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("1 + this"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("1 + this"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } TEST_F(AdvancedScorerTest, DocumentFunctionTypeError) { const double default_score = 0; - EXPECT_THAT( - AdvancedScorer::Create( - CreateAdvancedScoringSpec("documentScore(1)"), default_score, - kDefaultSemanticMetricType, document_store_.get(), - schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("documentScore(1)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT( AdvancedScorer::Create( CreateAdvancedScoringSpec("this.creationTimestamp(1)"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT( - AdvancedScorer::Create( - CreateAdvancedScoringSpec("this.usageCount()"), default_score, - kDefaultSemanticMetricType, document_store_.get(), - schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("this.usageCount()"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT( AdvancedScorer::Create( CreateAdvancedScoringSpec("usageLastUsedTimestamp(1, 1)"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT( - AdvancedScorer::Create( - CreateAdvancedScoringSpec("relevanceScore(1)"), default_score, - kDefaultSemanticMetricType, document_store_.get(), - schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("relevanceScore(1)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT( AdvancedScorer::Create( CreateAdvancedScoringSpec("documentScore(this)"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT( AdvancedScorer::Create( CreateAdvancedScoringSpec("that.documentScore()"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT( AdvancedScorer::Create( CreateAdvancedScoringSpec("this.this.creationTimestamp()"), default_score, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - EXPECT_THAT(AdvancedScorer::Create(CreateAdvancedScoringSpec("this.log(2)"), - default_score, kDefaultSemanticMetricType, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_), + EXPECT_THAT(AdvancedScorer::Create( + CreateAdvancedScoringSpec("this.log(2)"), default_score, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -1283,7 +1378,8 @@ kDefaultSemanticMetricType, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1294,7 +1390,8 @@ kDefaultSemanticMetricType, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1306,7 +1403,8 @@ kDefaultSemanticMetricType, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1318,7 +1416,8 @@ kDefaultSemanticMetricType, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1330,7 +1429,8 @@ kDefaultSemanticMetricType, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1342,7 +1442,8 @@ kDefaultSemanticMetricType, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1354,7 +1455,8 @@ kDefaultSemanticMetricType, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1383,7 +1485,8 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &embedding_query_results); + &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1395,7 +1498,8 @@ "getEmbeddingParameter(1), \"COSINE\"))"), /*default_score=*/0, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1407,7 +1511,8 @@ "getEmbeddingParameter(2)))"), /*default_score=*/0, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1425,7 +1530,8 @@ /*default_score=*/0, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1437,7 +1543,8 @@ "getEmbeddingParameter(pow(2, 50))))"), /*default_score=*/0, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1449,7 +1556,8 @@ "getEmbeddingParameter(0.5)))"), /*default_score=*/0, kDefaultSemanticMetricType, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1526,7 +1634,8 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results)); + /*join_children_fetcher=*/nullptr, &embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(doc_hit_info_0), DoubleNear(0.5, kEps)); EXPECT_THAT(scorer->GetScore(doc_hit_info_1), DoubleNear(0.6, kEps)); @@ -1541,7 +1650,8 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results)); + /*join_children_fetcher=*/nullptr, &embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(doc_hit_info_0), DoubleNear(0.1 + 0.2, kEps)); EXPECT_THAT(scorer->GetScore(doc_hit_info_1), DoubleNear(0.3 + 0.4, kEps)); @@ -1559,7 +1669,8 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results)); + /*join_children_fetcher=*/nullptr, &embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(doc_hit_info_0), DoubleNear(0.1 + 0.2 + 0.5 + 0.7, kEps)); EXPECT_THAT(scorer->GetScore(doc_hit_info_1), @@ -1575,7 +1686,8 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results)); + /*join_children_fetcher=*/nullptr, &embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(doc_hit_info_0), DoubleNear(0.1, kEps)); EXPECT_THAT(scorer->GetScore(doc_hit_info_1), DoubleNear(0.2, kEps)); @@ -1588,7 +1700,8 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results); + /*join_children_fetcher=*/nullptr, &embedding_query_results, + feature_flags_.get()); EXPECT_THAT(scorer_or, StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(scorer_or.status().error_message(), @@ -1633,7 +1746,8 @@ SearchSpecProto::EmbeddingQueryMetricType::COSINE, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results)); + /*join_children_fetcher=*/nullptr, &embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(doc_hit_info_0), DoubleNear(5, kEps)); // minOrDefault({4, 5, 2, 1, 3}, -100) = 1 @@ -1646,7 +1760,8 @@ SearchSpecProto::EmbeddingQueryMetricType::COSINE, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results)); + /*join_children_fetcher=*/nullptr, &embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(doc_hit_info_0), DoubleNear(1, kEps)); // maxOrDefault({}, 100) = 100 @@ -1659,7 +1774,8 @@ SearchSpecProto::EmbeddingQueryMetricType::COSINE, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results)); + /*join_children_fetcher=*/nullptr, &embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(doc_hit_info_0), DoubleNear(100, kEps)); // minOrDefault({}, -100) = -100 @@ -1672,20 +1788,22 @@ SearchSpecProto::EmbeddingQueryMetricType::COSINE, document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results)); + /*join_children_fetcher=*/nullptr, &embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scorer->GetScore(doc_hit_info_0), DoubleNear(-100, kEps)); // sum(filterByRange({4, 5, 2, 1, 3}, 2, 4)) = sum({4, 2, 3}) = 9 ICING_ASSERT_OK_AND_ASSIGN( - scorer, AdvancedScorer::Create( - CreateAdvancedScoringSpec( - "sum(filterByRange(this.matchedSemanticScores(" - "getEmbeddingParameter(0)), 2, 4))"), - kDefaultScore, /*default_semantic_metric_type=*/ - SearchSpecProto::EmbeddingQueryMetricType::COSINE, - document_store_.get(), schema_store_.get(), - fake_clock_.GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &embedding_query_results)); + scorer, + AdvancedScorer::Create(CreateAdvancedScoringSpec( + "sum(filterByRange(this.matchedSemanticScores(" + "getEmbeddingParameter(0)), 2, 4))"), + kDefaultScore, /*default_semantic_metric_type=*/ + SearchSpecProto::EmbeddingQueryMetricType::COSINE, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &embedding_query_results, feature_flags_.get())); EXPECT_THAT(scorer->GetScore(doc_hit_info_0), DoubleNear(9, kEps)); } @@ -1714,7 +1832,8 @@ document_store_.get(), schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results_)); + &empty_embedding_query_results_, + feature_flags_.get())); EXPECT_FALSE(scorer->is_constant()); scorer->PrepareToScore(/*query_term_iterators=*/{}); EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), @@ -1725,6 +1844,728 @@ ElementsAre(DoubleNear(4, kEps), DoubleNear(127, kEps))); } +TEST_F(AdvancedScorerTest, GetScorablePropertyWithFeatureDisabled) { + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"email\"))"); + + EXPECT_THAT( + AdvancedScorer::Create( + scoring_spec, /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("SCORABLE_PROPERTY_RANKING feature is not enabled."))); +} + +TEST_F(AdvancedScorerTest, GetScorableProperty_WrongParamsNumber) { + ScoringSpecProto scoring_spec_with_one_param = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"email\"))"); + scoring_spec_with_one_param.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + EXPECT_THAT( + AdvancedScorer::Create( + scoring_spec_with_one_param, /*default_score=*/10, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr( + "getScorableProperty must take exactly two string params"))); + + ScoringSpecProto scoring_spec_with_int_param = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"email\", 123))"); + scoring_spec_with_int_param.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + EXPECT_THAT( + AdvancedScorer::Create( + scoring_spec_with_int_param, /*default_score=*/10, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr( + "getScorableProperty must take exactly two string params"))); +} + +TEST_F(AdvancedScorerTest, GetScorableProperty_ParamsMustBeString) { + ScoringSpecProto scoring_spec_with_int_param = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"email\", 123))"); + scoring_spec_with_int_param.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + EXPECT_THAT( + AdvancedScorer::Create( + scoring_spec_with_int_param, /*default_score=*/10, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr( + "getScorableProperty must take exactly two string params"))); +} + +TEST_F(AdvancedScorerTest, GetScorableProperty_SchemaNotExistInSchemaStore) { + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"aliasEmail\", \"frequencyScore\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasEmail", {"non_exist"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + EXPECT_THAT(AdvancedScorer::Create( + scoring_spec, /*default_score=*/10, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::OK)); +} + +TEST_F(AdvancedScorerTest, GetScorableProperty_PropertyNameNotScorable) { + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"aliasEmail\", \"subject\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasEmail", {"email"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + EXPECT_THAT(AdvancedScorer::Create( + scoring_spec, /*default_score=*/10, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("'subject' is not defined as a scorable " + "property under schema type"))); +} + +TEST_F(AdvancedScorerTest, GetScorableProperty_PropertyNameNotExist) { + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"aliasEmail\", \"non_exist\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasEmail", {"email"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + EXPECT_THAT(AdvancedScorer::Create( + scoring_spec, /*default_score=*/10, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("'non_exist' is not defined as a scorable " + "property under schema type"))); +} + +TEST_F(AdvancedScorerTest, GetScorableProperty_SomePropertiesNotScorable) { + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "100 * avg(getScorableProperty(\"aliasPerson\", \"isStarred\")) + " + "10 * max(getScorableProperty(\"aliasPerson\", \"frequencyScore\")) + " + "10 * sum(getScorableProperty(\"aliasPerson\", \"non_exist\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + EXPECT_THAT(AdvancedScorer::Create( + scoring_spec, /*default_score=*/10, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("'non_exist' is not defined as a scorable " + "property under schema type"))); +} + +TEST_F(AdvancedScorerTest, + GetScorableProperty_DocumentSchemaDifferentFromScoringSpecSchema) { + DocumentProto document = DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("email") + .SetScore(100) + .SetCreationTimestampMs(123) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + + // getScorableProperty("aliasPerson", "frequencyScore") will return an empty + // list because the schema type of the document is "email" instead of + // "person". + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"aliasPerson\", \"frequencyScore\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score = 100 + 0; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create( + scoring_spec, /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score, kEps)); +} + +TEST_F(AdvancedScorerTest, + GetScorableProperty_DocumentWithoutScorableProperties) { + DocumentProto document = DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("person") + .SetScore(100) + .SetCreationTimestampMs(123) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + + // getScorableProperty("aliasPerson", "frequencyScore") will return an empty + // list. + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"aliasPerson\", \"frequencyScore\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score = 100 + 0; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create( + scoring_spec, /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score, kEps)); +} + +TEST_F(AdvancedScorerTest, GetScorableProperty_WithDoubleList) { + DocumentProto document = + DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("person") + .SetScore(100) + .SetCreationTimestampMs(123) + .AddDoubleProperty("frequencyScore", 1.0, 2.0, 3.0) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "max(getScorableProperty(\"aliasPerson\", \"frequencyScore\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score = 100 + std::max({1.0, 2.0, 3.0}); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create(scoring_spec, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score, kEps)); +} + +TEST_F(AdvancedScorerTest, GetScorableProperty_WithInt64List) { + DocumentProto document = DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("person") + .SetScore(100) + .SetCreationTimestampMs(123) + .AddInt64Property("contactTimes", 10, 20, 30) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "min(getScorableProperty(\"aliasPerson\", \"contactTimes\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score = 100 + std::min({10, 20, 30}); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create(scoring_spec, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score, kEps)); +} + +TEST_F(AdvancedScorerTest, GetScorableProperty_WithBoolean) { + DocumentProto document = DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("person") + .SetScore(100) + .SetCreationTimestampMs(123) + .AddBooleanProperty("isStarred", true) + .AddInt64Property("contactTimes", 10, 20, 30) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "100 * avg(getScorableProperty(\"aliasPerson\", \"isStarred\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score = 100 + 100 * 1.0; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create(scoring_spec, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score, kEps)); +} + +TEST_F(AdvancedScorerTest, + ScoreWithScorableProperty_ScoringSpecWithMultipleProperties) { + DocumentProto document = + DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("person") + .SetScore(100) + .SetCreationTimestampMs(123) + .AddBooleanProperty("isStarred", false) + .AddInt64Property("contactTimes", 10, 20, 30) + .AddDoubleProperty("frequencyScore", 1.0, 2.0, 3.0) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "100 * avg(getScorableProperty(\"aliasPerson\", \"isStarred\")) + " + "10 * max(getScorableProperty(\"aliasPerson\", \"frequencyScore\")) + " + "10 * max(getScorableProperty(\"aliasPerson\", \"contactTimes\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score = 100 + 100 * 0 + 10 * std::max({1.0, 2.0, 3.0}) + + 10 * std::max({10, 20, 30}); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create(scoring_spec, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score, kEps)); +} + +TEST_F(AdvancedScorerTest, + ScoreWithScorableProperty_ScoringSpecWithMultipleSchemas) { + DocumentProto document = + DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("person") + .SetScore(100) + .SetCreationTimestampMs(123) + .AddBooleanProperty("isStarred", false) + .AddDoubleProperty("frequencyScore", 1.0, 2.0, 3.0) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "100 * avg(getScorableProperty(\"aliasPerson\", \"isStarred\")) + " + "10 * max(getScorableProperty(\"aliasPerson\", \"frequencyScore\")) + " + "10 * sum(getScorableProperty(\"aliasMessage\", \"frequencyScore\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + AddSchemaTypeAliasMap(&scoring_spec, "aliasMessage", {"message"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score = + 100 + 100 * 0 + 10 * std::max({1.0, 2.0, 3.0}) + 10 * 0; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create(scoring_spec, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score, kEps)); +} + +TEST_F(AdvancedScorerTest, + ScoreWithScorableProperty_WithMathExpressionTakeEmptyList) { + DocumentProto document = DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("email") + .SetScore(100) + .SetCreationTimestampMs(123) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + + // Expected score will fall back to the default score, because max() throws an + // error when it takes an empty list. + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "100 * avg(getScorableProperty(\"aliasPerson\", \"isStarred\")) + " + "10 * max(getScorableProperty(\"aliasPerson\", \"frequencyScore\"))"); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create(scoring_spec, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(10, kEps)); +} + +TEST_F(AdvancedScorerTest, + ScoreWithScorableProperty_MaxOrDefaultTakeEmptyList) { + DocumentProto document = DocumentBuilder() + .SetKey("namespace", "uri") + .SetSchema("email") + .SetScore(100) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "10 * maxOrDefault(getScorableProperty(\"aliasPerson\", " + "\"frequencyScore\"), " + "5)"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score = 100 + 10 * 5; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create(scoring_spec, + /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score, kEps)); +} + +TEST_F(AdvancedScorerTest, ScoreWithScorableProperty_WithNestedSchemas) { + SchemaProto schema_proto = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("person") + .AddProperty(PropertyConfigBuilder() + .SetName("id") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("age") + .SetDataType(PropertyConfigProto::DataType::INT64) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("isStarred") + .SetDataType(PropertyConfigProto::DataType::BOOLEAN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeDocument("person", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("receiver") + .SetDataTypeDocument("person", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("importanceBoolean") + .SetDataType(PropertyConfigProto::DataType::BOOLEAN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("scoreDouble") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("scoreInt64") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + ICING_ASSERT_OK(schema_store_->SetSchema( + schema_proto, /*ignore_errors_and_delete_documents=*/true, + /*allow_circular_schema_definitions=*/true)); + + DocumentProto document = + DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("email") + .SetScore(100) + .AddStringProperty("subject", "subject foo") + .AddBooleanProperty("importanceBoolean", true) + .AddInt64Property("scoreInt64", 1, 2, 3) + .AddDocumentProperty( + "receiver", + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("person") + .AddInt64Property("age", 30) + .AddDoubleProperty("income", 10000, 20000, 30000) + .AddBooleanProperty("isStarred", true) + .Build(), + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("person") + .AddInt64Property("age", 35) + .AddDoubleProperty("income", 10001, 20001, 30001) + .AddBooleanProperty("isStarred", false) + .Build()) + .AddDocumentProperty( + "sender", DocumentBuilder() + .SetKey("namespace", "uri3") + .SetSchema("person") + .AddInt64Property("age", 50) + .AddDoubleProperty("income", 21001, 21002, 21003) + .AddBooleanProperty("isStarred", true) + .Build()) + .SetCreationTimestampMs(0) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + document_store_->Put(document)); + DocHitInfo docHitInfo(put_result.new_document_id); + ScoringSpecProto scoring_spec1 = CreateAdvancedScoringSpec( + "this.documentScore() + " + "10 * max(getScorableProperty(\"aliasEmail\", \"scoreInt64\"))"); + AddSchemaTypeAliasMap(&scoring_spec1, "aliasEmail", {"email"}); + double expected_score1 = 100 + 10 * std::max({1, 2, 3}); + scoring_spec1.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<AdvancedScorer> scorer, + AdvancedScorer::Create(scoring_spec1, + /*default_score=*/0, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, + feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score1, kEps)); + + ScoringSpecProto scoring_spec2 = CreateAdvancedScoringSpec( + "this.documentScore() + " + "10 * min(getScorableProperty(\"aliasEmail\", \"receiver.age\"))"); + AddSchemaTypeAliasMap(&scoring_spec2, "aliasEmail", {"email"}); + scoring_spec2.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score2 = 100 + 10 * std::min({30, 35}); + ICING_ASSERT_OK_AND_ASSIGN( + scorer, AdvancedScorer::Create( + scoring_spec2, + /*default_score=*/0, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score2, kEps)); + + ScoringSpecProto scoring_spec3 = CreateAdvancedScoringSpec( + "this.documentScore() + " + "50 * min(getScorableProperty(\"aliasEmail\", \"importanceBoolean\")) + " + "10 * min(getScorableProperty(\"aliasEmail\", \"receiver.age\")) + " + "20 * min(getScorableProperty(\"aliasEmail\", \"sender.isStarred\")) + " + "5 * max(getScorableProperty(\"aliasEmail\", \"receiver.income\"))"); + AddSchemaTypeAliasMap(&scoring_spec3, "aliasEmail", {"email"}); + scoring_spec3.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + double expected_score3 = + 100 + 50 * 1 + 10 * std::min({30, 35}) + 20 * 1 + + 5 * std::max({10000, 20000, 30000, 10001, 20001, 30001}); + ICING_ASSERT_OK_AND_ASSIGN( + scorer, AdvancedScorer::Create( + scoring_spec3, + /*default_score=*/0, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results_, feature_flags_.get())); + scorer->PrepareToScore(/*query_term_iterators=*/{}); + EXPECT_THAT(scorer->GetScore(docHitInfo, /*query_it=*/nullptr), + DoubleNear(expected_score3, kEps)); +} + +TEST_F(AdvancedScorerTest, SchemaTypeAliasMap_AliasSchemaTypeNotMatched) { + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"aliasEmail\", \"frequencyScore\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "aliasPerson", {"person"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + EXPECT_THAT( + AdvancedScorer::Create( + scoring_spec, /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("The alias schema type in the score expression is not " + "found in the schema_type_alias_map"))); +} + +TEST_F(AdvancedScorerTest, + SchemaTypeAliasMap_PropertyNotScorableForSomeSchemaTypes) { + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("pkg1/db1/message") + .AddProperty( + PropertyConfigBuilder() + .SetName("frequencyScore") + .SetDataType(PropertyConfigProto::DataType::INT64) + .SetCardinality(CARDINALITY_REPEATED) + .SetScorableType(SCORABLE_TYPE_ENABLED))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("pkg2/db1/message") + .AddProperty( + PropertyConfigBuilder() + .SetName("frequencyScore") + .SetDataType(PropertyConfigProto::DataType::INT64) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + ICING_ASSERT_OK(schema_store_->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/true, + /*allow_circular_schema_definitions=*/false)); + + ScoringSpecProto scoring_spec = CreateAdvancedScoringSpec( + "this.documentScore() + " + "sum(getScorableProperty(\"message\", \"frequencyScore\"))"); + AddSchemaTypeAliasMap(&scoring_spec, "message", + {"pkg1/db1/message", "pkg2/db1/message"}); + scoring_spec.add_scoring_feature_types_enabled( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING); + + EXPECT_THAT( + AdvancedScorer::Create( + scoring_spec, /*default_score=*/10, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results_, + feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("'frequencyScore' is not defined as a scorable " + "property under schema type 1"))); +} + } // namespace } // namespace lib
diff --git a/icing/scoring/advanced_scoring/score-expression-util.h b/icing/scoring/advanced_scoring/score-expression-util.h new file mode 100644 index 0000000..1756585 --- /dev/null +++ b/icing/scoring/advanced_scoring/score-expression-util.h
@@ -0,0 +1,111 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_SCORING_ADVANCED_SCORING_SCORE_EXPRESSION_UTIL_H_ +#define ICING_SCORING_ADVANCED_SCORING_SCORE_EXPRESSION_UTIL_H_ + +#include <cstdint> +#include <memory> +#include <string_view> +#include <unordered_set> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/feature-flags.h" +#include "icing/index/embed/embedding-query-results.h" +#include "icing/join/join-children-fetcher.h" +#include "icing/proto/scoring.pb.h" +#include "icing/query/advanced_query_parser/abstract-syntax-tree.h" +#include "icing/query/advanced_query_parser/lexer.h" +#include "icing/query/advanced_query_parser/parser.h" +#include "icing/schema/schema-store.h" +#include "icing/scoring/advanced_scoring/score-expression.h" +#include "icing/scoring/advanced_scoring/scoring-visitor.h" +#include "icing/scoring/bm25f-calculator.h" +#include "icing/scoring/section-weights.h" +#include "icing/store/document-store.h" +#include "icing/util/status-macros.h" + +namespace icing { +namespace lib { +namespace score_expression_util { + +// Returns a ScoreExpression instance for the given scoring expression. +// +// join_children_fetcher, embedding_query_results, section_weights, +// bm25f_calculator, schema_type_alias_map are allowed to be nullptr if the +// corresponding scoring expression does not use them. +// +// Returns: +// - A ScoreExpression instance on success. +// - Any syntax or semantics error from Lexer, Parser, or ScoringVisitor. +inline libtextclassifier3::StatusOr<std::unique_ptr<ScoreExpression>> +GetScoreExpression(std::string_view scoring_expression, double default_score, + SearchSpecProto::EmbeddingQueryMetricType::Code + default_semantic_metric_type, + const DocumentStore* document_store, + const SchemaStore* schema_store, int64_t current_time_ms, + const JoinChildrenFetcher* join_children_fetcher, + const EmbeddingQueryResults* embedding_query_results, + SectionWeights* section_weights, + Bm25fCalculator* bm25f_calculator, + const SchemaTypeAliasMap* schema_type_alias_map, + const FeatureFlags* feature_flags, + const std::unordered_set<ScoringFeatureType>* + scoring_feature_types_enabled) { + ICING_RETURN_ERROR_IF_NULL(document_store); + ICING_RETURN_ERROR_IF_NULL(schema_store); + ICING_RETURN_ERROR_IF_NULL(feature_flags); + + Lexer lexer(scoring_expression, Lexer::Language::SCORING); + ICING_ASSIGN_OR_RETURN(std::vector<Lexer::LexerToken> lexer_tokens, + std::move(lexer).ExtractTokens()); + Parser parser = Parser::Create(std::move(lexer_tokens)); + ICING_ASSIGN_OR_RETURN(std::unique_ptr<Node> tree_root, + parser.ConsumeScoring()); + ScoringVisitor visitor( + default_score, default_semantic_metric_type, document_store, schema_store, + section_weights, bm25f_calculator, join_children_fetcher, + embedding_query_results, schema_type_alias_map, feature_flags, + scoring_feature_types_enabled, current_time_ms); + tree_root->Accept(&visitor); + + ICING_ASSIGN_OR_RETURN(std::unique_ptr<ScoreExpression> expression, + std::move(visitor).Expression()); + if (expression->type() != ScoreExpressionType::kDouble) { + return absl_ports::InvalidArgumentError( + "The root scoring expression is not of double type."); + } + return expression; +} + +inline std::unique_ptr<std::unordered_set<ScoringFeatureType>> +GetEnabledScoringFeatureTypes(const ScoringSpecProto& scoring_spec) { + auto scoring_feature_types_enabled = + std::make_unique<std::unordered_set<ScoringFeatureType>>(); + for (int feature_type : scoring_spec.scoring_feature_types_enabled()) { + scoring_feature_types_enabled->insert( + static_cast<ScoringFeatureType>(feature_type)); + } + return scoring_feature_types_enabled; +} + +} // namespace score_expression_util + +} // namespace lib +} // namespace icing + +#endif // ICING_SCORING_ADVANCED_SCORING_SCORE_EXPRESSION_UTIL_H_
diff --git a/icing/scoring/advanced_scoring/score-expression-util_test.cc b/icing/scoring/advanced_scoring/score-expression-util_test.cc new file mode 100644 index 0000000..29ef915 --- /dev/null +++ b/icing/scoring/advanced_scoring/score-expression-util_test.cc
@@ -0,0 +1,283 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/scoring/advanced_scoring/score-expression-util.h" + +#include <cstdint> +#include <memory> +#include <string> +#include <string_view> +#include <unordered_set> +#include <utility> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/document-builder.h" +#include "icing/feature-flags.h" +#include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" +#include "icing/index/hit/doc-hit-info.h" +#include "icing/proto/document.pb.h" +#include "icing/proto/schema.pb.h" +#include "icing/proto/scoring.pb.h" +#include "icing/proto/usage.pb.h" +#include "icing/schema-builder.h" +#include "icing/schema/schema-store.h" +#include "icing/scoring/advanced_scoring/score-expression.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" +#include "icing/testing/common-matchers.h" +#include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" +#include "icing/testing/tmp-directory.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::DoubleEq; +using ::testing::HasSubstr; + +class ScoreExpressionUtilTest : public testing::Test { + protected: + ScoreExpressionUtilTest() + : test_dir_(GetTestTempDir() + "/icing"), + doc_store_dir_(test_dir_ + "/doc_store"), + schema_store_dir_(test_dir_ + "/schema_store") {} + + void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + scoring_feature_types_enabled_ = { + ScoringFeatureType::SCORABLE_PROPERTY_RANKING}; + filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); + filesystem_.CreateDirectoryRecursively(doc_store_dir_.c_str()); + filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()); + + ICING_ASSERT_OK_AND_ASSIGN( + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); + + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); + document_store_ = std::move(create_result.document_store); + + // Creates the schema + SchemaProto test_schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email").AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString( + TermMatchType::PREFIX, + StringIndexingConfig::TokenizerType::PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + ICING_ASSERT_OK(schema_store_->SetSchema( + std::move(test_schema), /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + } + + void TearDown() override { + document_store_.reset(); + schema_store_.reset(); + filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); + } + + std::unique_ptr<FeatureFlags> feature_flags_; + std::unordered_set<ScoringFeatureType> scoring_feature_types_enabled_; + Filesystem filesystem_; + FakeClock fake_clock_; + const std::string test_dir_; + const std::string doc_store_dir_; + const std::string schema_store_dir_; + std::unique_ptr<SchemaStore> schema_store_; + std::unique_ptr<DocumentStore> document_store_; +}; + +constexpr int kDefaultScore = 0; +constexpr SearchSpecProto::EmbeddingQueryMetricType::Code + kDefaultSemanticMetricType = + SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT; + +DocumentProto CreateDocument(const std::string& name_space, + const std::string& uri, int score, + int64_t creation_timestamp_ms) { + return DocumentBuilder() + .SetKey(name_space, uri) + .SetSchema("email") + .SetScore(score) + .SetCreationTimestampMs(creation_timestamp_ms) + .Build(); +} + +TEST_F(ScoreExpressionUtilTest, + FunctionsAreBannedIfMissingCorrespondingDependencies) { + EXPECT_THAT( + score_expression_util::GetScoreExpression( + "len(this.childrenRankingSignals())", kDefaultScore, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + feature_flags_.get(), &scoring_feature_types_enabled_), + StatusIs( + libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("childrenRankingSignals must only be used with join"))); + + EXPECT_THAT( + score_expression_util::GetScoreExpression( + "sum(this.matchedSemanticScores(getEmbeddingParameter(0)))", + kDefaultScore, kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + feature_flags_.get(), &scoring_feature_types_enabled_), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("matchedSemanticScores function is not " + "available in this context"))); + + EXPECT_THAT( + score_expression_util::GetScoreExpression( + "sum(this.propertyWeights())", kDefaultScore, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + feature_flags_.get(), &scoring_feature_types_enabled_), + StatusIs( + libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr( + "propertyWeights function is not available in this context"))); + + EXPECT_THAT( + score_expression_util::GetScoreExpression( + "this.relevanceScore()", kDefaultScore, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + feature_flags_.get(), &scoring_feature_types_enabled_), + StatusIs( + libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr( + "relevanceScore function is not available in this context"))); + + EXPECT_THAT( + score_expression_util::GetScoreExpression( + "sum(getScorableProperty(\"aliasPerson\", \"contactTimes\"))", + kDefaultScore, kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + feature_flags_.get(), &scoring_feature_types_enabled_), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, + HasSubstr("getScorableProperty function is not " + "available in this context"))); +} + +TEST_F(ScoreExpressionUtilTest, SimpleExpression) { + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::PutResult put_result, + document_store_->Put(CreateDocument("namespace", "uri", /*score=*/10, + /*creation_timestamp_ms=*/100))); + DocumentId document_id = put_result.new_document_id; + DocHitInfo doc_hit_info = DocHitInfo(document_id); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<ScoreExpression> score_expression, + score_expression_util::GetScoreExpression( + "1 + 1", kDefaultScore, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + feature_flags_.get(), &scoring_feature_types_enabled_)); + + EXPECT_THAT( + score_expression->EvaluateDouble(doc_hit_info, /*query_it=*/nullptr), + IsOkAndHolds(DoubleEq(2))); +} + +TEST_F(ScoreExpressionUtilTest, DocumentFunctionsWithoutOptionalDependencies) { + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::PutResult put_result, + document_store_->Put(CreateDocument("namespace", "uri", /*score=*/10, + /*creation_timestamp_ms=*/100))); + DocumentId document_id = put_result.new_document_id; + DocHitInfo doc_hit_info = DocHitInfo(document_id); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<ScoreExpression> score_expression, + score_expression_util::GetScoreExpression( + "this.documentScore()", kDefaultScore, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + feature_flags_.get(), &scoring_feature_types_enabled_)); + EXPECT_THAT( + score_expression->EvaluateDouble(doc_hit_info, /*query_it=*/nullptr), + IsOkAndHolds(DoubleEq(10))); + + ICING_ASSERT_OK_AND_ASSIGN( + score_expression, + score_expression_util::GetScoreExpression( + "this.creationTimestamp()", kDefaultScore, kDefaultSemanticMetricType, + document_store_.get(), schema_store_.get(), + fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + feature_flags_.get(), &scoring_feature_types_enabled_)); + EXPECT_THAT( + score_expression->EvaluateDouble(doc_hit_info, /*query_it=*/nullptr), + IsOkAndHolds(DoubleEq(100))); + + ICING_ASSERT_OK_AND_ASSIGN( + score_expression, + score_expression_util::GetScoreExpression( + "this.documentScore() + this.creationTimestamp()", kDefaultScore, + kDefaultSemanticMetricType, document_store_.get(), + schema_store_.get(), fake_clock_.GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + /*embedding_query_results=*/nullptr, /*section_weights=*/nullptr, + /*bm25f_calculator=*/nullptr, /*schema_type_alias_map=*/nullptr, + feature_flags_.get(), &scoring_feature_types_enabled_)); + EXPECT_THAT( + score_expression->EvaluateDouble(doc_hit_info, /*query_it=*/nullptr), + IsOkAndHolds(DoubleEq(10 + 100))); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/scoring/advanced_scoring/score-expression.cc b/icing/scoring/advanced_scoring/score-expression.cc index 8bd151e..8804605 100644 --- a/icing/scoring/advanced_scoring/score-expression.cc +++ b/icing/scoring/advanced_scoring/score-expression.cc
@@ -37,6 +37,9 @@ #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/join/join-children-fetcher.h" +#include "icing/legacy/core/icing-string-util.h" +#include "icing/proto/internal/scorable_property_set.pb.h" +#include "icing/schema/schema-store.h" #include "icing/schema/section.h" #include "icing/scoring/bm25f-calculator.h" #include "icing/scoring/scored-document-hit.h" @@ -47,6 +50,7 @@ #include "icing/store/document-store.h" #include "icing/util/embedding-util.h" #include "icing/util/logging.h" +#include "icing/util/scorable_property_set.h" #include "icing/util/status-macros.h" namespace icing { @@ -62,6 +66,23 @@ return libtextclassifier3::Status::OK; } +SchemaTypeId GetSchemaTypeId(DocumentId document_id, + const DocumentStore& document_store, + int64_t current_time_ms) { + auto filter_data_optional = + document_store.GetAliveDocumentFilterData(document_id, current_time_ms); + if (!filter_data_optional) { + // This should never happen. The only failure case for + // GetAliveDocumentFilterData is if the document_id is outside of the range + // of allocated document_ids, which shouldn't be possible since we're + // getting this document_id from the posting lists. + ICING_LOG(WARNING) << "No document filter data for document [" + << document_id << "]"; + return kInvalidSchemaTypeId; + } + return filter_data_optional.value().schema_type_id(); +} + } // namespace libtextclassifier3::StatusOr<std::unique_ptr<ScoreExpression>> @@ -586,7 +607,8 @@ std::unique_ptr<ChildrenRankingSignalsFunctionScoreExpression>> ChildrenRankingSignalsFunctionScoreExpression::Create( std::vector<std::unique_ptr<ScoreExpression>> args, - const JoinChildrenFetcher* join_children_fetcher) { + const DocumentStore& document_store, + const JoinChildrenFetcher* join_children_fetcher, int64_t current_time_ms) { if (args.size() != 1) { return absl_ports::InvalidArgumentError( "childrenRankingSignals must have 1 argument."); @@ -600,12 +622,11 @@ if (join_children_fetcher == nullptr) { return absl_ports::InvalidArgumentError( "childrenRankingSignals must only be used with join, but " - "JoinChildrenFetcher " - "is not provided."); + "JoinChildrenFetcher is not provided."); } return std::unique_ptr<ChildrenRankingSignalsFunctionScoreExpression>( new ChildrenRankingSignalsFunctionScoreExpression( - *join_children_fetcher)); + document_store, *join_children_fetcher, current_time_ms)); } libtextclassifier3::StatusOr<std::vector<double>> @@ -648,7 +669,8 @@ const DocHitInfo& hit_info, const DocHitInfoIterator*) const { std::vector<double> weights; SectionIdMask sections = hit_info.hit_section_ids_mask(); - SchemaTypeId schema_type_id = GetSchemaTypeId(hit_info.document_id()); + SchemaTypeId schema_type_id = GetSchemaTypeId( + hit_info.document_id(), document_store_, current_time_ms_); while (sections != 0) { SectionId section_id = __builtin_ctzll(sections); @@ -659,22 +681,6 @@ return weights; } -SchemaTypeId PropertyWeightsFunctionScoreExpression::GetSchemaTypeId( - DocumentId document_id) const { - auto filter_data_optional = - document_store_.GetAliveDocumentFilterData(document_id, current_time_ms_); - if (!filter_data_optional) { - // This should never happen. The only failure case for - // GetAliveDocumentFilterData is if the document_id is outside of the range - // of allocated document_ids, which shouldn't be possible since we're - // getting this document_id from the posting lists. - ICING_LOG(WARNING) << "No document filter data for document [" - << document_id << "]"; - return kInvalidSchemaTypeId; - } - return filter_data_optional.value().schema_type_id(); -} - libtextclassifier3::StatusOr<std::unique_ptr<ScoreExpression>> GetEmbeddingParameterFunctionScoreExpression::Create( std::vector<std::unique_ptr<ScoreExpression>> args) { @@ -795,5 +801,139 @@ return *scores; } +GetScorablePropertyFunctionScoreExpression:: + GetScorablePropertyFunctionScoreExpression( + const DocumentStore* document_store, const SchemaStore* schema_store, + int64_t current_time_ms, + std::unordered_set<SchemaTypeId>&& schema_type_ids, + std::string_view property_path) + : document_store_(*document_store), + schema_store_(*schema_store), + current_time_ms_(current_time_ms), + schema_type_ids_(std::move(schema_type_ids)), + property_path_(property_path) {} + +libtextclassifier3::StatusOr<std::unordered_set<SchemaTypeId>> +GetScorablePropertyFunctionScoreExpression::GetAndValidateSchemaTypeIds( + std::string_view alias_schema_type, std::string_view property_path, + const SchemaTypeAliasMap& schema_type_alias_map, + const SchemaStore& schema_store) { + auto alias_map_iter = schema_type_alias_map.find(alias_schema_type.data()); + if (alias_map_iter == schema_type_alias_map.end()) { + return absl_ports::InvalidArgumentError(absl_ports::StrCat( + "The alias schema type in the score expression is not found in the " + "schema_type_alias_map: ", + alias_schema_type)); + } + + std::unordered_set<SchemaTypeId> schema_type_ids; + for (std::string_view schema_type : alias_map_iter->second) { + // First, verify that the schema type has a valid schema type id in the + // schema store. + libtextclassifier3::StatusOr<SchemaTypeId> schema_type_id_or = + schema_store.GetSchemaTypeId(schema_type); + if (!schema_type_id_or.ok()) { + if (absl_ports::IsNotFound(schema_type_id_or.status())) { + // Ignores the schema type if it is not found in the schema store. + continue; + } + return schema_type_id_or.status(); + } + SchemaTypeId schema_type_id = schema_type_id_or.ValueOrDie(); + + // Then, calls GetScorablePropertyIndex() here to validate if the property + // path is scorable under the schema type, no need to check the returned + // index value. + libtextclassifier3::StatusOr<std::optional<int>> + scorable_property_index_or = schema_store.GetScorablePropertyIndex( + schema_type_id, property_path); + if (!scorable_property_index_or.ok()) { + return scorable_property_index_or.status(); + } + if (!scorable_property_index_or.ValueOrDie().has_value()) { + return absl_ports::InvalidArgumentError(IcingStringUtil::StringPrintf( + "'%s' is not defined as a scorable property under schema type %d", + property_path.data(), schema_type_id)); + } + schema_type_ids.insert(schema_type_id); + } + return schema_type_ids; +} + +libtextclassifier3::StatusOr< + std::unique_ptr<GetScorablePropertyFunctionScoreExpression>> +GetScorablePropertyFunctionScoreExpression::Create( + std::vector<std::unique_ptr<ScoreExpression>> args, + const DocumentStore* document_store, const SchemaStore* schema_store, + const SchemaTypeAliasMap& schema_type_alias_map, int64_t current_time_ms) { + ICING_RETURN_IF_ERROR(CheckChildrenNotNull(args)); + + if (args.size() != 2 || args[0]->type() != ScoreExpressionType::kString || + args[1]->type() != ScoreExpressionType::kString) { + return absl_ports::InvalidArgumentError(absl_ports::StrCat( + kFunctionName, " must take exactly two string params")); + } + + // Validate schema type. + ICING_ASSIGN_OR_RETURN(std::string_view alias_schema_type, + args[0]->EvaluateString()); + ICING_ASSIGN_OR_RETURN(std::string_view property_path, + args[1]->EvaluateString()); + ICING_ASSIGN_OR_RETURN( + std::unordered_set<SchemaTypeId> schema_type_ids, + GetAndValidateSchemaTypeIds(alias_schema_type, property_path, + schema_type_alias_map, *schema_store)); + + return std::unique_ptr<GetScorablePropertyFunctionScoreExpression>( + new GetScorablePropertyFunctionScoreExpression( + document_store, schema_store, current_time_ms, + std::move(schema_type_ids), property_path)); +} + +libtextclassifier3::StatusOr<std::vector<double>> +GetScorablePropertyFunctionScoreExpression::EvaluateList( + const DocHitInfo& hit_info, const DocHitInfoIterator* query_it) const { + SchemaTypeId doc_schema_type_id = GetSchemaTypeId( + hit_info.document_id(), document_store_, current_time_ms_); + if (schema_type_ids_.find(doc_schema_type_id) == schema_type_ids_.end()) { + return std::vector<double>(); + } + + std::unique_ptr<ScorablePropertySet> scorable_property_set = + document_store_.GetScorablePropertySet(hit_info.document_id(), + current_time_ms_); + // It should never happen. + if (scorable_property_set == nullptr) { + return absl_ports::InternalError(IcingStringUtil::StringPrintf( + "Failed to retrieve ScorablePropertySet for document %d", + hit_info.document_id())); + } + + const ScorablePropertyProto* scorable_property_proto = + scorable_property_set->GetScorablePropertyProto(property_path_); + // It should never happen as icing generates a default value for each scorable + // property when the document is created. + if (scorable_property_proto == nullptr) { + return absl_ports::InternalError(IcingStringUtil::StringPrintf( + "Failed to retrieve ScorablePropertyProto for document %d, and " + "property path %s", + hit_info.document_id(), property_path_.c_str())); + } + + // Converts ScorablePropertyProto to a vector of doubles. + if (scorable_property_proto->int64_values_size() > 0) { + return std::vector<double>(scorable_property_proto->int64_values().begin(), + scorable_property_proto->int64_values().end()); + } else if (scorable_property_proto->double_values_size() > 0) { + return std::vector<double>(scorable_property_proto->double_values().begin(), + scorable_property_proto->double_values().end()); + } else if (scorable_property_proto->boolean_values_size() > 0) { + return std::vector<double>( + scorable_property_proto->boolean_values().begin(), + scorable_property_proto->boolean_values().end()); + } + return std::vector<double>(); +} + } // namespace lib } // namespace icing
diff --git a/icing/scoring/advanced_scoring/score-expression.h b/icing/scoring/advanced_scoring/score-expression.h index 2bb5871..b61ac7b 100644 --- a/icing/scoring/advanced_scoring/score-expression.h +++ b/icing/scoring/advanced_scoring/score-expression.h
@@ -30,6 +30,7 @@ #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/join/join-children-fetcher.h" +#include "icing/schema/schema-store.h" #include "icing/scoring/bm25f-calculator.h" #include "icing/scoring/section-weights.h" #include "icing/store/document-filter-data.h" @@ -50,6 +51,10 @@ kString, }; +// A map from alias schema type to a set of Icing schema types. +using SchemaTypeAliasMap = + std::unordered_map<std::string, std::unordered_set<std::string>>; + class ScoreExpression { public: virtual ~ScoreExpression() = default; @@ -372,7 +377,9 @@ static libtextclassifier3::StatusOr< std::unique_ptr<ChildrenRankingSignalsFunctionScoreExpression>> Create(std::vector<std::unique_ptr<ScoreExpression>> args, - const JoinChildrenFetcher* join_children_fetcher); + const DocumentStore& document_store, + const JoinChildrenFetcher* join_children_fetcher, + int64_t current_time_ms); libtextclassifier3::StatusOr<std::vector<double>> EvaluateList( const DocHitInfo& hit_info, @@ -384,9 +391,15 @@ private: explicit ChildrenRankingSignalsFunctionScoreExpression( - const JoinChildrenFetcher& join_children_fetcher) - : join_children_fetcher_(join_children_fetcher) {} - const JoinChildrenFetcher& join_children_fetcher_; + const DocumentStore& document_store, + const JoinChildrenFetcher& join_children_fetcher, int64_t current_time_ms) + : document_store_(document_store), + join_children_fetcher_(join_children_fetcher), + current_time_ms_(current_time_ms) {} + + const DocumentStore& document_store_; // Does not own. + const JoinChildrenFetcher& join_children_fetcher_; // Does not own. + int64_t current_time_ms_; }; class PropertyWeightsFunctionScoreExpression : public ScoreExpression { @@ -410,8 +423,6 @@ return ScoreExpressionType::kDoubleList; } - SchemaTypeId GetSchemaTypeId(DocumentId document_id) const; - private: explicit PropertyWeightsFunctionScoreExpression( const DocumentStore* document_store, @@ -488,6 +499,63 @@ const EmbeddingQueryResults& embedding_query_results_; }; +class GetScorablePropertyFunctionScoreExpression : public ScoreExpression { + public: + static constexpr std::string_view kFunctionName = "getScorableProperty"; + + // Returns: + // - FAILED_PRECONDITION on any null pointer in children. + // - INVALID_ARGUMENT on + // - |args| type errors. + // - alias_schema_type in the scoring expression does not match to any + // schema type in the |schema_type_alias_map|. + // - any matched schema type not having the specified property_path as a + // scorable property. + static libtextclassifier3::StatusOr< + std::unique_ptr<GetScorablePropertyFunctionScoreExpression>> + Create(std::vector<std::unique_ptr<ScoreExpression>> args, + const DocumentStore* document_store, const SchemaStore* schema_store, + const SchemaTypeAliasMap& schema_type_alias_map, + int64_t current_time_ms); + + ScoreExpressionType type() const override { + return ScoreExpressionType::kDoubleList; + } + + libtextclassifier3::StatusOr<std::vector<double>> EvaluateList( + const DocHitInfo& hit_info, + const DocHitInfoIterator* query_it) const override; + + private: + explicit GetScorablePropertyFunctionScoreExpression( + const DocumentStore* document_store, const SchemaStore* schema_store, + int64_t current_time_ms, + std::unordered_set<SchemaTypeId>&& schema_type_ids, + std::string_view property_path); + + // Returns a set of schema type ids that are matched to the + // |alias_schema_type| in the scoring expression, based on the + // |schema_type_alias_map|. + // + // For each of the schema type in the returned set, this function also + // validates that: + // - The schema type is valid in the schema store. + // - The |property_path| is defined as scorable under the schema type. + static libtextclassifier3::StatusOr<std::unordered_set<SchemaTypeId>> + GetAndValidateSchemaTypeIds(std::string_view alias_schema_type, + std::string_view property_path, + const SchemaTypeAliasMap& schema_type_alias_map, + const SchemaStore& schema_store); + + const DocumentStore& document_store_; + const SchemaStore& schema_store_; + int64_t current_time_ms_; + // A doc hit is evaluated by this function only if its schema type id is in + // this set. + std::unordered_set<SchemaTypeId> schema_type_ids_; + std::string property_path_; +}; + } // namespace lib } // namespace icing
diff --git a/icing/scoring/advanced_scoring/scoring-visitor.cc b/icing/scoring/advanced_scoring/scoring-visitor.cc index 4d38c0e..eeab302 100644 --- a/icing/scoring/advanced_scoring/scoring-visitor.cc +++ b/icing/scoring/advanced_scoring/scoring-visitor.cc
@@ -106,18 +106,31 @@ } else if (function_name == RelevanceScoreFunctionScoreExpression::kFunctionName) { // relevanceScore function - expression = RelevanceScoreFunctionScoreExpression::Create( - std::move(args), &bm25f_calculator_, default_score_); + if (bm25f_calculator_ != nullptr) { + expression = RelevanceScoreFunctionScoreExpression::Create( + std::move(args), bm25f_calculator_, default_score_); + } else { + expression = absl_ports::InvalidArgumentError( + "relevanceScore function is not available in this context."); + } + } else if (function_name == ChildrenRankingSignalsFunctionScoreExpression::kFunctionName) { // childrenRankingSignals function expression = ChildrenRankingSignalsFunctionScoreExpression::Create( - std::move(args), join_children_fetcher_); + std::move(args), document_store_, join_children_fetcher_, + current_time_ms_); } else if (function_name == PropertyWeightsFunctionScoreExpression::kFunctionName) { // propertyWeights function - expression = PropertyWeightsFunctionScoreExpression::Create( - std::move(args), &document_store_, §ion_weights_, current_time_ms_); + if (section_weights_ != nullptr) { + expression = PropertyWeightsFunctionScoreExpression::Create( + std::move(args), &document_store_, section_weights_, + current_time_ms_); + } else { + expression = absl_ports::InvalidArgumentError( + "propertyWeights function is not available in this context."); + } } else if (MathFunctionScoreExpression::kFunctionNames.find(function_name) != MathFunctionScoreExpression::kFunctionNames.end()) { // Math functions @@ -132,16 +145,40 @@ ListOperationFunctionScoreExpression::kFunctionNames.at(function_name), std::move(args)); } else if (function_name == - GetEmbeddingParameterFunctionScoreExpression::kFunctionName) { + GetEmbeddingParameterFunctionScoreExpression::kFunctionName) { // getEmbeddingParameter function expression = GetEmbeddingParameterFunctionScoreExpression::Create(std::move(args)); } else if (function_name == MatchedSemanticScoresFunctionScoreExpression::kFunctionName) { // matchedSemanticScores function - expression = MatchedSemanticScoresFunctionScoreExpression::Create( - std::move(args), default_semantic_metric_type_, - &embedding_query_results_); + if (embedding_query_results_ != nullptr) { + expression = MatchedSemanticScoresFunctionScoreExpression::Create( + std::move(args), default_semantic_metric_type_, + embedding_query_results_); + } else { + expression = absl_ports::InvalidArgumentError( + "matchedSemanticScores function is not available in this context."); + } + } else if (function_name == + GetScorablePropertyFunctionScoreExpression::kFunctionName) { + if (!feature_flags_.enable_scorable_properties()) { + expression = absl_ports::InvalidArgumentError( + "getScorableProperty function is not enabled."); + } else if (scoring_feature_types_enabled_.find( + ScoringFeatureType::SCORABLE_PROPERTY_RANKING) == + scoring_feature_types_enabled_.end()) { + expression = absl_ports::InvalidArgumentError( + "SCORABLE_PROPERTY_RANKING feature is not enabled."); + } else if (schema_type_alias_map_ == nullptr) { + expression = absl_ports::InvalidArgumentError( + "getScorableProperty function is not available in this context."); + } else { + // getScorableProperty function + expression = GetScorablePropertyFunctionScoreExpression::Create( + std::move(args), &document_store_, &schema_store_, + *schema_type_alias_map_, current_time_ms_); + } } if (!expression.ok()) {
diff --git a/icing/scoring/advanced_scoring/scoring-visitor.h b/icing/scoring/advanced_scoring/scoring-visitor.h index 3c4a374..44a4d80 100644 --- a/icing/scoring/advanced_scoring/scoring-visitor.h +++ b/icing/scoring/advanced_scoring/scoring-visitor.h
@@ -17,15 +17,18 @@ #include <cstdint> #include <memory> +#include <unordered_set> #include <utility> #include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-query-results.h" #include "icing/join/join-children-fetcher.h" #include "icing/legacy/core/icing-string-util.h" +#include "icing/proto/scoring.pb.h" #include "icing/query/advanced_query_parser/abstract-syntax-tree.h" #include "icing/schema/schema-store.h" #include "icing/scoring/advanced_scoring/score-expression.h" @@ -38,6 +41,9 @@ class ScoringVisitor : public AbstractSyntaxTreeVisitor { public: + // join_children_fetcher, embedding_query_results, section_weights, + // bm25f_calculator, schema_type_alias_map are allowed to be nullptr if the + // corresponding scoring expression does not use them. explicit ScoringVisitor(double default_score, SearchSpecProto::EmbeddingQueryMetricType::Code default_semantic_metric_type, @@ -47,15 +53,22 @@ Bm25fCalculator* bm25f_calculator, const JoinChildrenFetcher* join_children_fetcher, const EmbeddingQueryResults* embedding_query_results, + const SchemaTypeAliasMap* schema_type_alias_map, + const FeatureFlags* feature_flags, + const std::unordered_set<ScoringFeatureType>* + scoring_feature_types_enabled, int64_t current_time_ms) : default_score_(default_score), default_semantic_metric_type_(default_semantic_metric_type), document_store_(*document_store), schema_store_(*schema_store), - section_weights_(*section_weights), - bm25f_calculator_(*bm25f_calculator), + section_weights_(section_weights), + bm25f_calculator_(bm25f_calculator), join_children_fetcher_(join_children_fetcher), - embedding_query_results_(*embedding_query_results), + embedding_query_results_(embedding_query_results), + schema_type_alias_map_(schema_type_alias_map), + feature_flags_(*feature_flags), + scoring_feature_types_enabled_(*scoring_feature_types_enabled), current_time_ms_(current_time_ms) {} void VisitString(const StringNode* node) override; @@ -104,13 +117,20 @@ double default_score_; const SearchSpecProto::EmbeddingQueryMetricType::Code default_semantic_metric_type_; - const DocumentStore& document_store_; - const SchemaStore& schema_store_; - SectionWeights& section_weights_; - Bm25fCalculator& bm25f_calculator_; + const DocumentStore& document_store_; // Does not own. + const SchemaStore& schema_store_; // Does not own. + + SectionWeights* section_weights_; // nullable, does not own. + Bm25fCalculator* bm25f_calculator_; // nullable, does not own. // A non-null join_children_fetcher_ indicates scoring in a join. - const JoinChildrenFetcher* join_children_fetcher_; // Does not own. - const EmbeddingQueryResults& embedding_query_results_; + const JoinChildrenFetcher* join_children_fetcher_; // nullable, does not own. + const EmbeddingQueryResults* + embedding_query_results_; // nullable, does not own. + const SchemaTypeAliasMap* schema_type_alias_map_; // nullable, does not own. + + const FeatureFlags& feature_flags_; // Does not own. + const std::unordered_set<ScoringFeatureType>& + scoring_feature_types_enabled_; // Does not own. libtextclassifier3::Status pending_error_; std::vector<std::unique_ptr<ScoreExpression>> stack_;
diff --git a/icing/scoring/bm25f-calculator.cc b/icing/scoring/bm25f-calculator.cc index a80ef34..e32d4e6 100644 --- a/icing/scoring/bm25f-calculator.cc +++ b/icing/scoring/bm25f-calculator.cc
@@ -14,19 +14,26 @@ #include "icing/scoring/bm25f-calculator.h" +#include <cmath> #include <cstdint> -#include <cstdlib> +#include <memory> #include <string> -#include <unordered_set> +#include <string_view> +#include <unordered_map> #include <vector> #include "icing/index/hit/doc-hit-info.h" +#include "icing/index/hit/hit.h" #include "icing/index/iterator/doc-hit-info-iterator.h" +#include "icing/schema/section.h" +#include "icing/scoring/section-weights.h" #include "icing/store/corpus-associated-scoring-data.h" #include "icing/store/corpus-id.h" #include "icing/store/document-associated-score-data.h" #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" +#include "icing/store/document-store.h" +#include "icing/util/logging.h" namespace icing { namespace lib { @@ -150,6 +157,11 @@ uint32_t num_docs = csdata.num_docs(); uint32_t nqi = corpus_nqi_map_[corpus_term_info.value]; + if (nqi > num_docs) { + ICING_LOG(ERROR) << "nqi > num_docs when calculating idf for corpus " + << corpus_id << " term " << term; + return 0; + } float idf = nqi != 0 ? log(1.0f + (num_docs - nqi + 0.5f) / (nqi + 0.5f)) : 0.0f; corpus_idf_map_.insert({corpus_term_info.value, idf});
diff --git a/icing/scoring/bm25f-calculator.h b/icing/scoring/bm25f-calculator.h index 36f9c68..d3f4354 100644 --- a/icing/scoring/bm25f-calculator.h +++ b/icing/scoring/bm25f-calculator.h
@@ -16,14 +16,19 @@ #define ICING_SCORING_BM25F_CALCULATOR_H_ #include <cstdint> +#include <memory> #include <string> -#include <unordered_set> -#include <vector> +#include <string_view> +#include <unordered_map> +#include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/legacy/index/icing-bit-util.h" #include "icing/scoring/section-weights.h" #include "icing/store/corpus-id.h" +#include "icing/store/document-associated-score-data.h" +#include "icing/store/document-filter-data.h" +#include "icing/store/document-id.h" #include "icing/store/document-store.h" namespace icing { @@ -92,13 +97,13 @@ // Compact representation of <CorpusId, TermId> for use as a key in a // hash_map. struct CorpusTermInfo { - // Layout bits: 16 bit CorpusId + 16 bit TermId - using Value = uint32_t; + // Layout bits: 16 bit padding + 32 bit CorpusId + 16 bit TermId + using Value = uint64_t; Value value; - static constexpr int kCorpusIdBits = sizeof(CorpusId); - static constexpr int kTermIdBits = sizeof(TermId); + static constexpr int kCorpusIdBits = sizeof(CorpusId) * 8; + static constexpr int kTermIdBits = sizeof(TermId) * 8; explicit CorpusTermInfo(CorpusId corpus_id, TermId term_id) : value(0) { BITFIELD_OR(value, kTermIdBits, kCorpusIdBits,
diff --git a/icing/scoring/score-and-rank_benchmark.cc b/icing/scoring/score-and-rank_benchmark.cc index b203b6f..57c02ef 100644 --- a/icing/scoring/score-and-rank_benchmark.cc +++ b/icing/scoring/score-and-rank_benchmark.cc
@@ -25,6 +25,7 @@ #include "testing/base/public/benchmark.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-query-results.h" @@ -42,6 +43,7 @@ #include "icing/store/document-id.h" #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/util/clock.h" @@ -96,17 +98,18 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, const SchemaStore* schema_store) { + const Clock* clock, const SchemaStore* schema_store, + const FeatureFlags& feature_flags) { return DocumentStore::Create( - filesystem, base_dir, clock, schema_store, + filesystem, base_dir, clock, schema_store, &feature_flags, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr); } void BM_ScoreAndRankDocumentHitsByDocumentScore(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); const std::string base_dir = GetTestTempDir() + "/score_and_rank_benchmark"; const std::string document_store_dir = base_dir + "/document_store"; const std::string schema_store_dir = base_dir + "/schema_store"; @@ -119,14 +122,14 @@ ASSERT_TRUE(filesystem.CreateDirectoryRecursively(schema_store_dir.c_str())); Clock clock; - ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_store_dir, &clock)); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem, schema_store_dir, + &clock, &feature_flags)); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -144,7 +147,7 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT, document_store.get(), schema_store.get(), clock.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results)); + &empty_embedding_query_results, &feature_flags)); int num_to_score = state.range(0); int num_of_documents = state.range(1); @@ -217,6 +220,7 @@ ->ArgPair(10000, 20000); void BM_ScoreAndRankDocumentHitsByCreationTime(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); const std::string base_dir = GetTestTempDir() + "/score_and_rank_benchmark"; const std::string document_store_dir = base_dir + "/document_store"; const std::string schema_store_dir = base_dir + "/schema_store"; @@ -229,14 +233,14 @@ ASSERT_TRUE(filesystem.CreateDirectoryRecursively(schema_store_dir.c_str())); Clock clock; - ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_store_dir, &clock)); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem, schema_store_dir, + &clock, &feature_flags)); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -255,7 +259,7 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT, document_store.get(), schema_store.get(), clock.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results)); + &empty_embedding_query_results, &feature_flags)); int num_to_score = state.range(0); int num_of_documents = state.range(1); @@ -330,6 +334,7 @@ ->ArgPair(10000, 20000); void BM_ScoreAndRankDocumentHitsNoScoring(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); const std::string base_dir = GetTestTempDir() + "/score_and_rank_benchmark"; const std::string document_store_dir = base_dir + "/document_store"; const std::string schema_store_dir = base_dir + "/schema_store"; @@ -342,14 +347,14 @@ ASSERT_TRUE(filesystem.CreateDirectoryRecursively(schema_store_dir.c_str())); Clock clock; - ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_store_dir, &clock)); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem, schema_store_dir, + &clock, &feature_flags)); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -367,7 +372,7 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT, document_store.get(), schema_store.get(), clock.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results)); + &empty_embedding_query_results, &feature_flags)); int num_to_score = state.range(0); int num_of_documents = state.range(1); @@ -437,6 +442,7 @@ ->ArgPair(10000, 20000); void BM_ScoreAndRankDocumentHitsByRelevanceScoring(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); const std::string base_dir = GetTestTempDir() + "/score_and_rank_benchmark"; const std::string document_store_dir = base_dir + "/document_store"; const std::string schema_store_dir = base_dir + "/schema_store"; @@ -449,14 +455,14 @@ ASSERT_TRUE(filesystem.CreateDirectoryRecursively(schema_store_dir.c_str())); Clock clock; - ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem, schema_store_dir, &clock)); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem, schema_store_dir, + &clock, &feature_flags)); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -474,7 +480,7 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT, document_store.get(), schema_store.get(), clock.GetSystemTimeMilliseconds(), /*join_children_fetcher=*/nullptr, - &empty_embedding_query_results)); + &empty_embedding_query_results, &feature_flags)); int num_to_score = state.range(0); int num_of_documents = state.range(1);
diff --git a/icing/scoring/scorer-factory.cc b/icing/scoring/scorer-factory.cc index 7fc9efb..afda649 100644 --- a/icing/scoring/scorer-factory.cc +++ b/icing/scoring/scorer-factory.cc
@@ -23,6 +23,7 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-query-results.h" #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator.h" @@ -184,10 +185,12 @@ default_semantic_metric_type, const DocumentStore* document_store, const SchemaStore* schema_store, int64_t current_time_ms, const JoinChildrenFetcher* join_children_fetcher, - const EmbeddingQueryResults* embedding_query_results) { + const EmbeddingQueryResults* embedding_query_results, + const FeatureFlags* feature_flags) { ICING_RETURN_ERROR_IF_NULL(document_store); ICING_RETURN_ERROR_IF_NULL(schema_store); ICING_RETURN_ERROR_IF_NULL(embedding_query_results); + ICING_RETURN_ERROR_IF_NULL(feature_flags); if (scoring_spec.rank_by() != ScoringSpecProto::RankingStrategy::ADVANCED_SCORING_EXPRESSION) { @@ -243,7 +246,7 @@ return AdvancedScorer::Create( scoring_spec, default_score, default_semantic_metric_type, document_store, schema_store, current_time_ms, join_children_fetcher, - embedding_query_results); + embedding_query_results, feature_flags); case ScoringSpecProto::RankingStrategy::JOIN_AGGREGATE_SCORE: // Use join aggregate score to rank. Since the aggregation score is // calculated by child documents after joining (in JoinProcessor), we can
diff --git a/icing/scoring/scorer-factory.h b/icing/scoring/scorer-factory.h index f5766b3..7687e98 100644 --- a/icing/scoring/scorer-factory.h +++ b/icing/scoring/scorer-factory.h
@@ -19,6 +19,7 @@ #include <memory> #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-query-results.h" #include "icing/join/join-children-fetcher.h" #include "icing/schema/schema-store.h" @@ -46,7 +47,8 @@ default_semantic_metric_type, const DocumentStore* document_store, const SchemaStore* schema_store, int64_t current_time_ms, const JoinChildrenFetcher* join_children_fetcher, - const EmbeddingQueryResults* embedding_query_results); + const EmbeddingQueryResults* embedding_query_results, + const FeatureFlags* feature_flags); } // namespace scorer_factory
diff --git a/icing/scoring/scorer_test.cc b/icing/scoring/scorer_test.cc index 097d0c7..a027a94 100644 --- a/icing/scoring/scorer_test.cc +++ b/icing/scoring/scorer_test.cc
@@ -24,6 +24,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-query-results.h" @@ -40,6 +41,7 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" namespace icing { @@ -56,6 +58,7 @@ schema_store_dir_(test_dir_ + "/schema_store") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(doc_store_dir_.c_str()); filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()); @@ -65,18 +68,19 @@ ICING_ASSERT_OK_AND_ASSIGN( schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock1_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock1_, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, doc_store_dir_, &fake_clock1_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock1_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); // Creates a simple email schema @@ -116,7 +120,7 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT; EmbeddingQueryResults empty_embedding_query_results; - private: + std::unique_ptr<FeatureFlags> feature_flags_; const std::string test_dir_; const std::string doc_store_dir_; const std::string schema_store_dir_; @@ -146,7 +150,8 @@ /*default_score=*/0, default_semantic_metric_type, /*document_store=*/nullptr, schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } @@ -157,7 +162,8 @@ ScoringSpecProto::RankingStrategy::DOCUMENT_SCORE, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), /*schema_store=*/nullptr, fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } @@ -169,7 +175,8 @@ ScoringSpecProto::RankingStrategy::DOCUMENT_SCORE, GetParam()), /*default_score=*/10, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); // Non existent document id DocHitInfo docHitInfo = DocHitInfo(/*document_id_in=*/1); @@ -197,7 +204,8 @@ ScoringSpecProto::RankingStrategy::DOCUMENT_SCORE, GetParam()), /*default_score=*/10, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(0)); @@ -224,7 +232,8 @@ ScoringSpecProto::RankingStrategy::DOCUMENT_SCORE, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(5)); @@ -253,7 +262,8 @@ ScoringSpecProto::RankingStrategy::RELEVANCE_SCORE, GetParam()), /*default_score=*/10, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); EXPECT_THAT(scorer->GetScore(docHitInfo), Eq(10)); @@ -291,7 +301,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo1 = DocHitInfo(document_id1); DocHitInfo docHitInfo2 = DocHitInfo(document_id2); @@ -322,7 +333,8 @@ ScoringSpecProto::RankingStrategy::USAGE_TYPE1_COUNT, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer2, scorer_factory::Create( @@ -330,7 +342,8 @@ ScoringSpecProto::RankingStrategy::USAGE_TYPE2_COUNT, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer3, scorer_factory::Create( @@ -338,7 +351,8 @@ ScoringSpecProto::RankingStrategy::USAGE_TYPE3_COUNT, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); EXPECT_THAT(scorer1->GetScore(docHitInfo), Eq(0)); EXPECT_THAT(scorer2->GetScore(docHitInfo), Eq(0)); @@ -376,7 +390,8 @@ ScoringSpecProto::RankingStrategy::USAGE_TYPE1_COUNT, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer2, scorer_factory::Create( @@ -384,7 +399,8 @@ ScoringSpecProto::RankingStrategy::USAGE_TYPE2_COUNT, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer3, scorer_factory::Create( @@ -392,7 +408,8 @@ ScoringSpecProto::RankingStrategy::USAGE_TYPE3_COUNT, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); EXPECT_THAT(scorer1->GetScore(docHitInfo), Eq(0)); EXPECT_THAT(scorer2->GetScore(docHitInfo), Eq(0)); @@ -430,7 +447,8 @@ ScoringSpecProto::RankingStrategy::USAGE_TYPE1_COUNT, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer2, scorer_factory::Create( @@ -438,7 +456,8 @@ ScoringSpecProto::RankingStrategy::USAGE_TYPE2_COUNT, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer3, scorer_factory::Create( @@ -446,7 +465,8 @@ ScoringSpecProto::RankingStrategy::USAGE_TYPE3_COUNT, GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); EXPECT_THAT(scorer1->GetScore(docHitInfo), Eq(0)); EXPECT_THAT(scorer2->GetScore(docHitInfo), Eq(0)); @@ -486,7 +506,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer2, scorer_factory::Create( @@ -496,7 +517,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer3, scorer_factory::Create( @@ -506,7 +528,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); EXPECT_THAT(scorer1->GetScore(docHitInfo), Eq(0)); EXPECT_THAT(scorer2->GetScore(docHitInfo), Eq(0)); @@ -562,7 +585,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer2, scorer_factory::Create( @@ -572,7 +596,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer3, scorer_factory::Create( @@ -582,7 +607,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); EXPECT_THAT(scorer1->GetScore(docHitInfo), Eq(0)); EXPECT_THAT(scorer2->GetScore(docHitInfo), Eq(0)); @@ -638,7 +664,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer2, scorer_factory::Create( @@ -648,7 +675,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Scorer> scorer3, scorer_factory::Create( @@ -658,7 +686,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); EXPECT_THAT(scorer1->GetScore(docHitInfo), Eq(0)); EXPECT_THAT(scorer2->GetScore(docHitInfo), Eq(0)); @@ -699,7 +728,8 @@ ScoringSpecProto::RankingStrategy::NONE, GetParam()), /*default_score=*/3, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo1 = DocHitInfo(/*document_id_in=*/0); DocHitInfo docHitInfo2 = DocHitInfo(/*document_id_in=*/1); @@ -715,7 +745,8 @@ ScoringSpecProto::RankingStrategy::NONE, GetParam()), /*default_score=*/111, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); docHitInfo1 = DocHitInfo(/*document_id_in=*/4); docHitInfo2 = DocHitInfo(/*document_id_in=*/5); @@ -747,7 +778,8 @@ GetParam()), /*default_score=*/0, default_semantic_metric_type, document_store(), schema_store(), fake_clock1().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); DocHitInfo docHitInfo = DocHitInfo(document_id); // Create usage report for the maximum allowable timestamp.
diff --git a/icing/scoring/scoring-processor.cc b/icing/scoring/scoring-processor.cc index db38a6f..4cba8a6 100644 --- a/icing/scoring/scoring-processor.cc +++ b/icing/scoring/scoring-processor.cc
@@ -24,6 +24,7 @@ #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-query-results.h" #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator.h" @@ -53,22 +54,23 @@ const SchemaStore* schema_store, int64_t current_time_ms, const JoinChildrenFetcher* join_children_fetcher, - const EmbeddingQueryResults* embedding_query_results) { + const EmbeddingQueryResults* embedding_query_results, + const FeatureFlags* feature_flags) { ICING_RETURN_ERROR_IF_NULL(document_store); ICING_RETURN_ERROR_IF_NULL(schema_store); ICING_RETURN_ERROR_IF_NULL(embedding_query_results); + ICING_RETURN_ERROR_IF_NULL(feature_flags); - bool is_descending_order = - scoring_spec.order_by() == ScoringSpecProto::Order::DESC; - + double default_score = + (scoring_spec.order_by() == ScoringSpecProto::Order::DESC) + ? kDefaultScoreInDescendingOrder + : kDefaultScoreInAscendingOrder; ICING_ASSIGN_OR_RETURN( std::unique_ptr<Scorer> scorer, scorer_factory::Create( - scoring_spec, - is_descending_order ? kDefaultScoreInDescendingOrder - : kDefaultScoreInAscendingOrder, - default_semantic_metric_type, document_store, schema_store, - current_time_ms, join_children_fetcher, embedding_query_results)); + scoring_spec, default_score, default_semantic_metric_type, + document_store, schema_store, current_time_ms, join_children_fetcher, + embedding_query_results, feature_flags)); // Using `new` to access a non-public constructor. return std::unique_ptr<ScoringProcessor>( new ScoringProcessor(std::move(scorer)));
diff --git a/icing/scoring/scoring-processor.h b/icing/scoring/scoring-processor.h index de0b95c..40a5912 100644 --- a/icing/scoring/scoring-processor.h +++ b/icing/scoring/scoring-processor.h
@@ -23,6 +23,7 @@ #include <vector> #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/feature-flags.h" #include "icing/index/embed/embedding-query-results.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/join/join-children-fetcher.h" @@ -52,7 +53,8 @@ default_semantic_metric_type, const DocumentStore* document_store, const SchemaStore* schema_store, int64_t current_time_ms, const JoinChildrenFetcher* join_children_fetcher, - const EmbeddingQueryResults* embedding_query_results); + const EmbeddingQueryResults* embedding_query_results, + const FeatureFlags* feature_flags); // Assigns scores to DocHitInfos from the given DocHitInfoIterator and returns // a vector of ScoredDocumentHits. The size of results is no more than
diff --git a/icing/scoring/scoring-processor_test.cc b/icing/scoring/scoring-processor_test.cc index 0e46464..7949cdf 100644 --- a/icing/scoring/scoring-processor_test.cc +++ b/icing/scoring/scoring-processor_test.cc
@@ -26,6 +26,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-query-results.h" @@ -46,6 +47,7 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/util/status-macros.h" @@ -68,25 +70,26 @@ schema_store_dir_(test_dir_ + "/schema_store") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); // Creates file directories filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(doc_store_dir_.c_str()); filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, doc_store_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); // Creates a simple email schema @@ -132,7 +135,7 @@ SearchSpecProto::EmbeddingQueryMetricType::DOT_PRODUCT; EmbeddingQueryResults empty_embedding_query_results; - private: + std::unique_ptr<FeatureFlags> feature_flags_; const std::string test_dir_; const std::string doc_store_dir_; const std::string schema_store_dir_; @@ -209,12 +212,13 @@ TEST_F(ScoringProcessorTest, CreationWithNullDocumentStoreShouldFail) { ScoringSpecProto spec_proto; - EXPECT_THAT( - ScoringProcessor::Create( - spec_proto, default_semantic_metric_type, /*document_store=*/nullptr, - schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results), - StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + EXPECT_THAT(ScoringProcessor::Create( + spec_proto, default_semantic_metric_type, + /*document_store=*/nullptr, schema_store(), + fake_clock().GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, + &empty_embedding_query_results, feature_flags_.get()), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } TEST_F(ScoringProcessorTest, CreationWithNullSchemaStoreShouldFail) { @@ -223,7 +227,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), /*schema_store=*/nullptr, fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } @@ -233,7 +238,8 @@ ICING_EXPECT_OK(ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); } TEST_P(ScoringProcessorTest, ShouldHandleEmptyDocHitIterator) { @@ -251,7 +257,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scoring_processor->Score(std::move(doc_hit_info_iterator), /*num_to_score=*/5), @@ -281,7 +288,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scoring_processor->Score(std::move(doc_hit_info_iterator), /*num_to_score=*/-1), @@ -314,7 +322,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scoring_processor->Score(std::move(doc_hit_info_iterator), /*num_to_score=*/2), @@ -349,7 +358,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scoring_processor->Score(std::move(doc_hit_info_iterator), /*num_to_score=*/3), @@ -410,7 +420,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> query_term_iterators; @@ -485,7 +496,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> query_term_iterators; @@ -564,7 +576,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> query_term_iterators; @@ -586,6 +599,232 @@ EqualsScoredDocumentHit(expected_scored_doc_hit3))); } +TEST_P(ScoringProcessorTest, ShouldScoreByRelevanceScore_MultipleQueryTerms) { + DocumentProto document1 = + CreateDocument("icing", "email/1", kDefaultScore, + /*creation_timestamp_ms=*/kDefaultCreationTimestampMs); + DocumentProto document2 = + CreateDocument("icing", "email/2", kDefaultScore, + /*creation_timestamp_ms=*/kDefaultCreationTimestampMs); + DocumentProto document3 = + CreateDocument("icing", "email/3", kDefaultScore, + /*creation_timestamp_ms=*/kDefaultCreationTimestampMs); + + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::PutResult put_result1, + document_store()->Put(document1, /*num_tokens=*/20)); + DocumentId document_id1 = put_result1.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::PutResult put_result2, + document_store()->Put(document2, /*num_tokens=*/20)); + DocumentId document_id2 = put_result2.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::PutResult put_result3, + document_store()->Put(document3, /*num_tokens=*/20)); + DocumentId document_id3 = put_result3.new_document_id; + + // Index 5 terms with total frequencies: + // {"foo": 7, "bar": 5, "baz": 2, "qux": 1, "lux": 4} + // Document 1 term-frequencies: {"foo": 3, "bar": 3, "baz": 2, "qux": 0, + // "lux": 3} + DocHitInfoTermFrequencyPair foo_doc_hit_info1 = DocHitInfo(document_id1); + DocHitInfoTermFrequencyPair bar_doc_hit_info1 = DocHitInfo(document_id1); + DocHitInfoTermFrequencyPair baz_doc_hit_info1 = DocHitInfo(document_id1); + DocHitInfoTermFrequencyPair lux_doc_hit_info1 = DocHitInfo(document_id1); + foo_doc_hit_info1.UpdateSection(/*section_id*/ 0, /*hit_term_frequency=*/3); + bar_doc_hit_info1.UpdateSection(/*section_id*/ 1, /*hit_term_frequency=*/3); + baz_doc_hit_info1.UpdateSection(/*section_id*/ 0, /*hit_term_frequency=*/2); + lux_doc_hit_info1.UpdateSection(/*section_id*/ 1, /*hit_term_frequency=*/3); + // Document 2 term-frequencies: {"foo": 3, "bar": 2, "baz": 0, "qux": 0, + // "lux": 0} + DocHitInfoTermFrequencyPair foo_doc_hit_info2 = DocHitInfo(document_id2); + DocHitInfoTermFrequencyPair bar_doc_hit_info2 = DocHitInfo(document_id2); + foo_doc_hit_info2.UpdateSection(/*section_id*/ 0, /*hit_term_frequency=*/3); + bar_doc_hit_info2.UpdateSection(/*section_id*/ 1, /*hit_term_frequency=*/2); + // Document 3 term-frequencies: {"foo": 1, "bar": 0, "baz": 0, "qux": 1, + // "lux": 1} + DocHitInfoTermFrequencyPair foo_doc_hit_info3 = DocHitInfo(document_id3); + DocHitInfoTermFrequencyPair qux_doc_hit_info3 = DocHitInfo(document_id3); + DocHitInfoTermFrequencyPair lux_doc_hit_info3 = DocHitInfo(document_id3); + foo_doc_hit_info3.UpdateSection(/*section_id*/ 0, /*hit_term_frequency=*/1); + qux_doc_hit_info3.UpdateSection(/*section_id*/ 1, /*hit_term_frequency=*/1); + lux_doc_hit_info3.UpdateSection(/*section_id*/ 1, /*hit_term_frequency=*/1); + + // Creates input doc_hit_infos and expected output scored_document_hits + std::vector<DocHitInfoTermFrequencyPair> foo_doc_hit_infos = { + foo_doc_hit_info1, foo_doc_hit_info2, foo_doc_hit_info3}; + std::vector<DocHitInfoTermFrequencyPair> bar_doc_hit_infos = { + bar_doc_hit_info1, bar_doc_hit_info2}; + std::vector<DocHitInfoTermFrequencyPair> baz_doc_hit_infos = { + baz_doc_hit_info1}; + std::vector<DocHitInfoTermFrequencyPair> qux_doc_hit_infos = { + qux_doc_hit_info3}; + std::vector<DocHitInfoTermFrequencyPair> lux_doc_hit_infos = { + lux_doc_hit_info1, lux_doc_hit_info3}; + + // Creates a dummy DocHitInfoIterator with the results for the query words + std::unique_ptr<DocHitInfoIterator> foo_doc_hit_info_iterator = + std::make_unique<DocHitInfoIteratorDummy>(foo_doc_hit_infos, "foo"); + std::unique_ptr<DocHitInfoIterator> bar_doc_hit_info_iterator = + std::make_unique<DocHitInfoIteratorDummy>(bar_doc_hit_infos, "bar"); + std::unique_ptr<DocHitInfoIterator> baz_doc_hit_info_iterator = + std::make_unique<DocHitInfoIteratorDummy>(baz_doc_hit_infos, "baz"); + std::unique_ptr<DocHitInfoIterator> qux_doc_hit_info_iterator = + std::make_unique<DocHitInfoIteratorDummy>(qux_doc_hit_infos, "qux"); + std::unique_ptr<DocHitInfoIterator> lux_doc_hit_info_iterator = + std::make_unique<DocHitInfoIteratorDummy>(lux_doc_hit_infos, "lux"); + + ScoringSpecProto spec_proto = CreateScoringSpecForRankingStrategy( + ScoringSpecProto::RankingStrategy::RELEVANCE_SCORE, GetParam()); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<ScoringProcessor> scoring_processor, + ScoringProcessor::Create( + spec_proto, default_semantic_metric_type, document_store(), + schema_store(), fake_clock().GetSystemTimeMilliseconds(), + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); + + SectionIdMask kSectionIdMask1 = 0b00000001; + SectionIdMask kSectionIdMask2 = 0b00000010; + + { + std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> + query_term_iterators; + query_term_iterators["foo"] = + std::make_unique<DocHitInfoIteratorDummy>(foo_doc_hit_infos, "foo"); + query_term_iterators["bar"] = + std::make_unique<DocHitInfoIteratorDummy>(bar_doc_hit_infos, "bar"); + query_term_iterators["baz"] = + std::make_unique<DocHitInfoIteratorDummy>(baz_doc_hit_infos, "baz"); + query_term_iterators["qux"] = + std::make_unique<DocHitInfoIteratorDummy>(qux_doc_hit_infos, "qux"); + query_term_iterators["lux"] = + std::make_unique<DocHitInfoIteratorDummy>(lux_doc_hit_infos, "lux"); + + // Score "foo". + ScoredDocumentHit foo_expected_scored_doc_hit1(document_id1, + kSectionIdMask1, + /*score=*/0.19672); + ScoredDocumentHit foo_expected_scored_doc_hit2(document_id2, + kSectionIdMask1, + /*score=*/0.19672); + ScoredDocumentHit foo_expected_scored_doc_hit3(document_id3, + kSectionIdMask1, + /*score=*/0.118455); + EXPECT_THAT( + scoring_processor->Score(std::move(foo_doc_hit_info_iterator), + /*num_to_score=*/3, &query_term_iterators), + ElementsAre(EqualsScoredDocumentHit(foo_expected_scored_doc_hit1), + EqualsScoredDocumentHit(foo_expected_scored_doc_hit2), + EqualsScoredDocumentHit(foo_expected_scored_doc_hit3))); + } + + { + std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> + query_term_iterators; + query_term_iterators["foo"] = + std::make_unique<DocHitInfoIteratorDummy>(foo_doc_hit_infos, "foo"); + query_term_iterators["bar"] = + std::make_unique<DocHitInfoIteratorDummy>(bar_doc_hit_infos, "bar"); + query_term_iterators["baz"] = + std::make_unique<DocHitInfoIteratorDummy>(baz_doc_hit_infos, "baz"); + query_term_iterators["qux"] = + std::make_unique<DocHitInfoIteratorDummy>(qux_doc_hit_infos, "qux"); + query_term_iterators["lux"] = + std::make_unique<DocHitInfoIteratorDummy>(lux_doc_hit_infos, "lux"); + + // Score "bar" + ScoredDocumentHit bar_expected_scored_doc_hit1(document_id1, + kSectionIdMask2, + /*score=*/0.692416); + ScoredDocumentHit bar_expected_scored_doc_hit2(document_id2, + kSectionIdMask2, + /*score=*/0.594257); + EXPECT_THAT( + scoring_processor->Score(std::move(bar_doc_hit_info_iterator), + /*num_to_score=*/3, &query_term_iterators), + ElementsAre(EqualsScoredDocumentHit(bar_expected_scored_doc_hit1), + EqualsScoredDocumentHit(bar_expected_scored_doc_hit2))); + } + + { + std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> + query_term_iterators; + query_term_iterators["foo"] = + std::make_unique<DocHitInfoIteratorDummy>(foo_doc_hit_infos, "foo"); + query_term_iterators["bar"] = + std::make_unique<DocHitInfoIteratorDummy>(bar_doc_hit_infos, "bar"); + query_term_iterators["baz"] = + std::make_unique<DocHitInfoIteratorDummy>(baz_doc_hit_infos, "baz"); + query_term_iterators["qux"] = + std::make_unique<DocHitInfoIteratorDummy>(qux_doc_hit_infos, "qux"); + query_term_iterators["lux"] = + std::make_unique<DocHitInfoIteratorDummy>(lux_doc_hit_infos, "lux"); + + // Score "baz" + ScoredDocumentHit baz_expected_scored_doc_hit1(document_id1, + kSectionIdMask1, + /*score=*/1.240129); + EXPECT_THAT( + scoring_processor->Score(std::move(baz_doc_hit_info_iterator), + /*num_to_score=*/3, &query_term_iterators), + ElementsAre(EqualsScoredDocumentHit(baz_expected_scored_doc_hit1))); + } + + { + std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> + query_term_iterators; + query_term_iterators["foo"] = + std::make_unique<DocHitInfoIteratorDummy>(foo_doc_hit_infos, "foo"); + query_term_iterators["bar"] = + std::make_unique<DocHitInfoIteratorDummy>(bar_doc_hit_infos, "bar"); + query_term_iterators["baz"] = + std::make_unique<DocHitInfoIteratorDummy>(baz_doc_hit_infos, "baz"); + query_term_iterators["qux"] = + std::make_unique<DocHitInfoIteratorDummy>(qux_doc_hit_infos, "qux"); + query_term_iterators["lux"] = + std::make_unique<DocHitInfoIteratorDummy>(lux_doc_hit_infos, "lux"); + + // Score "qux" + ScoredDocumentHit qux_expected_scored_doc_hit1(document_id3, + kSectionIdMask2, + /*score=*/0.87009); + EXPECT_THAT( + scoring_processor->Score(std::move(qux_doc_hit_info_iterator), + /*num_to_score=*/3, &query_term_iterators), + ElementsAre(EqualsScoredDocumentHit(qux_expected_scored_doc_hit1))); + } + + { + std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> + query_term_iterators; + query_term_iterators["foo"] = + std::make_unique<DocHitInfoIteratorDummy>(foo_doc_hit_infos, "foo"); + query_term_iterators["bar"] = + std::make_unique<DocHitInfoIteratorDummy>(bar_doc_hit_infos, "bar"); + query_term_iterators["baz"] = + std::make_unique<DocHitInfoIteratorDummy>(baz_doc_hit_infos, "baz"); + query_term_iterators["qux"] = + std::make_unique<DocHitInfoIteratorDummy>(qux_doc_hit_infos, "qux"); + query_term_iterators["lux"] = + std::make_unique<DocHitInfoIteratorDummy>(lux_doc_hit_infos, "lux"); + + // Score "lux" + ScoredDocumentHit lux_expected_scored_doc_hit1(document_id1, + kSectionIdMask2, + /*score=*/0.692416); + ScoredDocumentHit lux_expected_scored_doc_hit2(document_id3, + kSectionIdMask2, + /*score=*/0.416939); + EXPECT_THAT( + scoring_processor->Score(std::move(lux_doc_hit_info_iterator), + /*num_to_score=*/3, &query_term_iterators), + ElementsAre(EqualsScoredDocumentHit(lux_expected_scored_doc_hit1), + EqualsScoredDocumentHit(lux_expected_scored_doc_hit2))); + } +} + TEST_P(ScoringProcessorTest, ShouldScoreByRelevanceScore_HitTermWithZeroFrequency) { DocumentProto document1 = @@ -617,7 +856,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> query_term_iterators; @@ -687,7 +927,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> query_term_iterators; @@ -762,7 +1003,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> query_term_iterators; @@ -827,7 +1069,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); ScoringSpecProto spec_proto_with_weights = CreateScoringSpecForRankingStrategy( @@ -846,7 +1089,8 @@ spec_proto_with_weights, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> query_term_iterators; @@ -937,7 +1181,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); std::unordered_map<std::string, std::unique_ptr<DocHitInfoIterator>> query_term_iterators; @@ -1005,7 +1250,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scoring_processor->Score(std::move(doc_hit_info_iterator), /*num_to_score=*/3), @@ -1071,7 +1317,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scoring_processor->Score(std::move(doc_hit_info_iterator), /*num_to_score=*/3), @@ -1137,7 +1384,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scoring_processor->Score(std::move(doc_hit_info_iterator), /*num_to_score=*/3), @@ -1176,7 +1424,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scoring_processor->Score(std::move(doc_hit_info_iterator), /*num_to_score=*/4), ElementsAre(EqualsScoredDocumentHit(scored_document_hit_default), @@ -1231,7 +1480,8 @@ ScoringProcessor::Create( spec_proto, default_semantic_metric_type, document_store(), schema_store(), fake_clock().GetSystemTimeMilliseconds(), - /*join_children_fetcher=*/nullptr, &empty_embedding_query_results)); + /*join_children_fetcher=*/nullptr, &empty_embedding_query_results, + feature_flags_.get())); EXPECT_THAT(scoring_processor->Score(std::move(doc_hit_info_iterator), /*num_to_score=*/3),
diff --git a/icing/scoring/section-weights_test.cc b/icing/scoring/section-weights_test.cc index 28b1797..c90ca4a 100644 --- a/icing/scoring/section-weights_test.cc +++ b/icing/scoring/section-weights_test.cc
@@ -15,15 +15,19 @@ #include "icing/scoring/section-weights.h" #include <cfloat> +#include <memory> #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "icing/feature-flags.h" #include "icing/proto/schema.pb.h" #include "icing/proto/scoring.pb.h" #include "icing/proto/term.pb.h" #include "icing/schema-builder.h" +#include "icing/schema/schema-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" namespace icing { @@ -39,13 +43,15 @@ schema_store_dir_(test_dir_ + "/schema_store") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + // Creates file directories filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, test_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, test_dir_, + &fake_clock_, feature_flags_.get())); SchemaTypeConfigProto sender_schema = SchemaTypeConfigBuilder() @@ -100,6 +106,7 @@ SchemaStore *schema_store() { return schema_store_.get(); } private: + std::unique_ptr<FeatureFlags> feature_flags_; const std::string test_dir_; const std::string schema_store_dir_; Filesystem filesystem_;
diff --git a/icing/store/blob-store.cc b/icing/store/blob-store.cc index ca943af..cb26761 100644 --- a/icing/store/blob-store.cc +++ b/icing/store/blob-store.cc
@@ -14,14 +14,18 @@ #include "icing/store/blob-store.h" +#include <fcntl.h> + #include <algorithm> #include <array> +#include <cerrno> #include <cstdint> #include <iterator> #include <limits> #include <memory> #include <string> #include <string_view> +#include <unordered_map> #include <unordered_set> #include <utility> #include <vector> @@ -30,11 +34,11 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" -#include "icing/file/destructible-directory.h" +#include "icing/file/constants.h" #include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" +#include "icing/proto/blob.pb.h" #include "icing/proto/document.pb.h" -#include "icing/store/dynamic-trie-key-mapper.h" -#include "icing/store/key-mapper.h" #include "icing/util/clock.h" #include "icing/util/encode-util.h" #include "icing/util/logging.h" @@ -44,19 +48,27 @@ namespace icing { namespace lib { -static constexpr std::string_view kKeyMapperDir = "key_mapper"; -// - Key: sha 256 digest (32 bytes) -// - Value: BlobInfo (24 bytes) -// Allow max 1M of blob info entries. -static constexpr int32_t kBlobInfoMapperMaxSize = 56 * 1024 * 1024; // 56 MiB +static constexpr std::string_view kBlobFileDir = "blob_files"; +static constexpr std::string_view kBlobInfoProtoLogFileName = + "blob_info_proto_file"; static constexpr int32_t kSha256LengthBytes = 32; static constexpr int32_t kReadBufferSize = 8192; -std::string MakeKeyMapperDir(const std::string& base_dir) { - return absl_ports::StrCat(base_dir, "/", kKeyMapperDir); +namespace { + +std::string MakeBlobInfoProtoLogFileName(const std::string& base_dir) { + return absl_ports::StrCat(base_dir, "/", kBlobInfoProtoLogFileName); } -namespace { +std::string MakeBlobFileDir(const std::string& base_dir) { + return absl_ports::StrCat(base_dir, "/", kBlobFileDir); +} + +std::string MakeBlobFileName(const std::string& base_dir, + int64_t creation_time_ms) { + return absl_ports::StrCat(MakeBlobFileDir(base_dir), "/", + std::to_string(creation_time_ms)); +} libtextclassifier3::Status ValidateBlobHandle( const PropertyProto::BlobHandleProto& blob_handle) { @@ -64,26 +76,63 @@ return absl_ports::InvalidArgumentError( "Invalid blob handle. The digest is not sha 256 digest."); } + if (blob_handle.namespace_().empty()) { + return absl_ports::InvalidArgumentError( + "Invalid blob handle. The namespace is empty."); + } return libtextclassifier3::Status::OK; } +libtextclassifier3::StatusOr<std::unordered_map<std::string, int32_t>> +LoadBlobHandleToOffsetMapper( + PortableFileBackedProtoLog<BlobInfoProto>* blob_info_log) { + std::unordered_map<std::string, int32_t> blob_handle_to_offset; + auto itr = blob_info_log->GetIterator(); + while (itr.Advance().ok()) { + auto blob_info_proto_or = blob_info_log->ReadProto(itr.GetOffset()); + if (!blob_info_proto_or.ok()) { + if (absl_ports::IsNotFound(blob_info_proto_or.status())) { + // Skip erased proto. + continue; + } + + // Return real error. + return std::move(blob_info_proto_or).status(); + } + BlobInfoProto blob_info_proto = std::move(blob_info_proto_or).ValueOrDie(); + + std::string blob_handle_str = + BlobStore::BuildBlobHandleStr(blob_info_proto.blob_handle()); + blob_handle_to_offset.insert({std::move(blob_handle_str), itr.GetOffset()}); + } + return blob_handle_to_offset; +} + } // namespace +/* static */ std::string BlobStore::BuildBlobHandleStr( + const PropertyProto::BlobHandleProto& blob_handle) { + return encode_util::EncodeStringToCString(blob_handle.digest() + + blob_handle.namespace_()); +} + libtextclassifier3::StatusOr<BlobStore> BlobStore::Create( const Filesystem* filesystem, std::string base_dir, const Clock* clock, - int64_t orphan_blob_time_to_live_ms) { + int64_t orphan_blob_time_to_live_ms, int32_t compression_level) { ICING_RETURN_ERROR_IF_NULL(filesystem); ICING_RETURN_ERROR_IF_NULL(clock); - ICING_ASSIGN_OR_RETURN( - std::unique_ptr<KeyMapper<BlobInfo>> blob_info_mapper, - DynamicTrieKeyMapper<BlobInfo>::Create( - *filesystem, MakeKeyMapperDir(base_dir), kBlobInfoMapperMaxSize)); + + // Make sure the blob file directory exists. + if (!filesystem->CreateDirectoryRecursively( + MakeBlobFileDir(base_dir).c_str())) { + return absl_ports::InternalError( + absl_ports::StrCat("Could not create blob file directory.")); + } // Load existing file names (excluding the directory of key mapper). std::vector<std::string> file_names; - std::unordered_set<std::string> excludes = {kKeyMapperDir.data()}; - if (!filesystem->ListDirectory(base_dir.c_str(), excludes, - /*recursive=*/false, &file_names)) { + if (!filesystem->ListDirectory(MakeBlobFileDir(base_dir).c_str(), + &file_names)) { return absl_ports::InternalError("Failed to list directory."); } std::unordered_set<std::string> known_file_names( @@ -92,51 +141,111 @@ if (orphan_blob_time_to_live_ms <= 0) { orphan_blob_time_to_live_ms = std::numeric_limits<int64_t>::max(); } - return BlobStore(filesystem, std::move(base_dir), clock, - orphan_blob_time_to_live_ms, std::move(blob_info_mapper), - std::move(known_file_names)); + + std::string blob_info_proto_file_name = + MakeBlobInfoProtoLogFileName(base_dir); + + ICING_ASSIGN_OR_RETURN( + PortableFileBackedProtoLog<BlobInfoProto>::CreateResult log_create_result, + PortableFileBackedProtoLog<BlobInfoProto>::Create( + filesystem, blob_info_proto_file_name, + PortableFileBackedProtoLog<BlobInfoProto>::Options( + /*compress_in=*/true, constants::kMaxProtoSize, + compression_level))); + + std::unordered_map<std::string, int> blob_handle_to_offset; + ICING_ASSIGN_OR_RETURN( + blob_handle_to_offset, + LoadBlobHandleToOffsetMapper(log_create_result.proto_log.get())); + + return BlobStore( + filesystem, std::move(base_dir), clock, orphan_blob_time_to_live_ms, + compression_level, std::move(log_create_result.proto_log), + std::move(blob_handle_to_offset), std::move(known_file_names)); } libtextclassifier3::StatusOr<int> BlobStore::OpenWrite( const PropertyProto::BlobHandleProto& blob_handle) { ICING_RETURN_IF_ERROR(ValidateBlobHandle(blob_handle)); - std::string blob_handle_str = - encode_util::EncodeStringToCString(blob_handle.digest()) + - blob_handle.label(); - ICING_ASSIGN_OR_RETURN(BlobInfo blob_info, - GetOrCreateBlobInfo(blob_handle_str)); - if (blob_info.is_committed) { - return absl_ports::AlreadyExistsError( - "Rewriting the committed blob is not allowed."); + std::string blob_handle_str = BuildBlobHandleStr(blob_handle); + + auto blob_info_itr = blob_handle_to_offset_.find(blob_handle_str); + if (blob_info_itr != blob_handle_to_offset_.end()) { + ICING_ASSIGN_OR_RETURN(BlobInfoProto blob_info, + blob_info_log_->ReadProto(blob_info_itr->second)); + if (blob_info.is_committed()) { + // The blob is already committed, return error. + return absl_ports::AlreadyExistsError(absl_ports::StrCat( + "Rewriting the committed blob is not allowed for blob handle: ", + blob_handle.digest())); + } } - std::string file_name = absl_ports::StrCat( - base_dir_, "/", std::to_string(blob_info.creation_time_ms)); + + // Create a new blob info and blob file. + ICING_ASSIGN_OR_RETURN(BlobInfoProto blob_info, + GetOrCreateBlobInfo(blob_handle_str, blob_handle)); + + std::string file_name = + MakeBlobFileName(base_dir_, blob_info.creation_time_ms()); int file_descriptor = filesystem_.OpenForWrite(file_name.c_str()); if (file_descriptor < 0) { return absl_ports::InternalError(absl_ports::StrCat( - "Failed to open blob file for handle: ", blob_handle.label())); + "Failed to open blob file for handle: ", blob_handle.digest())); } return file_descriptor; } +libtextclassifier3::Status BlobStore::RemoveBlob( + const PropertyProto::BlobHandleProto& blob_handle) { + ICING_RETURN_IF_ERROR(ValidateBlobHandle(blob_handle)); + std::string blob_handle_str = BuildBlobHandleStr(blob_handle); + + auto blob_info_itr = blob_handle_to_offset_.find(blob_handle_str); + if (blob_info_itr == blob_handle_to_offset_.end()) { + return absl_ports::NotFoundError(absl_ports::StrCat( + "Cannot find the blob for handle: ", blob_handle.digest())); + } + + int64_t blob_info_offset = blob_info_itr->second; + ICING_ASSIGN_OR_RETURN(BlobInfoProto blob_info, + blob_info_log_->ReadProto(blob_info_offset)); + + std::string file_name = + MakeBlobFileName(base_dir_, blob_info.creation_time_ms()); + if (!filesystem_.DeleteFile(file_name.c_str())) { + return absl_ports::InternalError(absl_ports::StrCat( + "Failed to abandon blob file for handle: ", blob_handle.digest())); + } + ICING_RETURN_IF_ERROR(blob_info_log_->EraseProto(blob_info_offset)); + blob_handle_to_offset_.erase(blob_info_itr); + + has_mutated_ = true; + return libtextclassifier3::Status::OK; +} + libtextclassifier3::StatusOr<int> BlobStore::OpenRead( const PropertyProto::BlobHandleProto& blob_handle) { ICING_RETURN_IF_ERROR(ValidateBlobHandle(blob_handle)); - std::string blob_handle_str = - encode_util::EncodeStringToCString(blob_handle.digest()) + - blob_handle.label(); - ICING_ASSIGN_OR_RETURN(BlobInfo blob_info, - blob_info_mapper_->Get(blob_handle_str)); - if (!blob_info.is_committed) { - return absl_ports::NotFoundError("Cannot find blob file to read."); + std::string blob_handle_str = BuildBlobHandleStr(blob_handle); + auto itr = blob_handle_to_offset_.find(blob_handle_str); + if (itr == blob_handle_to_offset_.end()) { + return absl_ports::NotFoundError(absl_ports::StrCat( + "Cannot find the blob for handle: ", blob_handle.digest())); + } + ICING_ASSIGN_OR_RETURN(BlobInfoProto blob_info, + blob_info_log_->ReadProto(itr->second)); + if (!blob_info.is_committed()) { + // The blob is not committed, return error. + return absl_ports::NotFoundError(absl_ports::StrCat( + "Cannot find the blob for handle: ", blob_handle.digest())); } - std::string file_name = absl_ports::StrCat( - base_dir_, "/", std::to_string(blob_info.creation_time_ms)); + std::string file_name = + MakeBlobFileName(base_dir_, blob_info.creation_time_ms()); int file_descriptor = filesystem_.OpenForRead(file_name.c_str()); if (file_descriptor < 0) { return absl_ports::InternalError(absl_ports::StrCat( - "Failed to open blob file for handle: ", blob_handle.label())); + "Failed to open blob file for handle: ", blob_handle.digest())); } return file_descriptor; } @@ -144,96 +253,137 @@ libtextclassifier3::Status BlobStore::CommitBlob( const PropertyProto::BlobHandleProto& blob_handle) { ICING_RETURN_IF_ERROR(ValidateBlobHandle(blob_handle)); - std::string blob_handle_str = - encode_util::EncodeStringToCString(blob_handle.digest()) + - blob_handle.label(); - ICING_ASSIGN_OR_RETURN(BlobInfo blob_info, - blob_info_mapper_->Get(blob_handle_str)); - if (blob_info.is_committed) { + + std::string blob_handle_str = BuildBlobHandleStr(blob_handle); + + auto pending_blob_info_itr = blob_handle_to_offset_.find(blob_handle_str); + if (pending_blob_info_itr == blob_handle_to_offset_.end()) { + return absl_ports::NotFoundError(absl_ports::StrCat( + "Cannot find the blob for handle: ", blob_handle.digest())); + } + int64_t pending_blob_info_offset = pending_blob_info_itr->second; + + ICING_ASSIGN_OR_RETURN(BlobInfoProto blob_info_proto, + blob_info_log_->ReadProto(pending_blob_info_offset)); + + // Check if the blob is already committed. + if (blob_info_proto.is_committed()) { return absl_ports::AlreadyExistsError(absl_ports::StrCat( - "The blob is already committed for handle: ", blob_handle.label())); + "The blob is already committed for handle: ", blob_handle.digest())); } // Read the file and verify the digest. - std::string file_name = absl_ports::StrCat( - base_dir_, "/", std::to_string(blob_info.creation_time_ms)); - ScopedFd sfd(filesystem_.OpenForRead(file_name.c_str())); - if (!sfd.is_valid()) { - return absl_ports::InternalError(absl_ports::StrCat( - "Failed to open blob file for handle: ", blob_handle.label())); - } - int64_t file_size = filesystem_.GetFileSize(sfd.get()); - if (file_size == Filesystem::kBadFileSize) { - return absl_ports::InternalError(absl_ports::StrCat( - "Failed to get file size for handle: ", blob_handle.label())); - } - + std::string file_name = + MakeBlobFileName(base_dir_, blob_info_proto.creation_time_ms()); Sha256 sha256; - // Read 8 KiB per iteration - int64_t prev_total_read_size = 0; - auto buffer = std::make_unique<uint8_t[]>(kReadBufferSize); - while (prev_total_read_size < file_size) { - int32_t size_to_read = - std::min<int32_t>(kReadBufferSize, file_size - prev_total_read_size); - if (!filesystem_.Read(sfd.get(), buffer.get(), size_to_read)) { + { + ScopedFd sfd(filesystem_.OpenForRead(file_name.c_str())); + if (!sfd.is_valid()) { return absl_ports::InternalError(absl_ports::StrCat( - "Failed to read blob file for handle: ", blob_handle.label())); + "Failed to open blob file for handle: ", blob_handle.digest())); } - sha256.Update(buffer.get(), size_to_read); - prev_total_read_size += size_to_read; - } - std::array<uint8_t, 32> hash = std::move(sha256).Finalize(); + int64_t file_size = filesystem_.GetFileSize(sfd.get()); + if (file_size == Filesystem::kBadFileSize) { + return absl_ports::InternalError(absl_ports::StrCat( + "Failed to get file size for handle: ", blob_handle.digest())); + } + // Read 8 KiB per iteration + int64_t prev_total_read_size = 0; + uint8_t buffer[kReadBufferSize]; + while (prev_total_read_size < file_size) { + int32_t size_to_read = + std::min<int32_t>(kReadBufferSize, file_size - prev_total_read_size); + if (!filesystem_.Read(sfd.get(), buffer, size_to_read)) { + return absl_ports::InternalError(absl_ports::StrCat( + "Failed to read blob file for handle: ", blob_handle.digest())); + } + + sha256.Update(buffer, size_to_read); + prev_total_read_size += size_to_read; + } + } + + std::array<uint8_t, 32> hash = std::move(sha256).Finalize(); const std::string& digest = blob_handle.digest(); + + // Close active file descriptor and cached pending blob info for the pending + // blob handle before we verify the digest. This is needed anyway. We will add + // cached pending blob info back if the digest is valid. + + ICING_RETURN_IF_ERROR(blob_info_log_->EraseProto(pending_blob_info_offset)); + if (digest.length() != hash.size() || digest.compare(0, digest.length(), reinterpret_cast<const char*>(hash.data()), hash.size()) != 0) { + // The blob content doesn't match to the digest. Delete this corrupted blob. + if (!filesystem_.DeleteFile(file_name.c_str())) { + return absl_ports::InternalError(absl_ports::StrCat( + "Failed to delete corrupted blob file for handle: ", + blob_handle.digest())); + } return absl_ports::InvalidArgumentError( "The blob content doesn't match to the digest."); } - // Mark the blob is committed - blob_info.is_committed = true; has_mutated_ = true; - ICING_RETURN_IF_ERROR(blob_info_mapper_->Put(blob_handle_str, blob_info)); + blob_info_proto.set_is_committed(true); + auto blob_info_offset_or = blob_info_log_->WriteProto(blob_info_proto); + if (!blob_info_offset_or.ok()) { + ICING_LOG(ERROR) << blob_info_offset_or.status().error_message() + << "Failed to write blob info"; + return blob_info_offset_or.status(); + } + blob_handle_to_offset_[blob_handle_str] = blob_info_offset_or.ValueOrDie(); return libtextclassifier3::Status::OK; } libtextclassifier3::Status BlobStore::PersistToDisk() { if (has_mutated_) { - ICING_RETURN_IF_ERROR(blob_info_mapper_->PersistToDisk()); + ICING_RETURN_IF_ERROR(blob_info_log_->PersistToDisk()); has_mutated_ = false; } return libtextclassifier3::Status::OK; } -libtextclassifier3::StatusOr<BlobStore::BlobInfo> -BlobStore::GetOrCreateBlobInfo(const std::string& blob_handle_str) { - libtextclassifier3::StatusOr<BlobInfo> blob_info_or = - blob_info_mapper_->Get(blob_handle_str); - - if (absl_ports::IsNotFound(blob_info_or.status())) { - // Create a new blob info, we are using creation time as the unique file - // name. - int64_t timestamp = clock_.GetSystemTimeMilliseconds(); - std::string file_name = std::to_string(timestamp); - while (known_file_names_.find(file_name) != known_file_names_.end()) { - ++timestamp; - file_name = std::to_string(timestamp); - } - known_file_names_.insert(file_name); - - BlobInfo blob_info = {timestamp, /*is_committed=*/false}; - ICING_RETURN_IF_ERROR(blob_info_mapper_->Put(blob_handle_str, blob_info)); - has_mutated_ = true; - return blob_info; +libtextclassifier3::StatusOr<BlobInfoProto> BlobStore::GetOrCreateBlobInfo( + const std::string& blob_handle_str, + const PropertyProto::BlobHandleProto& blob_handle) { + auto itr = blob_handle_to_offset_.find(blob_handle_str); + if (itr != blob_handle_to_offset_.end()) { + return blob_info_log_->ReadProto(itr->second); } - return blob_info_or; + // Create a new blob info, we are using creation time as the unique file + // name. + int64_t timestamp = clock_.GetSystemTimeMilliseconds(); + std::string file_name = std::to_string(timestamp); + while (known_file_names_.find(file_name) != known_file_names_.end()) { + ++timestamp; + file_name = std::to_string(timestamp); + } + known_file_names_.insert(file_name); + + BlobInfoProto blob_info_proto; + *blob_info_proto.mutable_blob_handle() = blob_handle; + blob_info_proto.set_creation_time_ms(timestamp); + blob_info_proto.set_is_committed(false); + + auto blob_info_offset_or = blob_info_log_->WriteProto(blob_info_proto); + if (!blob_info_offset_or.ok()) { + ICING_LOG(ERROR) << blob_info_offset_or.status().error_message() + << "Failed to write blob info"; + return blob_info_offset_or.status(); + } + + has_mutated_ = true; + blob_handle_to_offset_[blob_handle_str] = blob_info_offset_or.ValueOrDie(); + + return blob_info_proto; } std::unordered_set<std::string> @@ -243,15 +393,18 @@ // Nothing to optimize, return empty set. return std::unordered_set<std::string>(); } - int64_t expired_threshold = - clock_.GetSystemTimeMilliseconds() - orphan_blob_time_to_live_ms_; - std::unique_ptr<typename KeyMapper<BlobInfo>::Iterator> itr = - blob_info_mapper_->GetIterator(); - + int64_t expired_threshold = current_time_ms - orphan_blob_time_to_live_ms_; std::unordered_set<std::string> expired_blob_handles; - while (itr->Advance()) { - if (itr->GetValue().creation_time_ms < expired_threshold) { - expired_blob_handles.insert(std::string(itr->GetKey())); + auto itr = blob_info_log_->GetIterator(); + while (itr.Advance().ok()) { + auto blob_info_proto_or = blob_info_log_->ReadProto(itr.GetOffset()); + if (!blob_info_proto_or.ok()) { + continue; + } + BlobInfoProto blob_info_proto = std::move(blob_info_proto_or).ValueOrDie(); + if (blob_info_proto.creation_time_ms() < expired_threshold) { + expired_blob_handles.insert( + BuildBlobHandleStr(blob_info_proto.blob_handle())); } } return expired_blob_handles; @@ -259,64 +412,133 @@ libtextclassifier3::Status BlobStore::Optimize( const std::unordered_set<std::string>& dead_blob_handles) { - if (dead_blob_handles.empty()) { - // nothing to optimize, return early. - return libtextclassifier3::Status::OK; + // Create the temp blob info log file. + std::string temp_blob_info_proto_file_name = + absl_ports::StrCat(MakeBlobInfoProtoLogFileName(base_dir_), "_temp"); + if (!filesystem_.DeleteFile(temp_blob_info_proto_file_name.c_str())) { + return absl_ports::InternalError( + "Unable to delete temp file to prepare to build new blob proto file."); } - // Create the temp blob store directory. - std::string temp_blob_store_dir_path = base_dir_ + "_temp"; - if (!filesystem_.DeleteDirectoryRecursively( - temp_blob_store_dir_path.c_str())) { - ICING_LOG(ERROR) << "Recursively deleting " - << temp_blob_store_dir_path.c_str(); - return absl_ports::InternalError( - "Unable to delete temp directory to prepare to build new blob store."); - } - DestructibleDirectory temp_blob_store_dir(&filesystem_, - temp_blob_store_dir_path); - if (!temp_blob_store_dir.is_valid()) { - return absl_ports::InternalError( - "Unable to create temp directory to build new blob store."); - } + ICING_ASSIGN_OR_RETURN(PortableFileBackedProtoLog<BlobInfoProto>::CreateResult + temp_log_create_result, + PortableFileBackedProtoLog<BlobInfoProto>::Create( + &filesystem_, temp_blob_info_proto_file_name, + PortableFileBackedProtoLog<BlobInfoProto>::Options( + /*compress_in=*/true, constants::kMaxProtoSize, + compression_level_))); + std::unique_ptr<PortableFileBackedProtoLog<BlobInfoProto>> new_blob_info_log = + std::move(temp_log_create_result.proto_log); - // Destroy the old blob info mapper and replace it with the new one. - std::string new_key_mapper_dir = MakeKeyMapperDir(temp_blob_store_dir_path); - ICING_ASSIGN_OR_RETURN( - std::unique_ptr<KeyMapper<BlobInfo>> new_blob_info_mapper, - DynamicTrieKeyMapper<BlobInfo>::Create(filesystem_, new_key_mapper_dir, - kBlobInfoMapperMaxSize)); - std::unique_ptr<typename KeyMapper<BlobInfo>::Iterator> itr = - blob_info_mapper_->GetIterator(); - while (itr->Advance()) { - if (dead_blob_handles.find(std::string(itr->GetKey())) == - dead_blob_handles.end()) { - ICING_RETURN_IF_ERROR( - new_blob_info_mapper->Put(itr->GetKey(), itr->GetValue())); - } else { - // Delete the file of dead blobs. - std::string file_name = absl_ports::StrCat( - base_dir_, "/", std::to_string(itr->GetValue().creation_time_ms)); + auto itr = blob_info_log_->GetIterator(); + std::unordered_map<std::string, int32_t> new_blob_handle_to_offset; + while (itr.Advance().ok()) { + auto blob_info_proto_or = blob_info_log_->ReadProto(itr.GetOffset()); + if (!blob_info_proto_or.ok()) { + if (absl_ports::IsNotFound(blob_info_proto_or.status())) { + // Skip erased proto. + continue; + } + + // Return real error. + return std::move(blob_info_proto_or).status(); + } + BlobInfoProto blob_info_proto = std::move(blob_info_proto_or).ValueOrDie(); + std::string blob_handle_str = + BuildBlobHandleStr(blob_info_proto.blob_handle()); + if (dead_blob_handles.find(blob_handle_str) != dead_blob_handles.end()) { + // Delete all dead blob files. + + std::string file_name = + MakeBlobFileName(base_dir_, blob_info_proto.creation_time_ms()); if (!filesystem_.DeleteFile(file_name.c_str())) { return absl_ports::InternalError( absl_ports::StrCat("Failed to delete blob file: ", file_name)); } + } else { + // Write the alive blob info to the new blob info log file. + ICING_ASSIGN_OR_RETURN(int32_t new_offset, + new_blob_info_log->WriteProto(blob_info_proto)); + new_blob_handle_to_offset[blob_handle_str] = new_offset; } } - new_blob_info_mapper.reset(); + new_blob_info_log->PersistToDisk(); + new_blob_info_log.reset(); + blob_info_log_.reset(); + std::string old_blob_info_proto_file_name = + MakeBlobInfoProtoLogFileName(base_dir_); // Then we swap the new key mapper directory with the old one. - if (!filesystem_.SwapFiles(MakeKeyMapperDir(base_dir_).c_str(), - new_key_mapper_dir.c_str())) { + if (!filesystem_.SwapFiles(old_blob_info_proto_file_name.c_str(), + temp_blob_info_proto_file_name.c_str())) { return absl_ports::InternalError( "Unable to apply new blob store due to failed swap!"); } - ICING_ASSIGN_OR_RETURN( - blob_info_mapper_, - DynamicTrieKeyMapper<BlobInfo>::Create( - filesystem_, MakeKeyMapperDir(base_dir_), kBlobInfoMapperMaxSize)); + // Delete the temp file, don't need to throw error if it fails, it will be + // deleted in the next run. + filesystem_.DeleteFile(temp_blob_info_proto_file_name.c_str()); + + ICING_ASSIGN_OR_RETURN( + PortableFileBackedProtoLog<BlobInfoProto>::CreateResult log_create_result, + PortableFileBackedProtoLog<BlobInfoProto>::Create( + &filesystem_, old_blob_info_proto_file_name, + PortableFileBackedProtoLog<BlobInfoProto>::Options( + /*compress_in=*/true, constants::kMaxProtoSize, + compression_level_))); + blob_info_log_ = std::move(log_create_result.proto_log); + blob_handle_to_offset_ = std::move(new_blob_handle_to_offset); return libtextclassifier3::Status::OK; } +libtextclassifier3::StatusOr<std::vector<NamespaceBlobStorageInfoProto>> +BlobStore::GetStorageInfo() const { + // Get the file size of each namespace offset. + std::unordered_map<std::string, NamespaceBlobStorageInfoProto> + namespace_to_storage_info; + auto itr = blob_info_log_->GetIterator(); + while (itr.Advance().ok()) { + auto blob_info_proto_or = blob_info_log_->ReadProto(itr.GetOffset()); + if (!blob_info_proto_or.ok()) { + if (absl_ports::IsNotFound(blob_info_proto_or.status())) { + // Skip erased proto. + continue; + } + + // Return real error. + return std::move(blob_info_proto_or).status(); + } + BlobInfoProto blob_info_proto = std::move(blob_info_proto_or).ValueOrDie(); + + std::string file_name = + MakeBlobFileName(base_dir_, blob_info_proto.creation_time_ms()); + int64_t file_size = filesystem_.GetFileSize(file_name.c_str()); + if (file_size == Filesystem::kBadFileSize) { + ICING_LOG(WARNING) << "Bad file size for blob file: " << file_name; + continue; + } + std::string name_space = blob_info_proto.blob_handle().namespace_(); + NamespaceBlobStorageInfoProto namespace_blob_storage_info = + namespace_to_storage_info[name_space]; + namespace_blob_storage_info.set_namespace_(name_space); + namespace_blob_storage_info.set_blob_size( + namespace_blob_storage_info.blob_size() + file_size); + namespace_blob_storage_info.set_num_blobs( + namespace_blob_storage_info.num_blobs() + 1); + namespace_to_storage_info[name_space] = + std::move(namespace_blob_storage_info); + } + + // Create the namespace blob storage info for each namespace. + std::vector<NamespaceBlobStorageInfoProto> namespace_blob_storage_infos; + namespace_blob_storage_infos.reserve(namespace_to_storage_info.size()); + for (const auto& [_, namespace_blob_storage_info] : + namespace_to_storage_info) { + namespace_blob_storage_infos.push_back( + std::move(namespace_blob_storage_info)); + } + + return namespace_blob_storage_infos; +} + } // namespace lib } // namespace icing
diff --git a/icing/store/blob-store.h b/icing/store/blob-store.h index 821f896..fb1a773 100644 --- a/icing/store/blob-store.h +++ b/icing/store/blob-store.h
@@ -16,17 +16,18 @@ #define ICING_STORE_BLOB_STORE_H_ #include <cstdint> -#include <memory> #include <string> -#include <string_view> +#include <unordered_map> #include <unordered_set> #include <utility> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" +#include "icing/proto/blob.pb.h" #include "icing/proto/document.pb.h" -#include "icing/store/key-mapper.h" +#include "icing/proto/storage.pb.h" #include "icing/util/clock.h" namespace icing { @@ -47,25 +48,10 @@ // The BlobStore is not thread-safe. class BlobStore { public: - // BlobInfo holds information about a blob. This struct will be stored as the - // value in the dynamic trie key mapper, so it must be packed to avoid - // padding (which potentially causes use-of-uninitialized-value errors). - struct BlobInfo { - // The creation time of the blob. This is used to determine when to delete - // the orphaned blobs. - // We are using creation_time_ms to be file name of the blob, so this field - // is unique for each blob. - int64_t creation_time_ms; - bool is_committed; - - // The param needed for dynamic trie, we shouldn't call this constructor - // directly. - BlobInfo() : BlobInfo(/*creation_time_ms=*/-1, /*is_committed=*/false) {} - - BlobInfo(int64_t creation_time_ms, bool is_committed) - : creation_time_ms(creation_time_ms), is_committed(is_committed) {} - } __attribute__((packed)); - static_assert(sizeof(BlobInfo) == 9, "Invalid BlobInfo size"); + // Builds a string representation of a blob handle. + // The string is used as the key in the key mapper. + static std::string BuildBlobHandleStr( + const PropertyProto::BlobHandleProto& blob_handle); // Factory function to create a BlobStore instance. The base directory is // used to persist blobs. If a blob store was previously created with @@ -75,31 +61,50 @@ // // Returns: // A BlobStore on success - // FAILED_PRECONDITION on any null pointer input + // FAILED_PRECONDITION_ERROR on any null pointer input // INTERNAL_ERROR on I/O error static libtextclassifier3::StatusOr<BlobStore> Create( const Filesystem* filesystem, std::string base_dir, const Clock* clock, - int64_t orphan_blob_time_to_live_ms); + int64_t orphan_blob_time_to_live_ms, int32_t compression_level); // Gets or creates a file for write only purpose for the given blob handle. // To mark the blob is completed written, CommitBlob must be called. Once // CommitBlob is called, the blob is sealed and rewrite is not allowed. // + // It is the user's responsibility to close the file descriptor after writing + // is done and should operate on the file descriptor after commit or remove + // it. + // // Returns: // File descriptor (writable) on success - // INVALID_ARGUMENT on invalid blob handle - // ALREADY_EXISTS if the blob has already been committed + // INVALID_ARGUMENT_ERROR on invalid blob handle + // FAILED_PRECONDITION_ERROR on blob is already opened for write + // ALREADY_EXISTS_ERROR if the blob has already been committed // INTERNAL_ERROR on IO error libtextclassifier3::StatusOr<int> OpenWrite( const PropertyProto::BlobHandleProto& blob_handle); + // Removes a blob file and blob handle from the blob store. + // + // This will remove the blob on any state. No matter it's committed or not or + // it has reference document links or not. + // + // Returns: + // INVALID_ARGUMENT_ERROR on invalid blob handle + // NOT_FOUND_ERROR on blob is not found + // INTERNAL_ERROR on IO error + libtextclassifier3::Status RemoveBlob( + const PropertyProto::BlobHandleProto& blob_handle); + // Gets a file for read only purpose for the given blob handle. // Will only succeed for blobs that were committed by calling CommitBlob. // + // It is the user's responsibility to close the file descriptor after reading. + // // Returns: // File descriptor (read only) on success - // INVALID_ARGUMENT on invalid blob handle - // NOT_FOUND on blob is not found or is not committed + // INVALID_ARGUMENT_ERROR on invalid blob handle + // NOT_FOUND_ERROR on blob is not found or is not committed libtextclassifier3::StatusOr<int> OpenRead( const PropertyProto::BlobHandleProto& blob_handle); @@ -110,10 +115,10 @@ // // Returns: // OK on the blob is successfully committed. - // ALREADY_EXISTS on the blob is already committed, this is no op. - // INVALID_ARGUMENT on invalid blob handle or digest is mismatch with + // ALREADY_EXISTS_ERROR on the blob is already committed, this is no op. + // INVALID_ARGUMENT_ERROR on invalid blob handle or digest is mismatch with // file content. - // NOT_FOUND on blob is not found. + // NOT_FOUND_ERROR on blob is not found. libtextclassifier3::Status CommitBlob( const PropertyProto::BlobHandleProto& blob_handle); @@ -140,28 +145,52 @@ libtextclassifier3::Status Optimize( const std::unordered_set<std::string>& dead_blob_handles); - private: - explicit BlobStore(const Filesystem* filesystem, std::string base_dir, - const Clock* clock, int64_t orphan_blob_time_to_live_ms, - std::unique_ptr<KeyMapper<BlobInfo>> blob_info_mapper, - std::unordered_set<std::string> known_file_names) + // Calculates the StorageInfo for the Blob Store. + // + // Returns: + // Vector of NamespaceBlobStorageInfoProto contains size of each namespace. + // INTERNAL_ERROR on I/O error + libtextclassifier3::StatusOr<std::vector<NamespaceBlobStorageInfoProto>> + GetStorageInfo() const; + +private: + explicit BlobStore( + const Filesystem* filesystem, std::string base_dir, const Clock* clock, + int64_t orphan_blob_time_to_live_ms, int32_t compression_level, + std::unique_ptr<PortableFileBackedProtoLog<BlobInfoProto>> blob_info_log, + std::unordered_map<std::string, int32_t> blob_handle_to_offset, + std::unordered_set<std::string> known_file_names) : filesystem_(*filesystem), base_dir_(std::move(base_dir)), clock_(*clock), orphan_blob_time_to_live_ms_(orphan_blob_time_to_live_ms), - blob_info_mapper_(std::move(blob_info_mapper)), + compression_level_(compression_level), + blob_info_log_(std::move(blob_info_log)), + blob_handle_to_offset_(std::move(blob_handle_to_offset)), known_file_names_(std::move(known_file_names)) {} - libtextclassifier3::StatusOr<BlobStore::BlobInfo> GetOrCreateBlobInfo( - const std::string& blob_handle_str); + libtextclassifier3::StatusOr<BlobInfoProto> GetOrCreateBlobInfo( + const std::string& blob_handle_str, + const PropertyProto::BlobHandleProto& blob_handle); const Filesystem& filesystem_; std::string base_dir_; const Clock& clock_; int64_t orphan_blob_time_to_live_ms_; + int32_t compression_level_; - std::unique_ptr<KeyMapper<BlobInfo>> blob_info_mapper_; + // The ground truth blob info log file, which is used to read/write/erase + // BlobInfoProto. + std::unique_ptr<PortableFileBackedProtoLog<BlobInfoProto>> blob_info_log_; + + // The map for BlobHandle string to the offset of BlobInfoProto in the + // BlobInfoProto log file. + // The keys are the Encoded CString from BlobHandleProto. + std::unordered_map<std::string, int32_t> blob_handle_to_offset_; + + // The set of used file names to store blobs in the blob store. std::unordered_set<std::string> known_file_names_; + bool has_mutated_ = false; };
diff --git a/icing/store/document-associated-score-data.h b/icing/store/document-associated-score-data.h index 9a711c8..c7ee663 100644 --- a/icing/store/document-associated-score-data.h +++ b/icing/store/document-associated-score-data.h
@@ -33,21 +33,27 @@ // 3. Document creation timestamp. Unix timestamp of when the document is // created and inserted into Icing. // 4. Document length in number of tokens. +// 5. Index of the ScorablePropertySetProto data of a document in the scorable +// property cache, which is owned by the document-store. class DocumentAssociatedScoreData { public: explicit DocumentAssociatedScoreData(CorpusId corpus_id, int document_score, int64_t creation_timestamp_ms, + int32_t scorable_property_cache_index, int length_in_tokens = 0) : creation_timestamp_ms_(creation_timestamp_ms), corpus_id_(corpus_id), document_score_(document_score), - length_in_tokens_(length_in_tokens) {} + length_in_tokens_(length_in_tokens), + scorable_property_cache_index_(scorable_property_cache_index) {} bool operator==(const DocumentAssociatedScoreData& other) const { return document_score_ == other.document_score() && creation_timestamp_ms_ == other.creation_timestamp_ms() && length_in_tokens_ == other.length_in_tokens() && - corpus_id_ == other.corpus_id(); + corpus_id_ == other.corpus_id() && + scorable_property_cache_index_ == + other.scorable_property_cache_index(); } CorpusId corpus_id() const { return corpus_id_; } @@ -58,15 +64,25 @@ int length_in_tokens() const { return length_in_tokens_; } + int32_t scorable_property_cache_index() const { + return scorable_property_cache_index_; + } + + void set_scorable_property_cache_index( + int32_t scorable_property_cache_index) { + scorable_property_cache_index_ = scorable_property_cache_index; + } + private: int64_t creation_timestamp_ms_; CorpusId corpus_id_; int document_score_; int length_in_tokens_; + int32_t scorable_property_cache_index_; } __attribute__((packed)); -static_assert(sizeof(DocumentAssociatedScoreData) == 20, - "Size of DocumentAssociatedScoreData should be 20"); +static_assert(sizeof(DocumentAssociatedScoreData) == 24, + "Size of DocumentAssociatedScoreData should be 24"); static_assert(icing_is_packed_pod<DocumentAssociatedScoreData>::value, "go/icing-ubsan");
diff --git a/icing/store/document-filter-data.h b/icing/store/document-filter-data.h index 3970132..471ac60 100644 --- a/icing/store/document-filter-data.h +++ b/icing/store/document-filter-data.h
@@ -30,20 +30,25 @@ class DocumentFilterData { public: explicit DocumentFilterData(NamespaceId namespace_id, + uint64_t uri_fingerprint, SchemaTypeId schema_type_id, int64_t expiration_timestamp_ms) : expiration_timestamp_ms_(expiration_timestamp_ms), + uri_fingerprint_(uri_fingerprint), namespace_id_(namespace_id), schema_type_id_(schema_type_id) {} bool operator==(const DocumentFilterData& other) const { return namespace_id_ == other.namespace_id() && + uri_fingerprint_ == other.uri_fingerprint() && schema_type_id_ == other.schema_type_id() && expiration_timestamp_ms_ == other.expiration_timestamp_ms(); } NamespaceId namespace_id() const { return namespace_id_; } + uint64_t uri_fingerprint() const { return uri_fingerprint_; } + SchemaTypeId schema_type_id() const { return schema_type_id_; } void set_schema_type_id(SchemaTypeId schema_type_id) { schema_type_id_ = schema_type_id; @@ -53,11 +58,12 @@ private: int64_t expiration_timestamp_ms_; + uint64_t uri_fingerprint_; NamespaceId namespace_id_; SchemaTypeId schema_type_id_; } __attribute__((packed)); -static_assert(sizeof(DocumentFilterData) == 12, ""); +static_assert(sizeof(DocumentFilterData) == 20, ""); static_assert(icing_is_packed_pod<DocumentFilterData>::value, "go/icing-ubsan"); } // namespace lib
diff --git a/icing/store/document-store.cc b/icing/store/document-store.cc index e5ed547..449b005 100644 --- a/icing/store/document-store.cc +++ b/icing/store/document-store.cc
@@ -27,19 +27,21 @@ #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" -#include "icing/text_classifier/lib3/utils/hash/farmhash.h" #include "icing/absl_ports/annotate.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" +#include "icing/feature-flags.h" #include "icing/file/file-backed-proto-log.h" #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" +#include "icing/file/memory-mapped-file-backed-proto-log.h" #include "icing/file/memory-mapped-file.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/legacy/core/icing-string-util.h" #include "icing/proto/debug.pb.h" #include "icing/proto/document.pb.h" #include "icing/proto/document_wrapper.pb.h" +#include "icing/proto/internal/scorable_property_set.pb.h" #include "icing/proto/logging.pb.h" #include "icing/proto/optimize.pb.h" #include "icing/proto/persist.pb.h" @@ -48,6 +50,8 @@ #include "icing/proto/usage.pb.h" #include "icing/schema/property-util.h" #include "icing/schema/schema-store.h" +#include "icing/schema/scorable_property_manager.h" +#include "icing/store/blob-store.h" #include "icing/store/corpus-associated-scoring-data.h" #include "icing/store/corpus-id.h" #include "icing/store/document-associated-score-data.h" @@ -56,7 +60,7 @@ #include "icing/store/document-log-creator.h" #include "icing/store/dynamic-trie-key-mapper.h" #include "icing/store/key-mapper.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/store/persistent-hash-map-key-mapper.h" #include "icing/store/usage-store.h" @@ -64,9 +68,9 @@ #include "icing/util/clock.h" #include "icing/util/crc32.h" #include "icing/util/data-loss.h" -#include "icing/util/encode-util.h" #include "icing/util/fingerprint-util.h" #include "icing/util/logging.h" +#include "icing/util/scorable_property_set.h" #include "icing/util/status-macros.h" #include "icing/util/tokenized-document.h" @@ -77,10 +81,12 @@ // Used in DocumentId mapper to mark a document as deleted constexpr int64_t kDocDeletedFlag = -1; +constexpr int32_t kInvalidScorablePropertyCacheIndex = -1; constexpr char kDocumentIdMapperFilename[] = "document_id_mapper"; constexpr char kUriHashMapperWorkingPath[] = "uri_mapper"; constexpr char kDocumentStoreHeaderFilename[] = "document_store_header"; constexpr char kScoreCacheFilename[] = "score_cache"; +constexpr char kScorablePropertyCacheFilename[] = "scorable_property_cache"; constexpr char kCorpusScoreCache[] = "corpus_score_cache"; constexpr char kFilterCacheFilename[] = "filter_cache"; constexpr char kNamespaceMapperFilename[] = "namespace_mapper"; @@ -126,6 +132,10 @@ return absl_ports::StrCat(base_dir, "/", kScoreCacheFilename); } +std::string MakeScorablePropertyCacheFilename(const std::string& base_dir) { + return absl_ports::StrCat(base_dir, "/", kScorablePropertyCacheFilename); +} + std::string MakeCorpusScoreCache(const std::string& base_dir) { return absl_ports::StrCat(base_dir, "/", kCorpusScoreCache); } @@ -244,6 +254,10 @@ // Find the existing blob handles in the given document and remove them from the // dead_blob_handles set. Those are the blob handles that are still in use. // +// This method is flag-guarded by the flag enable_blob_store. If the flag is +// disabled, the dead_blob_handles must be empty and this method will be a +// no-op. +// // The type_blob_map is a map from schema type to a set of blob property names. void RemoveAliveBlobHandles( const DocumentProto& document, @@ -265,9 +279,7 @@ if (content_or.ok()) { for (const PropertyProto::BlobHandleProto& blob_handle : content_or.ValueOrDie()) { - dead_blob_handles.erase( - encode_util::EncodeStringToCString(blob_handle.digest()) + - blob_handle.label()); + dead_blob_handles.erase(BlobStore::BuildBlobHandleStr(blob_handle)); } } } @@ -275,35 +287,19 @@ } // namespace -std::string DocumentStore::MakeFingerprint( - NamespaceId namespace_id, std::string_view namespace_, - std::string_view uri_or_schema) const { - if (!namespace_id_fingerprint_) { - // Using a 64-bit fingerprint to represent the key could lead to collisions. - // But, even with 200K unique keys, the probability of collision is about - // one-in-a-billion (https://en.wikipedia.org/wiki/Birthday_attack). - uint64_t fprint = tc3farmhash::Fingerprint64( - absl_ports::StrCat(namespace_, uri_or_schema)); - return fingerprint_util::GetFingerprintString(fprint); - } else { - return NamespaceFingerprintIdentifier(namespace_id, uri_or_schema) - .EncodeToCString(); - } -} - DocumentStore::DocumentStore(const Filesystem* filesystem, const std::string_view base_dir, const Clock* clock, const SchemaStore* schema_store, - bool namespace_id_fingerprint, + const FeatureFlags* feature_flags, bool pre_mapping_fbv, bool use_persistent_hash_map, int32_t compression_level) : filesystem_(filesystem), base_dir_(base_dir), clock_(*clock), + feature_flags_(*feature_flags), schema_store_(schema_store), document_validator_(schema_store), - namespace_id_fingerprint_(namespace_id_fingerprint), pre_mapping_fbv_(pre_mapping_fbv), use_persistent_hash_map_(use_persistent_hash_map), compression_level_(compression_level) {} @@ -333,16 +329,18 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> DocumentStore::Create( const Filesystem* filesystem, const std::string& base_dir, const Clock* clock, const SchemaStore* schema_store, - bool force_recovery_and_revalidate_documents, bool namespace_id_fingerprint, - bool pre_mapping_fbv, bool use_persistent_hash_map, - int32_t compression_level, InitializeStatsProto* initialize_stats) { + const FeatureFlags* feature_flags, + bool force_recovery_and_revalidate_documents, bool pre_mapping_fbv, + bool use_persistent_hash_map, int32_t compression_level, + InitializeStatsProto* initialize_stats) { ICING_RETURN_ERROR_IF_NULL(filesystem); ICING_RETURN_ERROR_IF_NULL(clock); ICING_RETURN_ERROR_IF_NULL(schema_store); + ICING_RETURN_ERROR_IF_NULL(feature_flags); auto document_store = std::unique_ptr<DocumentStore>(new DocumentStore( - filesystem, base_dir, clock, schema_store, namespace_id_fingerprint, - pre_mapping_fbv, use_persistent_hash_map, compression_level)); + filesystem, base_dir, clock, schema_store, feature_flags, pre_mapping_fbv, + use_persistent_hash_map, compression_level)); ICING_ASSIGN_OR_RETURN( InitializeResult initialize_result, document_store->Initialize(force_recovery_and_revalidate_documents, @@ -395,6 +393,11 @@ ICING_RETURN_IF_ERROR(FileBackedVector<CorpusAssociatedScoreData>::Delete( *filesystem, MakeCorpusScoreCache(base_dir))); + // Scorable Property Cache + ICING_RETURN_IF_ERROR( + MemoryMappedFileBackedProtoLog<ScorablePropertySetProto>::Delete( + *filesystem, MakeScorablePropertyCacheFilename(base_dir))); + return libtextclassifier3::Status::OK; } @@ -502,8 +505,7 @@ absl_ports::StrCat("Couldn't read: ", MakeHeaderFilename(base_dir_))); } - if (header.magic != - DocumentStore::Header::GetCurrentMagic(namespace_id_fingerprint_)) { + if (header.magic != DocumentStore::Header::kMagic) { return absl_ports::InternalError(absl_ports::StrCat( "Invalid header kMagic for file: ", MakeHeaderFilename(base_dir_))); } @@ -536,6 +538,11 @@ *filesystem_, MakeScoreCacheFilename(base_dir_), MemoryMappedFile::READ_WRITE_AUTO_SYNC)); + ICING_ASSIGN_OR_RETURN( + scorable_property_cache_, + MemoryMappedFileBackedProtoLog<ScorablePropertySetProto>::Create( + *filesystem_, MakeScorablePropertyCacheFilename(base_dir_))); + ICING_ASSIGN_OR_RETURN(filter_cache_, FileBackedVector<DocumentFilterData>::Create( *filesystem_, MakeFilterCacheFilename(base_dir_), @@ -585,6 +592,7 @@ ICING_RETURN_IF_ERROR(ResetDocumentKeyMapper()); ICING_RETURN_IF_ERROR(ResetDocumentIdMapper()); ICING_RETURN_IF_ERROR(ResetDocumentAssociatedScoreCache()); + ICING_RETURN_IF_ERROR(ResetScorablePropertyCache()); ICING_RETURN_IF_ERROR(ResetFilterCache()); ICING_RETURN_IF_ERROR(ResetNamespaceMapper()); ICING_RETURN_IF_ERROR(ResetCorpusMapper()); @@ -646,10 +654,10 @@ // Updates key mapper and document_id mapper with the new document DocumentId new_document_id = document_id_mapper_->num_elements(); + NamespaceIdFingerprint new_doc_nsid_uri_fingerprint( + namespace_id, document_wrapper.document().uri()); ICING_RETURN_IF_ERROR(document_key_mapper_->Put( - MakeFingerprint(namespace_id, document_wrapper.document().namespace_(), - document_wrapper.document().uri()), - new_document_id)); + new_doc_nsid_uri_fingerprint.EncodeToCString(), new_document_id)); ICING_RETURN_IF_ERROR( document_id_mapper_->Set(new_document_id, iterator.GetOffset())); @@ -673,26 +681,36 @@ } // Update corpus maps - std::string corpus = - MakeFingerprint(namespace_id, document_wrapper.document().namespace_(), - document_wrapper.document().schema()); - ICING_ASSIGN_OR_RETURN( - CorpusId corpusId, - corpus_mapper_->GetOrPut(corpus, corpus_mapper_->num_keys())); + NamespaceIdFingerprint corpus_nsid_schema_fingerprint( + namespace_id, document_wrapper.document().schema()); + ICING_ASSIGN_OR_RETURN(CorpusId corpus_id, + corpus_mapper_->GetOrPut( + corpus_nsid_schema_fingerprint.EncodeToCString(), + corpus_mapper_->num_keys())); ICING_ASSIGN_OR_RETURN(CorpusAssociatedScoreData scoring_data, - GetCorpusAssociatedScoreDataToUpdate(corpusId)); + GetCorpusAssociatedScoreDataToUpdate(corpus_id)); scoring_data.AddDocument( document_wrapper.document().internal_fields().length_in_tokens()); ICING_RETURN_IF_ERROR( - UpdateCorpusAssociatedScoreCache(corpusId, scoring_data)); + UpdateCorpusAssociatedScoreCache(corpus_id, scoring_data)); + + int32_t scorable_property_cache_index = kInvalidScorablePropertyCacheIndex; + // Swallow the error when schema_type_id is not found, and skip updating the + // scorable property cache. + if (schema_type_id != -1) { + ICING_ASSIGN_OR_RETURN(scorable_property_cache_index, + UpdateScorablePropertyCache( + document_wrapper.document(), schema_type_id)); + } ICING_RETURN_IF_ERROR(UpdateDocumentAssociatedScoreCache( new_document_id, DocumentAssociatedScoreData( - corpusId, document_wrapper.document().score(), + corpus_id, document_wrapper.document().score(), document_wrapper.document().creation_timestamp_ms(), + scorable_property_cache_index, document_wrapper.document().internal_fields().length_in_tokens()))); int64_t expiration_timestamp_ms = CalculateExpirationTimestampMs( @@ -700,8 +718,10 @@ document_wrapper.document().ttl_ms()); ICING_RETURN_IF_ERROR(UpdateFilterCache( - new_document_id, DocumentFilterData(namespace_id, schema_type_id, - expiration_timestamp_ms))); + new_document_id, + DocumentFilterData(namespace_id, + new_doc_nsid_uri_fingerprint.fingerprint(), + schema_type_id, expiration_timestamp_ms))); iterator_status = iterator.Advance(); } @@ -797,6 +817,18 @@ return libtextclassifier3::Status::OK; } +libtextclassifier3::Status DocumentStore::ResetScorablePropertyCache() { + scorable_property_cache_.reset(); + ICING_RETURN_IF_ERROR( + MemoryMappedFileBackedProtoLog<ScorablePropertySetProto>::Delete( + *filesystem_, MakeScorablePropertyCacheFilename(base_dir_))); + ICING_ASSIGN_OR_RETURN( + scorable_property_cache_, + MemoryMappedFileBackedProtoLog<ScorablePropertySetProto>::Create( + *filesystem_, MakeScorablePropertyCacheFilename(base_dir_))); + return libtextclassifier3::Status::OK; +} + libtextclassifier3::Status DocumentStore::ResetCorpusAssociatedScoreCache() { // TODO(b/139734457): Replace ptr.reset()->Delete->Create flow with Reset(). corpus_score_cache_.reset(); @@ -908,6 +940,14 @@ } Crc32 score_cache_checksum = std::move(checksum_or).ValueOrDie(); + checksum_or = scorable_property_cache_->GetChecksum(); + if (!checksum_or.ok()) { + ICING_LOG(ERROR) << checksum_or.status().error_message() + << "Failed to compute checksum of scorable property cache"; + return checksum_or.status(); + } + Crc32 scorable_property_cache_checksum = std::move(checksum_or).ValueOrDie(); + // TODO(b/144458732): Implement a more robust version of TC_ASSIGN_OR_RETURN // that can support error logging. checksum_or = filter_cache_->GetChecksum(); @@ -957,6 +997,7 @@ total_checksum.Append(std::to_string(document_key_mapper_checksum.Get())); total_checksum.Append(std::to_string(document_id_mapper_checksum.Get())); total_checksum.Append(std::to_string(score_cache_checksum.Get())); + total_checksum.Append(std::to_string(scorable_property_cache_checksum.Get())); total_checksum.Append(std::to_string(filter_cache_checksum.Get())); total_checksum.Append(std::to_string(namespace_mapper_checksum.Get())); total_checksum.Append(std::to_string(corpus_mapper_checksum.Get())); @@ -1007,6 +1048,14 @@ } Crc32 score_cache_checksum = std::move(checksum_or).ValueOrDie(); + checksum_or = scorable_property_cache_->UpdateChecksum(); + if (!checksum_or.ok()) { + ICING_LOG(ERROR) << checksum_or.status().error_message() + << "Failed to compute checksum of scorable property cache"; + return checksum_or.status(); + } + Crc32 scorable_property_cache_checksum = std::move(checksum_or).ValueOrDie(); + // TODO(b/144458732): Implement a more robust version of TC_ASSIGN_OR_RETURN // that can support error logging. checksum_or = filter_cache_->UpdateChecksum(); @@ -1056,6 +1105,7 @@ total_checksum.Append(std::to_string(document_key_mapper_checksum.Get())); total_checksum.Append(std::to_string(document_id_mapper_checksum.Get())); total_checksum.Append(std::to_string(score_cache_checksum.Get())); + total_checksum.Append(std::to_string(scorable_property_cache_checksum.Get())); total_checksum.Append(std::to_string(filter_cache_checksum.Get())); total_checksum.Append(std::to_string(namespace_mapper_checksum.Get())); total_checksum.Append(std::to_string(corpus_mapper_checksum.Get())); @@ -1063,8 +1113,7 @@ // Write the header DocumentStore::Header header; - header.magic = - DocumentStore::Header::GetCurrentMagic(namespace_id_fingerprint_); + header.magic = DocumentStore::Header::kMagic; header.checksum = total_checksum.Get(); // This should overwrite the header. @@ -1121,8 +1170,8 @@ // Update ground truth first // TODO(b/144458732): Implement a more robust version of TC_ASSIGN_OR_RETURN // that can support error logging. - auto offset_or = - document_log_->WriteProto(CreateDocumentWrapper(std::move(document))); + DocumentWrapper document_wrapper = CreateDocumentWrapper(std::move(document)); + auto offset_or = document_log_->WriteProto(document_wrapper); if (!offset_or.ok()) { ICING_LOG(ERROR) << offset_or.status().error_message() << "Failed to write document"; @@ -1152,41 +1201,49 @@ NamespaceId namespace_id, namespace_mapper_->GetOrPut(name_space, namespace_mapper_->num_keys())); + NamespaceIdFingerprint new_doc_nsid_uri_fingerprint(namespace_id, uri); + // Updates key mapper and document_id mapper ICING_RETURN_IF_ERROR(document_key_mapper_->Put( - MakeFingerprint(namespace_id, name_space, uri), new_document_id)); + new_doc_nsid_uri_fingerprint.EncodeToCString(), new_document_id)); ICING_RETURN_IF_ERROR(document_id_mapper_->Set(new_document_id, file_offset)); // Update corpus maps - ICING_ASSIGN_OR_RETURN(CorpusId corpusId, - corpus_mapper_->GetOrPut( - MakeFingerprint(namespace_id, name_space, schema), - corpus_mapper_->num_keys())); + NamespaceIdFingerprint corpus_nsid_schema_fingerprint(namespace_id, schema); + ICING_ASSIGN_OR_RETURN( + CorpusId corpus_id, + corpus_mapper_->GetOrPut(corpus_nsid_schema_fingerprint.EncodeToCString(), + corpus_mapper_->num_keys())); ICING_ASSIGN_OR_RETURN(CorpusAssociatedScoreData scoring_data, - GetCorpusAssociatedScoreDataToUpdate(corpusId)); + GetCorpusAssociatedScoreDataToUpdate(corpus_id)); scoring_data.AddDocument(length_in_tokens); ICING_RETURN_IF_ERROR( - UpdateCorpusAssociatedScoreCache(corpusId, scoring_data)); - - ICING_RETURN_IF_ERROR(UpdateDocumentAssociatedScoreCache( - new_document_id, - DocumentAssociatedScoreData(corpusId, document_score, - creation_timestamp_ms, length_in_tokens))); + UpdateCorpusAssociatedScoreCache(corpus_id, scoring_data)); ICING_ASSIGN_OR_RETURN(SchemaTypeId schema_type_id, schema_store_->GetSchemaTypeId(schema)); + ICING_ASSIGN_OR_RETURN( + int scorable_property_cache_index, + UpdateScorablePropertyCache(document_wrapper.document(), schema_type_id)); + + ICING_RETURN_IF_ERROR(UpdateDocumentAssociatedScoreCache( + new_document_id, DocumentAssociatedScoreData( + corpus_id, document_score, creation_timestamp_ms, + scorable_property_cache_index, length_in_tokens))); ICING_RETURN_IF_ERROR(UpdateFilterCache( - new_document_id, DocumentFilterData(namespace_id, schema_type_id, - expiration_timestamp_ms))); + new_document_id, + DocumentFilterData(namespace_id, + new_doc_nsid_uri_fingerprint.fingerprint(), + schema_type_id, expiration_timestamp_ms))); if (old_document_id_or.ok()) { - put_result.was_replacement = true; // The old document exists, copy over the usage scores and delete the old // document. DocumentId old_document_id = old_document_id_or.ValueOrDie(); + put_result.old_document_id = old_document_id; ICING_RETURN_IF_ERROR( usage_store_->CloneUsageScores(/*from_document_id=*/old_document_id, @@ -1288,14 +1345,58 @@ return std::move(*document_wrapper.mutable_document()); } +std::unique_ptr<ScorablePropertySet> DocumentStore::GetScorablePropertySet( + DocumentId document_id, int64_t current_time_ms) const { + if (!feature_flags_.enable_scorable_properties()) { + return nullptr; + } + + // Get scorable property cache index from the score_cache_ + libtextclassifier3::StatusOr<const DocumentAssociatedScoreData*> + score_data_or = score_cache_->Get(document_id); + if (!score_data_or.ok()) { + return nullptr; + } + if (score_data_or.ValueOrDie()->scorable_property_cache_index() == + kInvalidScorablePropertyCacheIndex) { + return nullptr; + } + + // Get ScorablePropertySetProto. + libtextclassifier3::StatusOr<ScorablePropertySetProto> + scorable_property_set_proto_or = scorable_property_cache_->Read( + score_data_or.ValueOrDie()->scorable_property_cache_index()); + if (!scorable_property_set_proto_or.ok()) { + return nullptr; + } + + // Get schema type id. + auto document_filter_data_optional = + GetAliveDocumentFilterData(document_id, current_time_ms); + if (!document_filter_data_optional) { + return nullptr; + } + + libtextclassifier3::StatusOr<std::unique_ptr<ScorablePropertySet>> + scorable_property_set_or = ScorablePropertySet::Create( + std::move(scorable_property_set_proto_or.ValueOrDie()), + document_filter_data_optional.value().schema_type_id(), + schema_store_); + if (!scorable_property_set_or.ok()) { + return nullptr; + } + return std::move(scorable_property_set_or.ValueOrDie()); +} + libtextclassifier3::StatusOr<DocumentId> DocumentStore::GetDocumentId( const std::string_view name_space, const std::string_view uri) const { auto namespace_id_or = namespace_mapper_->Get(name_space); libtextclassifier3::Status status = namespace_id_or.status(); if (status.ok()) { NamespaceId namespace_id = namespace_id_or.ValueOrDie(); - auto document_id_or = document_key_mapper_->Get( - MakeFingerprint(namespace_id, name_space, uri)); + NamespaceIdFingerprint doc_nsid_uri_fingerprint(namespace_id, uri); + auto document_id_or = + document_key_mapper_->Get(doc_nsid_uri_fingerprint.EncodeToCString()); status = document_id_or.status(); if (status.ok()) { // Guaranteed to have a DocumentId now @@ -1308,16 +1409,9 @@ } libtextclassifier3::StatusOr<DocumentId> DocumentStore::GetDocumentId( - const NamespaceFingerprintIdentifier& namespace_fingerprint_identifier) - const { - if (!namespace_id_fingerprint_) { - return absl_ports::FailedPreconditionError( - "Cannot lookup document id by namespace id + fingerprint without " - "enabling it on uri_mapper"); - } - + const NamespaceIdFingerprint& doc_namespace_id_uri_fingerprint) const { auto document_id_or = document_key_mapper_->Get( - namespace_fingerprint_identifier.EncodeToCString()); + doc_namespace_id_uri_fingerprint.EncodeToCString()); if (document_id_or.ok()) { return document_id_or.ValueOrDie(); } @@ -1365,6 +1459,27 @@ return GetNonExpiredDocumentFilterData(document_id, current_time_ms); } +std::optional<DocumentFilterData> +DocumentStore::GetNonDeletedDocumentFilterData(DocumentId document_id) const { + if (IsDeleted(document_id)) { + return std::nullopt; + } + + auto filter_data_or = filter_cache_->GetCopy(document_id); + if (!filter_data_or.ok()) { + // This would only happen if document_id is out of range of the + // filter_cache, meaning we got some invalid document_id. Callers should + // already have checked that their document_id is valid or used + // DoesDocumentExist(WithStatus). Regardless, return std::nullopt since the + // document doesn't exist. + return std::nullopt; + } + + // At this point, it's guaranteed that the document has not been deleted. It + // could still be expired, but the filter data is guaranteed to be valid here. + return std::move(filter_data_or).ValueOrDie(); +} + bool DocumentStore::IsDeleted(DocumentId document_id) const { auto file_offset_or = document_id_mapper_->Get(document_id); if (!file_offset_or.ok()) { @@ -1389,7 +1504,7 @@ // This would only happen if document_id is out of range of the // filter_cache, meaning we got some invalid document_id. Callers should // already have checked that their document_id is valid or used - // DoesDocumentExist(WithStatus). Regardless, return true since the + // DoesDocumentExist(WithStatus). Regardless, return std::nullopt since the // document doesn't exist. return std::nullopt; } @@ -1451,7 +1566,8 @@ const std::string_view name_space, const std::string_view schema) const { ICING_ASSIGN_OR_RETURN(NamespaceId namespace_id, namespace_mapper_->Get(name_space)); - return corpus_mapper_->Get(MakeFingerprint(namespace_id, name_space, schema)); + NamespaceIdFingerprint corpus_nsid_schema_fp(namespace_id, schema); + return corpus_mapper_->Get(corpus_nsid_schema_fp.EncodeToCString()); } libtextclassifier3::StatusOr<int32_t> DocumentStore::GetResultGroupingEntryId( @@ -1699,6 +1815,7 @@ ICING_RETURN_IF_ERROR(document_key_mapper_->PersistToDisk()); ICING_RETURN_IF_ERROR(document_id_mapper_->PersistToDisk()); ICING_RETURN_IF_ERROR(score_cache_->PersistToDisk()); + ICING_RETURN_IF_ERROR(scorable_property_cache_->PersistToDisk()); ICING_RETURN_IF_ERROR(filter_cache_->PersistToDisk()); ICING_RETURN_IF_ERROR(namespace_mapper_->PersistToDisk()); ICING_RETURN_IF_ERROR(usage_store_->PersistToDisk()); @@ -1725,6 +1842,8 @@ GetValueOrDefault(document_id_mapper_->GetDiskUsage(), -1)); storage_info.set_score_cache_size( GetValueOrDefault(score_cache_->GetDiskUsage(), -1)); + storage_info.set_scorable_property_cache_size( + GetValueOrDefault(scorable_property_cache_->GetDiskUsage(), -1)); storage_info.set_filter_cache_size( GetValueOrDefault(filter_cache_->GetDiskUsage(), -1)); storage_info.set_namespace_id_mapper_size( @@ -1965,6 +2084,44 @@ return libtextclassifier3::Status::OK; } +libtextclassifier3::Status DocumentStore::RegenerateScorablePropertyCache( + const std::unordered_set<SchemaTypeId>& schema_type_ids) { + if (schema_type_ids.empty()) { + return libtextclassifier3::Status::OK; + } + + int64_t current_time_ms = clock_.GetSystemTimeMilliseconds(); + for (DocumentId document_id = 0; + document_id < document_id_mapper_->num_elements(); ++document_id) { + if (!GetAliveDocumentFilterData(document_id, current_time_ms)) { + continue; + } + // Guaranteed that the document exists now. + ICING_ASSIGN_OR_RETURN(const DocumentFilterData* filter_data, + filter_cache_->Get(document_id)); + SchemaTypeId schema_type_id = filter_data->schema_type_id(); + if (schema_type_ids.find(schema_type_id) == schema_type_ids.end()) { + continue; + } + + ICING_ASSIGN_OR_RETURN(DocumentProto document, Get(document_id)); + int32_t scorable_property_cache_index = kInvalidScorablePropertyCacheIndex; + ICING_ASSIGN_OR_RETURN( + scorable_property_cache_index, + UpdateScorablePropertyCache(document, schema_type_id)); + + // Update the score_cache_ with the new scorable property cache index. + ICING_ASSIGN_OR_RETURN( + typename FileBackedVector<DocumentAssociatedScoreData>::MutableView + doc_score_data_view, + score_cache_->GetMutable(document_id)); + doc_score_data_view.Get().set_scorable_property_cache_index( + scorable_property_cache_index); + } + + return libtextclassifier3::Status::OK; +} + // TODO(b/121227117): Implement Optimize() libtextclassifier3::Status DocumentStore::Optimize() { return libtextclassifier3::Status::OK; @@ -1983,11 +2140,11 @@ ICING_ASSIGN_OR_RETURN( auto doc_store_create_result, - DocumentStore::Create(filesystem_, new_directory, &clock_, schema_store_, - /*force_recovery_and_revalidate_documents=*/false, - namespace_id_fingerprint_, pre_mapping_fbv_, - use_persistent_hash_map_, compression_level_, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + filesystem_, new_directory, &clock_, schema_store_, &feature_flags_, + /*force_recovery_and_revalidate_documents=*/false, pre_mapping_fbv_, + use_persistent_hash_map_, compression_level_, + /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> new_doc_store = std::move(doc_store_create_result.document_store); @@ -1998,18 +2155,23 @@ UsageStore::UsageScores default_usage; OptimizeResult result; result.document_id_old_to_new.resize(document_cnt, kInvalidDocumentId); - result.dead_blob_handles = std::move(potentially_optimizable_blob_handles); - // Get the blob property map from the schema store. - auto type_blob_property_map_or = schema_store_->ConstructBlobPropertyMap(); - if (num_documents() == 0) { - // If we fail to retrieve this map when there *are* documents in - // doc store, then something is seriously wrong. Return error. - return result; - } + result.dead_blob_handles = std::move(potentially_optimizable_blob_handles); std::unordered_map<std::string, std::vector<std::string>> - type_blob_property_map = - std::move(type_blob_property_map_or).ValueOrDie(); + type_blob_property_map; + if (!result.dead_blob_handles.empty()) { + // Get the blob property map from the schema store. + if (num_documents() == 0) { + return result; + } + auto type_blob_property_map_or = schema_store_->ConstructBlobPropertyMap(); + if (!type_blob_property_map_or.ok()) { + // If we fail to retrieve this map when there *are* documents in + // doc store, then something is seriously wrong. Return error. + return type_blob_property_map_or.status(); + } + type_blob_property_map = std::move(type_blob_property_map_or).ValueOrDie(); + } int64_t current_time_ms = clock_.GetSystemTimeMilliseconds(); for (DocumentId document_id = 0; document_id < document_cnt; document_id++) { @@ -2164,6 +2326,8 @@ document_id_mapper_->GetElementsFileSize()); ICING_ASSIGN_OR_RETURN(const int64_t score_cache_file_size, score_cache_->GetElementsFileSize()); + ICING_ASSIGN_OR_RETURN(const int64_t scorable_property_cache_file_size, + scorable_property_cache_->GetElementsFileSize()); ICING_ASSIGN_OR_RETURN(const int64_t filter_cache_file_size, filter_cache_->GetElementsFileSize()); ICING_ASSIGN_OR_RETURN(const int64_t corpus_score_cache_file_size, @@ -2186,6 +2350,7 @@ int64_t total_size = document_log_file_size + document_key_mapper_size + document_id_mapper_file_size + score_cache_file_size + + scorable_property_cache_file_size + filter_cache_file_size + corpus_score_cache_file_size + usage_store_file_size; @@ -2219,15 +2384,20 @@ // Resets the score cache entry ICING_RETURN_IF_ERROR(UpdateDocumentAssociatedScoreCache( - document_id, DocumentAssociatedScoreData(kInvalidCorpusId, - /*document_score=*/-1, - /*creation_timestamp_ms=*/-1, - /*length_in_tokens=*/0))); + document_id, + DocumentAssociatedScoreData( + kInvalidCorpusId, + /*document_score=*/-1, + /*creation_timestamp_ms=*/-1, + /*scorable_property_cache_index=*/kInvalidScorablePropertyCacheIndex, + /*length_in_tokens=*/0))); // Resets the filter cache entry ICING_RETURN_IF_ERROR(UpdateFilterCache( - document_id, DocumentFilterData(kInvalidNamespaceId, kInvalidSchemaTypeId, - /*expiration_timestamp_ms=*/-1))); + document_id, + DocumentFilterData(kInvalidNamespaceId, /*uri_fingerprint=*/0, + kInvalidSchemaTypeId, + /*expiration_timestamp_ms=*/-1))); // Clears the usage scores. return usage_store_->DeleteUsageScores(document_id); @@ -2296,5 +2466,28 @@ return debug_info; } +libtextclassifier3::StatusOr<int> DocumentStore::UpdateScorablePropertyCache( + const DocumentProto& document, SchemaTypeId schema_type_id) { + if (!feature_flags_.enable_scorable_properties()) { + return kInvalidScorablePropertyCacheIndex; + } + ICING_ASSIGN_OR_RETURN( + const std::vector<ScorablePropertyManager::ScorablePropertyInfo>* + ordered_scorable_property_info, + schema_store_->GetOrderedScorablePropertyInfo(schema_type_id)); + if (ordered_scorable_property_info == nullptr || + ordered_scorable_property_info->empty()) { + // No scorable property defined under the schema config of the + // schema_type_id. + return kInvalidScorablePropertyCacheIndex; + } + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<ScorablePropertySet> scorable_property_set, + ScorablePropertySet::Create(document, schema_type_id, schema_store_)); + + return scorable_property_cache_->Write( + scorable_property_set->GetScorablePropertySetProto()); +} + } // namespace lib } // namespace icing
diff --git a/icing/store/document-store.h b/icing/store/document-store.h index 0caead0..7246235 100644 --- a/icing/store/document-store.h +++ b/icing/store/document-store.h
@@ -17,6 +17,7 @@ #include <cstdint> #include <memory> +#include <optional> #include <string> #include <string_view> #include <unordered_set> @@ -24,12 +25,15 @@ #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/feature-flags.h" #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" +#include "icing/file/memory-mapped-file-backed-proto-log.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/proto/debug.pb.h" #include "icing/proto/document.pb.h" #include "icing/proto/document_wrapper.pb.h" +#include "icing/proto/internal/scorable_property_set.pb.h" #include "icing/proto/logging.pb.h" #include "icing/proto/optimize.pb.h" #include "icing/proto/persist.pb.h" @@ -43,7 +47,7 @@ #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" #include "icing/store/key-mapper.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/store/usage-store.h" #include "icing/tokenization/language-segmenter.h" @@ -52,6 +56,7 @@ #include "icing/util/data-loss.h" #include "icing/util/document-validator.h" #include "icing/util/fingerprint-util.h" +#include "icing/util/scorable_property_set.h" namespace icing { namespace lib { @@ -60,19 +65,15 @@ class DocumentStore { public: struct Header { - static int32_t GetCurrentMagic(bool namespace_id_fingerprint) { - return namespace_id_fingerprint ? kNewMagic : kOldMagic; - } + // Previously used magic numbers, please avoid reusing those: + // [0x1b99c8b0, 0x3e005b5e] + static constexpr int32_t kMagic = 0x8a32cd1f; // Holds the magic as a quick sanity check against file corruption. int32_t magic; // Checksum of the DocumentStore's sub-component's checksums. uint32_t checksum; - - private: - static constexpr int32_t kOldMagic = 0x746f7265; - static constexpr int32_t kNewMagic = 0x1b99c8b0; }; struct OptimizeInfo { @@ -147,8 +148,8 @@ static libtextclassifier3::StatusOr<DocumentStore::CreateResult> Create( const Filesystem* filesystem, const std::string& base_dir, const Clock* clock, const SchemaStore* schema_store, - bool force_recovery_and_revalidate_documents, - bool namespace_id_fingerprint, bool pre_mapping_fbv, + const FeatureFlags* feature_flags, + bool force_recovery_and_revalidate_documents, bool pre_mapping_fbv, bool use_persistent_hash_map, int32_t compression_level, InitializeStatsProto* initialize_stats); @@ -182,16 +183,20 @@ // // Returns: // - On success, a PutResult with the DocumentId of the newly added document - // and a bool indicating whether this is a new document or a replacement - // of an existing document. + // and the old DocumentId before replacement. If this is a new document, + // then old DocumentId will be kInvalidDocumentId. // - RESOURCE_EXHAUSTED if exceeds maximum number of allowed documents // - FAILED_PRECONDITION if schema hasn't been set yet // - NOT_FOUND if the schema_type or a property config of the document // doesn't exist in schema // - INTERNAL_ERROR on IO error struct PutResult { + DocumentId old_document_id = kInvalidDocumentId; DocumentId new_document_id = kInvalidDocumentId; - bool was_replacement = false; + + bool was_replacement() const { + return old_document_id != kInvalidDocumentId; + } }; libtextclassifier3::StatusOr<PutResult> Put( const DocumentProto& document, int32_t num_tokens = 0, @@ -225,6 +230,19 @@ libtextclassifier3::StatusOr<DocumentProto> Get( DocumentId document_id, bool clear_internal_fields = true) const; + // Returns the ScorablePropertySet of the document specified by the + // DocumentId. + // + // Returns: + // - ScorablePropertySet on success + // - nullptr when the ScorablePropertySet fails to be created, it could be + // due to that: + // - |document_id| is invalid, or + // - no ScorablePropertySetProto is found for the document in the cache + // - internal IO error + std::unique_ptr<ScorablePropertySet> GetScorablePropertySet( + DocumentId document_id, int64_t current_time_ms) const; + // Returns all namespaces which have at least 1 active document (not deleted // or expired). Order of namespaces is undefined. std::vector<std::string> GetAllNamespaces() const; @@ -283,7 +301,7 @@ std::string_view name_space, std::string_view uri) const; // Helper method to find a DocumentId that is associated with the given - // NamespaceFingerprintIdentifier. + // NamespaceIdFingerprint. // // NOTE: The DocumentId may refer to a invalid document (deleted // or expired). Callers can call DoesDocumentExist(document_id) to ensure it @@ -294,8 +312,7 @@ // NOT_FOUND if the key doesn't exist // INTERNAL_ERROR on IO error libtextclassifier3::StatusOr<DocumentId> GetDocumentId( - const NamespaceFingerprintIdentifier& namespace_fingerprint_identifier) - const; + const NamespaceIdFingerprint& doc_namespace_id_uri_fingerprint) const; // Returns the CorpusId associated with the given namespace and schema. // @@ -359,8 +376,8 @@ libtextclassifier3::StatusOr<CorpusAssociatedScoreData> GetCorpusAssociatedScoreData(CorpusId corpus_id) const; - // Gets the document filter data if a document exists. Otherwise, will get a - // false optional. + // Gets the document filter data if a document exists and is not expired. + // Otherwise, will get a false optional. // // Existence means it hasn't been deleted and it hasn't expired yet. // @@ -370,6 +387,32 @@ std::optional<DocumentFilterData> GetAliveDocumentFilterData( DocumentId document_id, int64_t current_time_ms) const; + // Gets the document filter data if a document has not been deleted. If the + // document is expired but not deleted, will still return a valid document + // filter data. Otherwise, will get a false optional. + // + // Returns: + // True:DocumentFilterData if the given document exists. + // False if the given document has been deleted. + std::optional<DocumentFilterData> GetNonDeletedDocumentFilterData( + DocumentId document_id) const; + + // Gets the SchemaTypeId of a document. + // + // Returns: + // SchemaTypeId on success + // kInvalidSchemaTypeId if the document is deleted or expired. + SchemaTypeId GetSchemaTypeId(DocumentId document_id, + int64_t current_time_ms) const { + std::optional<DocumentFilterData> document_filter_data_optional = + GetAliveDocumentFilterData(document_id, current_time_ms); + if (document_filter_data_optional) { + return document_filter_data_optional.value().schema_type_id(); + } else { + return kInvalidSchemaTypeId; + } + } + // Gets the usage scores of a document. // // Returns: @@ -454,6 +497,15 @@ const SchemaStore* schema_store, const SchemaStore::SetSchemaResult& set_schema_result); + // Re-generates the scorable property cache for documents with the given + // schema types. + // + // Returns: + // OK on success + // INTERNAL_ERROR on IO error + libtextclassifier3::Status RegenerateScorablePropertyCache( + const std::unordered_set<SchemaTypeId>& schema_type_ids); + // Reduces internal file sizes by reclaiming space of deleted documents and // regenerating derived files. // @@ -544,13 +596,14 @@ explicit DocumentStore(const Filesystem* filesystem, std::string_view base_dir, const Clock* clock, const SchemaStore* schema_store, - bool namespace_id_fingerprint, bool pre_mapping_fbv, - bool use_persistent_hash_map, + const FeatureFlags* feature_flags, + bool pre_mapping_fbv, bool use_persistent_hash_map, int32_t compression_level); const Filesystem* const filesystem_; const std::string base_dir_; const Clock& clock_; + const FeatureFlags& feature_flags_; // Does not own. // Handles the ground truth schema and all of the derived data off of the // schema @@ -559,10 +612,6 @@ // Used to validate incoming documents DocumentValidator document_validator_; - // Whether to use namespace id or namespace name to build up fingerprint for - // document_key_mapper_ and corpus_mapper_. - bool namespace_id_fingerprint_; - // Flag indicating whether memory map max possible file size for underlying // FileBackedVector before growing the actual file size. bool pre_mapping_fbv_; @@ -592,8 +641,14 @@ // - Document score // - Document creation timestamp in seconds // - Document length in number of tokens + // - Index of the ScorablePropertySetProto at the scorable_property_cache_ std::unique_ptr<FileBackedVector<DocumentAssociatedScoreData>> score_cache_; + // A cache of document scorable properties. The ground truth of the data is + // DocumentProto stored in document_log_. + std::unique_ptr<MemoryMappedFileBackedProtoLog<ScorablePropertySetProto>> + scorable_property_cache_; + // A cache of data, indexed by DocumentId, used to filter documents. Currently // contains: // - NamespaceId @@ -688,6 +743,12 @@ // Returns OK or any IO errors. libtextclassifier3::Status ResetDocumentAssociatedScoreCache(); + // Resets the unique_ptr to the |scorable_property_cache_|, deletes the + // underlying file, and re-creates a new instance of it. + // + // Returns OK or any IO errors. + libtextclassifier3::Status ResetScorablePropertyCache(); + // Resets the unique_ptr to the corpus_score_cache, deletes the underlying // file, and re-creates a new instance of the corpus_score_cache. // @@ -823,12 +884,19 @@ google::protobuf::RepeatedPtrField<DocumentDebugInfoProto::CorpusInfo>> CollectCorpusInfo() const; - // Build fingerprint for the keys of document_key_mapper_ and corpus_mapper_. - // Note that namespace_id_fingerprint_ controls the way that a fingerprint is - // built. - std::string MakeFingerprint(NamespaceId namespace_id, - std::string_view namespace_, - std::string_view uri_or_schema) const; + // Extracts the ScorablePropertySetProto from the |document| and add it to + // the |scorable_property_cache_|. + // + // Returns: + // - Index of the newly inserted ScorablePropertySetProto in the + // |scorable_property_cache_|. + // - kInvalidScorablePropertyCacheIndex if the schema contains no + // scorable properties. + // - INVALID_ARGUMENT if |schema_type_id| is invalid, or the converted + // ScorablePropertySetProto exceeds the size limit of 16MiB. + // - INTERNAL_ERROR on IO error. + libtextclassifier3::StatusOr<int> UpdateScorablePropertyCache( + const DocumentProto& document, SchemaTypeId schema_type_id); }; } // namespace lib
diff --git a/icing/store/document-store_benchmark.cc b/icing/store/document-store_benchmark.cc index 3523e14..08f3e13 100644 --- a/icing/store/document-store_benchmark.cc +++ b/icing/store/document-store_benchmark.cc
@@ -30,7 +30,9 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" #include "icing/proto/document.pb.h" #include "icing/proto/persist.pb.h" #include "icing/proto/schema.pb.h" @@ -39,6 +41,7 @@ #include "icing/schema/schema-store.h" #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/util/clock.h" @@ -108,13 +111,14 @@ .Build(); } -std::unique_ptr<SchemaStore> CreateSchemaStore(Filesystem filesystem, - const std::string directory, - const Clock* clock) { +std::unique_ptr<SchemaStore> CreateSchemaStore( + Filesystem filesystem, const std::string directory, const Clock* clock, + const FeatureFlags& feature_flags) { const std::string schema_store_dir = directory + "/schema"; filesystem.CreateDirectoryRecursively(schema_store_dir.data()); std::unique_ptr<SchemaStore> schema_store = - SchemaStore::Create(&filesystem, schema_store_dir, clock).ValueOrDie(); + SchemaStore::Create(&filesystem, schema_store_dir, clock, &feature_flags) + .ValueOrDie(); auto set_schema_status = schema_store->SetSchema( CreateSchema(), /*ignore_errors_and_delete_documents=*/false, @@ -128,17 +132,18 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( const Filesystem* filesystem, const std::string& base_dir, - const Clock* clock, const SchemaStore* schema_store) { + const Clock* clock, const SchemaStore* schema_store, + const FeatureFlags& feature_flags) { return DocumentStore::Create( - filesystem, base_dir, clock, schema_store, + filesystem, base_dir, clock, schema_store, &feature_flags, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/true, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/true, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/true, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr); } void BM_DoesDocumentExistBenchmark(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); Filesystem filesystem; Clock clock; @@ -147,13 +152,13 @@ std::string document_store_dir = directory + "/store"; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, directory, &clock); + CreateSchemaStore(filesystem, directory, &clock, feature_flags); filesystem.CreateDirectoryRecursively(document_store_dir.data()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -181,6 +186,7 @@ BENCHMARK(BM_DoesDocumentExistBenchmark); void BM_Put(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); Filesystem filesystem; Clock clock; @@ -189,13 +195,13 @@ std::string document_store_dir = directory + "/store"; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, directory, &clock); + CreateSchemaStore(filesystem, directory, &clock, feature_flags); filesystem.CreateDirectoryRecursively(document_store_dir.data()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -210,6 +216,7 @@ BENCHMARK(BM_Put); void BM_GetSameDocument(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); Filesystem filesystem; Clock clock; @@ -218,13 +225,13 @@ std::string document_store_dir = directory + "/store"; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, directory, &clock); + CreateSchemaStore(filesystem, directory, &clock, feature_flags); filesystem.CreateDirectoryRecursively(document_store_dir.data()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -237,6 +244,7 @@ BENCHMARK(BM_GetSameDocument); void BM_Delete(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); Filesystem filesystem; Clock clock; @@ -245,13 +253,13 @@ std::string document_store_dir = directory + "/store"; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, directory, &clock); + CreateSchemaStore(filesystem, directory, &clock, feature_flags); filesystem.CreateDirectoryRecursively(document_store_dir.data()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -269,6 +277,7 @@ BENCHMARK(BM_Delete); void BM_Create(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); Filesystem filesystem; Clock clock; @@ -276,7 +285,7 @@ std::string document_store_dir = directory + "/store"; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, directory, &clock); + CreateSchemaStore(filesystem, directory, &clock, feature_flags); // Create an initial document store and put some data in. { @@ -286,7 +295,7 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -301,13 +310,15 @@ filesystem.CreateDirectoryRecursively(document_store_dir.data()); for (auto s : state) { - benchmark::DoNotOptimize(CreateDocumentStore( - &filesystem, document_store_dir, &clock, schema_store.get())); + benchmark::DoNotOptimize( + CreateDocumentStore(&filesystem, document_store_dir, &clock, + schema_store.get(), feature_flags)); } } BENCHMARK(BM_Create); void BM_UpdateChecksum(benchmark::State& state) { + FeatureFlags feature_flags = GetTestFeatureFlags(); Filesystem filesystem; Clock clock; @@ -316,13 +327,13 @@ std::string document_store_dir = directory + "/store"; std::unique_ptr<SchemaStore> schema_store = - CreateSchemaStore(filesystem, directory, &clock); + CreateSchemaStore(filesystem, directory, &clock, feature_flags); filesystem.CreateDirectoryRecursively(document_store_dir.data()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem, document_store_dir, &clock, - schema_store.get())); + schema_store.get(), feature_flags)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store);
diff --git a/icing/store/document-store_test.cc b/icing/store/document-store_test.cc index 776f19f..775a4e0 100644 --- a/icing/store/document-store_test.cc +++ b/icing/store/document-store_test.cc
@@ -23,10 +23,12 @@ #include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" +#include "icing/text_classifier/lib3/utils/hash/farmhash.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/absl_ports/str_cat.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" #include "icing/file/memory-mapped-file.h" @@ -36,6 +38,7 @@ #include "icing/proto/debug.pb.h" #include "icing/proto/document.pb.h" #include "icing/proto/document_wrapper.pb.h" +#include "icing/proto/internal/scorable_property_set.pb.h" #include "icing/proto/logging.pb.h" #include "icing/proto/schema.pb.h" #include "icing/proto/storage.pb.h" @@ -45,19 +48,22 @@ #include "icing/schema/schema-store.h" #include "icing/store/corpus-associated-scoring-data.h" #include "icing/store/corpus-id.h" +#include "icing/store/document-associated-score-data.h" #include "icing/store/document-filter-data.h" #include "icing/store/document-id.h" #include "icing/store/document-log-creator.h" -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include "icing/store/namespace-id.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/util/crc32.h" +#include "icing/util/icu-data-file-helper.h" +#include "icing/util/scorable_property_set.h" #include "unicode/uloc.h" namespace icing { @@ -75,7 +81,9 @@ using ::testing::IsEmpty; using ::testing::IsFalse; using ::testing::IsTrue; +using ::testing::Ne; using ::testing::Not; +using ::testing::Pointee; using ::testing::Return; using ::testing::UnorderedElementsAre; @@ -123,16 +131,36 @@ sizeof(PortableFileBackedProtoLog<DocumentWrapper>::Header)); } +ScorablePropertyProto BuildScorablePropertyProtoFromBoolean( + bool boolean_value) { + ScorablePropertyProto scorable_property; + scorable_property.add_boolean_values(boolean_value); + return scorable_property; +} + +ScorablePropertyProto BuildScorablePropertyProtoFromInt64( + const std::vector<int64_t>& int64_values) { + ScorablePropertyProto scorable_property; + scorable_property.mutable_int64_values()->Add(int64_values.begin(), + int64_values.end()); + return scorable_property; +} + +ScorablePropertyProto BuildScorablePropertyProtoFromDouble( + const std::vector<double>& double_values) { + ScorablePropertyProto scorable_property; + scorable_property.mutable_double_values()->Add(double_values.begin(), + double_values.end()); + return scorable_property; +} + struct DocumentStoreTestParam { - bool namespace_id_fingerprint; bool pre_mapping_fbv; bool use_persistent_hash_map; - explicit DocumentStoreTestParam(bool namespace_id_fingerprint_in, - bool pre_mapping_fbv_in, + explicit DocumentStoreTestParam(bool pre_mapping_fbv_in, bool use_persistent_hash_map_in) - : namespace_id_fingerprint(namespace_id_fingerprint_in), - pre_mapping_fbv(pre_mapping_fbv_in), + : pre_mapping_fbv(pre_mapping_fbv_in), use_persistent_hash_map(use_persistent_hash_map_in) {} }; @@ -149,6 +177,7 @@ .SetSchema("email") .AddStringProperty("subject", "subject foo") .AddStringProperty("body", "body bar") + .AddDoubleProperty("score", 1.5, 2.5) .SetScore(document1_score_) .SetCreationTimestampMs( document1_creation_timestamp_) // A random timestamp @@ -160,6 +189,7 @@ .SetSchema("email") .AddStringProperty("subject", "subject foo 2") .AddStringProperty("body", "body bar 2") + .AddDoubleProperty("score", 3.5, 4.5) .SetScore(document2_score_) .SetCreationTimestampMs( document2_creation_timestamp_) // A random timestamp @@ -168,6 +198,7 @@ } void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { // If we've specified using the reverse-JNI method for segmentation (i.e. // not ICU), then we won't have the ICU data file included to set up. @@ -178,7 +209,7 @@ std::string icu_data_file_path = GetTestFilePath("icing/icu.dat"); ICING_ASSERT_OK( - icu_data_file_helper::SetUpICUDataFile(icu_data_file_path)); + icu_data_file_helper::SetUpIcuDataFile(icu_data_file_path)); } filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); @@ -200,11 +231,17 @@ .SetName("body") .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) - .SetCardinality(CARDINALITY_OPTIONAL))) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("score") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) .Build(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); ASSERT_THAT(schema_store_->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false), @@ -229,8 +266,7 @@ const std::string header_file = absl_ports::StrCat(document_store_dir_, "/document_store_header"); DocumentStore::Header header; - header.magic = DocumentStore::Header::GetCurrentMagic( - GetParam().namespace_id_fingerprint); + header.magic = DocumentStore::Header::kMagic; header.checksum = 10; // Arbitrary garbage checksum filesystem_.DeleteFile(header_file.c_str()); filesystem_.Write(header_file.c_str(), &header, sizeof(header)); @@ -240,14 +276,14 @@ const Filesystem* filesystem, const std::string& base_dir, const Clock* clock, const SchemaStore* schema_store) { return DocumentStore::Create( - filesystem, base_dir, clock, schema_store, + filesystem, base_dir, clock, schema_store, feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, - GetParam().use_persistent_hash_map, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + GetParam().pre_mapping_fbv, GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr); } + std::unique_ptr<FeatureFlags> feature_flags_; const Filesystem filesystem_; const std::string test_dir_; FakeClock fake_clock_; @@ -306,11 +342,13 @@ // Both documents have namespace of "icing" ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, doc_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, doc_store->Put(test_document2_)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); DocumentId document_id2 = put_result2.new_document_id; EXPECT_THAT(doc_store->Get(document_id1), @@ -341,11 +379,13 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, doc_store->Put(foo_document)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, doc_store->Put(DocumentProto(bar_document))); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); DocumentId document_id2 = put_result2.new_document_id; EXPECT_THAT(doc_store->Get(document_id1), @@ -370,12 +410,14 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, doc_store->Put(document1)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, doc_store->Put(document2)); DocumentId document_id2 = put_result2.new_document_id; - EXPECT_TRUE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(document_id1)); + EXPECT_TRUE(put_result2.was_replacement()); EXPECT_THAT(document_id1, Not(document_id2)); // document2 overrides document1, so document_id1 becomes invalid EXPECT_THAT(doc_store->Get(document_id1), @@ -388,8 +430,9 @@ document3.set_uri("another/uri/1"); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result3, doc_store->Put(document3)); - EXPECT_FALSE(put_result3.was_replacement); - EXPECT_THAT(put_result3.new_document_id, Not(document_id1)); + EXPECT_THAT(put_result3.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_THAT(put_result3.new_document_id, Ne(document_id1)); + EXPECT_FALSE(put_result3.was_replacement()); } TEST_P(DocumentStoreTest, IsDocumentExistingWithoutStatus) { @@ -402,11 +445,13 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, doc_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, doc_store->Put(test_document2_)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); DocumentId document_id2 = put_result2.new_document_id; EXPECT_TRUE(doc_store->GetAliveDocumentFilterData( @@ -498,7 +543,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(test_document1_)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; DocumentId invalid_document_id_negative = -1; @@ -767,7 +813,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, @@ -787,7 +834,8 @@ .Build(); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult email_put_result1, document_store->Put(email_document_1)); - EXPECT_FALSE(email_put_result1.was_replacement); + EXPECT_THAT(email_put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_put_result1.was_replacement()); DocumentId email_1_document_id = email_put_result1.new_document_id; DocumentProto email_document_2 = DocumentBuilder() @@ -797,7 +845,8 @@ .Build(); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult email_put_result2, document_store->Put(email_document_2)); - EXPECT_FALSE(email_put_result2.was_replacement); + EXPECT_THAT(email_put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_put_result2.was_replacement()); DocumentId email_2_document_id = email_put_result2.new_document_id; DocumentProto message_document = DocumentBuilder() @@ -807,7 +856,8 @@ .Build(); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult message_put_result, document_store->Put(message_document)); - EXPECT_FALSE(message_put_result.was_replacement); + EXPECT_THAT(message_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(message_put_result.was_replacement()); DocumentId message_document_id = message_put_result.new_document_id; DocumentProto person_document = DocumentBuilder() @@ -817,7 +867,8 @@ .Build(); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult person_put_result, document_store->Put(person_document)); - EXPECT_FALSE(person_put_result.was_replacement); + EXPECT_THAT(person_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(person_put_result.was_replacement()); DocumentId person_document_id = person_put_result.new_document_id; // Delete the "email" type and ensure that it works across both @@ -905,7 +956,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, @@ -936,11 +988,13 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult email_put_result, document_store->Put(email_document)); - EXPECT_FALSE(email_put_result.was_replacement); + EXPECT_THAT(email_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_put_result.was_replacement()); email_document_id = email_put_result.new_document_id; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult message_put_result, document_store->Put(message_document)); - EXPECT_FALSE(message_put_result.was_replacement); + EXPECT_THAT(message_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(message_put_result.was_replacement()); message_document_id = message_put_result.new_document_id; // Delete "email". "message" documents should still be retrievable. DocumentStore::DeleteByGroupResult group_result = @@ -1002,7 +1056,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, @@ -1033,11 +1088,13 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult email_put_result, document_store->Put(email_document)); - EXPECT_FALSE(email_put_result.was_replacement); + EXPECT_THAT(email_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_put_result.was_replacement()); email_document_id = email_put_result.new_document_id; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult message_put_result, document_store->Put(message_document)); - EXPECT_FALSE(message_put_result.was_replacement); + EXPECT_THAT(message_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(message_put_result.was_replacement()); message_document_id = message_put_result.new_document_id; // Delete "email". "message" documents should still be retrievable. @@ -1475,12 +1532,14 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, doc_store->Put(DocumentProto(test_document1_), /*num_tokens=*/4)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, doc_store->Put(DocumentProto(test_document2_), /*num_tokens=*/4)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); document_id2 = put_result2.new_document_id; EXPECT_THAT(doc_store->Get(document_id1), IsOkAndHolds(EqualsProto(test_document1_))); @@ -1489,18 +1548,34 @@ // Checks derived score cache EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id1), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document1_score_, document1_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT(doc_store->GetCorpusAssociatedScoreData(/*corpus_id=*/0), IsOkAndHolds(CorpusAssociatedScoreData( /*num_docs=*/2, /*sum_length_in_tokens=*/8))); + // Checks derived scorable property set cache + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc1 = + doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({1.5, 2.5})))); + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({3.5, 4.5})))); + // Delete document 1 EXPECT_THAT(doc_store->Delete("icing", "email/1", fake_clock_.GetSystemTimeMilliseconds()), @@ -1540,20 +1615,33 @@ DocumentFilterData doc_filter_data, doc_store->GetAliveDocumentFilterData( document_id2, fake_clock_.GetSystemTimeMilliseconds())); - EXPECT_THAT(doc_filter_data, - Eq(DocumentFilterData( - /*namespace_id=*/0, - /*schema_type_id=*/0, document2_expiration_timestamp_))); + EXPECT_THAT( + doc_filter_data, + Eq(DocumentFilterData( + /*namespace_id=*/0, tc3farmhash::Fingerprint64(test_document2_.uri()), + /*schema_type_id=*/0, document2_expiration_timestamp_))); // Checks derived score cache EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT(doc_store->GetCorpusAssociatedScoreData(/*corpus_id=*/0), IsOkAndHolds(CorpusAssociatedScoreData( /*num_docs=*/1, /*sum_length_in_tokens=*/4))); + + // Checks derived scorable property set cache + EXPECT_EQ(doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()), + nullptr); + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({3.5, 4.5})))); } TEST_P(DocumentStoreTest, ShouldRecoverFromCorruptDerivedFile) { @@ -1570,12 +1658,14 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, doc_store->Put(DocumentProto(test_document1_), /*num_tokens=*/4)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, doc_store->Put(DocumentProto(test_document2_), /*num_tokens=*/4)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); document_id2 = put_result2.new_document_id; EXPECT_THAT(doc_store->Get(document_id1), IsOkAndHolds(EqualsProto(test_document1_))); @@ -1584,17 +1674,33 @@ // Checks derived score cache EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id1), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document1_score_, document1_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT(doc_store->GetCorpusAssociatedScoreData(/*corpus_id=*/0), IsOkAndHolds(CorpusAssociatedScoreData( /*num_docs=*/2, /*sum_length_in_tokens=*/8))); + // Checks derived scorable property set cache + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc1 = + doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({1.5, 2.5})))); + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({3.5, 4.5})))); + // Delete document 1 EXPECT_THAT(doc_store->Delete("icing", "email/1", fake_clock_.GetSystemTimeMilliseconds()), @@ -1645,17 +1751,19 @@ DocumentFilterData doc_filter_data, doc_store->GetAliveDocumentFilterData( document_id2, fake_clock_.GetSystemTimeMilliseconds())); - EXPECT_THAT(doc_filter_data, - Eq(DocumentFilterData( - /*namespace_id=*/0, - /*schema_type_id=*/0, document2_expiration_timestamp_))); + EXPECT_THAT( + doc_filter_data, + Eq(DocumentFilterData( + /*namespace_id=*/0, tc3farmhash::Fingerprint64(test_document2_.uri()), + /*schema_type_id=*/0, document2_expiration_timestamp_))); // Checks derived score cache EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT(doc_store->GetCorpusAssociatedScoreData(/*corpus_id=*/0), IsOkAndHolds(CorpusAssociatedScoreData( /*num_docs=*/1, /*sum_length_in_tokens=*/4))); @@ -1669,6 +1777,17 @@ doc_store->GetUsageScores(document_id2, fake_clock_.GetSystemTimeMilliseconds())); EXPECT_THAT(actual_scores, Eq(expected_scores)); + + // Checks derived scorable property set cache + EXPECT_EQ(doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()), + nullptr); + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({3.5, 4.5})))); } TEST_P(DocumentStoreTest, ShouldRecoverFromDiscardDerivedFiles) { @@ -1685,12 +1804,14 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, doc_store->Put(DocumentProto(test_document1_), /*num_tokens=*/4)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, doc_store->Put(DocumentProto(test_document2_), /*num_tokens=*/4)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); document_id2 = put_result2.new_document_id; EXPECT_THAT(doc_store->Get(document_id1), IsOkAndHolds(EqualsProto(test_document1_))); @@ -1699,17 +1820,34 @@ // Checks derived score cache EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id1), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document1_score_, document1_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT(doc_store->GetCorpusAssociatedScoreData(/*corpus_id=*/0), IsOkAndHolds(CorpusAssociatedScoreData( /*num_docs=*/2, /*sum_length_in_tokens=*/8))); + + // Checks derived scorable property set cache + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc1 = + doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({1.5, 2.5})))); + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({3.5, 4.5})))); + // Delete document 1 EXPECT_THAT(doc_store->Delete("icing", "email/1", fake_clock_.GetSystemTimeMilliseconds()), @@ -1747,17 +1885,19 @@ DocumentFilterData doc_filter_data, doc_store->GetAliveDocumentFilterData( document_id2, fake_clock_.GetSystemTimeMilliseconds())); - EXPECT_THAT(doc_filter_data, - Eq(DocumentFilterData( - /*namespace_id=*/0, - /*schema_type_id=*/0, document2_expiration_timestamp_))); + EXPECT_THAT( + doc_filter_data, + Eq(DocumentFilterData( + /*namespace_id=*/0, tc3farmhash::Fingerprint64(test_document2_.uri()), + /*schema_type_id=*/0, document2_expiration_timestamp_))); // Checks derived score cache. EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT(doc_store->GetCorpusAssociatedScoreData(/*corpus_id=*/0), IsOkAndHolds(CorpusAssociatedScoreData( /*num_docs=*/1, /*sum_length_in_tokens=*/4))); @@ -1771,6 +1911,17 @@ doc_store->GetUsageScores(document_id2, fake_clock_.GetSystemTimeMilliseconds())); EXPECT_THAT(actual_scores, Eq(expected_scores)); + + // Checks derived scorable property set cache + EXPECT_EQ(doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()), + nullptr); + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({3.5, 4.5})))); } TEST_P(DocumentStoreTest, ShouldRecoverFromBadChecksum) { @@ -1787,12 +1938,14 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, doc_store->Put(DocumentProto(test_document1_), /*num_tokens=*/4)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, doc_store->Put(DocumentProto(test_document2_), /*num_tokens=*/4)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); document_id2 = put_result2.new_document_id; EXPECT_THAT(doc_store->Get(document_id1), IsOkAndHolds(EqualsProto(test_document1_))); @@ -1801,17 +1954,34 @@ // Checks derived score cache EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id1), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document1_score_, document1_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT(doc_store->GetCorpusAssociatedScoreData(/*corpus_id=*/0), IsOkAndHolds(CorpusAssociatedScoreData( /*num_docs=*/2, /*sum_length_in_tokens=*/8))); + + // Checks derived scorable property set cache + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc1 = + doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({1.5, 2.5})))); + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({3.5, 4.5})))); + EXPECT_THAT(doc_store->Delete("icing", "email/1", fake_clock_.GetSystemTimeMilliseconds()), IsOk()); @@ -1840,19 +2010,32 @@ DocumentFilterData doc_filter_data, doc_store->GetAliveDocumentFilterData( document_id2, fake_clock_.GetSystemTimeMilliseconds())); - EXPECT_THAT(doc_filter_data, - Eq(DocumentFilterData( - /*namespace_id=*/0, - /*schema_type_id=*/0, document2_expiration_timestamp_))); + EXPECT_THAT( + doc_filter_data, + Eq(DocumentFilterData( + /*namespace_id=*/0, tc3farmhash::Fingerprint64(test_document2_.uri()), + /*schema_type_id=*/0, document2_expiration_timestamp_))); // Checks derived score cache EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT(doc_store->GetCorpusAssociatedScoreData(/*corpus_id=*/0), IsOkAndHolds(CorpusAssociatedScoreData( /*num_docs=*/1, /*sum_length_in_tokens=*/4))); + + // Checks derived scorable property set cache + EXPECT_EQ(doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()), + nullptr); + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({3.5, 4.5})))); } TEST_P(DocumentStoreTest, GetStorageInfo) { @@ -1912,7 +2095,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, doc_store->Put(DocumentProto(test_document1_))); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id1 = put_result1.new_document_id; EXPECT_THAT(doc_store->last_added_document_id(), Eq(document_id1)); @@ -1923,7 +2107,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, doc_store->Put(DocumentProto(test_document2_))); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); DocumentId document_id2 = put_result2.new_document_id; EXPECT_THAT(doc_store->last_added_document_id(), Eq(document_id2)); } @@ -2176,24 +2361,28 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, doc_store->Put(DocumentProto(document1), /*num_tokens=*/5)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, doc_store->Put(DocumentProto(document2), /*num_tokens=*/7)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); DocumentId document_id2 = put_result2.new_document_id; EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id1), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document1_score_, document1_creation_timestamp_, - /*length_in_tokens=*/5))); + /*length_in_tokens=*/5, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/7))); + /*length_in_tokens=*/7, + /*has_valid_scorable_property_cache_index=*/true))); } TEST_P(DocumentStoreTest, GetDocumentAssociatedScoreDataDifferentCorpus) { @@ -2224,24 +2413,28 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result1, doc_store->Put(DocumentProto(document1), /*num_tokens=*/5)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result2, doc_store->Put(DocumentProto(document2), /*num_tokens=*/7)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); DocumentId document_id2 = put_result2.new_document_id; EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id1), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, document1_score_, document1_creation_timestamp_, - /*length_in_tokens=*/5))); + /*length_in_tokens=*/5, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT( doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/1, document2_score_, document2_creation_timestamp_, - /*length_in_tokens=*/7))); + /*length_in_tokens=*/7, + /*has_valid_scorable_property_cache_index=*/true))); } TEST_P(DocumentStoreTest, NonexistentDocumentAssociatedScoreDataNotFound) { @@ -2278,17 +2471,19 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(DocumentProto(test_document1_))); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( DocumentFilterData doc_filter_data, doc_store->GetAliveDocumentFilterData( document_id, fake_clock_.GetSystemTimeMilliseconds())); - EXPECT_THAT(doc_filter_data, - Eq(DocumentFilterData( - /*namespace_id=*/0, - /*schema_type_id=*/0, document1_expiration_timestamp_))); + EXPECT_THAT( + doc_filter_data, + Eq(DocumentFilterData( + /*namespace_id=*/0, tc3farmhash::Fingerprint64(test_document1_.uri()), + /*schema_type_id=*/0, document1_expiration_timestamp_))); ICING_ASSERT_OK(doc_store->Delete("icing", "email/1", fake_clock_.GetSystemTimeMilliseconds())); @@ -2308,25 +2503,50 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result, doc_store->Put(DocumentProto(test_document1_), /*num_tokens=*/4)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; EXPECT_THAT(doc_store->GetDocumentAssociatedScoreData(document_id), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, /*document_score=*/document1_score_, /*creation_timestamp_ms=*/document1_creation_timestamp_, - /*length_in_tokens=*/4))); + /*length_in_tokens=*/4, + /*has_valid_scorable_property_cache_index=*/true))); ICING_ASSERT_OK(doc_store->Delete("icing", "email/1", fake_clock_.GetSystemTimeMilliseconds())); // Associated entry of the deleted document is removed. - EXPECT_THAT( - doc_store->GetDocumentAssociatedScoreData(document_id), - IsOkAndHolds(DocumentAssociatedScoreData(kInvalidCorpusId, - /*document_score=*/-1, - /*creation_timestamp_ms=*/-1, - /*length_in_tokens=*/0))); + EXPECT_THAT(doc_store->GetDocumentAssociatedScoreData(document_id), + IsOkAndHolds(EqualsDocumentAssociatedScoreData( + kInvalidCorpusId, + /*document_score=*/-1, + /*creation_timestamp_ms=*/-1, + /*length_in_tokens=*/0, + /*has_valid_scorable_property_cache_index=*/false))); +} + +TEST_P(DocumentStoreTest, DeleteClearsScorablePropertyCache) { + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, + schema_store_.get())); + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, + doc_store->Put(test_document1_, /*num_tokens=*/4)); + DocumentId document_id = put_result1.new_document_id; + EXPECT_NE(doc_store->GetScorablePropertySet( + document_id, fake_clock_.GetSystemTimeMilliseconds()), + nullptr); + ICING_ASSERT_OK(doc_store->Delete("icing", "email/1", + fake_clock_.GetSystemTimeMilliseconds())); + // Scorable property set is not found for deleted documents. + EXPECT_EQ(doc_store->GetScorablePropertySet( + document_id, fake_clock_.GetSystemTimeMilliseconds()), + nullptr); } TEST_P(DocumentStoreTest, DeleteShouldPreventUsageScores) { @@ -2339,7 +2559,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(DocumentProto(test_document1_))); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; // Report usage with type 1. @@ -2390,7 +2611,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(document)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; // Some arbitrary time before the document's creation time (10) + ttl (100) @@ -2441,16 +2663,18 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(document)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( DocumentFilterData doc_filter_data, doc_store->GetAliveDocumentFilterData( document_id, fake_clock_.GetSystemTimeMilliseconds())); - EXPECT_THAT(doc_filter_data, Eq(DocumentFilterData( - /*namespace_id=*/0, - /*schema_type_id=*/0, - /*expiration_timestamp_ms=*/1100))); + EXPECT_THAT( + doc_filter_data, + Eq(DocumentFilterData( + /*namespace_id=*/0, tc3farmhash::Fingerprint64(document.uri()), + /*schema_type_id=*/0, /*expiration_timestamp_ms=*/1100))); } TEST_P(DocumentStoreTest, ExpirationTimestampIsInt64MaxIfTtlIsZero) { @@ -2470,7 +2694,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(document)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( @@ -2481,7 +2706,7 @@ EXPECT_THAT( doc_filter_data, Eq(DocumentFilterData( - /*namespace_id=*/0, + /*namespace_id=*/0, tc3farmhash::Fingerprint64(document.uri()), /*schema_type_id=*/0, /*expiration_timestamp_ms=*/std::numeric_limits<int64_t>::max()))); } @@ -2504,7 +2729,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(document)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( @@ -2515,7 +2741,7 @@ EXPECT_THAT( doc_filter_data, Eq(DocumentFilterData( - /*namespace_id=*/0, + /*namespace_id=*/0, tc3farmhash::Fingerprint64(document.uri()), /*schema_type_id=*/0, /*expiration_timestamp_ms=*/std::numeric_limits<int64_t>::max()))); } @@ -2542,7 +2768,8 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult put_result, doc_store->Put(document_without_creation_timestamp)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; ICING_ASSERT_OK_AND_ASSIGN(DocumentProto document_with_creation_timestamp, @@ -2576,24 +2803,28 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, doc_store->Put(document1)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, doc_store->Put(document2)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); DocumentId document_id2 = put_result2.new_document_id; EXPECT_THAT(doc_store->GetDocumentAssociatedScoreData(document_id1), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, /*document_score=*/0, /*creation_timestamp_ms=*/0, - /*length_in_tokens=*/0))); + /*length_in_tokens=*/0, + /*has_valid_scorable_property_cache_index=*/true))); EXPECT_THAT(doc_store->GetDocumentAssociatedScoreData(document_id2), - IsOkAndHolds(DocumentAssociatedScoreData( + IsOkAndHolds(EqualsDocumentAssociatedScoreData( /*corpus_id=*/0, /*document_score=*/5, /*creation_timestamp_ms=*/0, - /*length_in_tokens=*/0))); + /*length_in_tokens=*/0, + /*has_valid_scorable_property_cache_index=*/true))); } TEST_P(DocumentStoreTest, GetChecksumDoesntUpdateStoredChecksum) { @@ -2634,7 +2865,8 @@ &fake_clock_, schema_store_.get())); EXPECT_FALSE(create_result.derived_files_regenerated); - std::unique_ptr<DocumentStore> document_store_two = std::move(create_result.document_store); + std::unique_ptr<DocumentStore> document_store_two = + std::move(create_result.document_store); EXPECT_THAT(document_store_two->GetChecksum(), IsOkAndHolds(checksum)); EXPECT_THAT(document_store_two->UpdateChecksum(), IsOkAndHolds(checksum)); EXPECT_THAT(document_store_two->GetChecksum(), IsOkAndHolds(checksum)); @@ -2726,7 +2958,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("email")) @@ -2752,7 +2985,8 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult email_put_result, document_store->Put(DocumentProto(email_document))); - EXPECT_FALSE(email_put_result.was_replacement); + EXPECT_THAT(email_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_put_result.was_replacement()); email_document_id = email_put_result.new_document_id; EXPECT_THAT(document_store->Get(email_document_id), IsOkAndHolds(EqualsProto(email_document))); @@ -2760,6 +2994,8 @@ DocumentFilterData email_data, document_store->GetAliveDocumentFilterData( email_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(email_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(email_document.uri()))); EXPECT_THAT(email_data.schema_type_id(), Eq(email_schema_type_id)); email_namespace_id = email_data.namespace_id(); email_expiration_timestamp = email_data.expiration_timestamp_ms(); @@ -2768,7 +3004,8 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult message_put_result, document_store->Put(DocumentProto(message_document))); - EXPECT_FALSE(message_put_result.was_replacement); + EXPECT_THAT(message_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(message_put_result.was_replacement()); message_document_id = message_put_result.new_document_id; EXPECT_THAT(document_store->Get(message_document_id), IsOkAndHolds(EqualsProto(message_document))); @@ -2776,6 +3013,8 @@ DocumentFilterData message_data, document_store->GetAliveDocumentFilterData( message_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(message_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(message_document.uri()))); EXPECT_THAT(message_data.schema_type_id(), Eq(message_schema_type_id)); message_namespace_id = message_data.namespace_id(); message_expiration_timestamp = message_data.expiration_timestamp_ms(); @@ -2791,7 +3030,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); SchemaProto schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("email")) @@ -2822,6 +3062,8 @@ EXPECT_THAT(email_data.schema_type_id(), Eq(email_schema_type_id)); // Make sure that all the other fields are stll valid/the same EXPECT_THAT(email_data.namespace_id(), Eq(email_namespace_id)); + EXPECT_THAT(email_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(email_document.uri()))); EXPECT_THAT(email_data.expiration_timestamp_ms(), Eq(email_expiration_timestamp)); @@ -2835,6 +3077,8 @@ EXPECT_THAT(message_data.schema_type_id(), Eq(-1)); // Make sure that all the other fields are stll valid/the same EXPECT_THAT(message_data.namespace_id(), Eq(message_namespace_id)); + EXPECT_THAT(message_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(message_document.uri()))); EXPECT_THAT(message_data.expiration_timestamp_ms(), Eq(message_expiration_timestamp)); } @@ -2853,7 +3097,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_EXPECT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2886,23 +3131,29 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult email_put_result, document_store->Put(DocumentProto(email_document))); - EXPECT_FALSE(email_put_result.was_replacement); + EXPECT_THAT(email_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_put_result.was_replacement()); DocumentId email_document_id = email_put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( DocumentFilterData email_data, document_store->GetAliveDocumentFilterData( email_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(email_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(email_document.uri()))); EXPECT_THAT(email_data.schema_type_id(), Eq(old_email_schema_type_id)); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult message_put_result, document_store->Put(DocumentProto(message_document))); - EXPECT_FALSE(message_put_result.was_replacement); + EXPECT_THAT(message_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(message_put_result.was_replacement()); DocumentId message_document_id = message_put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( DocumentFilterData message_data, document_store->GetAliveDocumentFilterData( message_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(message_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(message_document.uri()))); EXPECT_THAT(message_data.schema_type_id(), Eq(old_message_schema_type_id)); // Rearrange the schema types. Since SchemaTypeId is assigned based on order, @@ -2932,12 +3183,16 @@ email_data, document_store->GetAliveDocumentFilterData( email_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(email_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(email_document.uri()))); EXPECT_THAT(email_data.schema_type_id(), Eq(new_email_schema_type_id)); ICING_ASSERT_HAS_VALUE_AND_ASSIGN( message_data, document_store->GetAliveDocumentFilterData( message_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(message_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(message_document.uri()))); EXPECT_THAT(message_data.schema_type_id(), Eq(new_message_schema_type_id)); } @@ -2958,7 +3213,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_EXPECT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -2990,7 +3246,9 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult email_without_subject_put_result, document_store->Put(DocumentProto(email_without_subject))); - EXPECT_FALSE(email_without_subject_put_result.was_replacement); + EXPECT_THAT(email_without_subject_put_result.old_document_id, + Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_without_subject_put_result.was_replacement()); DocumentId email_without_subject_document_id = email_without_subject_put_result.new_document_id; EXPECT_THAT(document_store->Get(email_without_subject_document_id), @@ -2999,7 +3257,9 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult email_with_subject_put_result, document_store->Put(DocumentProto(email_with_subject))); - EXPECT_FALSE(email_with_subject_put_result.was_replacement); + EXPECT_THAT(email_with_subject_put_result.old_document_id, + Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_with_subject_put_result.was_replacement()); DocumentId email_with_subject_document_id = email_with_subject_put_result.new_document_id; EXPECT_THAT(document_store->Get(email_with_subject_document_id), @@ -3040,7 +3300,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_EXPECT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3070,14 +3331,16 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult email_put_result, document_store->Put(email_document)); - EXPECT_FALSE(email_put_result.was_replacement); + EXPECT_THAT(email_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_put_result.was_replacement()); DocumentId email_document_id = email_put_result.new_document_id; EXPECT_THAT(document_store->Get(email_document_id), IsOkAndHolds(EqualsProto(email_document))); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult message_put_result, document_store->Put(message_document)); - EXPECT_FALSE(message_put_result.was_replacement); + EXPECT_THAT(message_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(message_put_result.was_replacement()); DocumentId message_document_id = message_put_result.new_document_id; EXPECT_THAT(document_store->Get(message_document_id), IsOkAndHolds(EqualsProto(message_document))); @@ -3117,7 +3380,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_EXPECT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3149,22 +3413,28 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult email_put_result, document_store->Put(email_document)); - EXPECT_FALSE(email_put_result.was_replacement); + EXPECT_THAT(email_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_put_result.was_replacement()); DocumentId email_document_id = email_put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( DocumentFilterData email_data, document_store->GetAliveDocumentFilterData( email_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(email_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(email_document.uri()))); EXPECT_THAT(email_data.schema_type_id(), Eq(old_email_schema_type_id)); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult message_put_result, document_store->Put(message_document)); - EXPECT_FALSE(message_put_result.was_replacement); + EXPECT_THAT(message_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(message_put_result.was_replacement()); DocumentId message_document_id = message_put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( DocumentFilterData message_data, document_store->GetAliveDocumentFilterData( message_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(message_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(message_document.uri()))); EXPECT_THAT(message_data.schema_type_id(), Eq(old_message_schema_type_id)); // Rearrange the schema types. Since SchemaTypeId is assigned based on order, @@ -3197,12 +3467,16 @@ email_data, document_store->GetAliveDocumentFilterData( email_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(email_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(email_document.uri()))); EXPECT_THAT(email_data.schema_type_id(), Eq(new_email_schema_type_id)); ICING_ASSERT_HAS_VALUE_AND_ASSIGN( message_data, document_store->GetAliveDocumentFilterData( message_document_id, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(message_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(message_document.uri()))); EXPECT_THAT(message_data.schema_type_id(), Eq(new_message_schema_type_id)); } @@ -3223,7 +3497,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_EXPECT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3255,7 +3530,9 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult email_without_subject_put_result, document_store->Put(email_without_subject)); - EXPECT_FALSE(email_without_subject_put_result.was_replacement); + EXPECT_THAT(email_without_subject_put_result.old_document_id, + Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_without_subject_put_result.was_replacement()); DocumentId email_without_subject_document_id = email_without_subject_put_result.new_document_id; EXPECT_THAT(document_store->Get(email_without_subject_document_id), @@ -3264,7 +3541,9 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::PutResult email_with_subject_put_result, document_store->Put(email_with_subject)); - EXPECT_FALSE(email_with_subject_put_result.was_replacement); + EXPECT_THAT(email_with_subject_put_result.old_document_id, + Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_with_subject_put_result.was_replacement()); DocumentId email_with_subject_document_id = email_with_subject_put_result.new_document_id; EXPECT_THAT(document_store->Get(email_with_subject_document_id), @@ -3308,7 +3587,8 @@ ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_EXPECT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false)); @@ -3338,14 +3618,16 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult email_put_result, document_store->Put(email_document)); - EXPECT_FALSE(email_put_result.was_replacement); + EXPECT_THAT(email_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(email_put_result.was_replacement()); DocumentId email_document_id = email_put_result.new_document_id; EXPECT_THAT(document_store->Get(email_document_id), IsOkAndHolds(EqualsProto(email_document))); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult message_put_result, document_store->Put(message_document)); - EXPECT_FALSE(message_put_result.was_replacement); + EXPECT_THAT(message_put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(message_put_result.was_replacement()); DocumentId message_document_id = message_put_result.new_document_id; EXPECT_THAT(document_store->Get(message_document_id), IsOkAndHolds(EqualsProto(message_document))); @@ -3509,7 +3791,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, document_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id = put_result1.new_document_id; // Report usage with type 1 and time 1. @@ -3603,7 +3886,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, document_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id = put_result1.new_document_id; // Report usage with type 1. @@ -3658,7 +3942,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, document_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); document_id = put_result1.new_document_id; // Report usage with type 1. @@ -3705,7 +3990,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, document_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); document_id = put_result1.new_document_id; // Report usage with type 1. @@ -3760,7 +4046,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, document_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id = put_result1.new_document_id; // Report usage with type 1. @@ -3780,7 +4067,8 @@ // Update the document. ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, document_store->Put(test_document1_)); - EXPECT_TRUE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(document_id)); + EXPECT_TRUE(put_result2.was_replacement()); DocumentId updated_document_id = put_result2.new_document_id; // We should get a different document id. ASSERT_THAT(updated_document_id, Not(Eq(document_id))); @@ -3803,11 +4091,13 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, document_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); DocumentId document_id1 = put_result1.new_document_id; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, document_store->Put(test_document2_)); - EXPECT_FALSE(put_result2.was_replacement); + EXPECT_THAT(put_result2.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result2.was_replacement()); DocumentId document_id2 = put_result2.new_document_id; ICING_ASSERT_OK(document_store->Delete( document_id1, fake_clock_.GetSystemTimeMilliseconds())); @@ -3863,7 +4153,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(test_document1_)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; EXPECT_THAT(doc_store->Get(document_id), IsOkAndHolds(EqualsProto(test_document1_))); @@ -3918,7 +4209,8 @@ ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(test_document1_)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; EXPECT_THAT(doc_store->Get(document_id), IsOkAndHolds(EqualsProto(test_document1_))); @@ -4004,10 +4296,10 @@ DocumentStore::CreateResult create_result, DocumentStore::Create( &filesystem_, document_store_dir_, &fake_clock_, schema_store_.get(), + feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, - GetParam().use_persistent_hash_map, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + GetParam().pre_mapping_fbv, GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, &initialize_stats)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -4153,7 +4445,8 @@ SchemaProto schema = SchemaBuilder().AddType(email_type_config).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ASSERT_THAT(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false), @@ -4184,13 +4477,16 @@ .Build(); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(doc)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); docid = put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( DocumentFilterData filter_data, doc_store->GetAliveDocumentFilterData( docid, fake_clock_.GetSystemTimeMilliseconds())); + ASSERT_THAT(filter_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(doc.uri()))); ASSERT_THAT(filter_data.schema_type_id(), Eq(0)); } @@ -4224,14 +4520,14 @@ InitializeStatsProto initialize_stats; ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, document_store_dir_, &fake_clock_, schema_store.get(), - /*force_recovery_and_revalidate_documents=*/true, - GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, - GetParam().use_persistent_hash_map, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - &initialize_stats)); + DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, + schema_store.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/true, + GetParam().pre_mapping_fbv, + GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + &initialize_stats)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -4240,6 +4536,8 @@ DocumentFilterData filter_data, doc_store->GetAliveDocumentFilterData( docid, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(filter_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(std::string("email/1")))); EXPECT_THAT(filter_data.schema_type_id(), Eq(1)); EXPECT_THAT(initialize_stats.document_store_recovery_cause(), Eq(InitializeStatsProto::SCHEMA_CHANGES_OUT_OF_SYNC)); @@ -4268,7 +4566,8 @@ SchemaProto schema = SchemaBuilder().AddType(email_type_config).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ASSERT_THAT(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false), @@ -4299,13 +4598,16 @@ .Build(); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(doc)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); docid = put_result.new_document_id; ICING_ASSERT_HAS_VALUE_AND_ASSIGN( DocumentFilterData filter_data, doc_store->GetAliveDocumentFilterData( docid, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(filter_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(doc.uri()))); ASSERT_THAT(filter_data.schema_type_id(), Eq(0)); } @@ -4348,6 +4650,8 @@ DocumentFilterData filter_data, doc_store->GetAliveDocumentFilterData( docid, fake_clock_.GetSystemTimeMilliseconds())); + EXPECT_THAT(filter_data.uri_fingerprint(), + Eq(tc3farmhash::Fingerprint64(std::string("email/1")))); ASSERT_THAT(filter_data.schema_type_id(), Eq(0)); } } @@ -4374,7 +4678,8 @@ SchemaProto schema = SchemaBuilder().AddType(email_type_config).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ASSERT_THAT(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false), @@ -4415,13 +4720,16 @@ DocumentId docid = kInvalidDocumentId; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_with_body_result, doc_store->Put(docWithBody)); - EXPECT_FALSE(put_with_body_result.was_replacement); + EXPECT_THAT(put_with_body_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_with_body_result.was_replacement()); docid = put_with_body_result.new_document_id; ASSERT_NE(docid, kInvalidDocumentId); docid = kInvalidDocumentId; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_without_body_result, doc_store->Put(docWithoutBody)); - EXPECT_FALSE(put_without_body_result.was_replacement); + EXPECT_THAT(put_without_body_result.old_document_id, + Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_without_body_result.was_replacement()); docid = put_without_body_result.new_document_id; ASSERT_NE(docid, kInvalidDocumentId); @@ -4453,14 +4761,14 @@ CorruptDocStoreHeaderChecksumFile(); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, document_store_dir_, &fake_clock_, schema_store.get(), - /*force_recovery_and_revalidate_documents=*/true, - GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, - GetParam().use_persistent_hash_map, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, + schema_store.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/true, + GetParam().pre_mapping_fbv, + GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -4494,7 +4802,8 @@ SchemaProto schema = SchemaBuilder().AddType(email_type_config).Build(); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_, + feature_flags_.get())); ASSERT_THAT(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false), @@ -4535,13 +4844,16 @@ DocumentId docid = kInvalidDocumentId; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_with_body_result, doc_store->Put(docWithBody)); - EXPECT_FALSE(put_with_body_result.was_replacement); + EXPECT_THAT(put_with_body_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_with_body_result.was_replacement()); docid = put_with_body_result.new_document_id; ASSERT_NE(docid, kInvalidDocumentId); docid = kInvalidDocumentId; ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_without_body_result, doc_store->Put(docWithoutBody)); - EXPECT_FALSE(put_without_body_result.was_replacement); + EXPECT_THAT(put_without_body_result.old_document_id, + Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_without_body_result.was_replacement()); docid = put_without_body_result.new_document_id; ASSERT_NE(docid, kInvalidDocumentId); @@ -4610,7 +4922,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ASSERT_THAT(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, @@ -4654,10 +4967,10 @@ DocumentStore::CreateResult create_result, DocumentStore::Create( &filesystem_, document_store_dir, &fake_clock_, schema_store.get(), + feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, GetParam().pre_mapping_fbv, GetParam().use_persistent_hash_map, - GetParam().namespace_id_fingerprint, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, &initialize_stats)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -4742,7 +5055,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK(schema_store->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, @@ -4847,7 +5161,8 @@ filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, @@ -4892,20 +5207,20 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, GetParam().use_persistent_hash_map, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, doc_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); document_id1 = put_result1.new_document_id; if (GetParam().use_persistent_hash_map) { @@ -4934,12 +5249,12 @@ DocumentStore::CreateResult create_result, DocumentStore::Create( &filesystem_, document_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, + GetParam().pre_mapping_fbv, /*use_persistent_hash_map=*/switch_key_mapper_flag, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, &initialize_stats)); EXPECT_THAT(initialize_stats.document_store_recovery_cause(), Eq(InitializeStatsProto::IO_ERROR)); @@ -4978,20 +5293,20 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, GetParam().use_persistent_hash_map, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, doc_store->Put(test_document1_)); - EXPECT_FALSE(put_result1.was_replacement); + EXPECT_THAT(put_result1.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result1.was_replacement()); document_id1 = put_result1.new_document_id; if (GetParam().use_persistent_hash_map) { @@ -5017,13 +5332,12 @@ ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, - schema_store_.get(), + schema_store_.get(), feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, GetParam().use_persistent_hash_map, PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, + DocumentWrapper>::kDefaultCompressionLevel, &initialize_stats)); EXPECT_THAT(initialize_stats.document_store_recovery_cause(), Eq(InitializeStatsProto::NONE)); @@ -5052,7 +5366,7 @@ } } -TEST_P(DocumentStoreTest, GetDocumentIdByNamespaceFingerprintIdentifier) { +TEST_P(DocumentStoreTest, GetDocumentIdByNamespaceIdFingerprint) { std::string dynamic_trie_uri_mapper_dir = document_store_dir_ + "/key_mapper_dir"; std::string persistent_hash_map_uri_mapper_dir = @@ -5061,65 +5375,657 @@ DocumentStore::CreateResult create_result, DocumentStore::Create( &filesystem_, document_store_dir_, &fake_clock_, schema_store_.get(), + feature_flags_.get(), /*force_recovery_and_revalidate_documents=*/false, - GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, - GetParam().use_persistent_hash_map, - PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + GetParam().pre_mapping_fbv, GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog<DocumentWrapper>::kDefaultCompressionLevel, /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, doc_store->Put(test_document1_)); - EXPECT_FALSE(put_result.was_replacement); + EXPECT_THAT(put_result.old_document_id, Eq(kInvalidDocumentId)); + EXPECT_FALSE(put_result.was_replacement()); DocumentId document_id = put_result.new_document_id; ICING_ASSERT_OK_AND_ASSIGN( NamespaceId namespace_id, doc_store->GetNamespaceId(test_document1_.namespace_())); - NamespaceFingerprintIdentifier ns_fingerprint( - namespace_id, - /*target_str=*/test_document1_.uri()); - if (GetParam().namespace_id_fingerprint) { - EXPECT_THAT(doc_store->GetDocumentId(ns_fingerprint), - IsOkAndHolds(document_id)); + NamespaceIdFingerprint nsid_uri_fingerprint( + namespace_id, /*target_str=*/test_document1_.uri()); + EXPECT_THAT(doc_store->GetDocumentId(nsid_uri_fingerprint), + IsOkAndHolds(document_id)); - NamespaceFingerprintIdentifier non_existing_ns_fingerprint( - namespace_id + 1, /*target_str=*/test_document1_.uri()); - EXPECT_THAT(doc_store->GetDocumentId(non_existing_ns_fingerprint), - StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); - } else { - EXPECT_THAT(doc_store->GetDocumentId(ns_fingerprint), - StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); - } + NamespaceIdFingerprint non_existing_nsid_uri_fingerprint( + namespace_id + 1, /*target_str=*/test_document1_.uri()); + EXPECT_THAT(doc_store->GetDocumentId(non_existing_nsid_uri_fingerprint), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); +} + +TEST_P(DocumentStoreTest, PutDocumentWithNoScorablePropertiesInSchema) { + const std::string schema_store_dir = test_dir_ + "_custom"; + filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); + filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("message").AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); + ICING_EXPECT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, + schema_store.get())); + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + + DocumentProto document = DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("message") + .AddStringProperty("subject", "subject foo") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + doc_store->Put(document)); + DocumentId document_id = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN( + DocumentAssociatedScoreData score_data, + doc_store->GetDocumentAssociatedScoreData(document_id)); + EXPECT_EQ(score_data.scorable_property_cache_index(), -1); + EXPECT_EQ(doc_store->GetScorablePropertySet( + document_id, fake_clock_.GetSystemTimeMilliseconds()), + nullptr); +} + +TEST_P(DocumentStoreTest, PutDocumentWithNoScorableProperties) { + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, + schema_store_.get())); + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + + DocumentProto document = DocumentBuilder() + .SetKey("icing", "email/1") + .SetSchema("email") + .AddStringProperty("subject", "subject foo") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + doc_store->Put(document)); + DocumentId document_id = put_result.new_document_id; + std::unique_ptr<ScorablePropertySet> scorable_property_set = + doc_store->GetScorablePropertySet( + document_id, fake_clock_.GetSystemTimeMilliseconds()); + // scorable property data for the document exists but is empty. + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({})))); +} + +TEST_P(DocumentStoreTest, PutDocumentWithScorablePropertyThenRead) { + const std::string schema_store_dir = test_dir_ + "_custom"; + filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); + filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); + + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("importance") + .SetDataType(PropertyConfigProto::DataType::BOOLEAN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("scoreDouble") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("scoreInt64") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); + ICING_EXPECT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, + schema_store.get())); + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + + DocumentProto document1 = DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("email") + .AddStringProperty("subject", "subject foo") + .AddBooleanProperty("importance", true) + .AddInt64Property("scoreInt64", 1) + .AddDoubleProperty("scoreDouble", 1.5, 2.5) + .SetCreationTimestampMs(0) + .Build(); + DocumentProto document2 = DocumentBuilder() + .SetKey("bar", "2") + .SetSchema("email") + .AddStringProperty("subject", "subject bar") + .AddBooleanProperty("importance", false) + .AddInt64Property("scoreInt64", 5) + .SetCreationTimestampMs(0) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, + doc_store->Put(document1)); + DocumentId document_id1 = put_result1.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, + doc_store->Put(document2)); + DocumentId document_id2 = put_result2.new_document_id; + + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc1 = + doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("importance"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromBoolean(true)))); + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("scoreInt64"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({1})))); + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("scoreDouble"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({1.5, 2.5})))); + + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("importance"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromBoolean(false)))); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("scoreInt64"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({5})))); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("scoreDouble"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({})))); + + // document3 is a copy of document1 with scorable property scoreDouble + // updated. + DocumentProto document3 = DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("email") + .AddStringProperty("subject", "subject foo") + .AddBooleanProperty("importance", true) + .AddInt64Property("scoreInt64", 1) + .AddDoubleProperty("scoreDouble", 0.5, 0.8) + .SetCreationTimestampMs(0) + .Build(); + // Add document3 to the document store, it will result in document1 being + // deleted. + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result3, + doc_store->Put(document3)); + DocumentId document_id3 = put_result3.new_document_id; + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc3 = + doc_store->GetScorablePropertySet( + document_id3, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc3->GetScorablePropertyProto("importance"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromBoolean(true)))); + + EXPECT_THAT( + scorable_property_set_doc3->GetScorablePropertyProto("scoreInt64"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({1})))); + EXPECT_THAT( + scorable_property_set_doc3->GetScorablePropertyProto("scoreDouble"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({0.5, 0.8})))); + + // document4 is a copy of document3 with scorable property scoreInt64 + // removed. + DocumentProto document4 = DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("email") + .AddStringProperty("subject", "subject foo") + .AddBooleanProperty("importance", true) + .AddDoubleProperty("scoreDouble", 0.5, 0.8) + .SetCreationTimestampMs(0) + .Build(); + // Add document4 to the document store, it will result in document3 being + // deleted. + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result4, + doc_store->Put(document4)); + DocumentId document_id4 = put_result4.new_document_id; + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc4 = + doc_store->GetScorablePropertySet( + document_id4, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc4->GetScorablePropertyProto("importance"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromBoolean(true)))); + EXPECT_THAT( + scorable_property_set_doc4->GetScorablePropertyProto("scoreInt64"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({})))); + EXPECT_THAT( + scorable_property_set_doc4->GetScorablePropertyProto("scoreDouble"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({0.5, 0.8})))); +} + +TEST_P(DocumentStoreTest, ReadScorablePropertyAfterOptimization) { + const std::string schema_store_dir = test_dir_ + "_custom"; + filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); + filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); + + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("importance") + .SetDataType(PropertyConfigProto::DataType::BOOLEAN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("scoreDouble") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("scoreInt64") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); + ICING_EXPECT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, + schema_store.get())); + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + + DocumentProto document1 = DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("email") + .AddStringProperty("subject", "subject foo") + .AddBooleanProperty("importance", true) + .AddInt64Property("scoreInt64", 1) + .AddDoubleProperty("scoreDouble", 1.5, 2.5) + .SetCreationTimestampMs(0) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result1, + doc_store->Put(document1)); + DocumentId document_id1 = put_result1.new_document_id; + + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc1 = + doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()); + + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("importance"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromBoolean(true)))); + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("scoreInt64"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({1})))); + EXPECT_THAT( + scorable_property_set_doc1->GetScorablePropertyProto("scoreDouble"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({1.5, 2.5})))); + + // document2 is a copy of document1 with scorable property scoreDouble + // updated. + DocumentProto document2 = DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("email") + .AddStringProperty("subject", "subject foo") + .AddBooleanProperty("importance", true) + .AddInt64Property("scoreInt64", 1) + .AddDoubleProperty("scoreDouble", 0.5, 0.8) + .SetCreationTimestampMs(0) + .Build(); + + // Add document2 to the document store, it will result in document1 being + // deleted. + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result2, + doc_store->Put(document2)); + DocumentId document_id2 = put_result2.new_document_id; + std::unique_ptr<ScorablePropertySet> scorable_property_set_doc2 = + doc_store->GetScorablePropertySet( + document_id2, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("importance"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromBoolean(true)))); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("scoreInt64"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({1})))); + EXPECT_THAT( + scorable_property_set_doc2->GetScorablePropertyProto("scoreDouble"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({0.5, 0.8})))); + + // Optimize the document store. + std::string optimized_dir = document_store_dir_ + "_optimize"; + ASSERT_TRUE(filesystem_.DeleteDirectoryRecursively(optimized_dir.c_str())); + ASSERT_TRUE(filesystem_.CreateDirectoryRecursively(optimized_dir.c_str())); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::OptimizeResult optimize_result, + doc_store->OptimizeInto( + optimized_dir, lang_segmenter_.get(), + /*expired_blob_handles=*/std::unordered_set<std::string>())); + + // Verify that the scorable property set is still correct after optimization. + ICING_ASSERT_OK_AND_ASSIGN(DocumentId doc_id_post_optimization, + doc_store->GetDocumentId("foo", "1")); + std::unique_ptr<ScorablePropertySet> scorable_property_set_post_optimization = + doc_store->GetScorablePropertySet( + doc_id_post_optimization, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set_post_optimization->GetScorablePropertyProto( + "importance"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromBoolean(true)))); + EXPECT_THAT(scorable_property_set_post_optimization->GetScorablePropertyProto( + "scoreInt64"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({1})))); + EXPECT_THAT( + scorable_property_set_post_optimization->GetScorablePropertyProto( + "scoreDouble"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({0.5, 0.8})))); +} + +TEST_P(DocumentStoreTest, + RegenerateScorablePropertyCacheFlipPropertyToScorableEnabled) { + const std::string schema_store_dir = test_dir_ + "_custom"; + filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); + filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); + ICING_EXPECT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, + schema_store.get())); + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddDoubleProperty("income", 10000, 20000) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + doc_store->Put(document0)); + DocumentId document_id = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN( + DocumentAssociatedScoreData score_data, + doc_store->GetDocumentAssociatedScoreData(document_id)); + EXPECT_EQ(score_data.scorable_property_cache_index(), -1); + EXPECT_EQ(doc_store->GetScorablePropertySet( + document_id, fake_clock_.GetSystemTimeMilliseconds()), + nullptr); + + // Update the schema to make "income" property scorable. + schema = SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + ICING_EXPECT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_EXPECT_OK(doc_store->UpdateSchemaStore(schema_store.get())); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaTypeId schema_type_id, + schema_store->GetSchemaTypeId("Person")); + ICING_EXPECT_OK(doc_store->RegenerateScorablePropertyCache({schema_type_id})); + + ICING_ASSERT_OK_AND_ASSIGN( + score_data, doc_store->GetDocumentAssociatedScoreData(document_id)); + EXPECT_NE(score_data.scorable_property_cache_index(), -1); + std::unique_ptr<ScorablePropertySet> scorable_property_set = + doc_store->GetScorablePropertySet( + document_id, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("income"), + Pointee(EqualsProto( + BuildScorablePropertyProtoFromDouble({10000, 20000})))); +} + +TEST_P(DocumentStoreTest, + RegenerateScorablePropertyCacheFlipPropertyToScorableDisabled) { + const std::string schema_store_dir = test_dir_ + "_custom"; + filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); + filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); + ICING_EXPECT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, + schema_store.get())); + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddDoubleProperty("income", 10000, 20000) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + doc_store->Put(document0)); + DocumentId document_id = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN( + DocumentAssociatedScoreData score_data, + doc_store->GetDocumentAssociatedScoreData(document_id)); + EXPECT_NE(score_data.scorable_property_cache_index(), -1); + std::unique_ptr<ScorablePropertySet> scorable_property_set = + doc_store->GetScorablePropertySet( + document_id, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("income"), + Pointee(EqualsProto( + BuildScorablePropertyProtoFromDouble({10000, 20000})))); + + // Update the schema to make "income" property non-scorable. + schema = SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + ICING_EXPECT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_EXPECT_OK(doc_store->UpdateSchemaStore(schema_store.get())); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaTypeId schema_type_id, + schema_store->GetSchemaTypeId("Person")); + ICING_EXPECT_OK(doc_store->RegenerateScorablePropertyCache({schema_type_id})); + EXPECT_EQ(doc_store->GetScorablePropertySet( + document_id, fake_clock_.GetSystemTimeMilliseconds()), + nullptr); + ICING_ASSERT_OK_AND_ASSIGN( + score_data, doc_store->GetDocumentAssociatedScoreData(document_id)); + EXPECT_EQ(score_data.scorable_property_cache_index(), -1); +} + +TEST_P(DocumentStoreTest, + RegenerateScorablePropertyCacheWithSchemaTypeIdChange) { + const std::string schema_store_dir = test_dir_ + "_custom"; + filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); + filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); + + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_REPEATED))) + .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( + PropertyConfigBuilder() + .SetName("score") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir, &fake_clock_, + feature_flags_.get())); + ICING_EXPECT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, + schema_store.get())); + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + + DocumentProto document0 = DocumentBuilder() + .SetKey("icing", "person0") + .SetSchema("Person") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddDoubleProperty("income", 10000, 20000) + .Build(); + DocumentProto document1 = DocumentBuilder() + .SetKey("icing", "message0") + .SetSchema("Message") + .SetScore(10) + .SetCreationTimestampMs(1) + .AddDoubleProperty("score", 10, 20) + .Build(); + ICING_ASSERT_OK_AND_ASSIGN(DocumentStore::PutResult put_result, + doc_store->Put(document0)); + DocumentId document_id0 = put_result.new_document_id; + ICING_ASSERT_OK_AND_ASSIGN(put_result, doc_store->Put(document1)); + DocumentId document_id1 = put_result.new_document_id; + + // Update the schema by rearranging the schema types. Since SchemaTypeId is + // assigned based on order, this should change the SchemaTypeIds. + schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( + PropertyConfigBuilder() + .SetName("score") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_REPEATED))) + .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + ICING_EXPECT_OK(schema_store->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + ICING_EXPECT_OK(doc_store->UpdateSchemaStore(schema_store.get())); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaTypeId person_schema_type_id, + schema_store->GetSchemaTypeId("Person")); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaTypeId message_schema_type_id, + schema_store->GetSchemaTypeId("Message")); + ICING_EXPECT_OK(doc_store->RegenerateScorablePropertyCache( + {person_schema_type_id, message_schema_type_id})); + + std::unique_ptr<ScorablePropertySet> scorable_property_set0 = + doc_store->GetScorablePropertySet( + document_id0, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT(scorable_property_set0->GetScorablePropertyProto("income"), + Pointee(EqualsProto( + BuildScorablePropertyProtoFromDouble({10000, 20000})))); + std::unique_ptr<ScorablePropertySet> scorable_property_set1 = + doc_store->GetScorablePropertySet( + document_id1, fake_clock_.GetSystemTimeMilliseconds()); + EXPECT_THAT( + scorable_property_set1->GetScorablePropertyProto("score"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({10, 20})))); } INSTANTIATE_TEST_SUITE_P( DocumentStoreTest, DocumentStoreTest, testing::Values( - DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/false, - /*pre_mapping_fbv_in=*/false, + DocumentStoreTestParam(/*pre_mapping_fbv_in=*/false, /*use_persistent_hash_map_in=*/false), - DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/true, - /*pre_mapping_fbv_in=*/false, - /*use_persistent_hash_map_in=*/false), - DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/false, - /*pre_mapping_fbv_in=*/true, - /*use_persistent_hash_map_in=*/false), - DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/true, - /*pre_mapping_fbv_in=*/true, - /*use_persistent_hash_map_in=*/false), - DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/false, - /*pre_mapping_fbv_in=*/false, + DocumentStoreTestParam(/*pre_mapping_fbv_in=*/false, /*use_persistent_hash_map_in=*/true), - DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/true, - /*pre_mapping_fbv_in=*/false, - /*use_persistent_hash_map_in=*/true), - DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/false, - /*pre_mapping_fbv_in=*/true, - /*use_persistent_hash_map_in=*/true), - DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/true, - /*pre_mapping_fbv_in=*/true, + DocumentStoreTestParam(/*pre_mapping_fbv_in=*/true, + /*use_persistent_hash_map_in=*/false), + DocumentStoreTestParam(/*pre_mapping_fbv_in=*/true, /*use_persistent_hash_map_in=*/true))); } // namespace
diff --git a/icing/store/key-mapper_test.cc b/icing/store/key-mapper_test.cc index fa7d1e8..229e519 100644 --- a/icing/store/key-mapper_test.cc +++ b/icing/store/key-mapper_test.cc
@@ -224,6 +224,122 @@ UnorderedElementsAre(Pair("foo", 1), Pair("bar", 2), Pair("baz", 3))); } +TEST_P(KeyMapperTest, DeleteByKey_singleEntry) { + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyMapper<DocumentId>> key_mapper, + CreateKeyMapper()); + EXPECT_THAT(GetAllKeyValuePairs(key_mapper.get()), IsEmpty()); + + ICING_ASSERT_OK(key_mapper->Put("foo", /*value=*/1)); + ASSERT_THAT(key_mapper->num_keys(), 1); + ASSERT_THAT(GetAllKeyValuePairs(key_mapper.get()), + UnorderedElementsAre(Pair("foo", 1))); + + // Delete "foo". + EXPECT_THAT(key_mapper->Delete("foo"), IsTrue()); + + // Verify num_keys(). + EXPECT_THAT(key_mapper->num_keys(), 0); + + // Verify Get() returns NOT_FOUND. + EXPECT_THAT(key_mapper->Get("foo"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); + + // Verify the iterator returns nothing. + EXPECT_THAT(GetAllKeyValuePairs(key_mapper.get()), IsEmpty()); + + // Add back "foo" a with different value and verify the key mapper. + EXPECT_THAT(key_mapper->Put("foo", /*value=*/12345), IsOk()); + EXPECT_THAT(key_mapper->num_keys(), 1); + EXPECT_THAT(key_mapper->Get("foo"), IsOkAndHolds(12345)); + EXPECT_THAT(GetAllKeyValuePairs(key_mapper.get()), + UnorderedElementsAre(Pair("foo", 12345))); +} + +TEST_P(KeyMapperTest, DeleteByKey_multipleEntries) { + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyMapper<DocumentId>> key_mapper, + CreateKeyMapper()); + EXPECT_THAT(GetAllKeyValuePairs(key_mapper.get()), IsEmpty()); + + ICING_ASSERT_OK(key_mapper->Put("foo", /*value=*/1)); + ICING_ASSERT_OK(key_mapper->Put("bar", /*value=*/2)); + ICING_ASSERT_OK(key_mapper->Put("baz", /*value=*/3)); + ASSERT_THAT(key_mapper->num_keys(), 3); + ASSERT_THAT( + GetAllKeyValuePairs(key_mapper.get()), + UnorderedElementsAre(Pair("foo", 1), Pair("bar", 2), Pair("baz", 3))); + + // Delete "foo". + EXPECT_THAT(key_mapper->Delete("foo"), IsTrue()); + + // Verify num_keys(). + EXPECT_THAT(key_mapper->num_keys(), 2); + + // Verify Get("foo") returns NOT_FOUND and Get("bar") and Get("baz") return + // the correct values. + EXPECT_THAT(key_mapper->Get("foo"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); + EXPECT_THAT(key_mapper->Get("bar"), IsOkAndHolds(2)); + EXPECT_THAT(key_mapper->Get("baz"), IsOkAndHolds(3)); + + // Verify the iterator returns "bar" and "baz". + EXPECT_THAT(GetAllKeyValuePairs(key_mapper.get()), + UnorderedElementsAre(Pair("bar", 2), Pair("baz", 3))); + + // Add back "foo" a with different value and verify the key mapper. + EXPECT_THAT(key_mapper->Put("foo", /*value=*/12345), IsOk()); + EXPECT_THAT(key_mapper->num_keys(), 3); + EXPECT_THAT(key_mapper->Get("foo"), IsOkAndHolds(12345)); + EXPECT_THAT(key_mapper->Get("bar"), IsOkAndHolds(2)); + EXPECT_THAT(key_mapper->Get("baz"), IsOkAndHolds(3)); + EXPECT_THAT( + GetAllKeyValuePairs(key_mapper.get()), + UnorderedElementsAre(Pair("foo", 12345), Pair("bar", 2), Pair("baz", 3))); +} + +TEST_P(KeyMapperTest, DeleteByKey_multipleEntriesDeleteAll) { + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyMapper<DocumentId>> key_mapper, + CreateKeyMapper()); + EXPECT_THAT(GetAllKeyValuePairs(key_mapper.get()), IsEmpty()); + + ICING_ASSERT_OK(key_mapper->Put("foo", /*value=*/1)); + ICING_ASSERT_OK(key_mapper->Put("bar", /*value=*/2)); + ICING_ASSERT_OK(key_mapper->Put("baz", /*value=*/3)); + ASSERT_THAT(key_mapper->num_keys(), 3); + ASSERT_THAT( + GetAllKeyValuePairs(key_mapper.get()), + UnorderedElementsAre(Pair("foo", 1), Pair("bar", 2), Pair("baz", 3))); + + // Delete all keys. + EXPECT_THAT(key_mapper->Delete("foo"), IsTrue()); + EXPECT_THAT(key_mapper->Delete("bar"), IsTrue()); + EXPECT_THAT(key_mapper->Delete("baz"), IsTrue()); + + // Verify num_keys(). + EXPECT_THAT(key_mapper->num_keys(), 0); + + // Verify Get("foo"), Get("bar") and Get("baz") return NOT_FOUND. + EXPECT_THAT(key_mapper->Get("foo"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); + EXPECT_THAT(key_mapper->Get("bar"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); + EXPECT_THAT(key_mapper->Get("baz"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); + + // Verify the iterator returns nothing. + EXPECT_THAT(GetAllKeyValuePairs(key_mapper.get()), IsEmpty()); + + // Add back "foo" a with different value and verify the key mapper. + EXPECT_THAT(key_mapper->Put("foo", /*value=*/12345), IsOk()); + EXPECT_THAT(key_mapper->num_keys(), 1); + EXPECT_THAT(key_mapper->Get("foo"), IsOkAndHolds(12345)); + EXPECT_THAT(key_mapper->Get("bar"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); + EXPECT_THAT(key_mapper->Get("baz"), + StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); + EXPECT_THAT(GetAllKeyValuePairs(key_mapper.get()), + UnorderedElementsAre(Pair("foo", 12345))); +} + INSTANTIATE_TEST_SUITE_P( KeyMapperTest, KeyMapperTest, testing::Values(KeyMapperTestParam(KeyMapperType::kDynamicTrie,
diff --git a/icing/store/namespace-fingerprint-identifier.h b/icing/store/namespace-fingerprint-identifier.h deleted file mode 100644 index d91ef94..0000000 --- a/icing/store/namespace-fingerprint-identifier.h +++ /dev/null
@@ -1,72 +0,0 @@ -// Copyright (C) 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ICING_STORE_NAMESPACE_FINGERPRINT_IDENTIFIER_H_ -#define ICING_STORE_NAMESPACE_FINGERPRINT_IDENTIFIER_H_ - -#include <cstdint> -#include <string> -#include <string_view> - -#include "icing/text_classifier/lib3/utils/base/statusor.h" -#include "icing/store/namespace-id.h" - -namespace icing { -namespace lib { - -class NamespaceFingerprintIdentifier { - public: - static constexpr int kEncodedNamespaceIdLength = 3; - static constexpr int kMinEncodedLength = kEncodedNamespaceIdLength + 1; - - static libtextclassifier3::StatusOr<NamespaceFingerprintIdentifier> - DecodeFromCString(std::string_view encoded_cstr); - - explicit NamespaceFingerprintIdentifier() - : namespace_id_(0), fingerprint_(0) {} - - explicit NamespaceFingerprintIdentifier(NamespaceId namespace_id, - uint64_t fingerprint) - : namespace_id_(namespace_id), fingerprint_(fingerprint) {} - - explicit NamespaceFingerprintIdentifier(NamespaceId namespace_id, - std::string_view target_str); - - std::string EncodeToCString() const; - - bool operator<(const NamespaceFingerprintIdentifier& other) const { - if (namespace_id_ != other.namespace_id_) { - return namespace_id_ < other.namespace_id_; - } - return fingerprint_ < other.fingerprint_; - } - - bool operator==(const NamespaceFingerprintIdentifier& other) const { - return namespace_id_ == other.namespace_id_ && - fingerprint_ == other.fingerprint_; - } - - NamespaceId namespace_id() const { return namespace_id_; } - uint64_t fingerprint() const { return fingerprint_; } - - private: - NamespaceId namespace_id_; - uint64_t fingerprint_; -} __attribute__((packed)); -static_assert(sizeof(NamespaceFingerprintIdentifier) == 10, ""); - -} // namespace lib -} // namespace icing - -#endif // ICING_STORE_NAMESPACE_FINGERPRINT_IDENTIFIER_H_
diff --git a/icing/store/namespace-fingerprint-identifier_test.cc b/icing/store/namespace-fingerprint-identifier_test.cc deleted file mode 100644 index 5f86156..0000000 --- a/icing/store/namespace-fingerprint-identifier_test.cc +++ /dev/null
@@ -1,148 +0,0 @@ -// Copyright (C) 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "icing/store/namespace-fingerprint-identifier.h" - -#include <cstdint> -#include <limits> -#include <string> - -#include "icing/text_classifier/lib3/utils/base/status.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "icing/store/namespace-id.h" -#include "icing/testing/common-matchers.h" - -namespace icing { -namespace lib { - -namespace { - -using ::testing::Eq; - -TEST(NamespaceFingerprintIdentifierTest, EncodeToCString) { - NamespaceFingerprintIdentifier identifier1(/*namespace_id=*/0, - /*fingerprint=*/0); - EXPECT_THAT(identifier1.EncodeToCString(), Eq("\x01\x01\x01\x01")); - - NamespaceFingerprintIdentifier identifier2(/*namespace_id=*/0, - /*fingerprint=*/1); - EXPECT_THAT(identifier2.EncodeToCString(), Eq("\x01\x01\x01\x02")); - - NamespaceFingerprintIdentifier identifier3( - /*namespace_id=*/0, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); - EXPECT_THAT(identifier3.EncodeToCString(), - Eq("\x01\x01\x01\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02")); - - NamespaceFingerprintIdentifier identifier4(/*namespace_id=*/1, - /*fingerprint=*/0); - EXPECT_THAT(identifier4.EncodeToCString(), Eq("\x02\x01\x01\x01")); - - NamespaceFingerprintIdentifier identifier5(/*namespace_id=*/1, - /*fingerprint=*/1); - EXPECT_THAT(identifier5.EncodeToCString(), Eq("\x02\x01\x01\x02")); - - NamespaceFingerprintIdentifier identifier6( - /*namespace_id=*/1, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); - EXPECT_THAT(identifier6.EncodeToCString(), - Eq("\x02\x01\x01\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02")); - - NamespaceFingerprintIdentifier identifier7( - /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), - /*fingerprint=*/0); - EXPECT_THAT(identifier7.EncodeToCString(), Eq("\x80\x80\x02\x01")); - - NamespaceFingerprintIdentifier identifier8( - /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), - /*fingerprint=*/1); - EXPECT_THAT(identifier8.EncodeToCString(), Eq("\x80\x80\x02\x02")); - - NamespaceFingerprintIdentifier identifier9( - /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), - /*fingerprint=*/std::numeric_limits<uint64_t>::max()); - EXPECT_THAT(identifier9.EncodeToCString(), - Eq("\x80\x80\x02\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02")); -} - -TEST(NamespaceFingerprintIdentifierTest, - MultipleCStringConversionsAreReversible) { - NamespaceFingerprintIdentifier identifier1(/*namespace_id=*/0, - /*fingerprint=*/0); - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString( - identifier1.EncodeToCString()), - IsOkAndHolds(identifier1)); - - NamespaceFingerprintIdentifier identifier2(/*namespace_id=*/0, - /*fingerprint=*/1); - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString( - identifier2.EncodeToCString()), - IsOkAndHolds(identifier2)); - - NamespaceFingerprintIdentifier identifier3( - /*namespace_id=*/0, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString( - identifier3.EncodeToCString()), - IsOkAndHolds(identifier3)); - - NamespaceFingerprintIdentifier identifier4(/*namespace_id=*/1, - /*fingerprint=*/0); - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString( - identifier4.EncodeToCString()), - IsOkAndHolds(identifier4)); - - NamespaceFingerprintIdentifier identifier5(/*namespace_id=*/1, - /*fingerprint=*/1); - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString( - identifier5.EncodeToCString()), - IsOkAndHolds(identifier5)); - - NamespaceFingerprintIdentifier identifier6( - /*namespace_id=*/1, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString( - identifier6.EncodeToCString()), - IsOkAndHolds(identifier6)); - - NamespaceFingerprintIdentifier identifier7( - /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), - /*fingerprint=*/0); - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString( - identifier7.EncodeToCString()), - IsOkAndHolds(identifier7)); - - NamespaceFingerprintIdentifier identifier8( - /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), - /*fingerprint=*/1); - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString( - identifier8.EncodeToCString()), - IsOkAndHolds(identifier8)); - - NamespaceFingerprintIdentifier identifier9( - /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), - /*fingerprint=*/std::numeric_limits<uint64_t>::max()); - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString( - identifier9.EncodeToCString()), - IsOkAndHolds(identifier9)); -} - -TEST(NamespaceFingerprintIdentifierTest, - DecodeFromCStringInvalidLengthShouldReturnError) { - std::string invalid_str = "\x01\x01\x01"; - EXPECT_THAT(NamespaceFingerprintIdentifier::DecodeFromCString(invalid_str), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); -} - -} // namespace - -} // namespace lib -} // namespace icing
diff --git a/icing/store/namespace-fingerprint-identifier.cc b/icing/store/namespace-id-fingerprint.cc similarity index 83% rename from icing/store/namespace-fingerprint-identifier.cc rename to icing/store/namespace-id-fingerprint.cc index 3910105..9caafde 100644 --- a/icing/store/namespace-fingerprint-identifier.cc +++ b/icing/store/namespace-id-fingerprint.cc
@@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "icing/store/namespace-fingerprint-identifier.h" +#include "icing/store/namespace-id-fingerprint.h" #include <cstdint> #include <string> @@ -28,9 +28,8 @@ namespace icing { namespace lib { -/* static */ libtextclassifier3::StatusOr<NamespaceFingerprintIdentifier> -NamespaceFingerprintIdentifier::DecodeFromCString( - std::string_view encoded_cstr) { +/* static */ libtextclassifier3::StatusOr<NamespaceIdFingerprint> +NamespaceIdFingerprint::DecodeFromCString(std::string_view encoded_cstr) { if (encoded_cstr.size() < kMinEncodedLength) { return absl_ports::InvalidArgumentError("Invalid length"); } @@ -39,15 +38,15 @@ encoded_cstr.substr(0, kEncodedNamespaceIdLength)); uint64_t fingerprint = encode_util::DecodeIntFromCString( encoded_cstr.substr(kEncodedNamespaceIdLength)); - return NamespaceFingerprintIdentifier(namespace_id, fingerprint); + return NamespaceIdFingerprint(namespace_id, fingerprint); } -NamespaceFingerprintIdentifier::NamespaceFingerprintIdentifier( - NamespaceId namespace_id, std::string_view target_str) +NamespaceIdFingerprint::NamespaceIdFingerprint(NamespaceId namespace_id, + std::string_view target_str) : namespace_id_(namespace_id), fingerprint_(tc3farmhash::Fingerprint64(target_str)) {} -std::string NamespaceFingerprintIdentifier::EncodeToCString() const { +std::string NamespaceIdFingerprint::EncodeToCString() const { // encoded_namespace_id_str should be 1 to 3 bytes based on the value of // namespace_id. std::string encoded_namespace_id_str =
diff --git a/icing/store/namespace-id-fingerprint.h b/icing/store/namespace-id-fingerprint.h new file mode 100644 index 0000000..bae294e --- /dev/null +++ b/icing/store/namespace-id-fingerprint.h
@@ -0,0 +1,86 @@ +// Copyright (C) 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_STORE_NAMESPACE_ID_FINGERPRINT_H_ +#define ICING_STORE_NAMESPACE_ID_FINGERPRINT_H_ + +#include <cstddef> +#include <cstdint> +#include <functional> +#include <string> +#include <string_view> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/store/namespace-id.h" + +namespace icing { +namespace lib { + +// A wrapper class of namespace id and fingerprint. It is usually used as a +// unique identifier of a resource, e.g. document (namespace id + +// fingerprint(uri)), corpus (namespace id + fingerprint(schema_type_name)). +class NamespaceIdFingerprint { + public: + struct Hasher { + std::size_t operator()(const NamespaceIdFingerprint& nsid_fp) const { + return std::hash<NamespaceId>()(nsid_fp.namespace_id_) ^ + std::hash<uint64_t>()(nsid_fp.fingerprint_); + } + }; + + static constexpr int kEncodedNamespaceIdLength = 3; + static constexpr int kMinEncodedLength = kEncodedNamespaceIdLength + 1; + + static libtextclassifier3::StatusOr<NamespaceIdFingerprint> DecodeFromCString( + std::string_view encoded_cstr); + + explicit NamespaceIdFingerprint() + : namespace_id_(kInvalidNamespaceId), fingerprint_(0) {} + + explicit NamespaceIdFingerprint(NamespaceId namespace_id, + uint64_t fingerprint) + : namespace_id_(namespace_id), fingerprint_(fingerprint) {} + + explicit NamespaceIdFingerprint(NamespaceId namespace_id, + std::string_view target_str); + + std::string EncodeToCString() const; + + bool is_valid() const { return namespace_id_ >= 0; } + + bool operator<(const NamespaceIdFingerprint& other) const { + if (namespace_id_ != other.namespace_id_) { + return namespace_id_ < other.namespace_id_; + } + return fingerprint_ < other.fingerprint_; + } + + bool operator==(const NamespaceIdFingerprint& other) const { + return namespace_id_ == other.namespace_id_ && + fingerprint_ == other.fingerprint_; + } + + NamespaceId namespace_id() const { return namespace_id_; } + uint64_t fingerprint() const { return fingerprint_; } + + private: + NamespaceId namespace_id_; + uint64_t fingerprint_; +} __attribute__((packed)); +static_assert(sizeof(NamespaceIdFingerprint) == 10, ""); + +} // namespace lib +} // namespace icing + +#endif // ICING_STORE_NAMESPACE_ID_FINGERPRINT_H_
diff --git a/icing/store/namespace-id-fingerprint_test.cc b/icing/store/namespace-id-fingerprint_test.cc new file mode 100644 index 0000000..c11ed64 --- /dev/null +++ b/icing/store/namespace-id-fingerprint_test.cc
@@ -0,0 +1,199 @@ +// Copyright (C) 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/store/namespace-id-fingerprint.h" + +#include <cstdint> +#include <limits> +#include <string> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/store/namespace-id.h" +#include "icing/testing/common-matchers.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::Eq; +using ::testing::IsFalse; +using ::testing::IsTrue; + +TEST(NamespaceIdFingerprintTest, Invalid) { + NamespaceIdFingerprint identifier1(/*namespace_id=*/-1, /*fingerprint=*/0); + EXPECT_THAT(identifier1.is_valid(), IsFalse()); + + NamespaceIdFingerprint identifier2(/*namespace_id=*/-1, /*fingerprint=*/1); + EXPECT_THAT(identifier2.is_valid(), IsFalse()); + + NamespaceIdFingerprint identifier3(/*namespace_id=*/-2, /*fingerprint=*/1); + EXPECT_THAT(identifier3.is_valid(), IsFalse()); + + NamespaceIdFingerprint identifier4( + /*namespace_id=*/std::numeric_limits<NamespaceId>::min(), + /*fingerprint=*/1); + EXPECT_THAT(identifier4.is_valid(), IsFalse()); +} + +TEST(NamespaceIdFingerprintTest, DefaultInvalid) { + NamespaceIdFingerprint identifier; + EXPECT_THAT(identifier.is_valid(), IsFalse()); +} + +TEST(NamespaceIdFingerprintTest, Valid) { + NamespaceIdFingerprint identifier1(/*namespace_id=*/0, /*fingerprint=*/0); + EXPECT_THAT(identifier1.is_valid(), IsTrue()); + + NamespaceIdFingerprint identifier2(/*namespace_id=*/0, /*fingerprint=*/1); + EXPECT_THAT(identifier2.is_valid(), IsTrue()); + + NamespaceIdFingerprint identifier3( + /*namespace_id=*/0, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); + EXPECT_THAT(identifier3.is_valid(), IsTrue()); + + NamespaceIdFingerprint identifier4(/*namespace_id=*/1, /*fingerprint=*/0); + EXPECT_THAT(identifier4.is_valid(), IsTrue()); + + NamespaceIdFingerprint identifier5(/*namespace_id=*/1, /*fingerprint=*/1); + EXPECT_THAT(identifier5.is_valid(), IsTrue()); + + NamespaceIdFingerprint identifier6( + /*namespace_id=*/1, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); + EXPECT_THAT(identifier6.is_valid(), IsTrue()); + + NamespaceIdFingerprint identifier7( + /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), + /*fingerprint=*/0); + EXPECT_THAT(identifier7.is_valid(), IsTrue()); + + NamespaceIdFingerprint identifier8( + /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), + /*fingerprint=*/1); + EXPECT_THAT(identifier8.is_valid(), IsTrue()); + + NamespaceIdFingerprint identifier9( + /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), + /*fingerprint=*/std::numeric_limits<uint64_t>::max()); + EXPECT_THAT(identifier9.is_valid(), IsTrue()); +} + +TEST(NamespaceIdFingerprintTest, EncodeToCString) { + NamespaceIdFingerprint identifier1(/*namespace_id=*/0, /*fingerprint=*/0); + EXPECT_THAT(identifier1.EncodeToCString(), Eq("\x01\x01\x01\x01")); + + NamespaceIdFingerprint identifier2(/*namespace_id=*/0, /*fingerprint=*/1); + EXPECT_THAT(identifier2.EncodeToCString(), Eq("\x01\x01\x01\x02")); + + NamespaceIdFingerprint identifier3( + /*namespace_id=*/0, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); + EXPECT_THAT(identifier3.EncodeToCString(), + Eq("\x01\x01\x01\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02")); + + NamespaceIdFingerprint identifier4(/*namespace_id=*/1, /*fingerprint=*/0); + EXPECT_THAT(identifier4.EncodeToCString(), Eq("\x02\x01\x01\x01")); + + NamespaceIdFingerprint identifier5(/*namespace_id=*/1, /*fingerprint=*/1); + EXPECT_THAT(identifier5.EncodeToCString(), Eq("\x02\x01\x01\x02")); + + NamespaceIdFingerprint identifier6( + /*namespace_id=*/1, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); + EXPECT_THAT(identifier6.EncodeToCString(), + Eq("\x02\x01\x01\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02")); + + NamespaceIdFingerprint identifier7( + /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), + /*fingerprint=*/0); + EXPECT_THAT(identifier7.EncodeToCString(), Eq("\x80\x80\x02\x01")); + + NamespaceIdFingerprint identifier8( + /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), + /*fingerprint=*/1); + EXPECT_THAT(identifier8.EncodeToCString(), Eq("\x80\x80\x02\x02")); + + NamespaceIdFingerprint identifier9( + /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), + /*fingerprint=*/std::numeric_limits<uint64_t>::max()); + EXPECT_THAT(identifier9.EncodeToCString(), + Eq("\x80\x80\x02\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02")); +} + +TEST(NamespaceIdFingerprintTest, MultipleCStringConversionsAreReversible) { + NamespaceIdFingerprint identifier1(/*namespace_id=*/0, /*fingerprint=*/0); + EXPECT_THAT( + NamespaceIdFingerprint::DecodeFromCString(identifier1.EncodeToCString()), + IsOkAndHolds(identifier1)); + + NamespaceIdFingerprint identifier2(/*namespace_id=*/0, /*fingerprint=*/1); + EXPECT_THAT( + NamespaceIdFingerprint::DecodeFromCString(identifier2.EncodeToCString()), + IsOkAndHolds(identifier2)); + + NamespaceIdFingerprint identifier3( + /*namespace_id=*/0, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); + EXPECT_THAT( + NamespaceIdFingerprint::DecodeFromCString(identifier3.EncodeToCString()), + IsOkAndHolds(identifier3)); + + NamespaceIdFingerprint identifier4(/*namespace_id=*/1, /*fingerprint=*/0); + EXPECT_THAT( + NamespaceIdFingerprint::DecodeFromCString(identifier4.EncodeToCString()), + IsOkAndHolds(identifier4)); + + NamespaceIdFingerprint identifier5(/*namespace_id=*/1, /*fingerprint=*/1); + EXPECT_THAT( + NamespaceIdFingerprint::DecodeFromCString(identifier5.EncodeToCString()), + IsOkAndHolds(identifier5)); + + NamespaceIdFingerprint identifier6( + /*namespace_id=*/1, /*fingerprint=*/std::numeric_limits<uint64_t>::max()); + EXPECT_THAT( + NamespaceIdFingerprint::DecodeFromCString(identifier6.EncodeToCString()), + IsOkAndHolds(identifier6)); + + NamespaceIdFingerprint identifier7( + /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), + /*fingerprint=*/0); + EXPECT_THAT( + NamespaceIdFingerprint::DecodeFromCString(identifier7.EncodeToCString()), + IsOkAndHolds(identifier7)); + + NamespaceIdFingerprint identifier8( + /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), + /*fingerprint=*/1); + EXPECT_THAT( + NamespaceIdFingerprint::DecodeFromCString(identifier8.EncodeToCString()), + IsOkAndHolds(identifier8)); + + NamespaceIdFingerprint identifier9( + /*namespace_id=*/std::numeric_limits<NamespaceId>::max(), + /*fingerprint=*/std::numeric_limits<uint64_t>::max()); + EXPECT_THAT( + NamespaceIdFingerprint::DecodeFromCString(identifier9.EncodeToCString()), + IsOkAndHolds(identifier9)); +} + +TEST(NamespaceIdFingerprintTest, + DecodeFromCStringInvalidLengthShouldReturnError) { + std::string invalid_str = "\x01\x01\x01"; + EXPECT_THAT(NamespaceIdFingerprint::DecodeFromCString(invalid_str), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/testing/common-matchers.h b/icing/testing/common-matchers.h index fd117b8..aaa4f68 100644 --- a/icing/testing/common-matchers.h +++ b/icing/testing/common-matchers.h
@@ -37,6 +37,7 @@ #include "icing/proto/status.pb.h" #include "icing/schema/joinable-property.h" #include "icing/schema/schema-store.h" +#include "icing/schema/scorable_property_manager.h" #include "icing/schema/section.h" #include "icing/scoring/scored-document-hit.h" @@ -55,6 +56,17 @@ return true; } +MATCHER_P(EqualsNormalizedTerm, text, "") { + std::string arg_string(arg.text.data(), arg.text.length()); + if (arg.text != text) { + *result_listener << IcingStringUtil::StringPrintf( + "(Expected: text=\"%s\". Actual: text=\"%s\")", text, + arg_string.c_str()); + return false; + } + return true; +} + // Used to match a DocHitInfo MATCHER_P2(EqualsDocHitInfo, document_id, section_ids, "") { const DocHitInfo& actual = arg; @@ -105,6 +117,26 @@ actual.num_blocks_inspected == num_blocks_inspected; } +// Used to match a DocumentAssociatedScoreData +MATCHER_P5(EqualsDocumentAssociatedScoreData, corpus_id, document_score, + creation_timestamp_ms, length_in_tokens, + has_valid_scorable_property_cache_index, "") { + bool expected_has_valid_scorable_property_cache_index = + arg.scorable_property_cache_index() != -1; + return arg.corpus_id() == corpus_id && + arg.document_score() == document_score && + arg.creation_timestamp_ms() == creation_timestamp_ms && + arg.length_in_tokens() == length_in_tokens && + expected_has_valid_scorable_property_cache_index == + has_valid_scorable_property_cache_index; +} + +// Used to match a ScorablePropertyManager::ScorablePropertyInfo +MATCHER_P2(EqualsScorablePropertyInfo, property_path, data_type, "") { + const ScorablePropertyManager::ScorablePropertyInfo& actual = arg; + return actual.property_path == property_path && actual.data_type == data_type; +} + struct ExtractTermFrequenciesResult { std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies = {0}; SectionIdMask section_mask = kSectionIdMaskNone; @@ -288,7 +320,9 @@ actual.schema_types_index_incompatible_by_name == expected.schema_types_index_incompatible_by_name && actual.schema_types_join_incompatible_by_name == - expected.schema_types_join_incompatible_by_name) { + expected.schema_types_join_incompatible_by_name && + actual.schema_types_scorable_property_inconsistent_by_id == + expected.schema_types_scorable_property_inconsistent_by_id) { return true; } @@ -459,7 +493,13 @@ .term_match_type() && actual.numeric_match_type == expected_property_config_proto.integer_indexing_config() - .numeric_match_type(); + .numeric_match_type() && + actual.embedding_indexing_type == + expected_property_config_proto.embedding_indexing_config() + .embedding_indexing_type() && + actual.quantization_type == + expected_property_config_proto.embedding_indexing_config() + .quantization_type(); } MATCHER_P3(EqualsJoinablePropertyMetadata, expected_id, expected_property_path, @@ -468,7 +508,10 @@ return actual.id == expected_id && actual.path == expected_property_path && actual.data_type == expected_property_config_proto.data_type() && actual.value_type == - expected_property_config_proto.joinable_config().value_type(); + expected_property_config_proto.joinable_config().value_type() && + actual.delete_propagation_type == + expected_property_config_proto.joinable_config() + .delete_propagation_type(); } std::string StatusCodeToString(libtextclassifier3::StatusCode code);
diff --git a/icing/testing/embedding-test-utils.cc b/icing/testing/embedding-test-utils.cc new file mode 100644 index 0000000..23f4197 --- /dev/null +++ b/icing/testing/embedding-test-utils.cc
@@ -0,0 +1,87 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/testing/embedding-test-utils.h" + +#include <cstdint> +#include <memory> +#include <string_view> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/index/embed/embedding-hit.h" +#include "icing/index/embed/embedding-index.h" +#include "icing/index/embed/posting-list-embedding-hit-accessor.h" +#include "icing/index/embed/quantizer.h" +#include "icing/proto/document.pb.h" +#include "icing/util/status-macros.h" + +namespace icing { +namespace lib { + +libtextclassifier3::StatusOr<std::vector<EmbeddingHit>> +GetEmbeddingHitsFromIndex(const EmbeddingIndex* embedding_index, + uint32_t dimension, + std::string_view model_signature) { + std::vector<EmbeddingHit> hits; + + libtextclassifier3::StatusOr<std::unique_ptr<PostingListEmbeddingHitAccessor>> + pl_accessor_or = embedding_index->GetAccessor(dimension, model_signature); + if (absl_ports::IsNotFound(pl_accessor_or.status())) { + return hits; + } + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<PostingListEmbeddingHitAccessor> pl_accessor, + std::move(pl_accessor_or)); + + while (true) { + ICING_ASSIGN_OR_RETURN(std::vector<EmbeddingHit> batch, + pl_accessor->GetNextHitsBatch()); + if (batch.empty()) { + return hits; + } + hits.insert(hits.end(), batch.begin(), batch.end()); + } +} + +std::vector<float> GetRawEmbeddingDataFromIndex( + const EmbeddingIndex* embedding_index) { + ICING_ASSIGN_OR_RETURN(const float* data, + embedding_index->GetRawEmbeddingData(), + std::vector<float>()); + return std::vector<float>(data, data + embedding_index->GetTotalVectorSize()); +} + +libtextclassifier3::StatusOr<std::vector<float>> +GetAndRestoreQuantizedEmbeddingVectorFromIndex( + const EmbeddingIndex* embedding_index, const EmbeddingHit& hit, + uint32_t dimension) { + ICING_ASSIGN_OR_RETURN( + const char* data, + embedding_index->GetQuantizedEmbeddingVector(hit, dimension)); + Quantizer quantizer(data); + const uint8_t* quantized_vector = + reinterpret_cast<const uint8_t*>(data + sizeof(Quantizer)); + std::vector<float> result; + result.reserve(dimension); + for (int i = 0; i < dimension; ++i) { + result.push_back(quantizer.Dequantize(quantized_vector[i])); + } + return result; +} + +} // namespace lib +} // namespace icing
diff --git a/icing/testing/embedding-test-utils.h b/icing/testing/embedding-test-utils.h index 931953e..7f0b0c1 100644 --- a/icing/testing/embedding-test-utils.h +++ b/icing/testing/embedding-test-utils.h
@@ -15,9 +15,15 @@ #ifndef ICING_TESTING_EMBEDDING_TEST_UTILS_H_ #define ICING_TESTING_EMBEDDING_TEST_UTILS_H_ +#include <cstdint> #include <initializer_list> #include <string> +#include <string_view> +#include <vector> +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/index/embed/embedding-hit.h" +#include "icing/index/embed/embedding-index.h" #include "icing/proto/document.pb.h" namespace icing { @@ -39,6 +45,20 @@ return CreateVector(model_signature, values...); } +libtextclassifier3::StatusOr<std::vector<EmbeddingHit>> +GetEmbeddingHitsFromIndex(const EmbeddingIndex* embedding_index, + uint32_t dimension, std::string_view model_signature); + +std::vector<float> GetRawEmbeddingDataFromIndex( + const EmbeddingIndex* embedding_index); + +// Gets the quantized embedding vector from the index based on the given hit, +// and returns the dequantized version of the vector. +libtextclassifier3::StatusOr<std::vector<float>> +GetAndRestoreQuantizedEmbeddingVectorFromIndex( + const EmbeddingIndex* embedding_index, const EmbeddingHit& hit, + uint32_t dimension); + } // namespace lib } // namespace icing
diff --git a/icing/testing/iterator-test-utils.cc b/icing/testing/iterator-test-utils.cc new file mode 100644 index 0000000..0999530 --- /dev/null +++ b/icing/testing/iterator-test-utils.cc
@@ -0,0 +1,96 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/testing/iterator-test-utils.h" + +#include <string_view> +#include <vector> + +#include "icing/tokenization/language-segmenter.h" +#include "icing/util/character-iterator.h" + +namespace icing { +namespace lib { + +// Returns a vector containing all terms retrieved by Advancing on the iterator. +std::vector<std::string_view> GetAllTermsAdvance( + LanguageSegmenter::Iterator* itr) { + std::vector<std::string_view> terms; + while (itr->Advance()) { + terms.push_back(itr->GetTerm()); + } + return terms; +} + +// Returns a vector containing all terms retrieved by calling ResetAfter with +// the UTF-32 position of the current term start to simulate Advancing on the +// iterator. +std::vector<std::string_view> GetAllTermsResetAfterUtf32( + LanguageSegmenter::Iterator* itr) { + std::vector<std::string_view> terms; + // Calling ResetToTermStartingAfterUtf32 with -1 should get the first term in + // the sequence. + bool is_ok = itr->ResetToTermStartingAfterUtf32(-1).ok(); + while (is_ok) { + terms.push_back(itr->GetTerm()); + // Calling ResetToTermStartingAfterUtf32 with the current position should + // get the very next term in the sequence. + CharacterIterator char_itr = itr->CalculateTermStart().ValueOrDie(); + is_ok = itr->ResetToTermStartingAfterUtf32(char_itr.utf32_index()).ok(); + } + return terms; +} + +// Returns a vector containing all terms retrieved by alternating calls to +// Advance and calls to ResetAfter with the UTF-32 position of the current term +// start to simulate Advancing. +std::vector<std::string_view> GetAllTermsAdvanceAndResetAfterUtf32( + LanguageSegmenter::Iterator* itr) { + std::vector<std::string_view> terms; + bool is_ok = itr->Advance(); + while (is_ok) { + terms.push_back(itr->GetTerm()); + // Alternate between using Advance and ResetToTermAfter. + if (terms.size() % 2 == 0) { + is_ok = itr->Advance(); + } else { + // Calling ResetToTermStartingAfterUtf32 with the current position should + // get the very next term in the sequence. + CharacterIterator char_itr = itr->CalculateTermStart().ValueOrDie(); + is_ok = itr->ResetToTermStartingAfterUtf32(char_itr.utf32_index()).ok(); + } + } + return terms; +} + +// Returns a vector containing all terms retrieved by calling ResetBefore with +// the UTF-32 position of the current term start, starting at the end of the +// text. This vector should be in reverse order of GetAllTerms and missing the +// last term. +std::vector<std::string_view> GetAllTermsResetBeforeUtf32( + LanguageSegmenter::Iterator* itr) { + std::vector<std::string_view> terms; + bool is_ok = itr->ResetToTermEndingBeforeUtf32(1000).ok(); + while (is_ok) { + terms.push_back(itr->GetTerm()); + // Calling ResetToTermEndingBeforeUtf32 with the current position should get + // the previous term in the sequence. + CharacterIterator char_itr = itr->CalculateTermStart().ValueOrDie(); + is_ok = itr->ResetToTermEndingBeforeUtf32(char_itr.utf32_index()).ok(); + } + return terms; +} + +} // namespace lib +} // namespace icing
diff --git a/icing/testing/iterator-test-utils.h b/icing/testing/iterator-test-utils.h new file mode 100644 index 0000000..089143f --- /dev/null +++ b/icing/testing/iterator-test-utils.h
@@ -0,0 +1,52 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_TESTING_ITERATOR_TEST_UTILS_H_ +#define ICING_TESTING_ITERATOR_TEST_UTILS_H_ + +#include <string_view> +#include <vector> + +#include "icing/tokenization/language-segmenter.h" + +namespace icing { +namespace lib { + +// Returns a vector containing all terms retrieved by Advancing on the iterator. +std::vector<std::string_view> GetAllTermsAdvance( + LanguageSegmenter::Iterator* itr); + +// Returns a vector containing all terms retrieved by calling ResetAfter with +// the UTF-32 position of the current term start to simulate Advancing on the +// iterator. +std::vector<std::string_view> GetAllTermsResetAfterUtf32( + LanguageSegmenter::Iterator* itr); + +// Returns a vector containing all terms retrieved by alternating calls to +// Advance and calls to ResetAfter with the UTF-32 position of the current term +// start to simulate Advancing. +std::vector<std::string_view> GetAllTermsAdvanceAndResetAfterUtf32( + LanguageSegmenter::Iterator* itr); + +// Returns a vector containing all terms retrieved by calling ResetBefore with +// the UTF-32 position of the current term start, starting at the end of the +// text. This vector should be in reverse order of GetAllTerms and missing the +// last term. +std::vector<std::string_view> GetAllTermsResetBeforeUtf32( + LanguageSegmenter::Iterator* itr); + +} // namespace lib +} // namespace icing + +#endif // ICING_TESTING_ITERATOR_TEST_UTILS_H_
diff --git a/icing/testing/jni-test-helpers.h b/icing/testing/jni-test-helpers.h index 67a98c3..401b401 100644 --- a/icing/testing/jni-test-helpers.h +++ b/icing/testing/jni-test-helpers.h
@@ -24,19 +24,26 @@ #include <jni.h> extern JNIEnv* g_jenv; +extern char* g_icu_data_file_path; #define ICING_TEST_JNI_CACHE JniCache::Create(g_jenv).ValueOrDie() +#define ICING_TEST_ICU_DATA_FILE_PATH g_icu_data_file_path #else #define ICING_TEST_JNI_CACHE nullptr +#define ICING_TEST_ICU_DATA_FILE_PATH nullptr #endif // ICING_REVERSE_JNI_SEGMENTATION namespace icing { namespace lib { -std::unique_ptr<JniCache> GetTestJniCache() { return ICING_TEST_JNI_CACHE; } +inline std::unique_ptr<JniCache> GetTestJniCache() { + return ICING_TEST_JNI_CACHE; +} + +inline char* GetTestIcuDataFilePath() { return ICING_TEST_ICU_DATA_FILE_PATH; } } // namespace lib } // namespace icing
diff --git a/icing/testing/test-feature-flags.cc b/icing/testing/test-feature-flags.cc new file mode 100644 index 0000000..c32e95f --- /dev/null +++ b/icing/testing/test-feature-flags.cc
@@ -0,0 +1,29 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/testing/test-feature-flags.h" + +#include "icing/feature-flags.h" + +namespace icing { +namespace lib { + +FeatureFlags GetTestFeatureFlags() { + return FeatureFlags(/*enable_scorable_properties=*/true, + /*enable_embedding_quantization=*/true, + /*enable_repeated_field_joins=*/true); +} + +} // namespace lib +} // namespace icing
diff --git a/icing/testing/test-feature-flags.h b/icing/testing/test-feature-flags.h new file mode 100644 index 0000000..c84ec1a --- /dev/null +++ b/icing/testing/test-feature-flags.h
@@ -0,0 +1,28 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_TESTING_TEST_FEATURE_FLAGS_H_ +#define ICING_TESTING_TEST_FEATURE_FLAGS_H_ + +#include "icing/feature-flags.h" + +namespace icing { +namespace lib { + +FeatureFlags GetTestFeatureFlags(); + +} // namespace lib +} // namespace icing + +#endif // ICING_TESTING_TEST_FEATURE_FLAGS_H_
diff --git a/icing/text_classifier/lib3/utils/base/statusor.h b/icing/text_classifier/lib3/utils/base/statusor.h index aa1e598..4d9bb71 100644 --- a/icing/text_classifier/lib3/utils/base/statusor.h +++ b/icing/text_classifier/lib3/utils/base/statusor.h
@@ -15,6 +15,7 @@ #ifndef ICING_TEXT_CLASSIFIER_LIB3_UTILS_BASE_STATUSOR_H_ #define ICING_TEXT_CLASSIFIER_LIB3_UTILS_BASE_STATUSOR_H_ +#include <cstdlib> #include <type_traits> #include <utility>
diff --git a/icing/tokenization/combined-tokenizer_test.cc b/icing/tokenization/combined-tokenizer_test.cc index 0411748..04d56fe 100644 --- a/icing/tokenization/combined-tokenizer_test.cc +++ b/icing/tokenization/combined-tokenizer_test.cc
@@ -24,6 +24,7 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" #include "icing/index/embed/embedding-index.h" @@ -41,9 +42,9 @@ #include "icing/store/document-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" @@ -52,6 +53,7 @@ #include "icing/tokenization/tokenizer.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" #include "icing/util/status-macros.h" #include "unicode/uloc.h" @@ -76,6 +78,7 @@ embedding_index_dir_(test_dir_ + "/embedding_index") {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(index_dir_.c_str()); filesystem_.CreateDirectoryRecursively(store_dir_.c_str()); @@ -83,25 +86,25 @@ if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } jni_cache_ = GetTestJniCache(); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create( - &filesystem_, store_dir_, &fake_clock_, schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create(&filesystem_, store_dir_, &fake_clock_, + schema_store_.get(), feature_flags_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDefaultCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); Index::Options options(index_dir_, @@ -116,7 +119,8 @@ DummyNumericIndex<int64_t>::Create(filesystem_, numeric_index_dir_)); ICING_ASSERT_OK_AND_ASSIGN( embedding_index_, - EmbeddingIndex::Create(&filesystem_, embedding_index_dir_)); + EmbeddingIndex::Create(&filesystem_, embedding_index_dir_, &fake_clock_, + feature_flags_.get())); language_segmenter_factory::SegmenterOptions segmenter_options( ULOC_US, jni_cache_.get()); @@ -128,10 +132,11 @@ /*max_term_byte_size=*/1000)); ICING_ASSERT_OK_AND_ASSIGN( query_processor_, - QueryProcessor::Create(index_.get(), numeric_index_.get(), - embedding_index_.get(), lang_segmenter_.get(), - normalizer_.get(), document_store_.get(), - schema_store_.get(), &fake_clock_)); + QueryProcessor::Create( + index_.get(), numeric_index_.get(), embedding_index_.get(), + lang_segmenter_.get(), normalizer_.get(), document_store_.get(), + schema_store_.get(), /*join_children_fetcher=*/nullptr, + &fake_clock_, feature_flags_.get())); } libtextclassifier3::StatusOr<std::vector<std::string>> GetQueryTerms( @@ -154,6 +159,7 @@ return query_terms; } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; const std::string test_dir_; const std::string store_dir_;
diff --git a/icing/tokenization/icu/icu-language-segmenter-factory.cc b/icing/tokenization/icu/icu-language-segmenter-factory.cc index 7b095b4..4f8263b 100644 --- a/icing/tokenization/icu/icu-language-segmenter-factory.cc +++ b/icing/tokenization/icu/icu-language-segmenter-factory.cc
@@ -30,7 +30,7 @@ // // Returns: // A LanguageSegmenter on success -// INVALID_ARGUMENT if locale string is invalid +// INVALID_ARGUMENT_ERROR if locale string is invalid // // TODO(b/156383798): Figure out if we want to verify locale strings and notify // users. Right now illegal locale strings will be ignored by ICU. ICU
diff --git a/icing/tokenization/icu/icu-language-segmenter_test.cc b/icing/tokenization/icu/icu-language-segmenter_test.cc index a7f7419..664ccce 100644 --- a/icing/tokenization/icu/icu-language-segmenter_test.cc +++ b/icing/tokenization/icu/icu-language-segmenter_test.cc
@@ -23,13 +23,13 @@ #include "icing/jni/jni-cache.h" #include "icing/portable/platform.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/icu-i18n-test-utils.h" +#include "icing/testing/iterator-test-utils.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" -#include "icing/util/character-iterator.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -41,80 +41,6 @@ namespace { -language_segmenter_factory::SegmenterOptions GetSegmenterOptions( - const std::string& locale, const JniCache* jni_cache) { - return language_segmenter_factory::SegmenterOptions(locale, jni_cache); -} - -// Returns a vector containing all terms retrieved by Advancing on the iterator. -std::vector<std::string_view> GetAllTermsAdvance( - LanguageSegmenter::Iterator* itr) { - std::vector<std::string_view> terms; - while (itr->Advance()) { - terms.push_back(itr->GetTerm()); - } - return terms; -} - -// Returns a vector containing all terms retrieved by calling ResetAfter with -// the UTF-32 position of the current term start to simulate Advancing on the -// iterator. -std::vector<std::string_view> GetAllTermsResetAfterUtf32( - LanguageSegmenter::Iterator* itr) { - std::vector<std::string_view> terms; - // Calling ResetToTermStartingAfterUtf32 with -1 should get the first term in - // the sequence. - bool is_ok = itr->ResetToTermStartingAfterUtf32(-1).ok(); - while (is_ok) { - terms.push_back(itr->GetTerm()); - // Calling ResetToTermStartingAfterUtf32 with the current position should - // get the very next term in the sequence. - CharacterIterator char_itr = itr->CalculateTermStart().ValueOrDie(); - is_ok = itr->ResetToTermStartingAfterUtf32(char_itr.utf32_index()).ok(); - } - return terms; -} - -// Returns a vector containing all terms retrieved by alternating calls to -// Advance and calls to ResetAfter with the UTF-32 position of the current term -// start to simulate Advancing. -std::vector<std::string_view> GetAllTermsAdvanceAndResetAfterUtf32( - LanguageSegmenter::Iterator* itr) { - std::vector<std::string_view> terms; - bool is_ok = itr->Advance(); - while (is_ok) { - terms.push_back(itr->GetTerm()); - // Alternate between using Advance and ResetToTermAfter. - if (terms.size() % 2 == 0) { - is_ok = itr->Advance(); - } else { - // Calling ResetToTermStartingAfterUtf32 with the current position should - // get the very next term in the sequence. - CharacterIterator char_itr = itr->CalculateTermStart().ValueOrDie(); - is_ok = itr->ResetToTermStartingAfterUtf32(char_itr.utf32_index()).ok(); - } - } - return terms; -} - -// Returns a vector containing all terms retrieved by calling ResetBefore with -// the UTF-32 position of the current term start, starting at the end of the -// text. This vector should be in reverse order of GetAllTerms and missing the -// last term. -std::vector<std::string_view> GetAllTermsResetBeforeUtf32( - LanguageSegmenter::Iterator* itr) { - std::vector<std::string_view> terms; - bool is_ok = itr->ResetToTermEndingBeforeUtf32(1000).ok(); - while (is_ok) { - terms.push_back(itr->GetTerm()); - // Calling ResetToTermEndingBeforeUtf32 with the current position should get - // the previous term in the sequence. - CharacterIterator char_itr = itr->CalculateTermStart().ValueOrDie(); - is_ok = itr->ResetToTermEndingBeforeUtf32(char_itr.utf32_index()).ok(); - } - return terms; -} - class IcuLanguageSegmenterAllLocalesTest : public testing::TestWithParam<const char*> { protected: @@ -124,7 +50,7 @@ } ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -139,7 +65,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms(""), IsOkAndHolds(IsEmpty())); } @@ -147,7 +74,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("Hello World"), IsOkAndHolds(ElementsAre("Hello", " ", "World"))); } @@ -156,7 +84,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // ASCII punctuation marks are kept EXPECT_THAT( language_segmenter->GetAllTerms("Hello, World!!!"), @@ -173,7 +102,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // ASCII special characters are kept EXPECT_THAT(language_segmenter->GetAllTerms("Pay $1000"), IsOkAndHolds(ElementsAre("Pay", " ", "$", "1000"))); @@ -191,7 +121,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Full-width (non-ASCII) punctuation marks and special characters are left // out. EXPECT_THAT(language_segmenter->GetAllTerms("ăďź·Helloďź×"), @@ -202,7 +133,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("U.S.đĄ Bank"), IsOkAndHolds(ElementsAre("U.S", ".", "đĄ", " ", "Bank"))); EXPECT_THAT(language_segmenter->GetAllTerms("I.B.M."), @@ -217,7 +149,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // According to unicode word break rules // WB6(https://unicode.org/reports/tr29/#WB6), // WB7(https://unicode.org/reports/tr29/#WB7), and a few others, some @@ -326,7 +259,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("It's ok."), IsOkAndHolds(ElementsAre("It's", " ", "ok", "."))); EXPECT_THAT(language_segmenter->GetAllTerms("He'll be back."), @@ -349,7 +283,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("(Hello)"), IsOkAndHolds(ElementsAre("(", "Hello", ")"))); @@ -362,7 +297,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("\"Hello\""), IsOkAndHolds(ElementsAre("\"", "Hello", "\""))); @@ -375,7 +311,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Alphanumeric terms are allowed EXPECT_THAT(language_segmenter->GetAllTerms("Se7en A4 3a"), @@ -386,7 +323,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Alphanumeric terms are allowed EXPECT_THAT( @@ -404,7 +342,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("ďźďźďźďźďźďźďźďźďźďź"), IsOkAndHolds(ElementsAre("ďźďźďźďźďźďźďźďźďźďź"))); } @@ -413,7 +352,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Multiple continuous whitespaces are treated as one. const int kNumSeparators = 256; std::string text_with_spaces = @@ -440,7 +380,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // CJKT (Chinese, Japanese, Khmer, Thai) are the 4 main languages that don't // have whitespaces as word delimiter. @@ -465,7 +406,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("ÄÄÄ á¸á¸ á¸ÄÄç"), IsOkAndHolds(ElementsAre("ÄÄÄ á¸á¸ á¸ÄÄç"))); } @@ -474,7 +416,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Turkish EXPECT_THAT(language_segmenter->GetAllTerms("merhaba dünya"), IsOkAndHolds(ElementsAre("merhaba", " ", "dünya"))); @@ -488,7 +431,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"), IsOkAndHolds(ElementsAre("How", " ", "are", " ", "you", "ä˝ ĺĽ˝", "ĺ", "ă", "ĺ ć°", "ă§ă", "ă"))); @@ -502,7 +446,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Validates that the input strings are not copied const std::string text = "Hello World"; const char* word1_address = text.c_str(); @@ -520,11 +465,13 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, ResetToStartUtf32WordConnector) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "com.google.android is package"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "com.google.android is package" // ^ ^^ ^^ @@ -537,11 +484,13 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, NewIteratorResetToStartUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -554,11 +503,13 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, IteratorOneAdvanceResetToStartUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -572,11 +523,13 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, IteratorMultipleAdvancesResetToStartUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -592,11 +545,13 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, IteratorDoneResetToStartUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -611,11 +566,13 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, ResetToTermAfterUtf32WordConnector) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "package com.google.android name"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "package com.google.android name" // ^ ^^ ^^ @@ -634,11 +591,13 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, ResetToTermAfterUtf32OutOfBounds) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -656,25 +615,27 @@ } // Tests that ResetToTermAfter and Advance produce the same output. With the -// exception of the first term which is inacessible via ResetToTermAfter, -// the stream of terms produced by Advance calls should exacly match the +// exception of the first term which is inaccessible via ResetToTermAfter, +// the stream of terms produced by Advance calls should exactly match the // terms produced by ResetToTermAfter calls with the current position // provided as the argument. TEST_P(IcuLanguageSegmenterAllLocalesTest, MixedLanguagesResetToTermAfterUtf32EquivalentToAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How aređĄ youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> reset_terms = GetAllTermsResetAfterUtf32(reset_to_term_itr.get()); @@ -685,18 +646,20 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, ThaiResetToTermAfterUtf32EquivalentToAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> reset_terms = GetAllTermsResetAfterUtf32(reset_to_term_itr.get()); @@ -707,18 +670,20 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, KoreanResetToTermAfterUtf32EquivalentToAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKorean = "ëë ë§¤ěź ěśęˇźíŠëë¤."; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> reset_terms = GetAllTermsResetAfterUtf32(reset_to_term_itr.get()); @@ -733,18 +698,20 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, MixedLanguagesResetToTermAfterUtf32InteroperableWithAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How aređĄ youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_and_reset_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> advance_and_reset_terms = GetAllTermsAdvanceAndResetAfterUtf32(advance_and_reset_itr.get()); @@ -756,18 +723,20 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, ThaiResetToTermAfterUtf32InteroperableWithAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_and_reset_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> advance_and_reset_terms = GetAllTermsAdvanceAndResetAfterUtf32(advance_and_reset_itr.get()); @@ -779,18 +748,20 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, KoreanResetToTermAfterUtf32InteroperableWithAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKorean = "ëë ë§¤ěź ěśęˇźíŠëë¤."; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_and_reset_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> advance_and_reset_terms = GetAllTermsAdvanceAndResetAfterUtf32(advance_and_reset_itr.get()); @@ -804,7 +775,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> itr, language_segmenter->Segment("How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă")); @@ -841,7 +813,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Multiple continuous whitespaces are treated as one. constexpr std::string_view kTextWithSpace = "Hello World"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -880,7 +853,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // CJKT (Chinese, Japanese, Khmer, Thai) are the 4 main languages that // don't have whitespaces as word delimiter. Chinese constexpr std::string_view kChinese = "ććŻĺ¤Ščľ°čˇŻĺťä¸çă"; @@ -908,7 +882,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Japanese constexpr std::string_view kJapanese = "ç§ăŻćŻćĽäťäşăŤćŠăăŚăăžăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -935,7 +910,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKhmer = "ááťááážááá áááážááśáááśááááááá"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, language_segmenter->Segment(kKhmer)); @@ -961,7 +937,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Thai constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -987,11 +964,13 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, ResetToTermBeforeWordConnectorUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "package name com.google.android!"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "package name com.google.android!" // ^ ^^ ^^ ^ @@ -1010,11 +989,13 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, ResetToTermBeforeOutOfBoundsUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -1032,25 +1013,27 @@ } // Tests that ResetToTermBefore and Advance produce the same output. With the -// exception of the last term which is inacessible via ResetToTermBefore, -// the stream of terms produced by Advance calls should exacly match the +// exception of the last term which is inaccessible via ResetToTermBefore, +// the stream of terms produced by Advance calls should exactly match the // terms produced by ResetToTermBefore calls with the current position // provided as the argument (after their order has been reversed). TEST_P(IcuLanguageSegmenterAllLocalesTest, MixedLanguagesResetToTermBeforeEquivalentToAdvanceUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How aređĄ youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> reset_terms = GetAllTermsResetBeforeUtf32(reset_to_term_itr.get()); std::reverse(reset_terms.begin(), reset_terms.end()); @@ -1063,18 +1046,20 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, ThaiResetToTermBeforeEquivalentToAdvanceUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> reset_terms = GetAllTermsResetBeforeUtf32(reset_to_term_itr.get()); std::reverse(reset_terms.begin(), reset_terms.end()); @@ -1086,18 +1071,20 @@ TEST_P(IcuLanguageSegmenterAllLocalesTest, KoreanResetToTermBeforeEquivalentToAdvanceUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKorean = "ëë ë§¤ěź ěśęˇźíŠëë¤."; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> reset_terms = GetAllTermsResetBeforeUtf32(reset_to_term_itr.get()); std::reverse(reset_terms.begin(), reset_terms.end()); @@ -1111,7 +1098,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> itr, language_segmenter->Segment("How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă")); @@ -1149,7 +1137,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Multiple continuous whitespaces are treated as one. constexpr std::string_view kTextWithSpace = "Hello World"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -1187,7 +1176,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // CJKT (Chinese, Japanese, Khmer, Thai) are the 4 main languages that // don't have whitespaces as word delimiter. Chinese constexpr std::string_view kChinese = "ććŻĺ¤Ščľ°čˇŻĺťä¸çă"; @@ -1212,7 +1202,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Japanese constexpr std::string_view kJapanese = "ç§ăŻćŻćĽäťäşăŤćŠăăŚăăžăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -1236,7 +1227,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKhmer = "ááťááážááá áááážááśáááśááááááá"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, language_segmenter->Segment(kKhmer)); @@ -1259,7 +1251,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Thai constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -1286,7 +1279,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Validates that the input strings are not copied ICING_ASSERT_OK_AND_ASSIGN( std::vector<std::string_view> terms, @@ -1301,7 +1295,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> iterator_one,
diff --git a/icing/tokenization/language-segmenter-factory.h b/icing/tokenization/language-segmenter-factory.h index 2505a07..ff7b781 100644 --- a/icing/tokenization/language-segmenter-factory.h +++ b/icing/tokenization/language-segmenter-factory.h
@@ -17,6 +17,7 @@ #include <memory> #include <string_view> +#include <utility> #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/jni/jni-cache.h" @@ -29,20 +30,32 @@ struct SegmenterOptions { explicit SegmenterOptions(std::string locale, - const JniCache* jni_cache = nullptr) - : locale(std::move(locale)), jni_cache(jni_cache) {} + const JniCache* jni_cache = nullptr, + bool enable_icu_segmenter = false) + : locale(std::move(locale)), + jni_cache(jni_cache), + enable_icu_segmenter(enable_icu_segmenter) {} std::string locale; // Does not hold ownership. const JniCache* jni_cache; + + // Determines whether to use an ICU based language segmenter + // in icu-with-reverse-jni-language-segmenter-factory or not. + // The default value is false, which means that the fallback option of a + // Reverse JNI based language segmenter will be used. + // + // This variable is a no-op for all other segmenter factories because they + // only support one segmenter type. + bool enable_icu_segmenter; }; // Creates a language segmenter with the given locale. // // Returns: // A LanguageSegmenter on success -// INVALID_ARGUMENT if locale string is invalid +// INVALID_ARGUMENT_ERROR if locale string is invalid libtextclassifier3::StatusOr<std::unique_ptr<LanguageSegmenter>> Create( SegmenterOptions options);
diff --git a/icing/tokenization/language-segmenter-iterator_test.cc b/icing/tokenization/language-segmenter-iterator_test.cc index 3aff45c..d9d71c6 100644 --- a/icing/tokenization/language-segmenter-iterator_test.cc +++ b/icing/tokenization/language-segmenter-iterator_test.cc
@@ -17,11 +17,11 @@ #include "icing/absl_ports/str_cat.h" #include "icing/portable/platform.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -40,7 +40,7 @@ if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } }
diff --git a/icing/tokenization/language-segmenter_benchmark.cc b/icing/tokenization/language-segmenter_benchmark.cc index 748a322..1ddcfb8 100644 --- a/icing/tokenization/language-segmenter_benchmark.cc +++ b/icing/tokenization/language-segmenter_benchmark.cc
@@ -15,11 +15,11 @@ #include "testing/base/public/benchmark.h" #include "gmock/gmock.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" // Run on a Linux workstation: @@ -56,7 +56,7 @@ void BM_SegmentNoSpace(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -93,7 +93,7 @@ void BM_SegmentWithSpaces(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -133,7 +133,7 @@ void BM_SegmentCJK(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); }
diff --git a/icing/tokenization/plain-tokenizer_test.cc b/icing/tokenization/plain-tokenizer_test.cc index 6c426da..a0d187d 100644 --- a/icing/tokenization/plain-tokenizer_test.cc +++ b/icing/tokenization/plain-tokenizer_test.cc
@@ -20,12 +20,12 @@ #include "icing/absl_ports/str_cat.h" #include "icing/portable/platform.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/icu-i18n-test-utils.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/tokenizer-factory.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -40,7 +40,7 @@ if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } }
diff --git a/icing/tokenization/reverse_jni/reverse-jni-language-segmenter-factory.cc b/icing/tokenization/reverse_jni/reverse-jni-language-segmenter-factory.cc index a251f90..1581e13 100644 --- a/icing/tokenization/reverse_jni/reverse-jni-language-segmenter-factory.cc +++ b/icing/tokenization/reverse_jni/reverse-jni-language-segmenter-factory.cc
@@ -32,7 +32,7 @@ // // Returns: // A LanguageSegmenter on success -// INVALID_ARGUMENT if locale string is invalid +// INVALID_ARGUMENT_ERROR if locale string is invalid // // TODO(b/156383798): Figure out if we want to verify locale strings and notify // users. Right now illegal locale strings will be ignored by ICU. ICU
diff --git a/icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc b/icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc index 47a01fe..683c5f9 100644 --- a/icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc +++ b/icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc
@@ -24,10 +24,10 @@ #include "icing/jni/jni-cache.h" #include "icing/testing/common-matchers.h" #include "icing/testing/icu-i18n-test-utils.h" +#include "icing/testing/iterator-test-utils.h" #include "icing/testing/jni-test-helpers.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" -#include "icing/util/character-iterator.h" #include "unicode/uloc.h" namespace icing { @@ -41,80 +41,6 @@ namespace { -language_segmenter_factory::SegmenterOptions GetSegmenterOptions( - const std::string& locale, const JniCache* jni_cache) { - return language_segmenter_factory::SegmenterOptions(locale, jni_cache); -} - -// Returns a vector containing all terms retrieved by Advancing on the iterator. -std::vector<std::string_view> GetAllTermsAdvance( - LanguageSegmenter::Iterator* itr) { - std::vector<std::string_view> terms; - while (itr->Advance()) { - terms.push_back(itr->GetTerm()); - } - return terms; -} - -// Returns a vector containing all terms retrieved by calling ResetAfter with -// the UTF-32 position of the current term start to simulate Advancing on the -// iterator. -std::vector<std::string_view> GetAllTermsResetAfterUtf32( - LanguageSegmenter::Iterator* itr) { - std::vector<std::string_view> terms; - // Calling ResetToTermStartingAfterUtf32 with -1 should get the first term in - // the sequence. - bool is_ok = itr->ResetToTermStartingAfterUtf32(-1).ok(); - while (is_ok) { - terms.push_back(itr->GetTerm()); - // Calling ResetToTermStartingAfterUtf32 with the current position should - // get the very next term in the sequence. - CharacterIterator char_itr = itr->CalculateTermStart().ValueOrDie(); - is_ok = itr->ResetToTermStartingAfterUtf32(char_itr.utf32_index()).ok(); - } - return terms; -} - -// Returns a vector containing all terms retrieved by alternating calls to -// Advance and calls to ResetAfter with the UTF-32 position of the current term -// start to simulate Advancing. -std::vector<std::string_view> GetAllTermsAdvanceAndResetAfterUtf32( - LanguageSegmenter::Iterator* itr) { - std::vector<std::string_view> terms; - bool is_ok = itr->Advance(); - while (is_ok) { - terms.push_back(itr->GetTerm()); - // Alternate between using Advance and ResetToTermAfter. - if (terms.size() % 2 == 0) { - is_ok = itr->Advance(); - } else { - // Calling ResetToTermStartingAfterUtf32 with the current position should - // get the very next term in the sequence. - CharacterIterator char_itr = itr->CalculateTermStart().ValueOrDie(); - is_ok = itr->ResetToTermStartingAfterUtf32(char_itr.utf32_index()).ok(); - } - } - return terms; -} - -// Returns a vector containing all terms retrieved by calling ResetBefore with -// the UTF-32 position of the current term start, starting at the end of the -// text. This vector should be in reverse order of GetAllTerms and missing the -// last term. -std::vector<std::string_view> GetAllTermsResetBeforeUtf32( - LanguageSegmenter::Iterator* itr) { - std::vector<std::string_view> terms; - bool is_ok = itr->ResetToTermEndingBeforeUtf32(1000).ok(); - while (is_ok) { - terms.push_back(itr->GetTerm()); - // Calling ResetToTermEndingBeforeUtf32 with the current position should get - // the previous term in the sequence. - CharacterIterator char_itr = itr->CalculateTermStart().ValueOrDie(); - is_ok = itr->ResetToTermEndingBeforeUtf32(char_itr.utf32_index()).ok(); - } - return terms; -} - class ReverseJniLanguageSegmenterTest : public testing::TestWithParam<const char*> { protected: @@ -129,7 +55,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms(""), IsOkAndHolds(IsEmpty())); } @@ -137,7 +64,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("Hello World"), IsOkAndHolds(ElementsAre("Hello", " ", "World"))); } @@ -146,7 +74,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // ASCII punctuation marks are kept EXPECT_THAT( language_segmenter->GetAllTerms("Hello, World!!!"), @@ -163,7 +92,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // ASCII special characters are kept EXPECT_THAT(language_segmenter->GetAllTerms("Pay $1000"), IsOkAndHolds(ElementsAre("Pay", " ", "$", "1000"))); @@ -181,7 +111,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Full-width (non-ASCII) punctuation marks and special characters are left // out. EXPECT_THAT(language_segmenter->GetAllTerms("ăďź·Helloďź×"), @@ -192,7 +123,9 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get(), + /*use_icu_segmenter=*/true))); EXPECT_THAT(language_segmenter->GetAllTerms("U.S.đĄ Bank"), IsOkAndHolds(ElementsAre("U.S", ".", "đĄ", " ", "Bank"))); EXPECT_THAT(language_segmenter->GetAllTerms("I.B.M."), @@ -207,7 +140,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // According to unicode word break rules // WB6(https://unicode.org/reports/tr29/#WB6), // WB7(https://unicode.org/reports/tr29/#WB7), and a few others, some @@ -292,7 +226,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("It's ok."), IsOkAndHolds(ElementsAre("It's", " ", "ok", "."))); EXPECT_THAT(language_segmenter->GetAllTerms("He'll be back."), @@ -315,7 +250,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("(Hello)"), IsOkAndHolds(ElementsAre("(", "Hello", ")"))); @@ -328,7 +264,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("\"Hello\""), IsOkAndHolds(ElementsAre("\"", "Hello", "\""))); @@ -341,7 +278,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Alphanumeric terms are allowed EXPECT_THAT(language_segmenter->GetAllTerms("Se7en A4 3a"), @@ -352,7 +290,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Alphanumeric terms are allowed EXPECT_THAT( @@ -370,7 +309,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("ďźďźďźďźďźďźďźďźďźďź"), IsOkAndHolds(ElementsAre("ďź", "ďź", "ďź", "ďź", "ďź", "ďź", "ďź", @@ -381,7 +321,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Multiple continuous whitespaces are treated as one. const int kNumSeparators = 256; std::string text_with_spaces = @@ -408,7 +349,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // CJKT (Chinese, Japanese, Khmer, Thai) are the 4 main languages that don't // have whitespaces as word delimiter. @@ -433,7 +375,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("ÄÄÄ á¸á¸ á¸ÄÄç"), IsOkAndHolds(ElementsAre("ÄÄÄ á¸á¸ á¸ÄÄç"))); } @@ -442,7 +385,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Turkish EXPECT_THAT(language_segmenter->GetAllTerms("merhaba dünya"), IsOkAndHolds(ElementsAre("merhaba", " ", "dünya"))); @@ -456,7 +400,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); EXPECT_THAT(language_segmenter->GetAllTerms("How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"), IsOkAndHolds(ElementsAre("How", " ", "are", " ", "you", "ä˝ ĺĽ˝", "ĺ", "ă", "ĺ ć°", "ă§ă", "ă"))); @@ -470,7 +415,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Validates that the input strings are not copied const std::string text = "Hello World"; const char* word1_address = text.c_str(); @@ -488,11 +434,13 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToStartUtf32WordConnector) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "com:google:android is package"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "com:google:android is package" // ^ ^^ ^^ @@ -505,11 +453,13 @@ TEST_P(ReverseJniLanguageSegmenterTest, NewIteratorResetToStartUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -521,11 +471,13 @@ TEST_P(ReverseJniLanguageSegmenterTest, IteratorOneAdvanceResetToStartUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -539,11 +491,13 @@ TEST_P(ReverseJniLanguageSegmenterTest, IteratorMultipleAdvancesResetToStartUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -559,11 +513,13 @@ TEST_P(ReverseJniLanguageSegmenterTest, IteratorDoneResetToStartUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -578,11 +534,13 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToTermAfterUtf32WordConnector) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "package com:google:android name"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "package com:google:android name" // ^ ^^ ^^ @@ -601,11 +559,13 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToTermAfterUtf32OutOfBounds) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -623,25 +583,27 @@ } // Tests that ResetToTermAfter and Advance produce the same output. With the -// exception of the first term which is inacessible via ResetToTermAfter, -// the stream of terms produced by Advance calls should exacly match the +// exception of the first term which is inaccessible via ResetToTermAfter, +// the stream of terms produced by Advance calls should exactly match the // terms produced by ResetToTermAfter calls with the current position // provided as the argument. TEST_P(ReverseJniLanguageSegmenterTest, MixedLanguagesResetToTermAfterUtf32EquivalentToAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How aređĄ youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> reset_terms = GetAllTermsResetAfterUtf32(reset_to_term_itr.get()); @@ -652,18 +614,20 @@ TEST_P(ReverseJniLanguageSegmenterTest, ThaiResetToTermAfterUtf32EquivalentToAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> reset_terms = GetAllTermsResetAfterUtf32(reset_to_term_itr.get()); @@ -674,18 +638,20 @@ TEST_P(ReverseJniLanguageSegmenterTest, KoreanResetToTermAfterUtf32EquivalentToAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKorean = "ëë ë§¤ěź ěśęˇźíŠëë¤."; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> reset_terms = GetAllTermsResetAfterUtf32(reset_to_term_itr.get()); @@ -700,18 +666,20 @@ TEST_P(ReverseJniLanguageSegmenterTest, MixedLanguagesResetToTermAfterUtf32InteroperableWithAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How aređĄ youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_and_reset_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> advance_and_reset_terms = GetAllTermsAdvanceAndResetAfterUtf32(advance_and_reset_itr.get()); @@ -723,18 +691,20 @@ TEST_P(ReverseJniLanguageSegmenterTest, ThaiResetToTermAfterUtf32InteroperableWithAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_and_reset_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> advance_and_reset_terms = GetAllTermsAdvanceAndResetAfterUtf32(advance_and_reset_itr.get()); @@ -746,18 +716,20 @@ TEST_P(ReverseJniLanguageSegmenterTest, KoreanResetToTermAfterUtf32InteroperableWithAdvance) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKorean = "ëë ë§¤ěź ěśęˇźíŠëë¤."; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_and_reset_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> advance_and_reset_terms = GetAllTermsAdvanceAndResetAfterUtf32(advance_and_reset_itr.get()); @@ -770,7 +742,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> itr, language_segmenter->Segment("How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă")); @@ -807,7 +780,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Multiple continuous whitespaces are treated as one. constexpr std::string_view kTextWithSpace = "Hello World"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -846,7 +820,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // CJKT (Chinese, Japanese, Khmer, Thai) are the 4 main languages that // don't have whitespaces as word delimiter. Chinese constexpr std::string_view kChinese = "ććŻĺ¤Ščľ°čˇŻĺťä¸çă"; @@ -874,7 +849,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Japanese constexpr std::string_view kJapanese = "ç§ăŻćŻćĽäťäşăŤćŠăăŚăăžăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -901,7 +877,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKhmer = "ááťááážááá áááážááśáááśááááááá"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, language_segmenter->Segment(kKhmer)); @@ -927,7 +904,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Thai constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -952,11 +930,13 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToTermBeforeWordConnectorUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "package name com:google:android!"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "package name com:google:android!" // ^ ^^ ^^ ^ @@ -975,11 +955,13 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToTermBeforeOutOfBoundsUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); // String: "How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă" // ^ ^^ ^^ ^ ^ ^ ^ ^ ^ @@ -997,25 +979,27 @@ } // Tests that ResetToTermBefore and Advance produce the same output. With the -// exception of the last term which is inacessible via ResetToTermBefore, -// the stream of terms produced by Advance calls should exacly match the +// exception of the last term which is inaccessible via ResetToTermBefore, +// the stream of terms produced by Advance calls should exactly match the // terms produced by ResetToTermBefore calls with the current position // provided as the argument (after their order has been reversed). TEST_P(ReverseJniLanguageSegmenterTest, MixedLanguagesResetToTermBeforeEquivalentToAdvanceUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kText = "How aređĄ youä˝ ĺĽ˝ĺăĺ ć°ă§ăă"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kText)); + language_segmenter->Segment(kText)); std::vector<std::string_view> reset_terms = GetAllTermsResetBeforeUtf32(reset_to_term_itr.get()); std::reverse(reset_terms.begin(), reset_terms.end()); @@ -1028,18 +1012,20 @@ TEST_P(ReverseJniLanguageSegmenterTest, ThaiResetToTermBeforeEquivalentToAdvanceUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kThai)); + language_segmenter->Segment(kThai)); std::vector<std::string_view> reset_terms = GetAllTermsResetBeforeUtf32(reset_to_term_itr.get()); std::reverse(reset_terms.begin(), reset_terms.end()); @@ -1051,18 +1037,20 @@ TEST_P(ReverseJniLanguageSegmenterTest, KoreanResetToTermBeforeEquivalentToAdvanceUtf32) { ICING_ASSERT_OK_AND_ASSIGN( - auto segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + auto language_segmenter, + language_segmenter_factory::Create( + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKorean = "ëë ë§¤ěź ěśęˇźíŠëë¤."; ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> advance_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> advance_terms = GetAllTermsAdvance(advance_itr.get()); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr, - segmenter->Segment(kKorean)); + language_segmenter->Segment(kKorean)); std::vector<std::string_view> reset_terms = GetAllTermsResetBeforeUtf32(reset_to_term_itr.get()); std::reverse(reset_terms.begin(), reset_terms.end()); @@ -1075,7 +1063,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<LanguageSegmenter::Iterator> itr, language_segmenter->Segment("How are youä˝ ĺĽ˝ĺăĺ ć°ă§ăă")); @@ -1113,7 +1102,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Multiple continuous whitespaces are treated as one. constexpr std::string_view kTextWithSpace = "Hello World"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -1151,7 +1141,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // CJKT (Chinese, Japanese, Khmer, Thai) are the 4 main languages that // don't have whitespaces as word delimiter. Chinese constexpr std::string_view kChinese = "ććŻĺ¤Ščľ°čˇŻĺťä¸çă"; @@ -1176,7 +1167,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Japanese constexpr std::string_view kJapanese = "ç§ăŻćŻćĽäťäşăŤćŠăăŚăăžăă"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -1200,7 +1192,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); constexpr std::string_view kKhmer = "ááťááážááá áááážááśáááśááááááá"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, language_segmenter->Segment(kKhmer)); @@ -1223,7 +1216,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Thai constexpr std::string_view kThai = "ŕ¸ŕ¸ąŕ¸ŕšŕ¸ŕ¸´ŕ¸ŕšŕ¸ŕ¸ŕ¸łŕ¸ŕ¸˛ŕ¸ŕ¸ŕ¸¸ŕ¸ŕ¸§ŕ¸ąŕ¸"; ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr, @@ -1250,7 +1244,8 @@ ICING_ASSERT_OK_AND_ASSIGN( auto language_segmenter, language_segmenter_factory::Create( - GetSegmenterOptions(GetLocale(), jni_cache_.get()))); + language_segmenter_factory::SegmenterOptions( + GetLocale(), jni_cache_.get()))); // Validates that the input strings are not copied ICING_ASSERT_OK_AND_ASSIGN( std::vector<std::string_view> terms,
diff --git a/icing/tokenization/verbatim-tokenizer_test.cc b/icing/tokenization/verbatim-tokenizer_test.cc index bae69ff..8e4418b 100644 --- a/icing/tokenization/verbatim-tokenizer_test.cc +++ b/icing/tokenization/verbatim-tokenizer_test.cc
@@ -18,13 +18,13 @@ #include "gtest/gtest.h" #include "icing/portable/platform.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/jni-test-helpers.h" #include "icing/testing/test-data.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/token.h" #include "icing/tokenization/tokenizer-factory.h" #include "icing/util/character-iterator.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -40,7 +40,7 @@ if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); }
diff --git a/icing/transform/icu/icu-normalizer.cc b/icing/transform/icu/icu-normalizer.cc index 58d4956..74c240a 100644 --- a/icing/transform/icu/icu-normalizer.cc +++ b/icing/transform/icu/icu-normalizer.cc
@@ -24,6 +24,7 @@ #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" #include "icing/transform/normalizer.h" +#include "icing/util/character-iterator.h" #include "icing/util/i18n-utils.h" #include "icing/util/logging.h" #include "icing/util/status-macros.h" @@ -121,8 +122,9 @@ : term_transformer_(std::move(term_transformer)), max_term_byte_size_(max_term_byte_size) {} -std::string IcuNormalizer::NormalizeTerm(const std::string_view term) const { - std::string normalized_text; +Normalizer::NormalizedTerm IcuNormalizer::NormalizeTerm( + const std::string_view term) const { + NormalizedTerm normalized_text = {""}; if (term.empty()) { return normalized_text; @@ -138,17 +140,19 @@ // Normalize the prefix that can be transformed into ASCII. // This is a faster method to normalize Latin terms. NormalizeLatinResult result = NormalizeLatin(normalizer2, term); - normalized_text = std::move(result.text); + normalized_text.text = std::move(result.text); if (result.end_pos < term.length()) { // Some portion of term couldn't be normalized via NormalizeLatin. Use // term_transformer to handle this portion. std::string_view rest_term = term.substr(result.end_pos); - absl_ports::StrAppend(&normalized_text, - term_transformer_->Transform(rest_term)); + TermTransformer::TransformResult transform_result = + term_transformer_->Transform(rest_term); + absl_ports::StrAppend(&normalized_text.text, + transform_result.transformed_term); } - if (normalized_text.length() > max_term_byte_size_) { - i18n_utils::SafeTruncateUtf8(&normalized_text, max_term_byte_size_); + if (normalized_text.text.length() > max_term_byte_size_) { + i18n_utils::SafeTruncateUtf8(&normalized_text.text, max_term_byte_size_); } return normalized_text; @@ -208,12 +212,12 @@ } } -std::string IcuNormalizer::TermTransformer::Transform( - const std::string_view term) const { +IcuNormalizer::TermTransformer::TransformResult +IcuNormalizer::TermTransformer::Transform(const std::string_view term) const { auto utf16_term_or = i18n_utils::Utf8ToUtf16(term); if (!utf16_term_or.ok()) { ICING_VLOG(0) << "Failed to convert UTF8 term '" << term << "' to UTF16"; - return std::string(term); + return {std::string(term)}; } std::u16string utf16_term = std::move(utf16_term_or).ValueOrDie(); UErrorCode status = U_ZERO_ERROR; @@ -248,15 +252,16 @@ if (U_FAILURE(status)) { // Failed to transform, return its original form. ICING_LOG(WARNING) << "Failed to normalize UTF8 term: " << term; - return std::string(term); + return {std::string(term)}; } auto utf8_term_or = i18n_utils::Utf16ToUtf8(utf16_term); if (!utf8_term_or.ok()) { ICING_VLOG(0) << "Failed to convert UTF16 term '" << term << "' to UTF8"; - return std::string(term); + return {std::string(term)}; } - return std::move(utf8_term_or).ValueOrDie(); + std::string utf8_term = std::move(utf8_term_or).ValueOrDie(); + return {std::move(utf8_term)}; } bool IcuNormalizer::FindNormalizedLatinMatchEndPosition(
diff --git a/icing/transform/icu/icu-normalizer.h b/icing/transform/icu/icu-normalizer.h index f6f2b78..c87f0bc 100644 --- a/icing/transform/icu/icu-normalizer.h +++ b/icing/transform/icu/icu-normalizer.h
@@ -56,7 +56,8 @@ // // NOTE: Term should not mix Latin and non-Latin characters. Doing so may // result in the non-Latin characters not properly being normalized - std::string NormalizeTerm(std::string_view term) const override; + Normalizer::NormalizedTerm NormalizeTerm( + std::string_view term) const override; // Returns a CharacterIterator pointing to one past the end of the segment of // term that (once normalized) matches with normalized_term. @@ -86,7 +87,10 @@ ~TermTransformer(); // Transforms the text based on our rules described at top of this file - std::string Transform(std::string_view term) const; + struct TransformResult { + std::string transformed_term; + }; + TransformResult Transform(std::string_view term) const; // Returns a CharacterIterator pointing to one past the end of the segment // of a non-latin term that (once normalized) matches with normalized_term.
diff --git a/icing/transform/icu/icu-normalizer_benchmark.cc b/icing/transform/icu/icu-normalizer_benchmark.cc index 89d5f1e..d3d3074 100644 --- a/icing/transform/icu/icu-normalizer_benchmark.cc +++ b/icing/transform/icu/icu-normalizer_benchmark.cc
@@ -15,10 +15,10 @@ #include "testing/base/public/benchmark.h" #include "gmock/gmock.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" // Run on a Linux workstation: // $ blaze build -c opt --dynamic_mode=off --copt=-gmlt @@ -54,7 +54,7 @@ void BM_NormalizeUppercase(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -87,7 +87,7 @@ void BM_NormalizeAccent(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -124,7 +124,7 @@ void BM_NormalizeGreekAccent(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -161,7 +161,7 @@ void BM_NormalizeHiragana(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -198,7 +198,7 @@ void BM_UppercaseSubTokenLength(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -233,7 +233,7 @@ void BM_AccentSubTokenLength(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -273,7 +273,7 @@ void BM_HiraganaSubTokenLength(benchmark::State& state) { bool run_via_adb = absl::GetFlag(FLAGS_adb); if (!run_via_adb) { - ICING_ASSERT_OK(icu_data_file_helper::SetUpICUDataFile( + ICING_ASSERT_OK(icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); }
diff --git a/icing/transform/icu/icu-normalizer_test.cc b/icing/transform/icu/icu-normalizer_test.cc index 0df23fc..7f5af04 100644 --- a/icing/transform/icu/icu-normalizer_test.cc +++ b/icing/transform/icu/icu-normalizer_test.cc
@@ -17,11 +17,11 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/testing/common-matchers.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/icu-i18n-test-utils.h" #include "icing/testing/test-data.h" #include "icing/transform/normalizer-factory.h" #include "icing/transform/normalizer.h" +#include "icing/util/icu-data-file-helper.h" namespace icing { namespace lib { @@ -33,7 +33,7 @@ void SetUp() override { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); ICING_ASSERT_OK_AND_ASSIGN(normalizer_, normalizer_factory::Create( @@ -57,88 +57,109 @@ // Strings that are already normalized won't change if normalized again. TEST_F(IcuNormalizerTest, AlreadyNormalized) { - EXPECT_THAT(normalizer_->NormalizeTerm(""), Eq("")); - EXPECT_THAT(normalizer_->NormalizeTerm("hello world"), Eq("hello world")); - EXPECT_THAT(normalizer_->NormalizeTerm("ä˝ ĺĽ˝"), Eq("ä˝ ĺĽ˝")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăăŁăłăăš"), Eq("ăăŁăłăăš")); - EXPECT_THAT(normalizer_->NormalizeTerm("ěë íě¸ě"), Eq("ěë íě¸ě")); + EXPECT_THAT(normalizer_->NormalizeTerm(""), EqualsNormalizedTerm("")); + EXPECT_THAT(normalizer_->NormalizeTerm("hello world"), + EqualsNormalizedTerm("hello world")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăŁăłăăš"), + EqualsNormalizedTerm("ăăŁăłăăš")); + EXPECT_THAT(normalizer_->NormalizeTerm("ěë íě¸ě"), + EqualsNormalizedTerm("ěë íě¸ě")); } TEST_F(IcuNormalizerTest, UppercaseToLowercase) { - EXPECT_THAT(normalizer_->NormalizeTerm("MDI"), Eq("mdi")); - EXPECT_THAT(normalizer_->NormalizeTerm("Icing"), Eq("icing")); + EXPECT_THAT(normalizer_->NormalizeTerm("MDI"), EqualsNormalizedTerm("mdi")); + EXPECT_THAT(normalizer_->NormalizeTerm("Icing"), + EqualsNormalizedTerm("icing")); } TEST_F(IcuNormalizerTest, LatinLetterRemoveAccent) { - EXPECT_THAT(normalizer_->NormalizeTerm("Zürich"), Eq("zurich")); - EXPECT_THAT(normalizer_->NormalizeTerm("après-midi"), Eq("apres-midi")); - EXPECT_THAT(normalizer_->NormalizeTerm("Buenos días"), Eq("buenos dias")); + EXPECT_THAT(normalizer_->NormalizeTerm("Zürich"), + EqualsNormalizedTerm("zurich")); + EXPECT_THAT(normalizer_->NormalizeTerm("après-midi"), + EqualsNormalizedTerm("apres-midi")); + EXPECT_THAT(normalizer_->NormalizeTerm("Buenos días"), + EqualsNormalizedTerm("buenos dias")); EXPECT_THAT(normalizer_->NormalizeTerm("ÀÁÂÃÄÅÄÄÄá¸á¸àáâãäåÄÄÄ "), - Eq("aaaaaaaaaaaaaaaaaaaa")); - EXPECT_THAT(normalizer_->NormalizeTerm("á¸á¸á¸á¸á¸ á¸"), Eq("bbbbbb")); - EXPECT_THAT(normalizer_->NormalizeTerm("ÇÄÄÄÄá¸á¸ÄÄÄÄç"), Eq("cccccccccccc")); + EqualsNormalizedTerm("aaaaaaaaaaaaaaaaaaaa")); + EXPECT_THAT(normalizer_->NormalizeTerm("á¸á¸á¸á¸á¸ á¸"), + EqualsNormalizedTerm("bbbbbb")); + EXPECT_THAT(normalizer_->NormalizeTerm("ÇÄÄÄÄá¸á¸ÄÄÄÄç"), + EqualsNormalizedTerm("cccccccccccc")); EXPECT_THAT(normalizer_->NormalizeTerm("ÐÄÄá¸á¸á¸á¸á¸á¸á¸á¸á¸á¸ÄÄ"), - Eq("ddddddddddddddd")); + EqualsNormalizedTerm("ddddddddddddddd")); EXPECT_THAT(normalizer_->NormalizeTerm("ÈÉÊËÄÄÄÄá¸á¸á¸á¸á¸á¸á¸á¸á¸á¸èéêëÄÄÄÄÄ"), - Eq("eeeeeeeeeeeeeeeeeeeeeeeeeee")); - EXPECT_THAT(normalizer_->NormalizeTerm("á¸á¸"), Eq("ff")); - EXPECT_THAT(normalizer_->NormalizeTerm("ÄÄÄ Ä˘á¸ á¸ĄÄÄÄĄÄŁ"), Eq("gggggggggg")); - EXPECT_THAT(normalizer_->NormalizeTerm("ĤḢḤḌḨḪḣḼḧḊḍļáş"), Eq("hhhhhhhhhhhhh")); + EqualsNormalizedTerm("eeeeeeeeeeeeeeeeeeeeeeeeeee")); + EXPECT_THAT(normalizer_->NormalizeTerm("á¸á¸"), EqualsNormalizedTerm("ff")); + EXPECT_THAT(normalizer_->NormalizeTerm("ÄÄÄ Ä˘á¸ á¸ĄÄÄÄĄÄŁ"), + EqualsNormalizedTerm("gggggggggg")); + EXPECT_THAT(normalizer_->NormalizeTerm("ĤḢḤḌḨḪḣḼḧḊḍļáş"), + EqualsNormalizedTerm("hhhhhhhhhhhhh")); EXPECT_THAT(normalizer_->NormalizeTerm("ÌÍÎÏĨĪďḏá¸á¸ŻìíîïÄŠÄŤÄ"), - Eq("iiiiiiiiiiiiiiiii")); - EXPECT_THAT(normalizer_->NormalizeTerm("Ä´Äľ"), Eq("jj")); - EXPECT_THAT(normalizer_->NormalizeTerm("ĜḰḲḴḾḹḳġ"), Eq("kkkkkkkk")); - EXPECT_THAT(normalizer_->NormalizeTerm("ĚĝĽḜḸḟḡḚḝḽĺğĞ"), Eq("lllllllllllll")); - EXPECT_THAT(normalizer_->NormalizeTerm("Ḟášášá¸żášáš"), Eq("mmmmmm")); + EqualsNormalizedTerm("iiiiiiiiiiiiiiiii")); + EXPECT_THAT(normalizer_->NormalizeTerm("Ä´Äľ"), EqualsNormalizedTerm("jj")); + EXPECT_THAT(normalizer_->NormalizeTerm("ĜḰḲḴḾḹḳġ"), + EqualsNormalizedTerm("kkkkkkkk")); + EXPECT_THAT(normalizer_->NormalizeTerm("ĚĝĽḜḸḟḡḚḝḽĺğĞ"), + EqualsNormalizedTerm("lllllllllllll")); + EXPECT_THAT(normalizer_->NormalizeTerm("Ḟášášá¸żášáš"), + EqualsNormalizedTerm("mmmmmm")); EXPECT_THAT(normalizer_->NormalizeTerm("ÑĹĹ Ĺášášášášáš ášášášñĹĹĹ"), - Eq("nnnnnnnnnnnnnnnn")); + EqualsNormalizedTerm("nnnnnnnnnnnnnnnn")); EXPECT_THAT(normalizer_->NormalizeTerm("ĹĹĹÒÓÔÕÖášášášášášášášášòóôõöĹĹĹ"), - Eq("oooooooooooooooooooooooo")); - EXPECT_THAT(normalizer_->NormalizeTerm("ášášášáš"), Eq("pppp")); + EqualsNormalizedTerm("oooooooooooooooooooooooo")); + EXPECT_THAT(normalizer_->NormalizeTerm("ášášášáš"), EqualsNormalizedTerm("pppp")); EXPECT_THAT(normalizer_->NormalizeTerm("ĹĹĹášášášášášášášášĹĹĹ"), - Eq("rrrrrrrrrrrrrr")); + EqualsNormalizedTerm("rrrrrrrrrrrrrr")); EXPECT_THAT(normalizer_->NormalizeTerm("ĹĹĹŠČᚠᚢᚤᚌᚨᚥᚣᚼᚧᚊČĹĹĹš"), - Eq("ssssssssssssssssssss")); + EqualsNormalizedTerm("ssssssssssssssssssss")); EXPECT_THAT(normalizer_->NormalizeTerm("ŢŤČᚪᚏᚎᚰᚍášášŻášąČĹŁĹĽ"), - Eq("tttttttttttttt")); + EqualsNormalizedTerm("tttttttttttttt")); EXPECT_THAT(normalizer_->NormalizeTerm("ŨŪŏÙÚÛÜᚲᚴ᚜ᚸᚺᚳᚾᚡᚚ᚝ùúûüĹŠĹŤĹ"), - Eq("uuuuuuuuuuuuuuuuuuuuuuuu")); - EXPECT_THAT(normalizer_->NormalizeTerm("᚟᚞ᚽᚿ"), Eq("vvvv")); - EXPECT_THAT(normalizer_->NormalizeTerm("Ĺ´áşáşáşáşáşáşáşáş áşáşĹľ"), Eq("wwwwwwwwwwww")); - EXPECT_THAT(normalizer_->NormalizeTerm("áşáşáşáş"), Eq("xxxx")); - EXPECT_THAT(normalizer_->NormalizeTerm("ÝĹśŸáşáşĹˇýÿ"), Eq("yyyyyyyy")); - EXPECT_THAT(normalizer_->NormalizeTerm("ŚŝŽáşáşáşáşáşáşĹşĹźĹž"), Eq("zzzzzzzzzzzz")); - EXPECT_THAT(normalizer_->NormalizeTerm("BarÄąĹ"), Eq("baris")); + EqualsNormalizedTerm("uuuuuuuuuuuuuuuuuuuuuuuu")); + EXPECT_THAT(normalizer_->NormalizeTerm("᚟᚞ᚽᚿ"), EqualsNormalizedTerm("vvvv")); + EXPECT_THAT(normalizer_->NormalizeTerm("Ĺ´áşáşáşáşáşáşáşáş áşáşĹľ"), + EqualsNormalizedTerm("wwwwwwwwwwww")); + EXPECT_THAT(normalizer_->NormalizeTerm("áşáşáşáş"), EqualsNormalizedTerm("xxxx")); + EXPECT_THAT(normalizer_->NormalizeTerm("ÝĹśŸáşáşĹˇýÿ"), + EqualsNormalizedTerm("yyyyyyyy")); + EXPECT_THAT(normalizer_->NormalizeTerm("ŚŝŽáşáşáşáşáşáşĹşĹźĹž"), + EqualsNormalizedTerm("zzzzzzzzzzzz")); + EXPECT_THAT(normalizer_->NormalizeTerm("BarÄąĹ"), + EqualsNormalizedTerm("baris")); } TEST_F(IcuNormalizerTest, GreekLetterRemoveAccent) { - EXPECT_THAT(normalizer_->NormalizeTerm("kαλημÎρα"), Eq("kαλημερα")); - EXPECT_THAT(normalizer_->NormalizeTerm("εγγραφÎŽ"), Eq("εγγραφη")); - EXPECT_THAT(normalizer_->NormalizeTerm( - "áźáźáźáźáźáźáźáźážážážážážážážážáž¸ážšážşážťážźáźáźáźáźáźáź áźáźá˝°á˝ąážážážážážáž ážážáž°ážąáž˛ážłáž´ážśážˇ"), - Eq("αααααααααααααααααααααααααααααααααααααααααααααα")); + EXPECT_THAT(normalizer_->NormalizeTerm("kαλημÎρα"), + EqualsNormalizedTerm("kαλημερα")); + EXPECT_THAT(normalizer_->NormalizeTerm("εγγραφÎŽ"), + EqualsNormalizedTerm("εγγραφη")); + EXPECT_THAT( + normalizer_->NormalizeTerm( + "áźáźáźáźáźáźáźáźážážážážážážážážáž¸ážšážşážťážźáźáźáźáźáźáź áźáźá˝°á˝ąážážážážážáž ážážáž°ážąáž˛ážłáž´ážśážˇ"), + EqualsNormalizedTerm("αααααααααααααααααααααααααααααααααααααααααααααα")); EXPECT_THAT(normalizer_->NormalizeTerm("áźáźáźáźáźáźáżáżáźáźáźáźáźáźá˝˛á˝ł"), - Eq("εεεεεεεεεεεεεεεε")); + EqualsNormalizedTerm("εεεεεεεεεεεεεεεε")); EXPECT_THAT( normalizer_->NormalizeTerm("៨៊៪៍៏áźáźŽáźŻážážážážážážážážáżáżáżáź ៥២៣៤៼៌៧ὴ὾ážážážážážážážážáżáżáżáżáż"), - Eq("ηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηη")); + EqualsNormalizedTerm("ηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηηη")); EXPECT_THAT(normalizer_->NormalizeTerm("៸៚៺៝៟៽៞៿áżáżáżáżáź°áźąáź˛áźłáź´áźľáźśáźˇá˝śá˝ˇáżáżáżáżáżáż"), - Eq("ιιιιιιιιιιιιιιιιιιιιιιιιιιιι")); + EqualsNormalizedTerm("ιιιιιιιιιιιιιιιιιιιιιιιιιιιι")); EXPECT_THAT(normalizer_->NormalizeTerm("á˝á˝á˝á˝á˝á˝áż¸áżšá˝á˝á˝á˝á˝á˝ ὸ὚"), - Eq("οοοοοοοοοοοοοοοο")); + EqualsNormalizedTerm("οοοοοοοοοοοοοοοο")); EXPECT_THAT(normalizer_->NormalizeTerm("á˝á˝á˝á˝áż¨áżŠáżŞáżŤá˝á˝á˝á˝á˝á˝á˝á˝á˝şá˝ťáż ῥῢΰῌῧ"), - Eq("υυυυυυυυυυυυυυυυυυυυυυυυ")); + EqualsNormalizedTerm("υυυυυυυυυυυυυυυυυυυυυυυυ")); EXPECT_THAT( normalizer_->NormalizeTerm("ὨὊὪὍ὏á˝á˝Žá˝Żáž¨ážŠážŞážŤážŹážážŽážŻáżşáżťáżźá˝ ὥὢὣὤὼὌὧὟώហឥអឣឤូឌឧῲῳῴ῜ῡ"), - Eq("ωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωω")); - EXPECT_THAT(normalizer_->NormalizeTerm("῏ῤῼ"), Eq("ρρρ")); + EqualsNormalizedTerm("ωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωωω")); + EXPECT_THAT(normalizer_->NormalizeTerm("῏ῤῼ"), EqualsNormalizedTerm("ρρρ")); } // Accent / diacritic marks won't be removed in non-latin chars, e.g. in // Japanese TEST_F(IcuNormalizerTest, NonLatinLetterNotRemoveAccent) { // Katakana - EXPECT_THAT(normalizer_->NormalizeTerm("ăăă ăă"), Eq("ăăă ăă")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăă ăă"), + EqualsNormalizedTerm("ăăă ăă")); // Our current ICU rules can't handle Hebrew properly, e.g. the accents in // "×Ö¸×֜ף֞×ÖľÖź×ת עִ×ְרִ×" @@ -147,76 +168,98 @@ TEST_F(IcuNormalizerTest, FullWidthCharsToASCII) { // Full-width punctuation to ASCII punctuation - EXPECT_THAT(normalizer_->NormalizeTerm("‘’ďźďźďźďźďź“”"), Eq("''.,!?:\"\"")); + EXPECT_THAT(normalizer_->NormalizeTerm("‘’ďźďźďźďźďź“”"), + EqualsNormalizedTerm("''.,!?:\"\"")); // Full-width 0-9 EXPECT_THAT(normalizer_->NormalizeTerm("ďźďźďźďźďźďźďźďźďźďź"), - Eq("0123456789")); + EqualsNormalizedTerm("0123456789")); // Full-width A-Z EXPECT_THAT(normalizer_->NormalizeTerm( "ďźďźŽďźŻďź°ďźąďź˛ďźłďź´ďźľďźśďźˇďź¸ďźšďźş"), - Eq("abcdefghijklmnopqrstuvwxyz")); + EqualsNormalizedTerm("abcdefghijklmnopqrstuvwxyz")); // Full-width a-z EXPECT_THAT(normalizer_->NormalizeTerm( "ď˝ď˝ď˝ď˝ď˝ ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝"), - Eq("abcdefghijklmnopqrstuvwxyz")); + EqualsNormalizedTerm("abcdefghijklmnopqrstuvwxyz")); } TEST_F(IcuNormalizerTest, IdeographicToASCII) { ICING_ASSERT_OK_AND_ASSIGN(auto normalizer, normalizer_factory::Create( /*max_term_byte_size=*/1000)); - EXPECT_THAT(normalizer->NormalizeTerm("ďźă"), Eq(",.")); + EXPECT_THAT(normalizer->NormalizeTerm("ďźă"), EqualsNormalizedTerm(",.")); } // For Katakana, each character is normalized to its full-width version. TEST_F(IcuNormalizerTest, KatakanaHalfWidthToFullWidth) { - EXPECT_THAT(normalizer_->NormalizeTerm("｜"), Eq("ăŤ")); - EXPECT_THAT(normalizer_->NormalizeTerm("ｍ"), Eq("ăŠ")); - EXPECT_THAT(normalizer_->NormalizeTerm("｝"), Eq("ăľ")); - EXPECT_THAT(normalizer_->NormalizeTerm("ďž"), Eq("ă")); + EXPECT_THAT(normalizer_->NormalizeTerm("｜"), EqualsNormalizedTerm("ăŤ")); + EXPECT_THAT(normalizer_->NormalizeTerm("ｍ"), EqualsNormalizedTerm("ăŠ")); + EXPECT_THAT(normalizer_->NormalizeTerm("｝"), EqualsNormalizedTerm("ăľ")); + EXPECT_THAT(normalizer_->NormalizeTerm("ďž"), EqualsNormalizedTerm("ă")); } TEST_F(IcuNormalizerTest, HiraganaToKatakana) { - EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), Eq("ă˘ă¤ăŚă¨ăŞ")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), Eq("ăŤăăŻăąăł")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), Eq("ăľăˇăšăťă˝")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăăĄă¤ăŚă¨"), Eq("ăżăăăă")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăŞăŤăŹăăŽ"), Eq("ăăăăă")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăŻă˛ăľă¸ăť"), Eq("ăăăăă")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăžăżăăă"), Eq("ăăă ăĄă˘")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăăă"), Eq("ă¤ăŚă¨")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), Eq("ăŠăŞăŤăŹă")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăăăă"), Eq("ăŻă°ăąă˛")); - EXPECT_THAT(normalizer_->NormalizeTerm("ă"), Eq("ăł")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), Eq("ăŹăŽă°ă˛ă´")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), Eq("ăśă¸ăşăźăž")); - EXPECT_THAT(normalizer_->NormalizeTerm("ă ă˘ăĽă§ăŠ"), Eq("ăăă ăă")); - EXPECT_THAT(normalizer_->NormalizeTerm("ă°ăłăśăšăź"), Eq("ăăăăă")); - EXPECT_THAT(normalizer_->NormalizeTerm("ăąă´ăˇăşă˝"), Eq("ăăăăă")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ă˘ă¤ăŚă¨ăŞ")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăŤăăŻăąăł")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăľăˇăšăťă˝")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăĄă¤ăŚă¨"), + EqualsNormalizedTerm("ăżăăăă")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăŞăŤăŹăăŽ"), + EqualsNormalizedTerm("ăăăăă")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăŻă˛ăľă¸ăť"), + EqualsNormalizedTerm("ăăăăă")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăžăżăăă"), + EqualsNormalizedTerm("ăăă ăĄă˘")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăă"), + EqualsNormalizedTerm("ă¤ăŚă¨")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăŠăŞăŤăŹă")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăăă"), + EqualsNormalizedTerm("ăŻă°ăąă˛")); + EXPECT_THAT(normalizer_->NormalizeTerm("ă"), EqualsNormalizedTerm("ăł")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăŹăŽă°ă˛ă´")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăśă¸ăşăźăž")); + EXPECT_THAT(normalizer_->NormalizeTerm("ă ă˘ăĽă§ăŠ"), + EqualsNormalizedTerm("ăăă ăă")); + EXPECT_THAT(normalizer_->NormalizeTerm("ă°ăłăśăšăź"), + EqualsNormalizedTerm("ăăăăă")); + EXPECT_THAT(normalizer_->NormalizeTerm("ăąă´ăˇăşă˝"), + EqualsNormalizedTerm("ăăăăă")); +} + +TEST_F(IcuNormalizerTest, HanToPinyin) { + EXPECT_THAT(normalizer_->NormalizeTerm("ć´"), EqualsNormalizedTerm("ć´")); + EXPECT_THAT(normalizer_->NormalizeTerm("ć §"), EqualsNormalizedTerm("ć §")); } TEST_F(IcuNormalizerTest, SuperscriptAndSubscriptToASCII) { - EXPECT_THAT(normalizer_->NormalizeTerm("âš"), Eq("9")); - EXPECT_THAT(normalizer_->NormalizeTerm("â"), Eq("9")); + EXPECT_THAT(normalizer_->NormalizeTerm("âš"), EqualsNormalizedTerm("9")); + EXPECT_THAT(normalizer_->NormalizeTerm("â"), EqualsNormalizedTerm("9")); } TEST_F(IcuNormalizerTest, CircledCharsToASCII) { - EXPECT_THAT(normalizer_->NormalizeTerm("â "), Eq("1")); - EXPECT_THAT(normalizer_->NormalizeTerm("âś"), Eq("a")); + EXPECT_THAT(normalizer_->NormalizeTerm("â "), EqualsNormalizedTerm("1")); + EXPECT_THAT(normalizer_->NormalizeTerm("âś"), EqualsNormalizedTerm("a")); } TEST_F(IcuNormalizerTest, RotatedCharsToASCII) { - EXPECT_THAT(normalizer_->NormalizeTerm("︡"), Eq("{")); - EXPECT_THAT(normalizer_->NormalizeTerm("︸"), Eq("}")); + EXPECT_THAT(normalizer_->NormalizeTerm("︡"), EqualsNormalizedTerm("{")); + EXPECT_THAT(normalizer_->NormalizeTerm("︸"), EqualsNormalizedTerm("}")); } TEST_F(IcuNormalizerTest, SquaredCharsToASCII) { - EXPECT_THAT(normalizer_->NormalizeTerm("ă"), Eq("ă˘ăăźă")); + EXPECT_THAT(normalizer_->NormalizeTerm("ă"), + EqualsNormalizedTerm("ă˘ăăźă")); } TEST_F(IcuNormalizerTest, FractionsToASCII) { - EXPECT_THAT(normalizer_->NormalizeTerm("¼"), Eq(" 1/4")); - EXPECT_THAT(normalizer_->NormalizeTerm("â "), Eq(" 5/6")); + EXPECT_THAT(normalizer_->NormalizeTerm("¼"), EqualsNormalizedTerm(" 1/4")); + EXPECT_THAT(normalizer_->NormalizeTerm("â "), EqualsNormalizedTerm(" 5/6")); } TEST_F(IcuNormalizerTest, Truncate) { @@ -225,19 +268,22 @@ /*max_term_byte_size=*/5)); // Won't be truncated - EXPECT_THAT(normalizer->NormalizeTerm("hi"), Eq("hi")); - EXPECT_THAT(normalizer->NormalizeTerm("hello"), Eq("hello")); + EXPECT_THAT(normalizer->NormalizeTerm("hi"), EqualsNormalizedTerm("hi")); + EXPECT_THAT(normalizer->NormalizeTerm("hello"), + EqualsNormalizedTerm("hello")); // Truncated to length 5. - EXPECT_THAT(normalizer->NormalizeTerm("hello!"), Eq("hello")); + EXPECT_THAT(normalizer->NormalizeTerm("hello!"), + EqualsNormalizedTerm("hello")); // Each Japanese character has 3 bytes, so truncating to length 5 results in // only 1 character. - EXPECT_THAT(normalizer->NormalizeTerm("ăăŁăłăăš"), Eq("ă")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăŁăłăăš"), + EqualsNormalizedTerm("ă")); // Each Greek character has 2 bytes, so truncating to length 5 results in 2 // character. - EXPECT_THAT(normalizer->NormalizeTerm("αβγδε"), Eq("αβ")); + EXPECT_THAT(normalizer->NormalizeTerm("αβγδε"), EqualsNormalizedTerm("αβ")); } { @@ -245,7 +291,7 @@ /*max_term_byte_size=*/2)); // The Japanese character has 3 bytes, truncating it results in an empty // string. - EXPECT_THAT(normalizer->NormalizeTerm("ă"), Eq("")); + EXPECT_THAT(normalizer->NormalizeTerm("ă"), EqualsNormalizedTerm("")); } }
diff --git a/icing/transform/map/map-normalizer.cc b/icing/transform/map/map-normalizer.cc index 61fce65..0c10cce 100644 --- a/icing/transform/map/map-normalizer.cc +++ b/icing/transform/map/map-normalizer.cc
@@ -22,6 +22,7 @@ #include "icing/absl_ports/str_cat.h" #include "icing/transform/map/normalization-map.h" +#include "icing/transform/normalizer.h" #include "icing/util/character-iterator.h" #include "icing/util/i18n-utils.h" #include "icing/util/logging.h" @@ -68,7 +69,8 @@ } // namespace -std::string MapNormalizer::NormalizeTerm(std::string_view term) const { +Normalizer::NormalizedTerm MapNormalizer::NormalizeTerm( + std::string_view term) const { std::string normalized_text; normalized_text.reserve(term.length()); @@ -101,7 +103,7 @@ i18n_utils::SafeTruncateUtf8(&normalized_text, max_term_byte_size_); } - return normalized_text; + return {std::move(normalized_text)}; } CharacterIterator MapNormalizer::FindNormalizedMatchEndPosition(
diff --git a/icing/transform/map/map-normalizer.h b/icing/transform/map/map-normalizer.h index ed996ae..469d714 100644 --- a/icing/transform/map/map-normalizer.h +++ b/icing/transform/map/map-normalizer.h
@@ -38,7 +38,8 @@ // - Common diacritic Latin characters -> ASCII // // Read more mapping details in normalization-map.cc - std::string NormalizeTerm(std::string_view term) const override; + Normalizer::NormalizedTerm NormalizeTerm( + std::string_view term) const override; // Returns a CharacterIterator pointing to one past the end of the segment of // term that (once normalized) matches with normalized_term.
diff --git a/icing/transform/map/map-normalizer_test.cc b/icing/transform/map/map-normalizer_test.cc index adc5623..72b122c 100644 --- a/icing/transform/map/map-normalizer_test.cc +++ b/icing/transform/map/map-normalizer_test.cc
@@ -48,63 +48,80 @@ ICING_ASSERT_OK_AND_ASSIGN(auto normalizer, normalizer_factory::Create( /*max_term_byte_size=*/1000)); - EXPECT_THAT(normalizer->NormalizeTerm(""), Eq("")); - EXPECT_THAT(normalizer->NormalizeTerm("hello world"), Eq("hello world")); - EXPECT_THAT(normalizer->NormalizeTerm("ä˝ ĺĽ˝"), Eq("ä˝ ĺĽ˝")); - EXPECT_THAT(normalizer->NormalizeTerm("ăăŁăłăăš"), Eq("ăăŁăłăăš")); - EXPECT_THAT(normalizer->NormalizeTerm("ěë íě¸ě"), Eq("ěë íě¸ě")); + EXPECT_THAT(normalizer->NormalizeTerm(""), EqualsNormalizedTerm("")); + EXPECT_THAT(normalizer->NormalizeTerm("hello world"), + EqualsNormalizedTerm("hello world")); + EXPECT_THAT(normalizer->NormalizeTerm("ä˝ ĺĽ˝"), EqualsNormalizedTerm("ä˝ ĺĽ˝")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăŁăłăăš"), + EqualsNormalizedTerm("ăăŁăłăăš")); + EXPECT_THAT(normalizer->NormalizeTerm("ěë íě¸ě"), + EqualsNormalizedTerm("ěë íě¸ě")); } TEST(MapNormalizerTest, UppercaseToLowercase) { ICING_ASSERT_OK_AND_ASSIGN(auto normalizer, normalizer_factory::Create( /*max_term_byte_size=*/1000)); - EXPECT_THAT(normalizer->NormalizeTerm("MDI"), Eq("mdi")); - EXPECT_THAT(normalizer->NormalizeTerm("Icing"), Eq("icing")); + EXPECT_THAT(normalizer->NormalizeTerm("MDI"), EqualsNormalizedTerm("mdi")); + EXPECT_THAT(normalizer->NormalizeTerm("Icing"), + EqualsNormalizedTerm("icing")); } TEST(MapNormalizerTest, LatinLetterRemoveAccent) { ICING_ASSERT_OK_AND_ASSIGN(auto normalizer, normalizer_factory::Create( /*max_term_byte_size=*/1000)); - EXPECT_THAT(normalizer->NormalizeTerm("Zürich"), Eq("zurich")); - EXPECT_THAT(normalizer->NormalizeTerm("après-midi"), Eq("apres-midi")); - EXPECT_THAT(normalizer->NormalizeTerm("Buenos días"), Eq("buenos dias")); + EXPECT_THAT(normalizer->NormalizeTerm("Zürich"), + EqualsNormalizedTerm("zurich")); + EXPECT_THAT(normalizer->NormalizeTerm("après-midi"), + EqualsNormalizedTerm("apres-midi")); + EXPECT_THAT(normalizer->NormalizeTerm("Buenos días"), + EqualsNormalizedTerm("buenos dias")); EXPECT_THAT(normalizer->NormalizeTerm("ÀÁÂÃÄÅÄÄÄá¸á¸àáâãäåÄÄÄ "), - Eq("aaaaaaaaaaaaaaaaaaaa")); - EXPECT_THAT(normalizer->NormalizeTerm("á¸á¸á¸á¸á¸ á¸"), Eq("bbbbbb")); - EXPECT_THAT(normalizer->NormalizeTerm("ÇÄÄÄÄá¸á¸ÄÄÄÄç"), Eq("cccccccccccc")); + EqualsNormalizedTerm("aaaaaaaaaaaaaaaaaaaa")); + EXPECT_THAT(normalizer->NormalizeTerm("á¸á¸á¸á¸á¸ á¸"), + EqualsNormalizedTerm("bbbbbb")); + EXPECT_THAT(normalizer->NormalizeTerm("ÇÄÄÄÄá¸á¸ÄÄÄÄç"), + EqualsNormalizedTerm("cccccccccccc")); EXPECT_THAT(normalizer->NormalizeTerm("ÐÄÄá¸á¸á¸á¸á¸á¸á¸á¸á¸á¸ÄÄ"), - Eq("ddddddddddddddd")); + EqualsNormalizedTerm("ddddddddddddddd")); EXPECT_THAT(normalizer->NormalizeTerm("ÈÉÊËÄÄÄÄá¸á¸á¸á¸á¸á¸á¸á¸á¸á¸èéêëÄÄÄÄÄ"), - Eq("eeeeeeeeeeeeeeeeeeeeeeeeeee")); - EXPECT_THAT(normalizer->NormalizeTerm("á¸á¸"), Eq("ff")); - EXPECT_THAT(normalizer->NormalizeTerm("ÄÄÄ Ä˘á¸ á¸ĄÄÄÄĄÄŁ"), Eq("gggggggggg")); - EXPECT_THAT(normalizer->NormalizeTerm("ĤḢḤḌḨḪḣḼḧḊḍļáş"), Eq("hhhhhhhhhhhhh")); + EqualsNormalizedTerm("eeeeeeeeeeeeeeeeeeeeeeeeeee")); + EXPECT_THAT(normalizer->NormalizeTerm("á¸á¸"), EqualsNormalizedTerm("ff")); + EXPECT_THAT(normalizer->NormalizeTerm("ÄÄÄ Ä˘á¸ á¸ĄÄÄÄĄÄŁ"), + EqualsNormalizedTerm("gggggggggg")); + EXPECT_THAT(normalizer->NormalizeTerm("ĤḢḤḌḨḪḣḼḧḊḍļáş"), + EqualsNormalizedTerm("hhhhhhhhhhhhh")); EXPECT_THAT(normalizer->NormalizeTerm("ÌÍÎÏĨĪďḏá¸á¸ŻìíîïÄŠÄŤÄ"), - Eq("iiiiiiiiiiiiiiiii")); - EXPECT_THAT(normalizer->NormalizeTerm("Ä´Äľ"), Eq("jj")); - EXPECT_THAT(normalizer->NormalizeTerm("ĜḰḲḴḾḹḳġ"), Eq("kkkkkkkk")); - EXPECT_THAT(normalizer->NormalizeTerm("ĚĝĽḜḸḟḡḚḝḽĺğĞ"), Eq("lllllllllllll")); - EXPECT_THAT(normalizer->NormalizeTerm("Ḟášášá¸żášáš"), Eq("mmmmmm")); + EqualsNormalizedTerm("iiiiiiiiiiiiiiiii")); + EXPECT_THAT(normalizer->NormalizeTerm("Ä´Äľ"), EqualsNormalizedTerm("jj")); + EXPECT_THAT(normalizer->NormalizeTerm("ĜḰḲḴḾḹḳġ"), + EqualsNormalizedTerm("kkkkkkkk")); + EXPECT_THAT(normalizer->NormalizeTerm("ĚĝĽḜḸḟḡḚḝḽĺğĞ"), + EqualsNormalizedTerm("lllllllllllll")); + EXPECT_THAT(normalizer->NormalizeTerm("Ḟášášá¸żášáš"), + EqualsNormalizedTerm("mmmmmm")); EXPECT_THAT(normalizer->NormalizeTerm("ÑĹĹ Ĺášášášášáš ášášášñĹĹĹ"), - Eq("nnnnnnnnnnnnnnnn")); + EqualsNormalizedTerm("nnnnnnnnnnnnnnnn")); EXPECT_THAT(normalizer->NormalizeTerm("ĹĹĹÒÓÔÕÖášášášášášášášášòóôõöĹĹĹ"), - Eq("oooooooooooooooooooooooo")); - EXPECT_THAT(normalizer->NormalizeTerm("ášášášáš"), Eq("pppp")); + EqualsNormalizedTerm("oooooooooooooooooooooooo")); + EXPECT_THAT(normalizer->NormalizeTerm("ášášášáš"), EqualsNormalizedTerm("pppp")); EXPECT_THAT(normalizer->NormalizeTerm("ĹĹĹášášášášášášášášĹĹĹ"), - Eq("rrrrrrrrrrrrrr")); + EqualsNormalizedTerm("rrrrrrrrrrrrrr")); EXPECT_THAT(normalizer->NormalizeTerm("ĹĹĹŠČᚠᚢᚤᚌᚨᚥᚣᚼᚧᚊČĹĹĹš"), - Eq("ssssssssssssssssssss")); + EqualsNormalizedTerm("ssssssssssssssssssss")); EXPECT_THAT(normalizer->NormalizeTerm("ŢŤČᚪᚏᚎᚰᚍášášŻášąČĹŁĹĽ"), - Eq("tttttttttttttt")); + EqualsNormalizedTerm("tttttttttttttt")); EXPECT_THAT(normalizer->NormalizeTerm("ŨŪŏÙÚÛÜᚲᚴ᚜ᚸᚺᚳᚾᚡᚚ᚝ùúûüĹŠĹŤĹ"), - Eq("uuuuuuuuuuuuuuuuuuuuuuuu")); - EXPECT_THAT(normalizer->NormalizeTerm("᚟᚞ᚽᚿ"), Eq("vvvv")); - EXPECT_THAT(normalizer->NormalizeTerm("Ĺ´áşáşáşáşáşáşáşáş áşáşĹľ"), Eq("wwwwwwwwwwww")); - EXPECT_THAT(normalizer->NormalizeTerm("áşáşáşáş"), Eq("xxxx")); - EXPECT_THAT(normalizer->NormalizeTerm("ÝĹśŸáşáşĹˇýÿ"), Eq("yyyyyyyy")); - EXPECT_THAT(normalizer->NormalizeTerm("ŚŝŽáşáşáşáşáşáşĹşĹźĹž"), Eq("zzzzzzzzzzzz")); + EqualsNormalizedTerm("uuuuuuuuuuuuuuuuuuuuuuuu")); + EXPECT_THAT(normalizer->NormalizeTerm("᚟᚞ᚽᚿ"), EqualsNormalizedTerm("vvvv")); + EXPECT_THAT(normalizer->NormalizeTerm("Ĺ´áşáşáşáşáşáşáşáş áşáşĹľ"), + EqualsNormalizedTerm("wwwwwwwwwwww")); + EXPECT_THAT(normalizer->NormalizeTerm("áşáşáşáş"), EqualsNormalizedTerm("xxxx")); + EXPECT_THAT(normalizer->NormalizeTerm("ÝĹśŸáşáşĹˇýÿ"), + EqualsNormalizedTerm("yyyyyyyy")); + EXPECT_THAT(normalizer->NormalizeTerm("ŚŝŽáşáşáşáşáşáşĹşĹźĹž"), + EqualsNormalizedTerm("zzzzzzzzzzzz")); } // Accent / diacritic marks won't be removed in non-latin chars, e.g. in @@ -114,12 +131,16 @@ /*max_term_byte_size=*/1000)); // Katakana - EXPECT_THAT(normalizer->NormalizeTerm("ăăă ăă"), Eq("ăăă ăă")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăă ăă"), + EqualsNormalizedTerm("ăăă ăă")); // Greek - EXPECT_THAT(normalizer->NormalizeTerm("kαλημÎρα"), Eq("kαλημÎρα")); - EXPECT_THAT(normalizer->NormalizeTerm("εγγραφÎŽ"), Eq("εγγραφÎŽ")); + EXPECT_THAT(normalizer->NormalizeTerm("kαλημÎρα"), + EqualsNormalizedTerm("kαλημÎρα")); + EXPECT_THAT(normalizer->NormalizeTerm("εγγραφÎŽ"), + EqualsNormalizedTerm("εγγραφÎŽ")); // Hebrew - EXPECT_THAT(normalizer->NormalizeTerm("×Ö¸×֜ף֞×ÖľÖź×ת עִ×ְרִ×"), Eq("×Ö¸×֜ף֞×ÖľÖź×ת עִ×ְרִ×")); + EXPECT_THAT(normalizer->NormalizeTerm("×Ö¸×֜ף֞×ÖľÖź×ת עִ×ְרִ×"), + EqualsNormalizedTerm("×Ö¸×֜ף֞×ÖľÖź×ת עִ×ְרִ×")); } TEST(MapNormalizerTest, FullWidthCharsToASCII) { @@ -127,47 +148,63 @@ /*max_term_byte_size=*/1000)); // Full-width punctuation to ASCII punctuation - EXPECT_THAT(normalizer->NormalizeTerm("‘’ďźďźďźďźďź“”"), Eq("''.,!?:\"\"")); + EXPECT_THAT(normalizer->NormalizeTerm("‘’ďźďźďźďźďź“”"), + EqualsNormalizedTerm("''.,!?:\"\"")); // Full-width 0-9 EXPECT_THAT(normalizer->NormalizeTerm("ďźďźďźďźďźďźďźďźďźďź"), - Eq("0123456789")); + EqualsNormalizedTerm("0123456789")); // Full-width A-Z EXPECT_THAT(normalizer->NormalizeTerm( "ďźďźŽďźŻďź°ďźąďź˛ďźłďź´ďźľďźśďźˇďź¸ďźšďźş"), - Eq("abcdefghijklmnopqrstuvwxyz")); + EqualsNormalizedTerm("abcdefghijklmnopqrstuvwxyz")); // Full-width a-z EXPECT_THAT(normalizer->NormalizeTerm( "ď˝ď˝ď˝ď˝ď˝ ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝ď˝"), - Eq("abcdefghijklmnopqrstuvwxyz")); + EqualsNormalizedTerm("abcdefghijklmnopqrstuvwxyz")); } TEST(MapNormalizerTest, IdeographicToASCII) { ICING_ASSERT_OK_AND_ASSIGN(auto normalizer, normalizer_factory::Create( /*max_term_byte_size=*/1000)); - EXPECT_THAT(normalizer->NormalizeTerm("ďźă"), Eq(",.")); + EXPECT_THAT(normalizer->NormalizeTerm("ďźă"), EqualsNormalizedTerm(",.")); } TEST(MapNormalizerTest, HiraganaToKatakana) { ICING_ASSERT_OK_AND_ASSIGN(auto normalizer, normalizer_factory::Create( /*max_term_byte_size=*/1000)); - EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), Eq("ă˘ă¤ăŚă¨ăŞ")); - EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), Eq("ăŤăăŻăąăł")); - EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), Eq("ăľăˇăšăťă˝")); - EXPECT_THAT(normalizer->NormalizeTerm("ăăĄă¤ăŚă¨"), Eq("ăżăăăă")); - EXPECT_THAT(normalizer->NormalizeTerm("ăŞăŤăŹăăŽ"), Eq("ăăăăă")); - EXPECT_THAT(normalizer->NormalizeTerm("ăŻă˛ăľă¸ăť"), Eq("ăăăăă")); - EXPECT_THAT(normalizer->NormalizeTerm("ăžăżăăă"), Eq("ăăă ăĄă˘")); - EXPECT_THAT(normalizer->NormalizeTerm("ăăă"), Eq("ă¤ăŚă¨")); - EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), Eq("ăŠăŞăŤăŹă")); - EXPECT_THAT(normalizer->NormalizeTerm("ăăăă"), Eq("ăŻă°ăąă˛")); - EXPECT_THAT(normalizer->NormalizeTerm("ă"), Eq("ăł")); - EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), Eq("ăŹăŽă°ă˛ă´")); - EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), Eq("ăśă¸ăşăźăž")); - EXPECT_THAT(normalizer->NormalizeTerm("ă ă˘ăĽă§ăŠ"), Eq("ăăă ăă")); - EXPECT_THAT(normalizer->NormalizeTerm("ă°ăłăśăšăź"), Eq("ăăăăă")); - EXPECT_THAT(normalizer->NormalizeTerm("ăąă´ăˇăşă˝"), Eq("ăăăăă")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ă˘ă¤ăŚă¨ăŞ")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăŤăăŻăąăł")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăľăˇăšăťă˝")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăĄă¤ăŚă¨"), + EqualsNormalizedTerm("ăżăăăă")); + EXPECT_THAT(normalizer->NormalizeTerm("ăŞăŤăŹăăŽ"), + EqualsNormalizedTerm("ăăăăă")); + EXPECT_THAT(normalizer->NormalizeTerm("ăŻă˛ăľă¸ăť"), + EqualsNormalizedTerm("ăăăăă")); + EXPECT_THAT(normalizer->NormalizeTerm("ăžăżăăă"), + EqualsNormalizedTerm("ăăă ăĄă˘")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăă"), + EqualsNormalizedTerm("ă¤ăŚă¨")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăŠăŞăŤăŹă")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăăă"), + EqualsNormalizedTerm("ăŻă°ăąă˛")); + EXPECT_THAT(normalizer->NormalizeTerm("ă"), EqualsNormalizedTerm("ăł")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăŹăŽă°ă˛ă´")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăăăă"), + EqualsNormalizedTerm("ăśă¸ăşăźăž")); + EXPECT_THAT(normalizer->NormalizeTerm("ă ă˘ăĽă§ăŠ"), + EqualsNormalizedTerm("ăăă ăă")); + EXPECT_THAT(normalizer->NormalizeTerm("ă°ăłăśăšăź"), + EqualsNormalizedTerm("ăăăăă")); + EXPECT_THAT(normalizer->NormalizeTerm("ăąă´ăˇăşă˝"), + EqualsNormalizedTerm("ăăăăă")); } TEST(MapNormalizerTest, Truncate) { @@ -176,19 +213,22 @@ /*max_term_byte_size=*/5)); // Won't be truncated - EXPECT_THAT(normalizer->NormalizeTerm("hi"), Eq("hi")); - EXPECT_THAT(normalizer->NormalizeTerm("hello"), Eq("hello")); + EXPECT_THAT(normalizer->NormalizeTerm("hi"), EqualsNormalizedTerm("hi")); + EXPECT_THAT(normalizer->NormalizeTerm("hello"), + EqualsNormalizedTerm("hello")); // Truncated to length 5. - EXPECT_THAT(normalizer->NormalizeTerm("hello!"), Eq("hello")); + EXPECT_THAT(normalizer->NormalizeTerm("hello!"), + EqualsNormalizedTerm("hello")); // Each Japanese character has 3 bytes, so truncating to length 5 results in // only 1 character. - EXPECT_THAT(normalizer->NormalizeTerm("ăăŁăłăăš"), Eq("ă")); + EXPECT_THAT(normalizer->NormalizeTerm("ăăŁăłăăš"), + EqualsNormalizedTerm("ă")); // Each Greek character has 2 bytes, so truncating to length 5 results in 2 // character. - EXPECT_THAT(normalizer->NormalizeTerm("αβγδε"), Eq("αβ")); + EXPECT_THAT(normalizer->NormalizeTerm("αβγδε"), EqualsNormalizedTerm("αβ")); } { @@ -196,7 +236,7 @@ /*max_term_byte_size=*/2)); // The Japanese character has 3 bytes, truncating it results in an empty // string. - EXPECT_THAT(normalizer->NormalizeTerm("ă"), Eq("")); + EXPECT_THAT(normalizer->NormalizeTerm("ă"), EqualsNormalizedTerm("")); } }
diff --git a/icing/transform/normalizer.h b/icing/transform/normalizer.h index 2110f0f..0383f55 100644 --- a/icing/transform/normalizer.h +++ b/icing/transform/normalizer.h
@@ -39,7 +39,10 @@ // Normalizes the input term based on rules. See implementation classes for // specific transformation rules. - virtual std::string NormalizeTerm(std::string_view term) const = 0; + struct NormalizedTerm { + std::string text; + }; + virtual NormalizedTerm NormalizeTerm(std::string_view term) const = 0; // Returns a CharacterIterator pointing to one past the end of the segment of // term that (once normalized) matches with normalized_term.
diff --git a/icing/util/document-validator_test.cc b/icing/util/document-validator_test.cc index 2c366fd..2784522 100644 --- a/icing/util/document-validator_test.cc +++ b/icing/util/document-validator_test.cc
@@ -23,6 +23,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/proto/document.pb.h" #include "icing/proto/schema.pb.h" @@ -30,6 +31,7 @@ #include "icing/schema/schema-store.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" namespace icing { @@ -61,6 +63,8 @@ DocumentValidatorTest() {} void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + SchemaProto schema = SchemaBuilder() .AddType( @@ -133,8 +137,8 @@ schema_dir_ = GetTestTempDir() + "/schema_store"; ASSERT_TRUE(filesystem_.CreateDirectory(schema_dir_.c_str())); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_dir_, + &fake_clock_, feature_flags_.get())); ASSERT_THAT(schema_store_->SetSchema( schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false), @@ -182,6 +186,7 @@ SimpleEmailBuilder().Build()); } + std::unique_ptr<FeatureFlags> feature_flags_; std::string schema_dir_; Filesystem filesystem_; FakeClock fake_clock_; @@ -488,7 +493,8 @@ // Set a schema with only the 'Email' type ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, - SchemaStore::Create(&filesystem_, custom_schema_dir, &fake_clock_)); + SchemaStore::Create(&filesystem_, custom_schema_dir, &fake_clock_, + feature_flags_.get())); ASSERT_THAT(schema_store->SetSchema( email_schema, /*ignore_errors_and_delete_documents=*/false, /*allow_circular_schema_definitions=*/false),
diff --git a/icing/util/encode-util.cc b/icing/util/encode-util.cc index 32a2b09..c733117 100644 --- a/icing/util/encode-util.cc +++ b/icing/util/encode-util.cc
@@ -44,7 +44,7 @@ return value; } -std::string EncodeStringToCString(std::string input) { +std::string EncodeStringToCString(const std::string& input) { std::string encoded_str; // use uint32_t to store 4 bytes, using unsigned types to keep automatically // remove extra left most bits.
diff --git a/icing/util/encode-util.h b/icing/util/encode-util.h index ca0d8a9..6e258b2 100644 --- a/icing/util/encode-util.h +++ b/icing/util/encode-util.h
@@ -47,7 +47,7 @@ // string. // Eg2: This increases the size from 1-byte string to 2-bytes C string. // Eg3: This increases the size from 2-byte string to 3-bytes C string. -std::string EncodeStringToCString(std::string input); +std::string EncodeStringToCString(const std::string& input); } // namespace encode_util
diff --git a/icing/testing/icu-data-file-helper.cc b/icing/util/icu-data-file-helper.cc similarity index 90% rename from icing/testing/icu-data-file-helper.cc rename to icing/util/icu-data-file-helper.cc index aaeb738..a947e4d 100644 --- a/icing/testing/icu-data-file-helper.cc +++ b/icing/util/icu-data-file-helper.cc
@@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "icing/testing/icu-data-file-helper.h" +#include "icing/util/icu-data-file-helper.h" #include <sys/mman.h> @@ -22,9 +22,10 @@ #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/file/filesystem.h" +#ifdef CUSTOM_ICU_DAT_FILE #include "unicode/udata.h" #include "unicode/utypes.h" - +#endif // CUSTOM_ICU_DAT_FILE namespace icing { namespace lib { @@ -35,8 +36,9 @@ // segmentation fault errors. bool has_set_up_icu_data_file = false; -libtextclassifier3::Status SetUpICUDataFile( +libtextclassifier3::Status SetUpIcuDataFile( const std::string& icu_data_file_absolute_path) { +#ifdef CUSTOM_ICU_DAT_FILE if (has_set_up_icu_data_file) { return libtextclassifier3::Status::OK; } @@ -62,7 +64,7 @@ } has_set_up_icu_data_file = true; - +#endif // CUSTOM_ICU_DAT_FILE return libtextclassifier3::Status::OK; }
diff --git a/icing/testing/icu-data-file-helper.h b/icing/util/icu-data-file-helper.h similarity index 77% rename from icing/testing/icu-data-file-helper.h rename to icing/util/icu-data-file-helper.h index d0276e7..695229a 100644 --- a/icing/testing/icu-data-file-helper.h +++ b/icing/util/icu-data-file-helper.h
@@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ICING_TESTING_ICU_DATA_FILE_HELPER -#define ICING_TESTING_ICU_DATA_FILE_HELPER +#ifndef ICING_UTIL_ICU_DATA_FILE_HELPER +#define ICING_UTIL_ICU_DATA_FILE_HELPER #include "icing/text_classifier/lib3/utils/base/status.h" @@ -22,8 +22,8 @@ namespace icu_data_file_helper { -// The library binary doesn't contain any ICU data files, so we generate a .dat -// file at compile time and here make ICU use that file. +// Initializes ICU using the specified absolute path to the ICU data file. +// The data file can either be downloaded via MDD or generated at compile time. // // NOTE: This target does NOT contain the ICU .dat file. To use this helper // function, the calling target must include a data dependency on the .dat file @@ -32,7 +32,7 @@ // Returns: // Ok on success // INTERNAL_ERROR if failed on any errors -libtextclassifier3::Status SetUpICUDataFile( +libtextclassifier3::Status SetUpIcuDataFile( const std::string& icu_data_file_absolute_path); } // namespace icu_data_file_helper @@ -40,4 +40,4 @@ } // namespace lib } // namespace icing -#endif // ICING_TESTING_ICU_DATA_FILE_HELPER +#endif // ICING_UTIL_ICU_DATA_FILE_HELPER
diff --git a/icing/util/math-util.h b/icing/util/math-util.h index 3f2a69d..22f429f 100644 --- a/icing/util/math-util.h +++ b/icing/util/math-util.h
@@ -15,6 +15,7 @@ #ifndef ICING_UTIL_MATH_UTIL_H_ #define ICING_UTIL_MATH_UTIL_H_ +#include <cstdint> #include <limits> namespace icing { @@ -73,6 +74,23 @@ : (input_value - remainder + rounding_value); } +// Returns the next power of 2 given n (the smallest power of 2 which is +// greater or equal to n). +// +// REQUIRES: n <= 2^31, since 2^31 is the largest power of 2 that can be +// represented by an unsigned int. +inline uint32_t NextPowerOf2(uint32_t n) { + if (n == 0) { + return 1; + } + + if ((n & (n - 1)) != 0) { + // not 2's power + return UINT32_C(1) << (32 - __builtin_clz(n)); + } + return n; +} + } // namespace math_util } // namespace lib
diff --git a/icing/util/math-util_test.cc b/icing/util/math-util_test.cc new file mode 100644 index 0000000..ffc850e --- /dev/null +++ b/icing/util/math-util_test.cc
@@ -0,0 +1,53 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/util/math-util.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::Eq; + +TEST(MathUtilTest, NextPowerOf2) { + EXPECT_THAT(math_util::NextPowerOf2(0), Eq(1)); + EXPECT_THAT(math_util::NextPowerOf2(1), Eq(1)); + EXPECT_THAT(math_util::NextPowerOf2(2), Eq(2)); + EXPECT_THAT(math_util::NextPowerOf2(3), Eq(4)); + EXPECT_THAT(math_util::NextPowerOf2(4), Eq(4)); + EXPECT_THAT(math_util::NextPowerOf2(5), Eq(8)); + EXPECT_THAT(math_util::NextPowerOf2(6), Eq(8)); + EXPECT_THAT(math_util::NextPowerOf2(7), Eq(8)); + EXPECT_THAT(math_util::NextPowerOf2(8), Eq(8)); + EXPECT_THAT(math_util::NextPowerOf2(9), Eq(16)); + EXPECT_THAT(math_util::NextPowerOf2(16), Eq(16)); + EXPECT_THAT(math_util::NextPowerOf2(17), Eq(32)); + + // 2^31 - 1 + EXPECT_THAT(math_util::NextPowerOf2((UINT32_C(1) << 31) - 1), + Eq(UINT32_C(1) << 31)); + + // 2^31 + EXPECT_THAT(math_util::NextPowerOf2(UINT32_C(1) << 31), + Eq(UINT32_C(1) << 31)); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/util/scorable_property_set.cc b/icing/util/scorable_property_set.cc new file mode 100644 index 0000000..2f7ac24 --- /dev/null +++ b/icing/util/scorable_property_set.cc
@@ -0,0 +1,137 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/util/scorable_property_set.h" + +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/absl_ports/canonical_errors.h" +#include "icing/legacy/core/icing-string-util.h" +#include "icing/proto/internal/scorable_property_set.pb.h" +#include "icing/proto/schema.pb.h" +#include "icing/schema/property-util.h" +#include "icing/schema/schema-store.h" +#include "icing/schema/scorable_property_manager.h" +#include "icing/store/document-filter-data.h" +#include "icing/util/status-macros.h" + +namespace icing { +namespace lib { + +libtextclassifier3::StatusOr<std::unique_ptr<ScorablePropertySet>> +ScorablePropertySet::Create( + ScorablePropertySetProto&& scorable_property_set_proto, + SchemaTypeId schema_type_id, const SchemaStore* schema_store) { + ICING_ASSIGN_OR_RETURN( + const std::vector<ScorablePropertyManager::ScorablePropertyInfo>* + ordered_scorable_property_info, + schema_store->GetOrderedScorablePropertyInfo(schema_type_id)); + if (ordered_scorable_property_info == nullptr || + ordered_scorable_property_info->size() != + scorable_property_set_proto.properties_size()) { + return absl_ports::InvalidArgumentError(IcingStringUtil::StringPrintf( + "ScorablePropertySetProto data is inconsistent with the schema config " + "of type id %d", + schema_type_id)); + } + return std::unique_ptr<ScorablePropertySet>(new ScorablePropertySet( + std::move(scorable_property_set_proto), schema_type_id, schema_store)); +} + +libtextclassifier3::StatusOr<std::unique_ptr<ScorablePropertySet>> +ScorablePropertySet::Create(const DocumentProto& document, + SchemaTypeId schema_type_id, + const SchemaStore* schema_store) { + ICING_ASSIGN_OR_RETURN( + const std::vector<ScorablePropertyManager::ScorablePropertyInfo>* + ordered_scorable_property_info, + schema_store->GetOrderedScorablePropertyInfo(schema_type_id)); + if (ordered_scorable_property_info == nullptr) { + // It should never happen + return absl_ports::InternalError( + "SchemaStore::ordered_scorable_property_paths returned nullptr"); + } + if (ordered_scorable_property_info->empty()) { + return absl_ports::InvalidArgumentError(IcingStringUtil::StringPrintf( + "No scorable property defined under the config of type id %d", + schema_type_id)); + } + + ScorablePropertySetProto scorable_property_set_proto_to_build; + for (const ScorablePropertyManager::ScorablePropertyInfo& + scorable_property_info : *ordered_scorable_property_info) { + ScorablePropertyProto* new_property = + scorable_property_set_proto_to_build.add_properties(); + + if (scorable_property_info.data_type == + PropertyConfigProto::DataType::DOUBLE) { + libtextclassifier3::StatusOr<std::vector<double>> content_or = + property_util::ExtractPropertyValuesFromDocument<double>( + document, scorable_property_info.property_path); + if (content_or.ok()) { + new_property->mutable_double_values()->Add( + content_or.ValueOrDie().begin(), content_or.ValueOrDie().end()); + } + } else if (scorable_property_info.data_type == + PropertyConfigProto::DataType::INT64) { + libtextclassifier3::StatusOr<std::vector<int64_t>> content_or = + property_util::ExtractPropertyValuesFromDocument<int64_t>( + document, scorable_property_info.property_path); + if (content_or.ok()) { + new_property->mutable_int64_values()->Add( + content_or.ValueOrDie().begin(), content_or.ValueOrDie().end()); + } + } else if (scorable_property_info.data_type == + PropertyConfigProto::DataType::BOOLEAN) { + libtextclassifier3::StatusOr<std::vector<bool>> content_or = + property_util::ExtractPropertyValuesFromDocument<bool>( + document, scorable_property_info.property_path); + if (content_or.ok()) { + new_property->mutable_boolean_values()->Add( + content_or.ValueOrDie().begin(), content_or.ValueOrDie().end()); + } + } + } + return std::unique_ptr<ScorablePropertySet>( + new ScorablePropertySet(std::move(scorable_property_set_proto_to_build), + schema_type_id, schema_store)); +} + +const ScorablePropertyProto* ScorablePropertySet::GetScorablePropertyProto( + const std::string& property_path) const { + libtextclassifier3::StatusOr<std::optional<int>> index_or = + schema_store_->GetScorablePropertyIndex(schema_type_id_, property_path); + if (!index_or.ok() || !index_or.ValueOrDie().has_value()) { + return nullptr; + } + return &scorable_property_set_proto_.properties( + index_or.ValueOrDie().value()); +} + +ScorablePropertySet::ScorablePropertySet( + ScorablePropertySetProto&& scorable_property_set_proto, + SchemaTypeId schema_type_id, const SchemaStore* schema_store) + : scorable_property_set_proto_(std::move(scorable_property_set_proto)), + schema_type_id_(schema_type_id), + schema_store_(schema_store) {} + +} // namespace lib +} // namespace icing
diff --git a/icing/util/scorable_property_set.h b/icing/util/scorable_property_set.h new file mode 100644 index 0000000..a11919f --- /dev/null +++ b/icing/util/scorable_property_set.h
@@ -0,0 +1,94 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ICING_UTIL_SCORABLE_PROPERTY_SET_H_ +#define ICING_UTIL_SCORABLE_PROPERTY_SET_H_ + +#include <memory> +#include <string> + +#include "icing/text_classifier/lib3/utils/base/statusor.h" +#include "icing/proto/internal/scorable_property_set.pb.h" +#include "icing/proto/schema.pb.h" +#include "icing/schema/schema-store.h" +#include "icing/store/document-filter-data.h" + +namespace icing { +namespace lib { + +// A class that interprets and represents the data from the proto of +// ScorablePropertySetProto. +// +// This class serves as an utility to access the scorable property data. +class ScorablePropertySet { + public: + // Creates a class that represents the data from the proto of + // ScorablePropertySetProto. + // + // This function is light weight, and it doesn't perform any proto copy. + // + // Returns: + // - INVALID_ARGUMENT if |schema_type_id| is invalid, or the proto data is + // inconsistent with the schema config. + // - FAILED_PRECONDITION if the schema hasn't been set yet + static libtextclassifier3::StatusOr<std::unique_ptr<ScorablePropertySet>> + Create(ScorablePropertySetProto&& scorable_property_set_proto, + SchemaTypeId schema_type_id, const SchemaStore* schema_store); + + // Creates a class that represents the data from the proto of + // ScorablePropertySetProto. + // + // This function converts the input |document| to the ScorablePropertySetProto + // under the hood. + // + // Returns: + // - INVALID_ARGUMENT if |schema_type_id| is invalid, or that no scorable + // property is found under the schema config of |schema_type_id|. + // - FAILED_PRECONDITION if the schema hasn't been set yet + static libtextclassifier3::StatusOr<std::unique_ptr<ScorablePropertySet>> + Create(const DocumentProto& document, SchemaTypeId schema_type_id, + const SchemaStore* schema_store); + + // Delete copy constructor and assignment operator. + ScorablePropertySet(const ScorablePropertySet&) = delete; + ScorablePropertySet& operator=(const ScorablePropertySet&) = delete; + + // Returns the scorable property proto for the given property path. + // + // Return: + // - ScorablePropertyProto on success + // - nullptr if the property path is not found in the schema config + // as a scorable property. + const ScorablePropertyProto* GetScorablePropertyProto( + const std::string& property_path) const; + + // Returns the reference to the underlying ScorablePropertySetProto. + const ScorablePropertySetProto& GetScorablePropertySetProto() const { + return scorable_property_set_proto_; + } + + private: + explicit ScorablePropertySet( + ScorablePropertySetProto&& scorable_property_set_proto, + SchemaTypeId schema_type_id, const SchemaStore* schema_store); + + const ScorablePropertySetProto scorable_property_set_proto_; + const SchemaTypeId schema_type_id_; + const SchemaStore* schema_store_; +}; + +} // namespace lib +} // namespace icing + +#endif // ICING_UTIL_SCORABLE_PROPERTY_SET_H_
diff --git a/icing/util/scorable_property_set_test.cc b/icing/util/scorable_property_set_test.cc new file mode 100644 index 0000000..1dc9f64 --- /dev/null +++ b/icing/util/scorable_property_set_test.cc
@@ -0,0 +1,422 @@ +// Copyright (C) 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/util/scorable_property_set.h" + +#include <cstdint> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/document-builder.h" +#include "icing/feature-flags.h" +#include "icing/file/filesystem.h" +#include "icing/proto/internal/scorable_property_set.pb.h" +#include "icing/proto/schema.pb.h" +#include "icing/schema-builder.h" +#include "icing/schema/schema-store.h" +#include "icing/store/document-filter-data.h" +#include "icing/testing/common-matchers.h" +#include "icing/testing/fake-clock.h" +#include "icing/testing/test-feature-flags.h" +#include "icing/testing/tmp-directory.h" + +namespace icing { +namespace lib { + +namespace { + +using ::icing::lib::portable_equals_proto::EqualsProto; +using ::testing::Pointee; + +ScorablePropertyProto BuildScorablePropertyProtoFromBoolean( + const std::vector<bool>& boolean_values) { + ScorablePropertyProto scorable_property; + scorable_property.mutable_boolean_values()->Add(boolean_values.begin(), + boolean_values.end()); + return scorable_property; +} + +ScorablePropertyProto BuildScorablePropertyProtoFromInt64( + const std::vector<int64_t>& int64_values) { + ScorablePropertyProto scorable_property; + scorable_property.mutable_int64_values()->Add(int64_values.begin(), + int64_values.end()); + return scorable_property; +} + +ScorablePropertyProto BuildScorablePropertyProtoFromDouble( + const std::vector<double>& double_values) { + ScorablePropertyProto scorable_property; + scorable_property.mutable_double_values()->Add(double_values.begin(), + double_values.end()); + return scorable_property; +} + +class ScorablePropertySetTest : public ::testing::Test { + protected: + ScorablePropertySetTest() + : schema_store_dir_(GetTestTempDir() + "/schema_store") {} + + void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); + filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()); + + ICING_ASSERT_OK_AND_ASSIGN( + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); + + SchemaProto schema_proto = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("dummy").AddProperty( + PropertyConfigBuilder() + .SetName("id") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("person") + .AddProperty(PropertyConfigBuilder() + .SetName("id") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("income") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("age") + .SetDataType(PropertyConfigProto::DataType::INT64) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("isStarred") + .SetDataType(PropertyConfigProto::DataType::BOOLEAN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType("email") + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeDocument( + "person", /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("receiver") + .SetDataTypeDocument( + "person", /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("importanceBoolean") + .SetDataType(PropertyConfigProto::DataType::BOOLEAN) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("scoreDouble") + .SetDataType(PropertyConfigProto::DataType::DOUBLE) + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty(PropertyConfigBuilder() + .SetName("scoreInt64") + .SetScorableType(SCORABLE_TYPE_ENABLED) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + ICING_ASSERT_OK(schema_store_->SetSchema( + schema_proto, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + email_schema_type_id_ = + schema_store_->GetSchemaTypeId("email").ValueOrDie(); + person_schema_type_id_ = + schema_store_->GetSchemaTypeId("person").ValueOrDie(); + dummy_schema_type_id_ = + schema_store_->GetSchemaTypeId("dummy").ValueOrDie(); + } + + void TearDown() override { + schema_store_.reset(); + filesystem_.DeleteDirectoryRecursively(schema_store_dir_.c_str()); + } + + std::unique_ptr<FeatureFlags> feature_flags_; + Filesystem filesystem_; + std::string schema_store_dir_; + FakeClock fake_clock_; + SchemaTypeId email_schema_type_id_; + SchemaTypeId person_schema_type_id_; + SchemaTypeId dummy_schema_type_id_; + std::unique_ptr<SchemaStore> schema_store_; +}; + +TEST_F(ScorablePropertySetTest, + BuildFromScorablePropertySetProto_GetScorablePropertyProto) { + ScorablePropertyProto is_starred_scorable_property = + BuildScorablePropertyProtoFromBoolean({true}); + ScorablePropertyProto income_scorable_property = + BuildScorablePropertyProtoFromDouble({1.5, 2.5}); + ScorablePropertyProto age_scorable_property = + BuildScorablePropertyProtoFromInt64({45}); + + ScorablePropertySetProto scorable_property_set_proto; + *scorable_property_set_proto.add_properties() = age_scorable_property; + *scorable_property_set_proto.add_properties() = income_scorable_property; + *scorable_property_set_proto.add_properties() = is_starred_scorable_property; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<ScorablePropertySet> scorable_property_set, + ScorablePropertySet::Create(std::move(scorable_property_set_proto), + person_schema_type_id_, schema_store_.get())); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("age"), + Pointee(EqualsProto(age_scorable_property))); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("income"), + Pointee(EqualsProto(income_scorable_property))); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("isStarred"), + Pointee(EqualsProto(is_starred_scorable_property))); + + EXPECT_EQ( + scorable_property_set->GetScorablePropertyProto("non_exist_property"), + nullptr); + + EXPECT_EQ(scorable_property_set->GetScorablePropertyProto("subject"), + nullptr); +} + +TEST_F(ScorablePropertySetTest, + BuildFromScorablePropertySetProto_InconsistentWithSchemaConfig) { + ScorablePropertyProto scorable_property_proto_boolean = + BuildScorablePropertyProtoFromBoolean({true}); + + ScorablePropertySetProto scorable_property_set_proto; + *scorable_property_set_proto.add_properties() = + scorable_property_proto_boolean; + EXPECT_THAT( + ScorablePropertySet::Create(std::move(scorable_property_set_proto), + email_schema_type_id_, schema_store_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(ScorablePropertySetTest, + BuildFromScorablePropertySetProto_InvalidSchemaTypeId) { + ScorablePropertyProto scorable_property_proto_boolean = + BuildScorablePropertyProtoFromBoolean({true}); + + ScorablePropertySetProto scorable_property_set_proto; + *scorable_property_set_proto.add_properties() = + scorable_property_proto_boolean; + EXPECT_THAT( + ScorablePropertySet::Create(std::move(scorable_property_set_proto), + /*schema_type_id=*/1000, schema_store_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(ScorablePropertySetTest, + BuildFromScorablePropertySetProto_WithSomePropertiesNotPopulated) { + ScorablePropertyProto age_scorable_property; + ScorablePropertyProto is_starred_scorable_property; + ScorablePropertyProto income_scorable_property = + BuildScorablePropertyProtoFromInt64({1, 2, 3}); + + ScorablePropertySetProto scorable_property_set_proto; + *scorable_property_set_proto.add_properties() = age_scorable_property; + *scorable_property_set_proto.add_properties() = income_scorable_property; + *scorable_property_set_proto.add_properties() = is_starred_scorable_property; + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<ScorablePropertySet> scorable_property_set, + ScorablePropertySet::Create(std::move(scorable_property_set_proto), + person_schema_type_id_, schema_store_.get())); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("age"), + Pointee(EqualsProto(age_scorable_property))); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("isStarred"), + Pointee(EqualsProto(is_starred_scorable_property))); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("income"), + Pointee(EqualsProto(income_scorable_property))); +} + +TEST_F(ScorablePropertySetTest, BuildFromDocument_InvalidSchemaTypeId) { + DocumentProto document = + DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("email") + .AddStringProperty("subjectString", "subject foo") + .SetCreationTimestampMs(0) + .Build(); + + EXPECT_THAT( + ScorablePropertySet::Create(document, + /*schema_type_id=*/1000, schema_store_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(ScorablePropertySetTest, + BuildFromDocument_NoScorablePropertiesFromDocument) { + DocumentProto document = + DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("email") + .AddStringProperty("subjectString", "subject foo") + .SetCreationTimestampMs(0) + .Build(); + + ScorablePropertySetProto expected_scorable_property_set; + expected_scorable_property_set.add_properties(); // importanceBoolean + expected_scorable_property_set.add_properties(); // receiver.age + expected_scorable_property_set.add_properties(); // receiver.income + expected_scorable_property_set.add_properties(); // receiver.isStarred + expected_scorable_property_set.add_properties(); // scoreDouble + expected_scorable_property_set.add_properties(); // scoreInt64 + expected_scorable_property_set.add_properties(); // sender.age + expected_scorable_property_set.add_properties(); // sender.income + expected_scorable_property_set.add_properties(); // sender.isStarred + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<ScorablePropertySet> scorable_property_set, + ScorablePropertySet::Create(document, email_schema_type_id_, + schema_store_.get())); + EXPECT_THAT(scorable_property_set->GetScorablePropertySetProto(), + EqualsProto(expected_scorable_property_set)); +} + +TEST_F(ScorablePropertySetTest, + BuildFromDocument_NoScorablePropertiesFromSchema) { + DocumentProto document; + EXPECT_THAT(ScorablePropertySet::Create(document, dummy_schema_type_id_, + schema_store_.get()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(ScorablePropertySetTest, ScorablePropertySetFromNestedDocument) { + DocumentProto document = + DocumentBuilder() + .SetKey("foo", "1") + .SetSchema("email") + .AddStringProperty("subjectString", "subject foo") + .AddBooleanProperty("importanceBoolean", true) + .AddInt64Property("scoreInt64", 1, 2, 3) + .AddDocumentProperty( + "receiver", + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("person") + .AddInt64Property("age", 30) + .AddDoubleProperty("income", 10000, 20000, 30000) + .AddBooleanProperty("isStarred", true) + .Build(), + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("person") + .AddInt64Property("age", 35) + .AddDoubleProperty("income", 10001, 20001, 30001) + .AddBooleanProperty("isStarred", false) + .Build()) + .AddDocumentProperty( + "sender", DocumentBuilder() + .SetKey("namespace", "uri3") + .SetSchema("person") + .AddInt64Property("age", 50) + .AddDoubleProperty("income", 21001, 21002, 21003) + .AddBooleanProperty("isStarred", false) + .Build()) + .SetCreationTimestampMs(0) + .Build(); + + ScorablePropertySetProto expected_scorable_property_set; + // importanceBoolean + *expected_scorable_property_set.add_properties() = + BuildScorablePropertyProtoFromBoolean({true}); + // receiver.age + *expected_scorable_property_set.add_properties() = + BuildScorablePropertyProtoFromInt64({30, 35}); + // receiver.income + *expected_scorable_property_set.add_properties() = + BuildScorablePropertyProtoFromDouble( + {10000, 20000, 30000, 10001, 20001, 30001}); + // receiver.isStarred + *expected_scorable_property_set.add_properties() = + BuildScorablePropertyProtoFromBoolean({true, false}); + // scoreDouble + *expected_scorable_property_set.add_properties() = + BuildScorablePropertyProtoFromDouble({}); + // scoreInt64 + *expected_scorable_property_set.add_properties() = + BuildScorablePropertyProtoFromInt64({1, 2, 3}); + // sender.age + *expected_scorable_property_set.add_properties() = + BuildScorablePropertyProtoFromInt64({50}); + // sender.income + *expected_scorable_property_set.add_properties() = + BuildScorablePropertyProtoFromDouble({21001, 21002, 21003}); + // sender.isStarred + *expected_scorable_property_set.add_properties() = + BuildScorablePropertyProtoFromBoolean({false}); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<ScorablePropertySet> scorable_property_set, + ScorablePropertySet::Create(document, email_schema_type_id_, + schema_store_.get())); + EXPECT_THAT(scorable_property_set->GetScorablePropertySetProto(), + EqualsProto(expected_scorable_property_set)); + + // Test GetScorablePropertyProto() for each property path. + EXPECT_THAT( + scorable_property_set->GetScorablePropertyProto("importanceBoolean"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromBoolean({true})))); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("scoreDouble"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble({})))); + EXPECT_THAT( + scorable_property_set->GetScorablePropertyProto("scoreInt64"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({1, 2, 3})))); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("sender.age"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({50})))); + EXPECT_THAT(scorable_property_set->GetScorablePropertyProto("sender.income"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble( + {21001, 21002, 21003})))); + EXPECT_THAT( + scorable_property_set->GetScorablePropertyProto("sender.isStarred"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromBoolean({false})))); + EXPECT_THAT( + scorable_property_set->GetScorablePropertyProto("receiver.age"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromInt64({30, 35})))); + EXPECT_THAT( + scorable_property_set->GetScorablePropertyProto("receiver.income"), + Pointee(EqualsProto(BuildScorablePropertyProtoFromDouble( + {10000, 20000, 30000, 10001, 20001, 30001})))); + EXPECT_THAT( + scorable_property_set->GetScorablePropertyProto("receiver.isStarred"), + Pointee( + EqualsProto(BuildScorablePropertyProtoFromBoolean({true, false})))); +} + +} // namespace + +} // namespace lib +} // namespace icing
diff --git a/icing/util/tokenized-document_test.cc b/icing/util/tokenized-document_test.cc index ab7f4b9..2aa92d1 100644 --- a/icing/util/tokenized-document_test.cc +++ b/icing/util/tokenized-document_test.cc
@@ -23,6 +23,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" +#include "icing/feature-flags.h" #include "icing/file/filesystem.h" #include "icing/portable/platform.h" #include "icing/proto/document.pb.h" @@ -33,11 +34,12 @@ #include "icing/schema/section.h" #include "icing/testing/common-matchers.h" #include "icing/testing/fake-clock.h" -#include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" +#include "icing/testing/test-feature-flags.h" #include "icing/testing/tmp-directory.h" #include "icing/tokenization/language-segmenter-factory.h" #include "icing/tokenization/language-segmenter.h" +#include "icing/util/icu-data-file-helper.h" #include "unicode/uloc.h" namespace icing { @@ -84,41 +86,43 @@ const SectionMetadata kIndexableInteger1SectionMetadata( kIndexableInteger1SectionId, TYPE_INT64, TOKENIZER_NONE, TERM_MATCH_UNKNOWN, - NUMERIC_MATCH_RANGE, EMBEDDING_INDEXING_UNKNOWN, + NUMERIC_MATCH_RANGE, EMBEDDING_INDEXING_UNKNOWN, QUANTIZATION_TYPE_NONE, std::string(kIndexableIntegerProperty1)); const SectionMetadata kIndexableInteger2SectionMetadata( kIndexableInteger2SectionId, TYPE_INT64, TOKENIZER_NONE, TERM_MATCH_UNKNOWN, - NUMERIC_MATCH_RANGE, EMBEDDING_INDEXING_UNKNOWN, + NUMERIC_MATCH_RANGE, EMBEDDING_INDEXING_UNKNOWN, QUANTIZATION_TYPE_NONE, std::string(kIndexableIntegerProperty2)); const SectionMetadata kIndexableVector1SectionMetadata( kIndexableVector1SectionId, TYPE_VECTOR, TOKENIZER_NONE, TERM_MATCH_UNKNOWN, NUMERIC_MATCH_UNKNOWN, EMBEDDING_INDEXING_LINEAR_SEARCH, - std::string(kIndexableVectorProperty1)); + QUANTIZATION_TYPE_NONE, std::string(kIndexableVectorProperty1)); const SectionMetadata kIndexableVector2SectionMetadata( kIndexableVector2SectionId, TYPE_VECTOR, TOKENIZER_NONE, TERM_MATCH_UNKNOWN, NUMERIC_MATCH_UNKNOWN, EMBEDDING_INDEXING_LINEAR_SEARCH, - std::string(kIndexableVectorProperty2)); + QUANTIZATION_TYPE_QUANTIZE_8_BIT, std::string(kIndexableVectorProperty2)); const SectionMetadata kStringExactSectionMetadata( kStringExactSectionId, TYPE_STRING, TOKENIZER_PLAIN, TERM_MATCH_EXACT, - NUMERIC_MATCH_UNKNOWN, EMBEDDING_INDEXING_UNKNOWN, + NUMERIC_MATCH_UNKNOWN, EMBEDDING_INDEXING_UNKNOWN, QUANTIZATION_TYPE_NONE, std::string(kStringExactProperty)); const SectionMetadata kStringPrefixSectionMetadata( kStringPrefixSectionId, TYPE_STRING, TOKENIZER_PLAIN, TERM_MATCH_PREFIX, - NUMERIC_MATCH_UNKNOWN, EMBEDDING_INDEXING_UNKNOWN, + NUMERIC_MATCH_UNKNOWN, EMBEDDING_INDEXING_UNKNOWN, QUANTIZATION_TYPE_NONE, std::string(kStringPrefixProperty)); const JoinablePropertyMetadata kQualifiedId1JoinablePropertyMetadata( kQualifiedId1JoinablePropertyId, TYPE_STRING, - JOINABLE_VALUE_TYPE_QUALIFIED_ID, std::string(kQualifiedId1)); + JOINABLE_VALUE_TYPE_QUALIFIED_ID, DELETE_PROPAGATION_TYPE_PROPAGATE_FROM, + std::string(kQualifiedId1)); const JoinablePropertyMetadata kQualifiedId2JoinablePropertyMetadata( kQualifiedId2JoinablePropertyId, TYPE_STRING, - JOINABLE_VALUE_TYPE_QUALIFIED_ID, std::string(kQualifiedId2)); + JOINABLE_VALUE_TYPE_QUALIFIED_ID, DELETE_PROPAGATION_TYPE_NONE, + std::string(kQualifiedId2)); // Other non-indexable/joinable properties. constexpr std::string_view kUnindexedStringProperty = "unindexedString"; @@ -128,6 +132,7 @@ class TokenizedDocumentTest : public ::testing::Test { protected: void SetUp() override { + feature_flags_ = std::make_unique<FeatureFlags>(GetTestFeatureFlags()); test_dir_ = GetTestTempDir() + "/icing"; schema_store_dir_ = test_dir_ + "/schema_store"; filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()); @@ -135,7 +140,7 @@ if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { ICING_ASSERT_OK( // File generated via icu_data_file rule in //icing/BUILD. - icu_data_file_helper::SetUpICUDataFile( + icu_data_file_helper::SetUpIcuDataFile( GetTestFilePath("icing/icu.dat"))); } @@ -145,8 +150,8 @@ language_segmenter_factory::Create(std::move(options))); ICING_ASSERT_OK_AND_ASSIGN( - schema_store_, - SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + schema_store_, SchemaStore::Create(&filesystem_, schema_store_dir_, + &fake_clock_, feature_flags_.get())); SchemaProto schema = SchemaBuilder() @@ -181,7 +186,8 @@ .AddProperty( PropertyConfigBuilder() .SetName(kIndexableVectorProperty2) - .SetDataTypeVector(EMBEDDING_INDEXING_LINEAR_SEARCH) + .SetDataTypeVector(EMBEDDING_INDEXING_LINEAR_SEARCH, + QUANTIZATION_TYPE_QUANTIZE_8_BIT) .SetCardinality(CARDINALITY_OPTIONAL)) .AddProperty(PropertyConfigBuilder() .SetName(kStringExactProperty) @@ -196,12 +202,14 @@ .AddProperty(PropertyConfigBuilder() .SetName(kQualifiedId1) .SetDataTypeJoinableString( - JOINABLE_VALUE_TYPE_QUALIFIED_ID) + JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_PROPAGATE_FROM) .SetCardinality(CARDINALITY_OPTIONAL)) .AddProperty(PropertyConfigBuilder() .SetName(kQualifiedId2) .SetDataTypeJoinableString( - JOINABLE_VALUE_TYPE_QUALIFIED_ID) + JOINABLE_VALUE_TYPE_QUALIFIED_ID, + DELETE_PROPAGATION_TYPE_NONE) .SetCardinality(CARDINALITY_OPTIONAL))) .Build(); ICING_ASSERT_OK(schema_store_->SetSchema( @@ -223,6 +231,7 @@ ASSERT_TRUE(filesystem_.DeleteDirectoryRecursively(test_dir_.c_str())); } + std::unique_ptr<FeatureFlags> feature_flags_; Filesystem filesystem_; FakeClock fake_clock_; std::string test_dir_;
diff --git a/java/src/com/google/android/icing/IcingSearchEngine.java b/java/src/com/google/android/icing/IcingSearchEngine.java index c8d4516..0dddb2e 100644 --- a/java/src/com/google/android/icing/IcingSearchEngine.java +++ b/java/src/com/google/android/icing/IcingSearchEngine.java
@@ -109,6 +109,13 @@ @NonNull @Override + public GetSchemaResultProto getSchemaForDatabase(@NonNull String database) { + return IcingSearchEngineUtils.byteArrayToGetSchemaResultProto( + icingSearchEngineImpl.getSchemaForDatabase(database)); + } + + @NonNull + @Override public GetSchemaTypeResultProto getSchemaType(@NonNull String schemaType) { return IcingSearchEngineUtils.byteArrayToGetSchemaTypeResultProto( icingSearchEngineImpl.getSchemaType(schemaType)); @@ -161,7 +168,6 @@ icingSearchEngineImpl.getNextPage(nextPageToken)); } - @NonNull @Override public void invalidateNextPageToken(long nextPageToken) { icingSearchEngineImpl.invalidateNextPageToken(nextPageToken); @@ -176,6 +182,13 @@ @NonNull @Override + public BlobProto removeBlob(PropertyProto.BlobHandleProto blobHandle) { + return IcingSearchEngineUtils.byteArrayToBlobProto( + icingSearchEngineImpl.removeBlob(blobHandle.toByteArray())); + } + + @NonNull + @Override public BlobProto openReadBlob(PropertyProto.BlobHandleProto blobHandle) { return IcingSearchEngineUtils.byteArrayToBlobProto( icingSearchEngineImpl.openReadBlob(blobHandle.toByteArray()));
diff --git a/java/src/com/google/android/icing/IcingSearchEngineImpl.java b/java/src/com/google/android/icing/IcingSearchEngineImpl.java index 2a76d92..c06f547 100644 --- a/java/src/com/google/android/icing/IcingSearchEngineImpl.java +++ b/java/src/com/google/android/icing/IcingSearchEngineImpl.java
@@ -95,6 +95,12 @@ } @Nullable + public byte[] getSchemaForDatabase(@NonNull String database) { + throwIfClosed(); + return nativeGetSchemaForDatabase(this, database); + } + + @Nullable public byte[] getSchemaType(@NonNull String schemaType) { throwIfClosed(); return nativeGetSchemaType(this, schemaType); @@ -150,7 +156,6 @@ return nativeGetNextPage(this, nextPageToken, System.currentTimeMillis()); } - @NonNull public void invalidateNextPageToken(long nextPageToken) { throwIfClosed(); nativeInvalidateNextPageToken(this, nextPageToken); @@ -163,6 +168,12 @@ } @NonNull + public byte[] removeBlob(@NonNull byte[] blobHandleBytes) { + throwIfClosed(); + return nativeRemoveBlob(this, blobHandleBytes); + } + + @NonNull public byte[] openReadBlob(@NonNull byte[] blobHandleBytes) { throwIfClosed(); return nativeOpenReadBlob(this, blobHandleBytes); @@ -281,6 +292,9 @@ private static native byte[] nativeGetSchema(IcingSearchEngineImpl instance); + private static native byte[] nativeGetSchemaForDatabase( + IcingSearchEngineImpl instance, String database); + private static native byte[] nativeGetSchemaType( IcingSearchEngineImpl instance, String schemaType); @@ -310,6 +324,9 @@ private static native byte[] nativeOpenWriteBlob( IcingSearchEngineImpl instance, byte[] blobHandleBytes); + private static native byte[] nativeRemoveBlob( + IcingSearchEngineImpl instance, byte[] blobHandleBytes); + private static native byte[] nativeOpenReadBlob( IcingSearchEngineImpl instance, byte[] blobHandleBytes);
diff --git a/java/src/com/google/android/icing/IcingSearchEngineInterface.java b/java/src/com/google/android/icing/IcingSearchEngineInterface.java index 65b8085..dcd5c3e 100644 --- a/java/src/com/google/android/icing/IcingSearchEngineInterface.java +++ b/java/src/com/google/android/icing/IcingSearchEngineInterface.java
@@ -58,6 +58,14 @@ GetSchemaResultProto getSchema(); /** + * Gets the schema for the specified database for the icing instance. + * + * @param database an icing schema database name. The retrieved SchemaProto will only contain + * types that belong to the requested database. + */ + GetSchemaResultProto getSchemaForDatabase(String database); + + /** * Gets the schema for the icing instance. * * @param schemaType type of the schema. @@ -100,6 +108,9 @@ /** Gets a file descriptor to write blob data. */ BlobProto openWriteBlob(PropertyProto.BlobHandleProto blobHandle); + /** Removes a pending blob. */ + BlobProto removeBlob(PropertyProto.BlobHandleProto blobHandle); + /** Gets a file descriptor to read blob data. */ BlobProto openReadBlob(PropertyProto.BlobHandleProto blobHandle);
diff --git a/java/tests/instrumentation/src/com/google/android/icing/IcingSearchEngineTest.java b/java/tests/instrumentation/src/com/google/android/icing/IcingSearchEngineTest.java index e7b114f..fb17f44 100644 --- a/java/tests/instrumentation/src/com/google/android/icing/IcingSearchEngineTest.java +++ b/java/tests/instrumentation/src/com/google/android/icing/IcingSearchEngineTest.java
@@ -90,6 +90,7 @@ public final class IcingSearchEngineTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private static final String SCHEMA_DATABASE_DELIMITER = "/"; private static final String EMAIL_TYPE = "Email"; private File tempDir; @@ -120,6 +121,31 @@ .build(); } + private static SchemaTypeConfigProto createEmailTypeConfigWithDatabase(String database) { + return SchemaTypeConfigProto.newBuilder() + .setSchemaType(database + SCHEMA_DATABASE_DELIMITER + EMAIL_TYPE) + .setDatabase(database) + .addProperties( + PropertyConfigProto.newBuilder() + .setPropertyName("subject") + .setDataType(PropertyConfigProto.DataType.Code.STRING) + .setCardinality(PropertyConfigProto.Cardinality.Code.OPTIONAL) + .setStringIndexingConfig( + StringIndexingConfig.newBuilder() + .setTokenizerType(TokenizerType.Code.PLAIN) + .setTermMatchType(TermMatchType.Code.PREFIX))) + .addProperties( + PropertyConfigProto.newBuilder() + .setPropertyName("body") + .setDataType(PropertyConfigProto.DataType.Code.STRING) + .setCardinality(PropertyConfigProto.Cardinality.Code.OPTIONAL) + .setStringIndexingConfig( + StringIndexingConfig.newBuilder() + .setTokenizerType(TokenizerType.Code.PLAIN) + .setTermMatchType(TermMatchType.Code.PREFIX))) + .build(); + } + private static DocumentProto createEmailDocument(String namespace, String uri) { return DocumentProto.newBuilder() .setNamespace(namespace) @@ -164,6 +190,72 @@ } @Test + public void testSetAndGetSchema() throws Exception { + assertStatusOk(icingSearchEngine.initialize().getStatus()); + + SchemaTypeConfigProto emailTypeConfig = createEmailTypeConfig(); + SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); + SetSchemaResultProto setSchemaResultProto = + icingSearchEngine.setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false); + assertStatusOk(setSchemaResultProto.getStatus()); + + GetSchemaResultProto getSchemaResultProto = icingSearchEngine.getSchema(); + assertStatusOk(getSchemaResultProto.getStatus()); + assertThat(getSchemaResultProto.getSchema()).isEqualTo(schema); + + GetSchemaTypeResultProto getSchemaTypeResultProto = + icingSearchEngine.getSchemaType(emailTypeConfig.getSchemaType()); + assertStatusOk(getSchemaTypeResultProto.getStatus()); + assertThat(getSchemaTypeResultProto.getSchemaTypeConfig()).isEqualTo(emailTypeConfig); + } + + // TODO(b/337913932) re-enable this test after we preregister this API in jni + @Ignore + @Test + public void setAndGetSchemaWithDatabase_ok() throws Exception { + IcingSearchEngineOptions options = + IcingSearchEngineOptions.newBuilder() + .setBaseDir(tempDir.getCanonicalPath()) + .setEnableSchemaDatabase(true) + .build(); + IcingSearchEngine icingSearchEngine = new IcingSearchEngine(options); + assertStatusOk(icingSearchEngine.initialize().getStatus()); + + String db1 = "db1"; + String db2 = "db2"; + SchemaProto db1Schema = + SchemaProto.newBuilder().addTypes(createEmailTypeConfigWithDatabase(db1)).build(); + SchemaProto db2Schema = + SchemaProto.newBuilder().addTypes(createEmailTypeConfigWithDatabase(db2)).build(); + + SetSchemaResultProto setSchemaResultProto = + icingSearchEngine.setSchema(db1Schema, /* ignoreErrorsAndDeleteDocuments= */ false); + assertStatusOk(setSchemaResultProto.getStatus()); + setSchemaResultProto = + icingSearchEngine.setSchema(db2Schema, /* ignoreErrorsAndDeleteDocuments= */ false); + assertStatusOk(setSchemaResultProto.getStatus()); + + // Get schema for individual databases. + GetSchemaResultProto getSchemaResultProto = icingSearchEngine.getSchemaForDatabase(db1); + assertStatusOk(getSchemaResultProto.getStatus()); + assertThat(getSchemaResultProto.getSchema()).isEqualTo(db1Schema); + + getSchemaResultProto = icingSearchEngine.getSchemaForDatabase(db2); + assertStatusOk(getSchemaResultProto.getStatus()); + assertThat(getSchemaResultProto.getSchema()).isEqualTo(db2Schema); + + // The getSchema() API should still return the full schema. + SchemaProto fullSchema = + SchemaProto.newBuilder() + .addTypes(createEmailTypeConfigWithDatabase(db1)) + .addTypes(createEmailTypeConfigWithDatabase(db2)) + .build(); + getSchemaResultProto = icingSearchEngine.getSchema(); + assertStatusOk(getSchemaResultProto.getStatus()); + assertThat(getSchemaResultProto.getSchema()).isEqualTo(fullSchema); + } + + @Test public void testPutAndGetDocuments() throws Exception { assertStatusOk(icingSearchEngine.initialize().getStatus()); @@ -171,7 +263,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -194,7 +286,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -237,7 +329,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -313,8 +405,8 @@ byte[] digest = calculateDigest(data); PropertyProto.BlobHandleProto blobHandle = PropertyProto.BlobHandleProto.newBuilder() - .setLabel("label") .setDigest(ByteString.copyFrom(digest)) + .setNamespace("namespace") .build(); // 2 Act: write the blob and read it back. @@ -351,6 +443,52 @@ assertThat(output).isEqualTo(data); } + @Ignore // b/350530146 + @Test + public void removeBlob() throws Exception { + // 1 Arrange: set up IcingSearchEngine with and blob data + File tempDir = temporaryFolder.newFolder(); + IcingSearchEngineOptions options = + IcingSearchEngineOptions.newBuilder() + .setBaseDir(tempDir.getCanonicalPath()) + .setEnableBlobStore(true) + .build(); + IcingSearchEngine icing = new IcingSearchEngine(options); + assertStatusOk(icing.initialize().getStatus()); + + byte[] data = generateRandomBytes(100); // 10 Bytes + byte[] digest = calculateDigest(data); + PropertyProto.BlobHandleProto blobHandle = + PropertyProto.BlobHandleProto.newBuilder() + .setNamespace("ns") + .setDigest(ByteString.copyFrom(digest)) + .build(); + + // 2 Act: write the blob and read it back. + BlobProto openWriteBlobProto = icing.openWriteBlob(blobHandle); + assertStatusOk(openWriteBlobProto.getStatus()); + Field field = FileDescriptor.class.getDeclaredField("fd"); + field.setAccessible(true); // Make the field accessible + + // Create a new FileDescriptor object + FileDescriptor writeFd = new FileDescriptor(); + + // Set the file descriptor value using reflection + field.setInt(writeFd, openWriteBlobProto.getFileDescriptor()); + + try (FileOutputStream outputStream = new FileOutputStream(writeFd)) { + outputStream.write(data); + } + + // Remove the blob. + BlobProto removeBlobProto = icing.removeBlob(blobHandle); + assertStatusOk(removeBlobProto.getStatus()); + + // Commit will not found. + BlobProto commitBlobProto = icing.commitBlob(blobHandle); + assertThat(commitBlobProto.getStatus().getCode()).isEqualTo(StatusProto.Code.NOT_FOUND); + } + @Test public void testDelete() throws Exception { assertStatusOk(icingSearchEngine.initialize().getStatus()); @@ -359,7 +497,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -383,7 +521,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -408,7 +546,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -433,7 +571,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -488,7 +626,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -513,7 +651,7 @@ .build(); DeleteByQueryResultProto deleteResultProto = - icingSearchEngine.deleteByQuery(searchSpec, /*returnDeletedDocumentInfo=*/ true); + icingSearchEngine.deleteByQuery(searchSpec, /* returnDeletedDocumentInfo= */ true); assertStatusOk(deleteResultProto.getStatus()); DeleteByQueryResultProto.DocumentGroupInfo info = DeleteByQueryResultProto.DocumentGroupInfo.newBuilder() @@ -574,7 +712,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -603,7 +741,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -633,7 +771,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK); @@ -660,7 +798,9 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(createEmailTypeConfig()).build(); assertStatusOk( - icingSearchEngine.setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false).getStatus()); + icingSearchEngine + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) + .getStatus()); // String: "夊ćŻčç" // ^ ^^ ^ @@ -728,7 +868,9 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(createEmailTypeConfig()).build(); assertStatusOk( - icingSearchEngine.setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false).getStatus()); + icingSearchEngine + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) + .getStatus()); // String: "đđ đđ đ" // ^ ^ ^ @@ -797,7 +939,7 @@ SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); assertThat( icingSearchEngine - .setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false) + .setSchema(schema, /* ignoreErrorsAndDeleteDocuments= */ false) .getStatus() .getCode()) .isEqualTo(StatusProto.Code.OK);
diff --git a/proto/icing/proto/blob.proto b/proto/icing/proto/blob.proto index f2cfe0d..9ff4fc5 100644 --- a/proto/icing/proto/blob.proto +++ b/proto/icing/proto/blob.proto
@@ -16,6 +16,7 @@ package icing.lib; +import "icing/proto/document.proto"; import "icing/proto/status.proto"; option java_package = "com.google.android.icing.proto"; @@ -38,3 +39,20 @@ // The file decriptor of the blob file from Icing. optional int32 file_descriptor = 2; } + +// BlobInfo holds information about a blob. It is used to store in the blob +// info proto log file. +// Next tag: 4 +message BlobInfoProto { + // The blob handle that represents the blob. + optional PropertyProto.BlobHandleProto blob_handle = 1; + + // The creation time of the blob. This is used to determine when to delete + // the orphaned blobs. + // We are using creation_time_ms to be file name of the blob, so this field + // is unique for each blob. + optional int64 creation_time_ms = 2; + + // Whether the blob is committed and ready to be read. + optional bool is_committed = 3; +}
diff --git a/proto/icing/proto/document.proto b/proto/icing/proto/document.proto index 0fee0c0..18ba95f 100644 --- a/proto/icing/proto/document.proto +++ b/proto/icing/proto/document.proto
@@ -105,12 +105,15 @@ repeated VectorProto vector_values = 8; // Holds a blob handle field of the property. - // Next tag: 3 + // Next tag: 4 message BlobHandleProto { // The sha-256 hashed digest for a blob. optional bytes digest = 1; - // The label of the blob. - optional string label = 2; + + // The namespace of the blob. + optional string namespace = 3; + + reserved 2; } repeated BlobHandleProto blob_handle_values = 9; }
diff --git a/proto/icing/proto/initialize.proto b/proto/icing/proto/initialize.proto index 102a59d..35290d0 100644 --- a/proto/icing/proto/initialize.proto +++ b/proto/icing/proto/initialize.proto
@@ -23,7 +23,7 @@ option java_multiple_files = true; option objc_class_prefix = "ICNG"; -// Next tag: 7 +// Next tag: 8 message IcingSearchEngineFeatureInfoProto { // REQUIRED: Enum representing an IcingLib feature flagged using // IcingSearchEngineOptions @@ -41,6 +41,38 @@ // feature, and related metadata hits indexing used for property existence // check. FEATURE_HAS_PROPERTY_OPERATOR = 1; + + // Feature for flag IcingSearchEngineOptions::enable_embedding_index. + // + // This feature covers the embedding index. + FEATURE_EMBEDDING_INDEX = 2; + + // Feature for flag IcingSearchEngineOptions::enable_scorable_properties. + // + // This feature covers the scorable properties feature. + FEATURE_SCORABLE_PROPERTIES = 3; + + // Feature for flag IcingSearchEngineOptions::enable_embedding_quantization. + // + // This feature covers whether to enable quantization for embedding vectors. + FEATURE_EMBEDDING_QUANTIZATION = 4; + + // Feature for flag IcingSearchEngineOptions::enable_schema_database. + // + // This feature covers whether to enable the database field in the schema. + // Once enabled, SetSchema will only allow setting schema types from a + // single database field at a time. + FEATURE_SCHEMA_DATABASE = 5; + + // Feature for flag IcingSearchEngineOptions:: + // enable_qualified_id_join_index_v3_and_delete_propagate_from. + // + // This feature covers whether to enable the join index v3 and support + // delete propagation PROPAGATE_FROM. Once enabled, join index v3 will be + // rebuilt to replace v2, and deleteing a document will also delete its + // child document(s) which refer to it via a joinable property with delete + // propagation type PROPAGATE_FROM. + FEATURE_QUALIFIED_ID_JOIN_INDEX_V3_AND_DELETE_PROPAGATE_FROM = 6; } // Whether the feature requires the document store to be rebuilt. @@ -62,6 +94,10 @@ // Whether the feature requires the qualified id join index to be rebuilt. // The default value is false. optional bool needs_qualified_id_join_index_rebuild = 6; + + // Whether the feature requires the embedding index to be rebuilt. + // The default value is false. + optional bool needs_embedding_index_rebuild = 7; } // Next tag: 4 @@ -73,7 +109,7 @@ repeated IcingSearchEngineFeatureInfoProto enabled_features = 3; } -// Next tag: 18 +// Next tag: 26 message IcingSearchEngineOptions { // Directory to persist files for Icing. Required. // If Icing was previously initialized with this directory, it will reload @@ -174,6 +210,7 @@ // indexing latency. optional int32 lite_index_sort_size = 13 [default = 8192]; // 8 KiB + // DEPRECATED: qualified id join index v2 is fully rolled out. optional bool use_new_qualified_id_join_index = 14; // Whether to build the metadata hits used for property existence check, which @@ -189,7 +226,58 @@ // If set to 0, the blob will never be deleted. optional int64 orphan_blob_time_to_live_ms = 17; - reserved 2; + // Whether to enable schema database. + // + // If set to true, the schema database field will be used to store and + // retrieve schemas, and SetSchema will only allow setting schema types from + // a single database field at a time. + // + // Icing will automatically handle migrating the stored SchemaProto and + // populate the database field when this flag is flipped from false to true. + // + // TODO - b/337913932: Remove this flag once the schema database is fully + // rolled out. + optional bool enable_schema_database = 18; + + // Whether to enable embedding index. + // If set to false, the EmbeddingIndex will only be created with a header, and + // embedding properties will not be indexed. + // TODO(b/326656531): Change the default value to false when the flag is + // propagated from AppSearch. + optional bool enable_embedding_index = 19 [default = true]; + + // Feature flag guarding the new scorable properties feature. + // TODO - b/357105837: Remove this flag once the feature is fully rolled out. + optional bool enable_scorable_properties = 21; + + // Whether to enable quantization for embedding vectors. + // If set to false, all embedding vectors will not be quantized. Otherwise, + // quantization will be controlled by the quantization type specified in the + // schema. + optional bool enable_embedding_quantization = 22; + + // Level of blob_info_file_log store compression in blob store , + // NO_COMPRESSION = 0, BEST_SPEED = 1, BEST_COMPRESSION = 9 + // Valid values: [0, 9] + // Optional. + optional int32 blob_store_compression_level = 23 [default = 3]; + + // Whether to allow repeated fields to have a joinable value type. + optional bool enable_repeated_field_joins = 24; + + // Whether to use qualified id join index v3 and enable delete propagation + // PROPAGATE_FROM. + // + // - If set to true, qualified id join index v3 will be created and delete + // propagation PROPAGATE_FROM will be enabled. + // - Otherwise, qualified id join index v2 will be created and delete + // propagation will be disabled. + // + // The default value is false. + optional bool enable_qualified_id_join_index_v3_and_delete_propagate_from = + 25; + + reserved 2, 20; } // Result of a call to IcingSearchEngine.Initialize
diff --git a/proto/icing/proto/internal/scorable_property_set.proto b/proto/icing/proto/internal/scorable_property_set.proto index 13d1101..fd6f573 100644 --- a/proto/icing/proto/internal/scorable_property_set.proto +++ b/proto/icing/proto/internal/scorable_property_set.proto
@@ -34,38 +34,107 @@ // proto. This will be stored in the icing document store to allow faster // access of scorable properties during the scoring/ranking phase. // -// TODO(b/357105837): Always use the util class(link TBD after cl/660373640 is -// submitted) to interpret and populate this proto. +// Use icing::lib::scorable_property_set.h to build/interpret this proto. // // See go/appsearch-docjoin-dd for the design doc. // // Next tag: 2 message ScorablePropertySetProto { - // The order of the properties in this proto should match the order of the - // scorable properties in the schema proto. + // The order of the properties in this proto should match the lexicographical + // order of the scorable properties paths in the schema proto. + // + // All values under a property path will be merged into a list. // // For example, if you have a schema like: - // SchemaTypeConfigBuilder() - // .setType('email') - // .AddProperty('subject') // not scorable - // .AddProperty('importance') // scorable - // .AddProperty('title') // not scorable - // .AddProperty('score') // scorable + // SchemaBuilder() + // .AddType( + // SchemaTypeConfigBuilder() + // .setType('person') + // .AddProperty('name') // not scorable + // .AddProperty('age') // scorable + // .AddProperty('address') // not scorable + // .AddProperty('income')) // scorable + // .AddType( + // SchemaTypeConfigBuilder() + // .setType('email') + // .AddProperty('subject') // not scorable + // .AddProperty('importance') // scorable + // .AddProperty('title') // not scorable + // .AddProperty('score') // scorable + // .AddDocumentProperty('receiver')) // scorable, person type + // .AddDocumentProperty('sender')) // scorable, person type // // And an email document like: - // Document email_document = - // DocumentBuilder() - // .AddStringProperty('subject', 'foo') - // .AddDoubleProperty('score', 1.5, 2.5, 3.5) - // .Build(); + // Document email_document { + // schema: 'email' + // properties: { + // name: 'subject' + // string_values: 'foo' + // } + // properties: { + // name: 'score' + // double_values: [1.5, 2.5, 3.5] + // } + // properties: { + // name: 'receiver + // document_values: { + // properties: { + // name: 'age' + // int64_values: [30] + // } + // properties: { + // name: 'income' + // double_values: [10000, 20000] + // } + // properties: { + // name: 'isStarred' + // boolean_values: [true] + // } + // } + // document_values: { + // properties: { + // name: 'age' + // int64_values: [40] + // } + // properties: { + // name: 'income' + // double_values: [30000, 40000] + // } + // properties: { + // name: 'isStarred' + // boolean_values: [false] + // } + // } + // } + // } // // When email_document is put into the icing document store, the following // proto will be generated and stored in icing: // { - // properties: {} // empty entry for the 'importance' property + // # For the 'importance' property + // properties: {} + // # For the 'receiver.age' property // properties: { - // double_values: [1.5, 2.5, 3.5] // The 'score' property + // int64_values: [30, 40] // } + // # For the 'receiver.income' property + // properties: { + // double_values: [10000, 20000, 30000, 40000] + // } + // # For the 'receiver.isStarred' property + // properties: { + // boolean_values: [true, false] + // } + // # For the 'score' property + // properties: { + // double_values: [1.5, 2.5, 3.5] + // } + // # For the 'sender.age' property + // properties: {} + // # For the 'sender.income' property + // properties: {} + // # For the 'sender.isStarred' property + // properties: {} // } // // During the scoring/ranking phase, the implicit property order can be used
diff --git a/proto/icing/proto/schema.proto b/proto/icing/proto/schema.proto index e547c4e..2a461bf 100644 --- a/proto/icing/proto/schema.proto +++ b/proto/icing/proto/schema.proto
@@ -34,10 +34,10 @@ // TODO(cassiewang) Define a sample proto file that can be used by tests and for // documentation. // -// Next tag: 8 +// Next tag: 9 message SchemaTypeConfigProto { - // REQUIRED: Named type that uniquely identifies the structured, logical - // schema being defined. + // REQUIRED: Named type that identifies the structured, logical schema being + // defined. // // Recommended format: Human readable string that's one of the types defined // in http://schema.org. Eg: DigitalDocument, Message, Person, etc. @@ -50,6 +50,16 @@ // future retrieval. optional string description = 7; + // OPTIONAL: Identifies a database that the schema type belongs to. This + // groups schema types that are related to each other, which is useful for + // setting or retrieving a subset of schema types. If unset, the schema type + // will be considered as part of the default empty database group. + // + // NOTE: Only schemas from one database can be set in a single SetSchema call. + // Please use multiple SetSchema calls if you want to set schemas across + // multiple databases. + optional string database = 8; + // List of all properties that are supported by Documents of this type. // An Document should never have properties that are not listed here. // @@ -210,11 +220,29 @@ } } optional EmbeddingIndexingType.Code embedding_indexing_type = 1; + + // OPTIONAL: Indicates whether the vector contents of this property should be + // quantized. Quantization can reduce the size of the embedding search index, + // potentially leading to faster embedding search due to lower I/O bandwidth. + // + // Quantization is usually very reliable and in most cases will have a + // negligible impact on recall. Using quantization is strongly recommended. + // + // The default value is NONE. + message QuantizationType { + enum Code { + // Contents in this property will not be quantized. + NONE = 0; + // Contents in this property will be quantized to 8 bits. + QUANTIZE_8_BIT = 1; + } + } + optional QuantizationType.Code quantization_type = 2; } // Describes how a property can be used to join this document with another // document. See JoinSpecProto (in search.proto) for more details. -// Next tag: 3 +// Next tag: 4 message JoinableConfig { // OPTIONAL: Indicates what joinable type the content value of this property // is. @@ -234,10 +262,25 @@ } optional ValueType.Code value_type = 1; - // If the parent document a child document is joined to is deleted, delete the - // child document as well. This will only apply to children joined through - // QUALIFIED_ID, other (future) joinable value types won't use it. - optional bool propagate_delete = 2 [default = false]; + // OPTIONAL: Indicates how to propagate the deletion between the document and + // the (referenced) joinable document. + // + // The default value is NONE. + // + // If delete propagation is enabled (i.e. not NONE), then value_type must be + // QUALIFIED_ID. + message DeletePropagationType { + enum Code { + // No delete propagation. + NONE = 0; + + // Propagate delete from the referenced document to the doucument. + PROPAGATE_FROM = 1; + } + } + optional DeletePropagationType.Code delete_propagation_type = 3; + + reserved 2; } // Describes the schema of a single property of Documents that belong to a
diff --git a/proto/icing/proto/scoring.proto b/proto/icing/proto/scoring.proto index bd05a86..5b83d09 100644 --- a/proto/icing/proto/scoring.proto +++ b/proto/icing/proto/scoring.proto
@@ -25,7 +25,7 @@ // Encapsulates the configurations on how Icing should score and rank the search // results. // TODO(b/170347684): Change all timestamps to seconds. -// Next tag: 6 +// Next tag: 8 message ScoringSpecProto { // OPTIONAL: Indicates how the search results will be ranked. message RankingStrategy { @@ -113,6 +113,43 @@ // To set this field, the ranking strategy must be set to // ADVANCED_SCORING_EXPRESSION. repeated string additional_advanced_scoring_expressions = 5; + + // OPTIONAL: Specifies the schema type alias map for advanced scoring + // expression. + // + // The alias map must be provided for all alias schema types used in the + // getScorableProperty function of the advanced scoring expression. Otherwise, + // an error will be returned. + // + // For example, Icing clients can pass a SchemaTypeAliasMapProto to icing: + // schema_type_alias_for_advanced_scoring { + // alias_schema_type: "person" + // schema_types: "package1$database1/person" + // schema_types: "package2$database1/person" + // } + // schema_type_alias_for_advanced_scoring { + // alias_schema_type: "email" + // schema_types: "package3$database2/email" + // schema_types: "package4$database2/email" + // } + // + // When Icing processes the advanced scoring expression such as: + // "getScorableProperty('person', 'rfsScore')", + // Icing will + // 1. first look up documents with schema type "person". + // 2. if the SchemaTypeAliasMapProto is provided, Icing will also look up + // documents with schema types from: + // - "package1$database1/person" + // - "package2$database1/person" + // + // TODO(b/357105837): Consider moving this to a higher level proto. + // Currently, ScoringSpec from both parent and child queries need to provide + // this proto, thus introducing duplicates. We can consider moving this to a + // higher level proto and send it to downstreams. + repeated SchemaTypeAliasMapProto schema_type_alias_map_protos = 6; + + // Features enabled for scoring. + repeated ScoringFeatureType scoring_feature_types_enabled = 7 [packed = true]; } // Next tag: 3 @@ -173,3 +210,21 @@ // By default, a property is given a raw, pre-normalized weight of 1.0. optional double weight = 2; } + +// Proto that maps an alias schema type to a list of Icing schema types. +// +// Next tag: 3 +message SchemaTypeAliasMapProto { + // Alias schema type provided by Icing clients. + optional string alias_schema_type = 1; + // Schema types in Icing. + repeated string schema_types = 2; +} + +// Next tag: 2 +enum ScoringFeatureType { + SCORING_FEATURE_TYPE_UNKNOWN = 0; + + // Ranking with getScorableProperty(), in advanced scoring expression. + SCORABLE_PROPERTY_RANKING = 1; +}
diff --git a/proto/icing/proto/search.proto b/proto/icing/proto/search.proto index 6271ecd..45e3736 100644 --- a/proto/icing/proto/search.proto +++ b/proto/icing/proto/search.proto
@@ -27,7 +27,7 @@ option objc_class_prefix = "ICNG"; // Client-supplied specifications on what documents to retrieve. -// Next tag: 14 +// Next tag: 15 message SearchSpecProto { // REQUIRED: The "raw" query string that users may type. For example, "cat" // will search for documents with the term cat in it. @@ -73,6 +73,13 @@ // schema filters will not be expanded for polymorphism. repeated string schema_type_filters = 4; + // OPTIONAL: Only search for documents under the specified namespaces and + // uris. If unset, all documents will be searched. + // + // All given NamespaceDocumentUriGroup must specify at least one + // document_uri. To exclude a namespace, use namespace_filters instead. + repeated NamespaceDocumentUriGroup document_uri_filters = 14; + // Timestamp taken just before sending proto across the JNI boundary from java // to native side. optional int64 java_to_native_start_timestamp_ms = 5;
diff --git a/proto/icing/proto/storage.proto b/proto/icing/proto/storage.proto index e0323a1..266576d 100644 --- a/proto/icing/proto/storage.proto +++ b/proto/icing/proto/storage.proto
@@ -52,7 +52,7 @@ // LINT.ThenChange() } -// Next tag: 15 +// Next tag: 16 message DocumentStorageInfoProto { // Total number of alive documents. optional int32 num_alive_documents = 1; @@ -100,6 +100,10 @@ // is encountered while calculating this field. optional int64 namespace_id_mapper_size = 12; + // Size of the scorable properties cache in bytes. Will be set to -1 if an IO + // error is encountered while calculating this field. + optional int64 scorable_property_cache_size = 15; + // Number of namespaces seen from the current documents. // // TODO(cassiewang): This isn't technically needed anymore since clients can @@ -159,7 +163,19 @@ optional float min_free_fraction = 8; } -// Next tag: 5 +// Next tag: 4 +message NamespaceBlobStorageInfoProto { + // The package name of who own the blobs. + optional string namespace = 1; + + // Total size of blobs storage of this package in bytes. + optional int64 blob_size = 2; + + // Total number of blobs of this package. + optional int32 num_blobs = 3; +} + +// Next tag: 6 message StorageInfoProto { // Total size of Icing’s storage in bytes. Will be set to -1 if an IO error is // encountered while calculating this field. @@ -173,6 +189,9 @@ // Storage information of the index. optional IndexStorageInfoProto index_storage_info = 4; + + // Storage information of the BlobStore. + repeated NamespaceBlobStorageInfoProto namespace_blob_storage_info = 5; } // Next tag: 3
diff --git a/synced_AOSP_CL_number.txt b/synced_AOSP_CL_number.txt index 5b9c321..e32586a 100644 --- a/synced_AOSP_CL_number.txt +++ b/synced_AOSP_CL_number.txt
@@ -1 +1 @@ -set(synced_AOSP_CL_number=669068792) +set(synced_AOSP_CL_number=702081687)