blob: ebdfe9602c8a6eafe1bed8c2325343d33b33c743 [file] [log] [blame]
/*
* Copyright 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @exportToFramework:skipFile()
package androidx.appsearch.localstorage;
import static androidx.appsearch.app.AppSearchResult.throwableToFailedResult;
import android.os.SystemClock;
import android.util.Log;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.appsearch.app.AppSearchBatchResult;
import androidx.appsearch.app.AppSearchSession;
import androidx.appsearch.app.Capabilities;
import androidx.appsearch.app.GenericDocument;
import androidx.appsearch.app.GetByDocumentIdRequest;
import androidx.appsearch.app.GetSchemaResponse;
import androidx.appsearch.app.Migrator;
import androidx.appsearch.app.PackageIdentifier;
import androidx.appsearch.app.PutDocumentsRequest;
import androidx.appsearch.app.RemoveByDocumentIdRequest;
import androidx.appsearch.app.ReportUsageRequest;
import androidx.appsearch.app.SearchResults;
import androidx.appsearch.app.SearchSpec;
import androidx.appsearch.app.SetSchemaRequest;
import androidx.appsearch.app.SetSchemaResponse;
import androidx.appsearch.app.StorageInfo;
import androidx.appsearch.exceptions.AppSearchException;
import androidx.appsearch.localstorage.stats.OptimizeStats;
import androidx.appsearch.localstorage.stats.RemoveStats;
import androidx.appsearch.localstorage.stats.SchemaMigrationStats;
import androidx.appsearch.localstorage.stats.SetSchemaStats;
import androidx.appsearch.localstorage.util.FutureUtil;
import androidx.appsearch.util.SchemaMigrationUtil;
import androidx.collection.ArrayMap;
import androidx.collection.ArraySet;
import androidx.core.util.Preconditions;
import com.google.android.icing.proto.PersistType;
import com.google.common.util.concurrent.ListenableFuture;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
/**
* An implementation of {@link AppSearchSession} which stores data locally in the app's storage
* space using a bundled version of the search native library.
*
* <p>Queries are executed multi-threaded, but a single thread is used for mutate requests (put,
* delete, etc..).
*/
class SearchSessionImpl implements AppSearchSession {
private static final String TAG = "AppSearchSessionImpl";
private final AppSearchImpl mAppSearchImpl;
private final Executor mExecutor;
private final Capabilities mCapabilities;
private final String mPackageName;
private final String mDatabaseName;
private volatile boolean mIsMutated = false;
private volatile boolean mIsClosed = false;
@Nullable private final AppSearchLogger mLogger;
SearchSessionImpl(
@NonNull AppSearchImpl appSearchImpl,
@NonNull Executor executor,
@NonNull Capabilities capabilities,
@NonNull String packageName,
@NonNull String databaseName,
@Nullable AppSearchLogger logger) {
mAppSearchImpl = Preconditions.checkNotNull(appSearchImpl);
mExecutor = Preconditions.checkNotNull(executor);
mCapabilities = Preconditions.checkNotNull(capabilities);
mPackageName = packageName;
mDatabaseName = Preconditions.checkNotNull(databaseName);
mLogger = logger;
}
@Override
@NonNull
public ListenableFuture<SetSchemaResponse> setSchema(
@NonNull SetSchemaRequest request) {
Preconditions.checkNotNull(request);
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
ListenableFuture<SetSchemaResponse> future = execute(() -> {
long startMillis = SystemClock.elapsedRealtime();
// Convert the inner set into a List since Binder can't handle Set.
Map<String, Set<PackageIdentifier>> schemasVisibleToPackages =
request.getSchemasVisibleToPackagesInternal();
Map<String, List<PackageIdentifier>> copySchemasVisibleToPackages = new ArrayMap<>();
for (Map.Entry<String, Set<PackageIdentifier>> entry :
schemasVisibleToPackages.entrySet()) {
copySchemasVisibleToPackages.put(entry.getKey(),
new ArrayList<>(entry.getValue()));
}
SetSchemaStats.Builder setSchemaStatsBuilder = null;
if (mLogger != null) {
setSchemaStatsBuilder = new SetSchemaStats.Builder(mPackageName, mDatabaseName);
}
Map<String, Migrator> migrators = request.getMigrators();
// No need to trigger migration if user never set migrator.
if (migrators.size() == 0) {
SetSchemaResponse setSchemaResponse =
setSchemaNoMigrations(request, copySchemasVisibleToPackages,
setSchemaStatsBuilder);
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - startMillis));
mLogger.logStats(setSchemaStatsBuilder.build());
}
return setSchemaResponse;
}
// Migration process
// 1. Validate and retrieve all active migrators.
GetSchemaResponse getSchemaResponse =
mAppSearchImpl.getSchema(mPackageName, mDatabaseName);
int currentVersion = getSchemaResponse.getVersion();
int finalVersion = request.getVersion();
Map<String, Migrator> activeMigrators = SchemaMigrationUtil.getActiveMigrators(
getSchemaResponse.getSchemas(), migrators, currentVersion, finalVersion);
// No need to trigger migration if no migrator is active.
if (activeMigrators.size() == 0) {
SetSchemaResponse setSchemaResponse =
setSchemaNoMigrations(request, copySchemasVisibleToPackages,
setSchemaStatsBuilder);
if (setSchemaStatsBuilder != null) {
setSchemaStatsBuilder.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - startMillis));
mLogger.logStats(setSchemaStatsBuilder.build());
}
return setSchemaResponse;
}
// 2. SetSchema with forceOverride=false, to retrieve the list of incompatible/deleted
// types.
long firstSetSchemaLatencyStartMillis = SystemClock.elapsedRealtime();
SetSchemaResponse setSchemaResponse = mAppSearchImpl.setSchema(
mPackageName,
mDatabaseName,
new ArrayList<>(request.getSchemas()),
/*visibilityStore=*/ null,
new ArrayList<>(request.getSchemasNotDisplayedBySystem()),
copySchemasVisibleToPackages,
/*forceOverride=*/false,
request.getVersion(),
setSchemaStatsBuilder);
// 3. If forceOverride is false, check that all incompatible types will be migrated.
// If some aren't we must throw an error, rather than proceeding and deleting those
// types.
long queryAndTransformLatencyStartMillis = SystemClock.elapsedRealtime();
if (!request.isForceOverride()) {
SchemaMigrationUtil.checkDeletedAndIncompatibleAfterMigration(setSchemaResponse,
activeMigrators.keySet());
}
SchemaMigrationStats.Builder schemaMigrationStatsBuilder = null;
if (setSchemaStatsBuilder != null) {
schemaMigrationStatsBuilder = new SchemaMigrationStats.Builder();
}
try (AppSearchMigrationHelper migrationHelper = new AppSearchMigrationHelper(
mAppSearchImpl, mPackageName, mDatabaseName, request.getSchemas())) {
// 4. Trigger migration for all activity migrators.
migrationHelper.queryAndTransform(activeMigrators, currentVersion, finalVersion,
schemaMigrationStatsBuilder);
// 5. SetSchema a second time with forceOverride=true if the first attempted failed
// due to backward incompatible changes.
long secondSetSchemaLatencyStartMillis = SystemClock.elapsedRealtime();
if (!setSchemaResponse.getIncompatibleTypes().isEmpty()
|| !setSchemaResponse.getDeletedTypes().isEmpty()) {
setSchemaResponse = mAppSearchImpl.setSchema(
mPackageName,
mDatabaseName,
new ArrayList<>(request.getSchemas()),
/*visibilityStore=*/ null,
new ArrayList<>(request.getSchemasNotDisplayedBySystem()),
copySchemasVisibleToPackages,
/*forceOverride=*/ true,
request.getVersion(),
setSchemaStatsBuilder);
}
SetSchemaResponse.Builder responseBuilder = setSchemaResponse.toBuilder()
.addMigratedTypes(activeMigrators.keySet());
mIsMutated = true;
// 6. Put all the migrated documents into the index, now that the new schema is set.
long saveDocumentLatencyStartMillis = SystemClock.elapsedRealtime();
SetSchemaResponse finalSetSchemaResponse =
migrationHelper.readAndPutDocuments(responseBuilder,
schemaMigrationStatsBuilder);
if (schemaMigrationStatsBuilder != null) {
long endMillis = SystemClock.elapsedRealtime();
schemaMigrationStatsBuilder
.setSaveDocumentLatencyMillis(
(int) (endMillis - saveDocumentLatencyStartMillis))
.setGetSchemaLatencyMillis(
(int) (firstSetSchemaLatencyStartMillis - startMillis))
.setFirstSetSchemaLatencyMillis(
(int) (queryAndTransformLatencyStartMillis
- firstSetSchemaLatencyStartMillis))
.setQueryAndTransformLatencyMillis(
(int) (secondSetSchemaLatencyStartMillis
- queryAndTransformLatencyStartMillis))
.setSecondSetSchemaLatencyMillis(
(int) (saveDocumentLatencyStartMillis
- secondSetSchemaLatencyStartMillis));
setSchemaStatsBuilder
.setSchemaMigrationStats(
schemaMigrationStatsBuilder.build())
.setTotalLatencyMillis((int) (endMillis - startMillis));
mLogger.logStats(setSchemaStatsBuilder.build());
}
return finalSetSchemaResponse;
}
});
// setSchema will sync the schemas in the request to AppSearch, any existing schemas which
// is not included in the request will be delete if we force override incompatible schemas.
// And all documents of these types will be deleted as well. We should checkForOptimize for
// these deletion.
checkForOptimize();
return future;
}
@Override
@NonNull
public ListenableFuture<GetSchemaResponse> getSchema() {
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
return execute(() -> mAppSearchImpl.getSchema(mPackageName, mDatabaseName));
}
@NonNull
@Override
public ListenableFuture<Set<String>> getNamespaces() {
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
return execute(() -> {
List<String> namespaces = mAppSearchImpl.getNamespaces(mPackageName, mDatabaseName);
return new ArraySet<>(namespaces);
});
}
@Override
@NonNull
public ListenableFuture<AppSearchBatchResult<String, Void>> put(
@NonNull PutDocumentsRequest request) {
Preconditions.checkNotNull(request);
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
ListenableFuture<AppSearchBatchResult<String, Void>> future = execute(() -> {
AppSearchBatchResult.Builder<String, Void> resultBuilder =
new AppSearchBatchResult.Builder<>();
for (int i = 0; i < request.getGenericDocuments().size(); i++) {
GenericDocument document = request.getGenericDocuments().get(i);
try {
mAppSearchImpl.putDocument(mPackageName, mDatabaseName, document, mLogger);
resultBuilder.setSuccess(document.getId(), /*value=*/ null);
} catch (Throwable t) {
resultBuilder.setResult(document.getId(), throwableToFailedResult(t));
}
}
// Now that the batch has been written. Persist the newly written data.
mAppSearchImpl.persistToDisk(PersistType.Code.LITE);
mIsMutated = true;
return resultBuilder.build();
});
// The existing documents with same ID will be deleted, so there may be some resources that
// could be released after optimize().
checkForOptimize(/*mutateBatchSize=*/ request.getGenericDocuments().size());
return future;
}
@Override
@NonNull
public ListenableFuture<AppSearchBatchResult<String, GenericDocument>> getByDocumentId(
@NonNull GetByDocumentIdRequest request) {
Preconditions.checkNotNull(request);
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
return execute(() -> {
AppSearchBatchResult.Builder<String, GenericDocument> resultBuilder =
new AppSearchBatchResult.Builder<>();
Map<String, List<String>> typePropertyPaths = request.getProjectionsInternal();
for (String id : request.getIds()) {
try {
GenericDocument document =
mAppSearchImpl.getDocument(mPackageName, mDatabaseName,
request.getNamespace(), id, typePropertyPaths);
resultBuilder.setSuccess(id, document);
} catch (Throwable t) {
resultBuilder.setResult(id, throwableToFailedResult(t));
}
}
return resultBuilder.build();
});
}
@Override
@NonNull
public SearchResults search(
@NonNull String queryExpression,
@NonNull SearchSpec searchSpec) {
Preconditions.checkNotNull(queryExpression);
Preconditions.checkNotNull(searchSpec);
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
return new SearchResultsImpl(
mAppSearchImpl,
mExecutor,
mPackageName,
mDatabaseName,
queryExpression,
searchSpec,
mLogger);
}
@Override
@NonNull
public ListenableFuture<Void> reportUsage(@NonNull ReportUsageRequest request) {
Preconditions.checkNotNull(request);
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
return execute(() -> {
mAppSearchImpl.reportUsage(
mPackageName,
mDatabaseName,
request.getNamespace(),
request.getDocumentId(),
request.getUsageTimestampMillis(),
/*systemUsage=*/ false);
mIsMutated = true;
return null;
});
}
@Override
@NonNull
public ListenableFuture<AppSearchBatchResult<String, Void>> remove(
@NonNull RemoveByDocumentIdRequest request) {
Preconditions.checkNotNull(request);
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
ListenableFuture<AppSearchBatchResult<String, Void>> future = execute(() -> {
AppSearchBatchResult.Builder<String, Void> resultBuilder =
new AppSearchBatchResult.Builder<>();
for (String id : request.getIds()) {
RemoveStats.Builder removeStatsBuilder = null;
if (mLogger != null) {
removeStatsBuilder = new RemoveStats.Builder(mPackageName, mDatabaseName);
}
try {
mAppSearchImpl.remove(mPackageName, mDatabaseName, request.getNamespace(), id,
removeStatsBuilder);
resultBuilder.setSuccess(id, /*value=*/null);
} catch (Throwable t) {
resultBuilder.setResult(id, throwableToFailedResult(t));
} finally {
if (mLogger != null) {
mLogger.logStats(removeStatsBuilder.build());
}
}
}
// Now that the batch has been written. Persist the newly written data.
mAppSearchImpl.persistToDisk(PersistType.Code.LITE);
mIsMutated = true;
return resultBuilder.build();
});
checkForOptimize(/*mutateBatchSize=*/ request.getIds().size());
return future;
}
@Override
@NonNull
public ListenableFuture<Void> remove(
@NonNull String queryExpression, @NonNull SearchSpec searchSpec) {
Preconditions.checkNotNull(queryExpression);
Preconditions.checkNotNull(searchSpec);
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
ListenableFuture<Void> future = execute(() -> {
RemoveStats.Builder removeStatsBuilder = null;
if (mLogger != null) {
removeStatsBuilder = new RemoveStats.Builder(mPackageName, mDatabaseName);
}
mAppSearchImpl.removeByQuery(mPackageName, mDatabaseName, queryExpression,
searchSpec, removeStatsBuilder);
// Now that the batch has been written. Persist the newly written data.
mAppSearchImpl.persistToDisk(PersistType.Code.LITE);
mIsMutated = true;
if (mLogger != null) {
mLogger.logStats(removeStatsBuilder.build());
}
return null;
});
checkForOptimize();
return future;
}
@Override
@NonNull
public ListenableFuture<StorageInfo> getStorageInfo() {
Preconditions.checkState(!mIsClosed, "AppSearchSession has already been closed");
return execute(() -> mAppSearchImpl.getStorageInfoForDatabase(mPackageName, mDatabaseName));
}
@NonNull
@Override
public ListenableFuture<Void> requestFlush() {
return execute(() -> {
mAppSearchImpl.persistToDisk(PersistType.Code.FULL);
return null;
});
}
@NonNull
@Override
public Capabilities getCapabilities() {
return mCapabilities;
}
@Override
@SuppressWarnings("FutureReturnValueIgnored")
public void close() {
if (mIsMutated && !mIsClosed) {
// No future is needed here since the method is void.
FutureUtil.execute(mExecutor, () -> {
mAppSearchImpl.persistToDisk(PersistType.Code.FULL);
mIsClosed = true;
return null;
});
}
}
private <T> ListenableFuture<T> execute(Callable<T> callable) {
return FutureUtil.execute(mExecutor, callable);
}
/**
* Set schema to Icing for no-migration scenario.
*
* <p>We only need one time {@link #setSchema} call for no-migration scenario by using the
* forceoverride in the request.
*/
private SetSchemaResponse setSchemaNoMigrations(@NonNull SetSchemaRequest request,
@NonNull Map<String, List<PackageIdentifier>> copySchemasVisibleToPackages,
SetSchemaStats.Builder setSchemaStatsBuilder)
throws AppSearchException {
SetSchemaResponse setSchemaResponse = mAppSearchImpl.setSchema(
mPackageName,
mDatabaseName,
new ArrayList<>(request.getSchemas()),
/*visibilityStore=*/ null,
new ArrayList<>(request.getSchemasNotDisplayedBySystem()),
copySchemasVisibleToPackages,
request.isForceOverride(),
request.getVersion(),
setSchemaStatsBuilder);
if (!request.isForceOverride()) {
// check both deleted types and incompatible types are empty. That's the only case we
// swallowed in the AppSearchImpl#setSchema().
SchemaMigrationUtil.checkDeletedAndIncompatible(setSchemaResponse.getDeletedTypes(),
setSchemaResponse.getIncompatibleTypes());
}
mIsMutated = true;
return setSchemaResponse;
}
private void checkForOptimize(int mutateBatchSize) {
mExecutor.execute(() -> {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
OptimizeStats.Builder builder = null;
try {
if (mLogger != null) {
builder = new OptimizeStats.Builder();
}
mAppSearchImpl.checkForOptimize(mutateBatchSize, builder);
} catch (AppSearchException e) {
Log.w(TAG, "Error occurred when check for optimize", e);
} finally {
if (builder != null) {
OptimizeStats oStats = builder
.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis))
.build();
if (mLogger != null && oStats.getOriginalDocumentCount() > 0) {
// see if optimize has been run by checking originalDocumentCount
mLogger.logStats(oStats);
}
}
}
});
}
private void checkForOptimize() {
mExecutor.execute(() -> {
long totalLatencyStartMillis = SystemClock.elapsedRealtime();
OptimizeStats.Builder builder = null;
try {
if (mLogger != null) {
builder = new OptimizeStats.Builder();
}
mAppSearchImpl.checkForOptimize(builder);
} catch (AppSearchException e) {
Log.w(TAG, "Error occurred when check for optimize", e);
} finally {
if (builder != null) {
OptimizeStats oStats = builder
.setTotalLatencyMillis(
(int) (SystemClock.elapsedRealtime() - totalLatencyStartMillis))
.build();
if (mLogger != null && oStats.getOriginalDocumentCount() > 0) {
// see if optimize has been run by checking originalDocumentCount
mLogger.logStats(oStats);
}
}
}
});
}
}