| /* |
| * Copyright 2022 Google LLC |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package com.google.android.libraries.mobiledatadownload.internal; |
| |
| import static com.google.common.util.concurrent.Futures.immediateVoidFuture; |
| import static java.lang.Math.min; |
| |
| import android.content.Context; |
| import android.net.Uri; |
| import android.util.Pair; |
| import androidx.annotation.VisibleForTesting; |
| import com.google.android.libraries.mobiledatadownload.Flags; |
| import com.google.android.libraries.mobiledatadownload.SilentFeedback; |
| import com.google.android.libraries.mobiledatadownload.TimeSource; |
| import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; |
| import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; |
| import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; |
| import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; |
| import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; |
| import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; |
| import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; |
| import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; |
| import com.google.common.base.Optional; |
| import com.google.common.util.concurrent.ListenableFuture; |
| import com.google.mobiledatadownload.internal.MetadataProto.DataFile; |
| import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; |
| import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; |
| import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Set; |
| import java.util.concurrent.Executor; |
| import java.util.concurrent.atomic.AtomicInteger; |
| import javax.inject.Inject; |
| |
| /** |
| * A class that handles of the logic for file group expiration and file expiration. Expiration is |
| * determined by two sources: 1) when the active_expiration_date (set server-side by the client) has |
| * passed 2) when stale_lifetime_secs has passed since the group became stale. |
| */ |
| public class ExpirationHandler { |
| |
| private static final String TAG = "ExpirationHandler"; |
| |
| @VisibleForTesting |
| static final String MDD_EXPIRATION_HANDLER = "gms_icing_mdd_expiration_handler"; |
| |
| private final Context context; |
| private final FileGroupsMetadata fileGroupsMetadata; |
| private final SharedFileManager sharedFileManager; |
| private final SharedFilesMetadata sharedFilesMetadata; |
| private final EventLogger eventLogger; |
| private final TimeSource timeSource; |
| private final SynchronousFileStorage fileStorage; |
| private final Optional<String> instanceId; |
| private final SilentFeedback silentFeedback; |
| private final Executor sequentialControlExecutor; |
| private final Flags flags; |
| |
| @Inject |
| public ExpirationHandler( |
| @ApplicationContext Context context, |
| FileGroupsMetadata fileGroupsMetadata, |
| SharedFileManager sharedFileManager, |
| SharedFilesMetadata sharedFilesMetadata, |
| EventLogger eventLogger, |
| TimeSource timeSource, |
| SynchronousFileStorage fileStorage, |
| @InstanceId Optional<String> instanceId, |
| SilentFeedback silentFeedback, |
| @SequentialControlExecutor Executor sequentialControlExecutor, |
| Flags flags) { |
| this.context = context; |
| this.fileGroupsMetadata = fileGroupsMetadata; |
| this.sharedFileManager = sharedFileManager; |
| this.sharedFilesMetadata = sharedFilesMetadata; |
| this.eventLogger = eventLogger; |
| this.timeSource = timeSource; |
| this.fileStorage = fileStorage; |
| this.instanceId = instanceId; |
| this.silentFeedback = silentFeedback; |
| this.sequentialControlExecutor = sequentialControlExecutor; |
| this.flags = flags; |
| } |
| |
| // TODO(b/124072754): Change to package private once all code is refactored. |
| public ListenableFuture<Void> updateExpiration() { |
| return PropagatedFutures.transformAsync( |
| removeExpiredStaleGroups(), |
| voidArg0 -> |
| PropagatedFutures.transformAsync( |
| removeExpiredFreshGroups(), |
| voidArg1 -> removeUnaccountedFiles(), |
| sequentialControlExecutor), |
| sequentialControlExecutor); |
| } |
| |
| /** Returns a future that checks all File Groups and remove expired ones from FileGroupManager */ |
| private ListenableFuture<Void> removeExpiredFreshGroups() { |
| return PropagatedFutures.transformAsync( |
| fileGroupsMetadata.getAllFreshGroups(), |
| groups -> { |
| List<GroupKey> expiredGroupKeys = new ArrayList<>(); |
| for (Pair<GroupKey, DataFileGroupInternal> pair : groups) { |
| GroupKey groupKey = pair.first; |
| DataFileGroupInternal dataFileGroup = pair.second; |
| Long groupExpirationDateMillis = FileGroupUtil.getExpirationDateMillis(dataFileGroup); |
| LogUtil.d( |
| "%s: Checking group %s with expiration date %s", |
| TAG, dataFileGroup.getGroupName(), groupExpirationDateMillis); |
| if (FileGroupUtil.isExpired(groupExpirationDateMillis, timeSource)) { |
| eventLogger.logEventSampled( |
| 0, |
| dataFileGroup.getGroupName(), |
| dataFileGroup.getFileGroupVersionNumber(), |
| dataFileGroup.getBuildId(), |
| dataFileGroup.getVariantId()); |
| LogUtil.d( |
| "%s: Expired group %s with expiration date %s", |
| TAG, dataFileGroup.getGroupName(), groupExpirationDateMillis); |
| expiredGroupKeys.add(groupKey); |
| |
| // Remove Isolated structure if necessary. |
| if (FileGroupUtil.isIsolatedStructureAllowed(dataFileGroup)) { |
| FileGroupUtil.removeIsolatedFileStructure( |
| context, instanceId, dataFileGroup, fileStorage); |
| } |
| } |
| } |
| |
| return PropagatedFutures.transform( |
| fileGroupsMetadata.removeAllGroupsWithKeys(expiredGroupKeys), |
| removeSuccess -> { |
| if (!removeSuccess) { |
| eventLogger.logEventSampled(0); |
| LogUtil.e("%s: Failed to remove expired groups!", TAG); |
| } |
| return null; |
| }, |
| sequentialControlExecutor); |
| }, |
| sequentialControlExecutor); |
| } |
| |
| /** Check and update all stale File Groups; remove staled ones */ |
| private ListenableFuture<Void> removeExpiredStaleGroups() { |
| return PropagatedFutures.transformAsync( |
| fileGroupsMetadata.getAllStaleGroups(), |
| staleGroups -> { |
| List<DataFileGroupInternal> nonExpiredStaleGroups = new ArrayList<>(); |
| for (DataFileGroupInternal staleGroup : staleGroups) { |
| long groupStaleExpirationDateMillis = |
| FileGroupUtil.getStaleExpirationDateMillis(staleGroup); |
| long groupExpirationDateMillis = FileGroupUtil.getExpirationDateMillis(staleGroup); |
| long actualExpirationDateMillis = |
| min(groupStaleExpirationDateMillis, groupExpirationDateMillis); |
| |
| // Remove the group from this list if its expired. |
| if (FileGroupUtil.isExpired(actualExpirationDateMillis, timeSource)) { |
| eventLogger.logEventSampled( |
| 0, |
| staleGroup.getGroupName(), |
| staleGroup.getFileGroupVersionNumber(), |
| staleGroup.getBuildId(), |
| staleGroup.getVariantId()); |
| |
| // Remove Isolated structure if necessary. |
| if (FileGroupUtil.isIsolatedStructureAllowed(staleGroup)) { |
| FileGroupUtil.removeIsolatedFileStructure( |
| context, instanceId, staleGroup, fileStorage); |
| } |
| } else { |
| nonExpiredStaleGroups.add(staleGroup); |
| } |
| } |
| |
| // Empty the list of stale groups in the FGGC and write only the non-expired stale groups. |
| return PropagatedFutures.transformAsync( |
| fileGroupsMetadata.removeAllStaleGroups(), |
| voidArg -> |
| PropagatedFutures.transformAsync( |
| fileGroupsMetadata.writeStaleGroups(nonExpiredStaleGroups), |
| writeSuccess -> { |
| if (!writeSuccess) { |
| eventLogger.logEventSampled(0); |
| LogUtil.e("%s: Failed to write back stale groups!", TAG); |
| } |
| return immediateVoidFuture(); |
| }, |
| sequentialControlExecutor), |
| sequentialControlExecutor); |
| }, |
| sequentialControlExecutor); |
| } |
| |
| private ListenableFuture<Void> removeUnaccountedFiles() { |
| return PropagatedFutures.transformAsync( |
| getFileKeysReferencedByAnyGroup(), |
| // Remove all shared file metadata that are not referenced by any group. |
| fileKeysReferencedByAnyGroup -> |
| PropagatedFutures.transformAsync( |
| sharedFilesMetadata.getAllFileKeys(), |
| allFileKeys -> { |
| List<Uri> filesRequiredByMdd = new ArrayList<>(); |
| List<Uri> androidSharedFilesToBeReleased = new ArrayList<>(); |
| // Use AtomicInteger because variables captured by lambdas must be effectively |
| // final. |
| AtomicInteger removedMetadataCount = new AtomicInteger(0); |
| List<ListenableFuture<Void>> futures = new ArrayList<>(); |
| for (NewFileKey newFileKey : allFileKeys) { |
| if (!fileKeysReferencedByAnyGroup.contains(newFileKey)) { |
| ListenableFuture<Void> removeEntryFuture = |
| PropagatedFutures.transformAsync( |
| sharedFilesMetadata.read(newFileKey), |
| sharedFile -> { |
| if (sharedFile != null && sharedFile.getAndroidShared()) { |
| androidSharedFilesToBeReleased.add( |
| DirectoryUtil.getBlobUri( |
| context, sharedFile.getAndroidSharingChecksum())); |
| } |
| return PropagatedFutures.transform( |
| sharedFileManager.removeFileEntry(newFileKey), |
| success -> { |
| if (success) { |
| removedMetadataCount.getAndIncrement(); |
| } else { |
| eventLogger.logEventSampled(0); |
| LogUtil.e( |
| "%s: Unsubscribe from file %s failed!", |
| TAG, newFileKey); |
| } |
| return null; |
| }, |
| sequentialControlExecutor); |
| }, |
| sequentialControlExecutor); |
| futures.add(removeEntryFuture); |
| } else { |
| futures.add( |
| PropagatedFutures.transform( |
| sharedFileManager.getOnDeviceUri(newFileKey), |
| uri -> { |
| if (uri != null) { |
| filesRequiredByMdd.add(uri); |
| } |
| return null; |
| }, |
| sequentialControlExecutor)); |
| } |
| } |
| |
| // If isolated structure verification is enabled, include all individual isolated |
| // file uris referenced by fresh groups. This ensures any unaccounted isolated |
| // file uris are removed (i.e. verification is performed). |
| if (flags.enableIsolatedStructureVerification()) { |
| futures.add( |
| PropagatedFutures.transform( |
| getIsolatedFileUrisReferencedByFreshGroups(), |
| referencedIsolatedFileUris -> { |
| filesRequiredByMdd.addAll(referencedIsolatedFileUris); |
| return null; |
| }, |
| sequentialControlExecutor)); |
| } else { |
| // Isolated structure verification is disabled, include the base symlink |
| // directory as required so all isolated file uris under this directory are |
| // _not_ removed (i.e. verification is not performed). |
| filesRequiredByMdd.add( |
| DirectoryUtil.getBaseDownloadSymlinkDirectory(context, instanceId)); |
| } |
| return PropagatedFutures.whenAllComplete(futures) |
| .call( |
| () -> { |
| if (removedMetadataCount.get() > 0) { |
| eventLogger.logMddDataDownloadFileExpirationEvent( |
| 0, removedMetadataCount.get()); |
| } |
| Uri parentDirectory = |
| DirectoryUtil.getBaseDownloadDirectory(context, instanceId); |
| int releasedFiles = |
| releaseUnaccountedAndroidSharedFiles( |
| androidSharedFilesToBeReleased); |
| LogUtil.d( |
| "%s: Total %d unaccounted file released. ", TAG, releasedFiles); |
| |
| int unaccountedFileCount = |
| deleteUnaccountedFilesRecursively( |
| parentDirectory, filesRequiredByMdd); |
| LogUtil.d( |
| "%s: Total %d unaccounted file deleted. ", |
| TAG, unaccountedFileCount); |
| if (unaccountedFileCount > 0) { |
| eventLogger.logMddDataDownloadFileExpirationEvent( |
| 0, unaccountedFileCount); |
| } |
| if (releasedFiles > 0) { |
| eventLogger.logMddDataDownloadFileExpirationEvent(0, releasedFiles); |
| } |
| return null; |
| }, |
| sequentialControlExecutor); |
| }, |
| sequentialControlExecutor), |
| sequentialControlExecutor); |
| } |
| |
| private ListenableFuture<Set<NewFileKey>> getFileKeysReferencedByAnyGroup() { |
| return PropagatedFutures.transformAsync( |
| fileGroupsMetadata.getAllFreshGroups(), |
| allGroupsByKey -> { |
| Set<NewFileKey> fileKeysReferencedByAnyGroup = new HashSet<>(); |
| List<DataFileGroupInternal> dataFileGroups = new ArrayList<>(); |
| for (Pair<GroupKey, DataFileGroupInternal> dataFileGroupPair : allGroupsByKey) { |
| dataFileGroups.add(dataFileGroupPair.second); |
| } |
| return PropagatedFutures.transform( |
| fileGroupsMetadata.getAllStaleGroups(), |
| staleGroups -> { |
| dataFileGroups.addAll(staleGroups); |
| for (DataFileGroupInternal dataFileGroup : dataFileGroups) { |
| for (DataFile dataFile : dataFileGroup.getFileList()) { |
| fileKeysReferencedByAnyGroup.add( |
| SharedFilesMetadata.createKeyFromDataFileForCurrentVersion( |
| context, |
| dataFile, |
| dataFileGroup.getAllowedReadersEnum(), |
| silentFeedback)); |
| } |
| } |
| return fileKeysReferencedByAnyGroup; |
| }, |
| sequentialControlExecutor); |
| }, |
| sequentialControlExecutor); |
| } |
| |
| /** |
| * Get all isolated file uris that are referenced by any fresh groups. |
| * |
| * <p>Fresh groups are active/pending groups. Isolated file uris are expected when 1) the OS |
| * version supports symlinks (at least Lollipop (21)); and 2) The file group enables file |
| * isolation. |
| * |
| * @return ListenableFuture that resolves with List of isolated uris that are referenced by |
| * active/pending groups |
| */ |
| private ListenableFuture<List<Uri>> getIsolatedFileUrisReferencedByFreshGroups() { |
| List<Uri> referencedIsolatedFileUris = new ArrayList<>(); |
| return PropagatedFutures.transform( |
| fileGroupsMetadata.getAllFreshGroups(), |
| groupKeyAndGroupList -> { |
| for (Pair<GroupKey, DataFileGroupInternal> groupKeyAndGroup : groupKeyAndGroupList) { |
| DataFileGroupInternal freshGroup = groupKeyAndGroup.second; |
| // Skip any groups that don't support isolated structures |
| if (!FileGroupUtil.isIsolatedStructureAllowed(freshGroup)) { |
| continue; |
| } |
| |
| // Add the expected isolated file uris for each file |
| for (DataFile file : freshGroup.getFileList()) { |
| Uri isolatedFileUri = |
| FileGroupUtil.getIsolatedFileUri(context, instanceId, file, freshGroup); |
| referencedIsolatedFileUris.add(isolatedFileUri); |
| } |
| } |
| |
| return referencedIsolatedFileUris; |
| }, |
| sequentialControlExecutor); |
| } |
| |
| private int releaseUnaccountedAndroidSharedFiles(List<Uri> androidSharedFilesToBeReleased) { |
| int releasedFiles = 0; |
| for (Uri sharedFile : androidSharedFilesToBeReleased) { |
| try { |
| fileStorage.deleteFile(sharedFile); |
| releasedFiles += 1; |
| eventLogger.logEventSampled(0); |
| } catch (IOException e) { |
| eventLogger.logEventSampled(0); |
| LogUtil.e(e, "%s: Failed to release unaccounted file!", TAG); |
| } |
| } |
| return releasedFiles; |
| } |
| |
| // TODO(b/119622504) Fix nullness violation: incompatible types in argument. |
| @SuppressWarnings("nullness:argument") |
| private int deleteUnaccountedFilesRecursively(Uri directory, List<Uri> filesRequiredByMdd) { |
| int unaccountedFileCount = 0; |
| try { |
| if (!fileStorage.exists(directory)) { |
| return unaccountedFileCount; |
| } |
| |
| for (Uri uri : fileStorage.children(directory)) { |
| try { |
| if (isContainedInUriList(uri, filesRequiredByMdd)) { |
| continue; |
| } |
| if (fileStorage.isDirectory(uri)) { |
| unaccountedFileCount += deleteUnaccountedFilesRecursively(uri, filesRequiredByMdd); |
| } else { |
| LogUtil.d("%s: Deleted unaccounted file with uri %s!", TAG, uri.getPath()); |
| fileStorage.deleteFile(uri); |
| unaccountedFileCount++; |
| } |
| |
| } catch (IOException e) { |
| eventLogger.logEventSampled(0); |
| LogUtil.e(e, "%s: Failed to delete unaccounted file!", TAG); |
| } |
| } |
| |
| } catch (IOException e) { |
| eventLogger.logEventSampled(0); |
| LogUtil.e(e, "%s: Failed to delete unaccounted file!", TAG); |
| } |
| return unaccountedFileCount; |
| } |
| |
| /** |
| * Returns true if given uri is within the given uri list or is a child of any uri in the list. |
| * |
| * <p>Used by MDD's unaccounted file logic to filter out files that shouldn't be deleted. This is |
| * used in two cases: |
| * |
| * <ul> |
| * <li>files referred by any active MDD files. This includes internal MDD files, such as delta |
| * files of a full active file, which are stored using the active file name and a checksum |
| * suffix. |
| * <li>symlinks created for an isolated file structure. These symlinks will reference active |
| * files and their lifecycle is managed on the file group level, rather than as individual |
| * files. |
| * </ul> |
| */ |
| private boolean isContainedInUriList(Uri uri, List<Uri> uriList) { |
| for (Uri activeUri : uriList) { |
| if (uri.toString().startsWith(activeUri.toString())) { |
| return true; |
| } |
| } |
| return false; |
| } |
| } |
