| /* |
| * Copyright (C) 2018 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.android.server; |
| |
| import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig; |
| |
| import static java.lang.annotation.RetentionPolicy.SOURCE; |
| |
| import android.annotation.IntDef; |
| import android.annotation.Nullable; |
| import android.content.Context; |
| import android.content.pm.PackageInfo; |
| import android.content.pm.PackageManager; |
| import android.content.pm.VersionedPackage; |
| import android.crashrecovery.flags.Flags; |
| import android.net.ConnectivityModuleConnector; |
| import android.os.Environment; |
| import android.os.Handler; |
| import android.os.Looper; |
| import android.os.Process; |
| import android.os.SystemProperties; |
| import android.provider.DeviceConfig; |
| import android.sysprop.CrashRecoveryProperties; |
| import android.text.TextUtils; |
| import android.util.ArrayMap; |
| import android.util.ArraySet; |
| import android.util.AtomicFile; |
| import android.util.LongArrayQueue; |
| import android.util.Slog; |
| import android.util.Xml; |
| |
| import com.android.internal.annotations.GuardedBy; |
| import com.android.internal.annotations.VisibleForTesting; |
| import com.android.internal.os.BackgroundThread; |
| import com.android.internal.util.IndentingPrintWriter; |
| import com.android.internal.util.XmlUtils; |
| import com.android.modules.utils.TypedXmlPullParser; |
| import com.android.modules.utils.TypedXmlSerializer; |
| |
| import libcore.io.IoUtils; |
| |
| import org.xmlpull.v1.XmlPullParserException; |
| |
| import java.io.BufferedReader; |
| import java.io.BufferedWriter; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileNotFoundException; |
| import java.io.FileOutputStream; |
| import java.io.FileReader; |
| import java.io.FileWriter; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.ObjectInputStream; |
| import java.io.ObjectOutputStream; |
| import java.lang.annotation.Retention; |
| import java.lang.annotation.RetentionPolicy; |
| import java.util.ArrayList; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.NoSuchElementException; |
| import java.util.Set; |
| import java.util.concurrent.TimeUnit; |
| |
| /** |
| * Monitors the health of packages on the system and notifies interested observers when packages |
| * fail. On failure, the registered observer with the least user impacting mitigation will |
| * be notified. |
| */ |
| public class PackageWatchdog { |
| private static final String TAG = "PackageWatchdog"; |
| |
| static final String PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS = |
| "watchdog_trigger_failure_duration_millis"; |
| static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT = |
| "watchdog_trigger_failure_count"; |
| static final String PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED = |
| "watchdog_explicit_health_check_enabled"; |
| |
| // TODO: make the following values configurable via DeviceConfig |
| private static final long NATIVE_CRASH_POLLING_INTERVAL_MILLIS = |
| TimeUnit.SECONDS.toMillis(30); |
| private static final long NUMBER_OF_NATIVE_CRASH_POLLS = 10; |
| |
| |
| public static final int FAILURE_REASON_UNKNOWN = 0; |
| public static final int FAILURE_REASON_NATIVE_CRASH = 1; |
| public static final int FAILURE_REASON_EXPLICIT_HEALTH_CHECK = 2; |
| public static final int FAILURE_REASON_APP_CRASH = 3; |
| public static final int FAILURE_REASON_APP_NOT_RESPONDING = 4; |
| public static final int FAILURE_REASON_BOOT_LOOP = 5; |
| |
| @IntDef(prefix = { "FAILURE_REASON_" }, value = { |
| FAILURE_REASON_UNKNOWN, |
| FAILURE_REASON_NATIVE_CRASH, |
| FAILURE_REASON_EXPLICIT_HEALTH_CHECK, |
| FAILURE_REASON_APP_CRASH, |
| FAILURE_REASON_APP_NOT_RESPONDING, |
| FAILURE_REASON_BOOT_LOOP |
| }) |
| @Retention(RetentionPolicy.SOURCE) |
| public @interface FailureReasons {} |
| |
| // Duration to count package failures before it resets to 0 |
| @VisibleForTesting |
| static final int DEFAULT_TRIGGER_FAILURE_DURATION_MS = |
| (int) TimeUnit.MINUTES.toMillis(1); |
| // Number of package failures within the duration above before we notify observers |
| @VisibleForTesting |
| static final int DEFAULT_TRIGGER_FAILURE_COUNT = 5; |
| @VisibleForTesting |
| static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2); |
| // Sliding window for tracking how many mitigation calls were made for a package. |
| @VisibleForTesting |
| static final long DEFAULT_DEESCALATION_WINDOW_MS = TimeUnit.HOURS.toMillis(1); |
| // Whether explicit health checks are enabled or not |
| private static final boolean DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED = true; |
| |
| @VisibleForTesting |
| static final int DEFAULT_BOOT_LOOP_TRIGGER_COUNT = 5; |
| |
| static final long DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS = TimeUnit.MINUTES.toMillis(10); |
| |
| // Threshold level at which or above user might experience significant disruption. |
| private static final String MAJOR_USER_IMPACT_LEVEL_THRESHOLD = |
| "persist.device_config.configuration.major_user_impact_level_threshold"; |
| private static final int DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD = |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_71; |
| |
| private long mNumberOfNativeCrashPollsRemaining; |
| |
| private static final int DB_VERSION = 1; |
| private static final String TAG_PACKAGE_WATCHDOG = "package-watchdog"; |
| private static final String TAG_PACKAGE = "package"; |
| private static final String TAG_OBSERVER = "observer"; |
| private static final String ATTR_VERSION = "version"; |
| private static final String ATTR_NAME = "name"; |
| private static final String ATTR_DURATION = "duration"; |
| private static final String ATTR_EXPLICIT_HEALTH_CHECK_DURATION = "health-check-duration"; |
| private static final String ATTR_PASSED_HEALTH_CHECK = "passed-health-check"; |
| private static final String ATTR_MITIGATION_CALLS = "mitigation-calls"; |
| private static final String ATTR_MITIGATION_COUNT = "mitigation-count"; |
| |
| // A file containing information about the current mitigation count in the case of a boot loop. |
| // This allows boot loop information to persist in the case of an fs-checkpoint being |
| // aborted. |
| private static final String METADATA_FILE = "/metadata/watchdog/mitigation_count.txt"; |
| |
| @GuardedBy("PackageWatchdog.class") |
| private static PackageWatchdog sPackageWatchdog; |
| |
| private final Object mLock = new Object(); |
| // System server context |
| private final Context mContext; |
| // Handler to run short running tasks |
| private final Handler mShortTaskHandler; |
| // Handler for processing IO and long running tasks |
| private final Handler mLongTaskHandler; |
| // Contains (observer-name -> observer-handle) that have ever been registered from |
| // previous boots. Observers with all packages expired are periodically pruned. |
| // It is saved to disk on system shutdown and repouplated on startup so it survives reboots. |
| @GuardedBy("mLock") |
| private final ArrayMap<String, ObserverInternal> mAllObservers = new ArrayMap<>(); |
| // File containing the XML data of monitored packages /data/system/package-watchdog.xml |
| private final AtomicFile mPolicyFile; |
| private final ExplicitHealthCheckController mHealthCheckController; |
| private final ConnectivityModuleConnector mConnectivityModuleConnector; |
| private final Runnable mSyncRequests = this::syncRequests; |
| private final Runnable mSyncStateWithScheduledReason = this::syncStateWithScheduledReason; |
| private final Runnable mSaveToFile = this::saveToFile; |
| private final SystemClock mSystemClock; |
| private final BootThreshold mBootThreshold; |
| private final DeviceConfig.OnPropertiesChangedListener |
| mOnPropertyChangedListener = this::onPropertyChanged; |
| |
| // The set of packages that have been synced with the ExplicitHealthCheckController |
| @GuardedBy("mLock") |
| private Set<String> mRequestedHealthCheckPackages = new ArraySet<>(); |
| @GuardedBy("mLock") |
| private boolean mIsPackagesReady; |
| // Flag to control whether explicit health checks are supported or not |
| @GuardedBy("mLock") |
| private boolean mIsHealthCheckEnabled = DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED; |
| @GuardedBy("mLock") |
| private int mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_DURATION_MS; |
| @GuardedBy("mLock") |
| private int mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT; |
| // SystemClock#uptimeMillis when we last executed #syncState |
| // 0 if no prune is scheduled. |
| @GuardedBy("mLock") |
| private long mUptimeAtLastStateSync; |
| // If true, sync explicit health check packages with the ExplicitHealthCheckController. |
| @GuardedBy("mLock") |
| private boolean mSyncRequired = false; |
| |
| @FunctionalInterface |
| @VisibleForTesting |
| interface SystemClock { |
| long uptimeMillis(); |
| } |
| |
| private PackageWatchdog(Context context) { |
| // Needs to be constructed inline |
| this(context, new AtomicFile( |
| new File(new File(Environment.getDataDirectory(), "system"), |
| "package-watchdog.xml")), |
| new Handler(Looper.myLooper()), BackgroundThread.getHandler(), |
| new ExplicitHealthCheckController(context), |
| ConnectivityModuleConnector.getInstance(), |
| android.os.SystemClock::uptimeMillis); |
| } |
| |
| /** |
| * Creates a PackageWatchdog that allows injecting dependencies. |
| */ |
| @VisibleForTesting |
| PackageWatchdog(Context context, AtomicFile policyFile, Handler shortTaskHandler, |
| Handler longTaskHandler, ExplicitHealthCheckController controller, |
| ConnectivityModuleConnector connectivityModuleConnector, SystemClock clock) { |
| mContext = context; |
| mPolicyFile = policyFile; |
| mShortTaskHandler = shortTaskHandler; |
| mLongTaskHandler = longTaskHandler; |
| mHealthCheckController = controller; |
| mConnectivityModuleConnector = connectivityModuleConnector; |
| mSystemClock = clock; |
| mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS; |
| mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT, |
| DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS); |
| |
| loadFromFile(); |
| sPackageWatchdog = this; |
| } |
| |
| /** Creates or gets singleton instance of PackageWatchdog. */ |
| public static PackageWatchdog getInstance(Context context) { |
| synchronized (PackageWatchdog.class) { |
| if (sPackageWatchdog == null) { |
| new PackageWatchdog(context); |
| } |
| return sPackageWatchdog; |
| } |
| } |
| |
| /** |
| * Called during boot to notify when packages are ready on the device so we can start |
| * binding. |
| */ |
| public void onPackagesReady() { |
| synchronized (mLock) { |
| mIsPackagesReady = true; |
| mHealthCheckController.setCallbacks(packageName -> onHealthCheckPassed(packageName), |
| packages -> onSupportedPackages(packages), |
| this::onSyncRequestNotified); |
| setPropertyChangedListenerLocked(); |
| updateConfigs(); |
| registerConnectivityModuleHealthListener(); |
| } |
| } |
| |
| /** |
| * Registers {@code observer} to listen for package failures. Add a new ObserverInternal for |
| * this observer if it does not already exist. |
| * |
| * <p>Observers are expected to call this on boot. It does not specify any packages but |
| * it will resume observing any packages requested from a previous boot. |
| */ |
| public void registerHealthObserver(PackageHealthObserver observer) { |
| synchronized (mLock) { |
| ObserverInternal internalObserver = mAllObservers.get(observer.getName()); |
| if (internalObserver != null) { |
| internalObserver.registeredObserver = observer; |
| } else { |
| internalObserver = new ObserverInternal(observer.getName(), new ArrayList<>()); |
| internalObserver.registeredObserver = observer; |
| mAllObservers.put(observer.getName(), internalObserver); |
| syncState("added new observer"); |
| } |
| } |
| } |
| |
| /** |
| * Starts observing the health of the {@code packages} for {@code observer} and notifies |
| * {@code observer} of any package failures within the monitoring duration. |
| * |
| * <p>If monitoring a package supporting explicit health check, at the end of the monitoring |
| * duration if {@link #onHealthCheckPassed} was never called, |
| * {@link PackageHealthObserver#execute} will be called as if the package failed. |
| * |
| * <p>If {@code observer} is already monitoring a package in {@code packageNames}, |
| * the monitoring window of that package will be reset to {@code durationMs} and the health |
| * check state will be reset to a default depending on if the package is contained in |
| * {@link mPackagesWithExplicitHealthCheckEnabled}. |
| * |
| * <p>If {@code packageNames} is empty, this will be a no-op. |
| * |
| * <p>If {@code durationMs} is less than 1, a default monitoring duration |
| * {@link #DEFAULT_OBSERVING_DURATION_MS} will be used. |
| */ |
| public void startObservingHealth(PackageHealthObserver observer, List<String> packageNames, |
| long durationMs) { |
| if (packageNames.isEmpty()) { |
| Slog.wtf(TAG, "No packages to observe, " + observer.getName()); |
| return; |
| } |
| if (durationMs < 1) { |
| Slog.wtf(TAG, "Invalid duration " + durationMs + "ms for observer " |
| + observer.getName() + ". Not observing packages " + packageNames); |
| durationMs = DEFAULT_OBSERVING_DURATION_MS; |
| } |
| |
| List<MonitoredPackage> packages = new ArrayList<>(); |
| for (int i = 0; i < packageNames.size(); i++) { |
| // Health checks not available yet so health check state will start INACTIVE |
| MonitoredPackage pkg = newMonitoredPackage(packageNames.get(i), durationMs, false); |
| if (pkg != null) { |
| packages.add(pkg); |
| } else { |
| Slog.w(TAG, "Failed to create MonitoredPackage for pkg=" + packageNames.get(i)); |
| } |
| } |
| |
| if (packages.isEmpty()) { |
| return; |
| } |
| |
| // Sync before we add the new packages to the observers. This will #pruneObservers, |
| // causing any elapsed time to be deducted from all existing packages before we add new |
| // packages. This maintains the invariant that the elapsed time for ALL (new and existing) |
| // packages is the same. |
| mLongTaskHandler.post(() -> { |
| syncState("observing new packages"); |
| |
| synchronized (mLock) { |
| ObserverInternal oldObserver = mAllObservers.get(observer.getName()); |
| if (oldObserver == null) { |
| Slog.d(TAG, observer.getName() + " started monitoring health " |
| + "of packages " + packageNames); |
| mAllObservers.put(observer.getName(), |
| new ObserverInternal(observer.getName(), packages)); |
| } else { |
| Slog.d(TAG, observer.getName() + " added the following " |
| + "packages to monitor " + packageNames); |
| oldObserver.updatePackagesLocked(packages); |
| } |
| } |
| |
| // Register observer in case not already registered |
| registerHealthObserver(observer); |
| |
| // Sync after we add the new packages to the observers. We may have received packges |
| // requiring an earlier schedule than we are currently scheduled for. |
| syncState("updated observers"); |
| }); |
| |
| } |
| |
| /** |
| * Unregisters {@code observer} from listening to package failure. |
| * Additionally, this stops observing any packages that may have previously been observed |
| * even from a previous boot. |
| */ |
| public void unregisterHealthObserver(PackageHealthObserver observer) { |
| mLongTaskHandler.post(() -> { |
| synchronized (mLock) { |
| mAllObservers.remove(observer.getName()); |
| } |
| syncState("unregistering observer: " + observer.getName()); |
| }); |
| } |
| |
| /** |
| * Called when a process fails due to a crash, ANR or explicit health check. |
| * |
| * <p>For each package contained in the process, one registered observer with the least user |
| * impact will be notified for mitigation. |
| * |
| * <p>This method could be called frequently if there is a severe problem on the device. |
| */ |
| public void onPackageFailure(List<VersionedPackage> packages, |
| @FailureReasons int failureReason) { |
| if (packages == null) { |
| Slog.w(TAG, "Could not resolve a list of failing packages"); |
| return; |
| } |
| mLongTaskHandler.post(() -> { |
| synchronized (mLock) { |
| if (mAllObservers.isEmpty()) { |
| return; |
| } |
| boolean requiresImmediateAction = (failureReason == FAILURE_REASON_NATIVE_CRASH |
| || failureReason == FAILURE_REASON_EXPLICIT_HEALTH_CHECK); |
| if (requiresImmediateAction) { |
| handleFailureImmediately(packages, failureReason); |
| } else { |
| for (int pIndex = 0; pIndex < packages.size(); pIndex++) { |
| VersionedPackage versionedPackage = packages.get(pIndex); |
| // Observer that will receive failure for versionedPackage |
| PackageHealthObserver currentObserverToNotify = null; |
| int currentObserverImpact = Integer.MAX_VALUE; |
| MonitoredPackage currentMonitoredPackage = null; |
| |
| // Find observer with least user impact |
| for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { |
| ObserverInternal observer = mAllObservers.valueAt(oIndex); |
| PackageHealthObserver registeredObserver = observer.registeredObserver; |
| if (registeredObserver != null |
| && observer.onPackageFailureLocked( |
| versionedPackage.getPackageName())) { |
| MonitoredPackage p = observer.getMonitoredPackage( |
| versionedPackage.getPackageName()); |
| int mitigationCount = 1; |
| if (p != null) { |
| mitigationCount = p.getMitigationCountLocked() + 1; |
| } |
| int impact = registeredObserver.onHealthCheckFailed( |
| versionedPackage, failureReason, mitigationCount); |
| if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0 |
| && impact < currentObserverImpact) { |
| currentObserverToNotify = registeredObserver; |
| currentObserverImpact = impact; |
| currentMonitoredPackage = p; |
| } |
| } |
| } |
| |
| // Execute action with least user impact |
| if (currentObserverToNotify != null) { |
| int mitigationCount = 1; |
| if (currentMonitoredPackage != null) { |
| currentMonitoredPackage.noteMitigationCallLocked(); |
| mitigationCount = |
| currentMonitoredPackage.getMitigationCountLocked(); |
| } |
| if (Flags.recoverabilityDetection()) { |
| maybeExecute(currentObserverToNotify, versionedPackage, |
| failureReason, currentObserverImpact, mitigationCount); |
| } else { |
| currentObserverToNotify.execute(versionedPackage, |
| failureReason, mitigationCount); |
| } |
| } |
| } |
| } |
| } |
| }); |
| } |
| |
| /** |
| * For native crashes or explicit health check failures, call directly into each observer to |
| * mitigate the error without going through failure threshold logic. |
| */ |
| private void handleFailureImmediately(List<VersionedPackage> packages, |
| @FailureReasons int failureReason) { |
| VersionedPackage failingPackage = packages.size() > 0 ? packages.get(0) : null; |
| PackageHealthObserver currentObserverToNotify = null; |
| int currentObserverImpact = Integer.MAX_VALUE; |
| for (ObserverInternal observer: mAllObservers.values()) { |
| PackageHealthObserver registeredObserver = observer.registeredObserver; |
| if (registeredObserver != null) { |
| int impact = registeredObserver.onHealthCheckFailed( |
| failingPackage, failureReason, 1); |
| if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0 |
| && impact < currentObserverImpact) { |
| currentObserverToNotify = registeredObserver; |
| currentObserverImpact = impact; |
| } |
| } |
| } |
| if (currentObserverToNotify != null) { |
| if (Flags.recoverabilityDetection()) { |
| maybeExecute(currentObserverToNotify, failingPackage, failureReason, |
| currentObserverImpact, /*mitigationCount=*/ 1); |
| } else { |
| currentObserverToNotify.execute(failingPackage, failureReason, 1); |
| } |
| } |
| } |
| |
| private void maybeExecute(PackageHealthObserver currentObserverToNotify, |
| VersionedPackage versionedPackage, |
| @FailureReasons int failureReason, |
| int currentObserverImpact, |
| int mitigationCount) { |
| if (currentObserverImpact < getUserImpactLevelLimit()) { |
| currentObserverToNotify.execute(versionedPackage, failureReason, mitigationCount); |
| } |
| } |
| |
| |
| /** |
| * Called when the system server boots. If the system server is detected to be in a boot loop, |
| * query each observer and perform the mitigation action with the lowest user impact. |
| * |
| * Note: PackageWatchdog considers system_server restart loop as bootloop. Full reboots |
| * are not counted in bootloop. |
| */ |
| @SuppressWarnings("GuardedBy") |
| public void noteBoot() { |
| synchronized (mLock) { |
| // if boot count has reached threshold, start mitigation. |
| // We wait until threshold number of restarts only for the first time. Perform |
| // mitigations for every restart after that. |
| boolean mitigate = mBootThreshold.incrementAndTest(); |
| if (mitigate) { |
| if (!Flags.recoverabilityDetection()) { |
| mBootThreshold.reset(); |
| } |
| int mitigationCount = mBootThreshold.getMitigationCount() + 1; |
| PackageHealthObserver currentObserverToNotify = null; |
| ObserverInternal currentObserverInternal = null; |
| int currentObserverImpact = Integer.MAX_VALUE; |
| for (int i = 0; i < mAllObservers.size(); i++) { |
| final ObserverInternal observer = mAllObservers.valueAt(i); |
| PackageHealthObserver registeredObserver = observer.registeredObserver; |
| if (registeredObserver != null) { |
| int impact = Flags.recoverabilityDetection() |
| ? registeredObserver.onBootLoop( |
| observer.getBootMitigationCount() + 1) |
| : registeredObserver.onBootLoop(mitigationCount); |
| if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0 |
| && impact < currentObserverImpact) { |
| currentObserverToNotify = registeredObserver; |
| currentObserverInternal = observer; |
| currentObserverImpact = impact; |
| } |
| } |
| } |
| if (currentObserverToNotify != null) { |
| if (Flags.recoverabilityDetection()) { |
| int currentObserverMitigationCount = |
| currentObserverInternal.getBootMitigationCount() + 1; |
| currentObserverInternal.setBootMitigationCount( |
| currentObserverMitigationCount); |
| saveAllObserversBootMitigationCountToMetadata(METADATA_FILE); |
| currentObserverToNotify.executeBootLoopMitigation( |
| currentObserverMitigationCount); |
| } else { |
| mBootThreshold.setMitigationCount(mitigationCount); |
| mBootThreshold.saveMitigationCountToMetadata(); |
| currentObserverToNotify.executeBootLoopMitigation(mitigationCount); |
| } |
| } |
| } |
| } |
| } |
| |
| // TODO(b/120598832): Optimize write? Maybe only write a separate smaller file? Also |
| // avoid holding lock? |
| // This currently adds about 7ms extra to shutdown thread |
| /** Writes the package information to file during shutdown. */ |
| public void writeNow() { |
| synchronized (mLock) { |
| // Must only run synchronous tasks as this runs on the ShutdownThread and no other |
| // thread is guaranteed to run during shutdown. |
| if (!mAllObservers.isEmpty()) { |
| mLongTaskHandler.removeCallbacks(mSaveToFile); |
| pruneObserversLocked(); |
| saveToFile(); |
| Slog.i(TAG, "Last write to update package durations"); |
| } |
| } |
| } |
| |
| /** |
| * Enables or disables explicit health checks. |
| * <p> If explicit health checks are enabled, the health check service is started. |
| * <p> If explicit health checks are disabled, pending explicit health check requests are |
| * passed and the health check service is stopped. |
| */ |
| private void setExplicitHealthCheckEnabled(boolean enabled) { |
| synchronized (mLock) { |
| mIsHealthCheckEnabled = enabled; |
| mHealthCheckController.setEnabled(enabled); |
| mSyncRequired = true; |
| // Prune to update internal state whenever health check is enabled/disabled |
| syncState("health check state " + (enabled ? "enabled" : "disabled")); |
| } |
| } |
| |
| /** |
| * This method should be only called on mShortTaskHandler, since it modifies |
| * {@link #mNumberOfNativeCrashPollsRemaining}. |
| */ |
| private void checkAndMitigateNativeCrashes() { |
| mNumberOfNativeCrashPollsRemaining--; |
| // Check if native watchdog reported a crash |
| if ("1".equals(SystemProperties.get("sys.init.updatable_crashing"))) { |
| // We rollback all available low impact rollbacks when crash is unattributable |
| onPackageFailure(Collections.EMPTY_LIST, FAILURE_REASON_NATIVE_CRASH); |
| // we stop polling after an attempt to execute rollback, regardless of whether the |
| // attempt succeeds or not |
| } else { |
| if (mNumberOfNativeCrashPollsRemaining > 0) { |
| mShortTaskHandler.postDelayed(() -> checkAndMitigateNativeCrashes(), |
| NATIVE_CRASH_POLLING_INTERVAL_MILLIS); |
| } |
| } |
| } |
| |
| /** |
| * Since this method can eventually trigger a rollback, it should be called |
| * only once boot has completed {@code onBootCompleted} and not earlier, because the install |
| * session must be entirely completed before we try to rollback. |
| */ |
| public void scheduleCheckAndMitigateNativeCrashes() { |
| Slog.i(TAG, "Scheduling " + mNumberOfNativeCrashPollsRemaining + " polls to check " |
| + "and mitigate native crashes"); |
| mShortTaskHandler.post(()->checkAndMitigateNativeCrashes()); |
| } |
| |
| private int getUserImpactLevelLimit() { |
| return SystemProperties.getInt(MAJOR_USER_IMPACT_LEVEL_THRESHOLD, |
| DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD); |
| } |
| |
| /** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */ |
| @Retention(SOURCE) |
| @IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_LEVEL_0, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_10, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_20, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_30, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_50, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_70, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_71, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_75, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_80, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_90, |
| PackageHealthObserverImpact.USER_IMPACT_LEVEL_100}) |
| public @interface PackageHealthObserverImpact { |
| /** No action to take. */ |
| int USER_IMPACT_LEVEL_0 = 0; |
| /* Action has low user impact, user of a device will barely notice. */ |
| int USER_IMPACT_LEVEL_10 = 10; |
| /* Actions having medium user impact, user of a device will likely notice. */ |
| int USER_IMPACT_LEVEL_20 = 20; |
| int USER_IMPACT_LEVEL_30 = 30; |
| int USER_IMPACT_LEVEL_50 = 50; |
| int USER_IMPACT_LEVEL_70 = 70; |
| /* Action has high user impact, a last resort, user of a device will be very frustrated. */ |
| int USER_IMPACT_LEVEL_71 = 71; |
| int USER_IMPACT_LEVEL_75 = 75; |
| int USER_IMPACT_LEVEL_80 = 80; |
| int USER_IMPACT_LEVEL_90 = 90; |
| int USER_IMPACT_LEVEL_100 = 100; |
| } |
| |
| /** Register instances of this interface to receive notifications on package failure. */ |
| public interface PackageHealthObserver { |
| /** |
| * Called when health check fails for the {@code versionedPackage}. |
| * |
| * @param versionedPackage the package that is failing. This may be null if a native |
| * service is crashing. |
| * @param failureReason the type of failure that is occurring. |
| * @param mitigationCount the number of times mitigation has been called for this package |
| * (including this time). |
| * |
| * |
| * @return any one of {@link PackageHealthObserverImpact} to express the impact |
| * to the user on {@link #execute} |
| */ |
| @PackageHealthObserverImpact int onHealthCheckFailed( |
| @Nullable VersionedPackage versionedPackage, |
| @FailureReasons int failureReason, |
| int mitigationCount); |
| |
| /** |
| * Executes mitigation for {@link #onHealthCheckFailed}. |
| * |
| * @param versionedPackage the package that is failing. This may be null if a native |
| * service is crashing. |
| * @param failureReason the type of failure that is occurring. |
| * @param mitigationCount the number of times mitigation has been called for this package |
| * (including this time). |
| * @return {@code true} if action was executed successfully, {@code false} otherwise |
| */ |
| boolean execute(@Nullable VersionedPackage versionedPackage, |
| @FailureReasons int failureReason, int mitigationCount); |
| |
| |
| /** |
| * Called when the system server has booted several times within a window of time, defined |
| * by {@link #mBootThreshold} |
| * |
| * @param mitigationCount the number of times mitigation has been attempted for this |
| * boot loop (including this time). |
| */ |
| default @PackageHealthObserverImpact int onBootLoop(int mitigationCount) { |
| return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0; |
| } |
| |
| /** |
| * Executes mitigation for {@link #onBootLoop} |
| * @param mitigationCount the number of times mitigation has been attempted for this |
| * boot loop (including this time). |
| */ |
| default boolean executeBootLoopMitigation(int mitigationCount) { |
| return false; |
| } |
| |
| // TODO(b/120598832): Ensure uniqueness? |
| /** |
| * Identifier for the observer, should not change across device updates otherwise the |
| * watchdog may drop observing packages with the old name. |
| */ |
| String getName(); |
| |
| /** |
| * An observer will not be pruned if this is set, even if the observer is not explicitly |
| * monitoring any packages. |
| */ |
| default boolean isPersistent() { |
| return false; |
| } |
| |
| /** |
| * Returns {@code true} if this observer wishes to observe the given package, {@code false} |
| * otherwise |
| * |
| * <p> A persistent observer may choose to start observing certain failing packages, even if |
| * it has not explicitly asked to watch the package with {@link #startObservingHealth}. |
| */ |
| default boolean mayObservePackage(String packageName) { |
| return false; |
| } |
| } |
| |
| @VisibleForTesting |
| long getTriggerFailureCount() { |
| synchronized (mLock) { |
| return mTriggerFailureCount; |
| } |
| } |
| |
| @VisibleForTesting |
| long getTriggerFailureDurationMs() { |
| synchronized (mLock) { |
| return mTriggerFailureDurationMs; |
| } |
| } |
| |
| /** |
| * Serializes and syncs health check requests with the {@link ExplicitHealthCheckController}. |
| */ |
| private void syncRequestsAsync() { |
| mShortTaskHandler.removeCallbacks(mSyncRequests); |
| mShortTaskHandler.post(mSyncRequests); |
| } |
| |
| /** |
| * Syncs health check requests with the {@link ExplicitHealthCheckController}. |
| * Calls to this must be serialized. |
| * |
| * @see #syncRequestsAsync |
| */ |
| private void syncRequests() { |
| boolean syncRequired = false; |
| synchronized (mLock) { |
| if (mIsPackagesReady) { |
| Set<String> packages = getPackagesPendingHealthChecksLocked(); |
| if (mSyncRequired || !packages.equals(mRequestedHealthCheckPackages) |
| || packages.isEmpty()) { |
| syncRequired = true; |
| mRequestedHealthCheckPackages = packages; |
| } |
| } // else, we will sync requests when packages become ready |
| } |
| |
| // Call outside lock to avoid holding lock when calling into the controller. |
| if (syncRequired) { |
| Slog.i(TAG, "Syncing health check requests for packages: " |
| + mRequestedHealthCheckPackages); |
| mHealthCheckController.syncRequests(mRequestedHealthCheckPackages); |
| mSyncRequired = false; |
| } |
| } |
| |
| /** |
| * Updates the observers monitoring {@code packageName} that explicit health check has passed. |
| * |
| * <p> This update is strictly for registered observers at the time of the call |
| * Observers that register after this signal will have no knowledge of prior signals and will |
| * effectively behave as if the explicit health check hasn't passed for {@code packageName}. |
| * |
| * <p> {@code packageName} can still be considered failed if reported by |
| * {@link #onPackageFailureLocked} before the package expires. |
| * |
| * <p> Triggered by components outside the system server when they are fully functional after an |
| * update. |
| */ |
| private void onHealthCheckPassed(String packageName) { |
| Slog.i(TAG, "Health check passed for package: " + packageName); |
| boolean isStateChanged = false; |
| |
| synchronized (mLock) { |
| for (int observerIdx = 0; observerIdx < mAllObservers.size(); observerIdx++) { |
| ObserverInternal observer = mAllObservers.valueAt(observerIdx); |
| MonitoredPackage monitoredPackage = observer.getMonitoredPackage(packageName); |
| |
| if (monitoredPackage != null) { |
| int oldState = monitoredPackage.getHealthCheckStateLocked(); |
| int newState = monitoredPackage.tryPassHealthCheckLocked(); |
| isStateChanged |= oldState != newState; |
| } |
| } |
| } |
| |
| if (isStateChanged) { |
| syncState("health check passed for " + packageName); |
| } |
| } |
| |
| private void onSupportedPackages(List<PackageConfig> supportedPackages) { |
| boolean isStateChanged = false; |
| |
| Map<String, Long> supportedPackageTimeouts = new ArrayMap<>(); |
| Iterator<PackageConfig> it = supportedPackages.iterator(); |
| while (it.hasNext()) { |
| PackageConfig info = it.next(); |
| supportedPackageTimeouts.put(info.getPackageName(), info.getHealthCheckTimeoutMillis()); |
| } |
| |
| synchronized (mLock) { |
| Slog.d(TAG, "Received supported packages " + supportedPackages); |
| Iterator<ObserverInternal> oit = mAllObservers.values().iterator(); |
| while (oit.hasNext()) { |
| Iterator<MonitoredPackage> pit = oit.next().getMonitoredPackages() |
| .values().iterator(); |
| while (pit.hasNext()) { |
| MonitoredPackage monitoredPackage = pit.next(); |
| String packageName = monitoredPackage.getName(); |
| int oldState = monitoredPackage.getHealthCheckStateLocked(); |
| int newState; |
| |
| if (supportedPackageTimeouts.containsKey(packageName)) { |
| // Supported packages become ACTIVE if currently INACTIVE |
| newState = monitoredPackage.setHealthCheckActiveLocked( |
| supportedPackageTimeouts.get(packageName)); |
| } else { |
| // Unsupported packages are marked as PASSED unless already FAILED |
| newState = monitoredPackage.tryPassHealthCheckLocked(); |
| } |
| isStateChanged |= oldState != newState; |
| } |
| } |
| } |
| |
| if (isStateChanged) { |
| syncState("updated health check supported packages " + supportedPackages); |
| } |
| } |
| |
| private void onSyncRequestNotified() { |
| synchronized (mLock) { |
| mSyncRequired = true; |
| syncRequestsAsync(); |
| } |
| } |
| |
| @GuardedBy("mLock") |
| private Set<String> getPackagesPendingHealthChecksLocked() { |
| Set<String> packages = new ArraySet<>(); |
| Iterator<ObserverInternal> oit = mAllObservers.values().iterator(); |
| while (oit.hasNext()) { |
| ObserverInternal observer = oit.next(); |
| Iterator<MonitoredPackage> pit = |
| observer.getMonitoredPackages().values().iterator(); |
| while (pit.hasNext()) { |
| MonitoredPackage monitoredPackage = pit.next(); |
| String packageName = monitoredPackage.getName(); |
| if (monitoredPackage.isPendingHealthChecksLocked()) { |
| packages.add(packageName); |
| } |
| } |
| } |
| return packages; |
| } |
| |
| /** |
| * Syncs the state of the observers. |
| * |
| * <p> Prunes all observers, saves new state to disk, syncs health check requests with the |
| * health check service and schedules the next state sync. |
| */ |
| private void syncState(String reason) { |
| synchronized (mLock) { |
| Slog.i(TAG, "Syncing state, reason: " + reason); |
| pruneObserversLocked(); |
| |
| saveToFileAsync(); |
| syncRequestsAsync(); |
| |
| // Done syncing state, schedule the next state sync |
| scheduleNextSyncStateLocked(); |
| } |
| } |
| |
| private void syncStateWithScheduledReason() { |
| syncState("scheduled"); |
| } |
| |
| @GuardedBy("mLock") |
| private void scheduleNextSyncStateLocked() { |
| long durationMs = getNextStateSyncMillisLocked(); |
| mShortTaskHandler.removeCallbacks(mSyncStateWithScheduledReason); |
| if (durationMs == Long.MAX_VALUE) { |
| Slog.i(TAG, "Cancelling state sync, nothing to sync"); |
| mUptimeAtLastStateSync = 0; |
| } else { |
| mUptimeAtLastStateSync = mSystemClock.uptimeMillis(); |
| mShortTaskHandler.postDelayed(mSyncStateWithScheduledReason, durationMs); |
| } |
| } |
| |
| /** |
| * Returns the next duration in millis to sync the watchdog state. |
| * |
| * @returns Long#MAX_VALUE if there are no observed packages. |
| */ |
| @GuardedBy("mLock") |
| private long getNextStateSyncMillisLocked() { |
| long shortestDurationMs = Long.MAX_VALUE; |
| for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { |
| ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex) |
| .getMonitoredPackages(); |
| for (int pIndex = 0; pIndex < packages.size(); pIndex++) { |
| MonitoredPackage mp = packages.valueAt(pIndex); |
| long duration = mp.getShortestScheduleDurationMsLocked(); |
| if (duration < shortestDurationMs) { |
| shortestDurationMs = duration; |
| } |
| } |
| } |
| return shortestDurationMs; |
| } |
| |
| /** |
| * Removes {@code elapsedMs} milliseconds from all durations on monitored packages |
| * and updates other internal state. |
| */ |
| @GuardedBy("mLock") |
| private void pruneObserversLocked() { |
| long elapsedMs = mUptimeAtLastStateSync == 0 |
| ? 0 : mSystemClock.uptimeMillis() - mUptimeAtLastStateSync; |
| if (elapsedMs <= 0) { |
| Slog.i(TAG, "Not pruning observers, elapsed time: " + elapsedMs + "ms"); |
| return; |
| } |
| |
| Iterator<ObserverInternal> it = mAllObservers.values().iterator(); |
| while (it.hasNext()) { |
| ObserverInternal observer = it.next(); |
| Set<MonitoredPackage> failedPackages = |
| observer.prunePackagesLocked(elapsedMs); |
| if (!failedPackages.isEmpty()) { |
| onHealthCheckFailed(observer, failedPackages); |
| } |
| if (observer.getMonitoredPackages().isEmpty() && (observer.registeredObserver == null |
| || !observer.registeredObserver.isPersistent())) { |
| Slog.i(TAG, "Discarding observer " + observer.name + ". All packages expired"); |
| it.remove(); |
| } |
| } |
| } |
| |
| private void onHealthCheckFailed(ObserverInternal observer, |
| Set<MonitoredPackage> failedPackages) { |
| mLongTaskHandler.post(() -> { |
| synchronized (mLock) { |
| PackageHealthObserver registeredObserver = observer.registeredObserver; |
| if (registeredObserver != null) { |
| Iterator<MonitoredPackage> it = failedPackages.iterator(); |
| while (it.hasNext()) { |
| VersionedPackage versionedPkg = getVersionedPackage(it.next().getName()); |
| if (versionedPkg != null) { |
| Slog.i(TAG, |
| "Explicit health check failed for package " + versionedPkg); |
| registeredObserver.execute(versionedPkg, |
| PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK, 1); |
| } |
| } |
| } |
| } |
| }); |
| } |
| |
| /** |
| * Gets PackageInfo for the given package. Matches any user and apex. |
| * |
| * @throws PackageManager.NameNotFoundException if no such package is installed. |
| */ |
| private PackageInfo getPackageInfo(String packageName) |
| throws PackageManager.NameNotFoundException { |
| PackageManager pm = mContext.getPackageManager(); |
| try { |
| // The MATCH_ANY_USER flag doesn't mix well with the MATCH_APEX |
| // flag, so make two separate attempts to get the package info. |
| // We don't need both flags at the same time because we assume |
| // apex files are always installed for all users. |
| return pm.getPackageInfo(packageName, PackageManager.MATCH_ANY_USER); |
| } catch (PackageManager.NameNotFoundException e) { |
| return pm.getPackageInfo(packageName, PackageManager.MATCH_APEX); |
| } |
| } |
| |
| @Nullable |
| private VersionedPackage getVersionedPackage(String packageName) { |
| final PackageManager pm = mContext.getPackageManager(); |
| if (pm == null || TextUtils.isEmpty(packageName)) { |
| return null; |
| } |
| try { |
| final long versionCode = getPackageInfo(packageName).getLongVersionCode(); |
| return new VersionedPackage(packageName, versionCode); |
| } catch (PackageManager.NameNotFoundException e) { |
| return null; |
| } |
| } |
| |
| /** |
| * Loads mAllObservers from file. |
| * |
| * <p>Note that this is <b>not</b> thread safe and should only called be called |
| * from the constructor. |
| */ |
| private void loadFromFile() { |
| InputStream infile = null; |
| mAllObservers.clear(); |
| try { |
| infile = mPolicyFile.openRead(); |
| final TypedXmlPullParser parser = Xml.resolvePullParser(infile); |
| XmlUtils.beginDocument(parser, TAG_PACKAGE_WATCHDOG); |
| int outerDepth = parser.getDepth(); |
| while (XmlUtils.nextElementWithin(parser, outerDepth)) { |
| ObserverInternal observer = ObserverInternal.read(parser, this); |
| if (observer != null) { |
| mAllObservers.put(observer.name, observer); |
| } |
| } |
| } catch (FileNotFoundException e) { |
| // Nothing to monitor |
| } catch (IOException | NumberFormatException | XmlPullParserException e) { |
| Slog.wtf(TAG, "Unable to read monitored packages, deleting file", e); |
| mPolicyFile.delete(); |
| } finally { |
| IoUtils.closeQuietly(infile); |
| } |
| } |
| |
| private void onPropertyChanged(DeviceConfig.Properties properties) { |
| try { |
| updateConfigs(); |
| } catch (Exception ignore) { |
| Slog.w(TAG, "Failed to reload device config changes"); |
| } |
| } |
| |
| /** Adds a {@link DeviceConfig#OnPropertiesChangedListener}. */ |
| private void setPropertyChangedListenerLocked() { |
| DeviceConfig.addOnPropertiesChangedListener( |
| DeviceConfig.NAMESPACE_ROLLBACK, |
| mContext.getMainExecutor(), |
| mOnPropertyChangedListener); |
| } |
| |
| @VisibleForTesting |
| void removePropertyChangedListener() { |
| DeviceConfig.removeOnPropertiesChangedListener(mOnPropertyChangedListener); |
| } |
| |
| /** |
| * Health check is enabled or disabled after reading the flags |
| * from DeviceConfig. |
| */ |
| @VisibleForTesting |
| void updateConfigs() { |
| synchronized (mLock) { |
| mTriggerFailureCount = DeviceConfig.getInt( |
| DeviceConfig.NAMESPACE_ROLLBACK, |
| PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT, |
| DEFAULT_TRIGGER_FAILURE_COUNT); |
| if (mTriggerFailureCount <= 0) { |
| mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT; |
| } |
| |
| mTriggerFailureDurationMs = DeviceConfig.getInt( |
| DeviceConfig.NAMESPACE_ROLLBACK, |
| PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS, |
| DEFAULT_TRIGGER_FAILURE_DURATION_MS); |
| if (mTriggerFailureDurationMs <= 0) { |
| mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_DURATION_MS; |
| } |
| |
| setExplicitHealthCheckEnabled(DeviceConfig.getBoolean( |
| DeviceConfig.NAMESPACE_ROLLBACK, |
| PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED, |
| DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED)); |
| } |
| } |
| |
| private void registerConnectivityModuleHealthListener() { |
| // TODO: have an internal method to trigger a rollback by reporting high severity errors, |
| // and rely on ActivityManager to inform the watchdog of severe network stack crashes |
| // instead of having this listener in parallel. |
| mConnectivityModuleConnector.registerHealthListener( |
| packageName -> { |
| final VersionedPackage pkg = getVersionedPackage(packageName); |
| if (pkg == null) { |
| Slog.wtf(TAG, "NetworkStack failed but could not find its package"); |
| return; |
| } |
| final List<VersionedPackage> pkgList = Collections.singletonList(pkg); |
| onPackageFailure(pkgList, FAILURE_REASON_EXPLICIT_HEALTH_CHECK); |
| }); |
| } |
| |
| /** |
| * Persists mAllObservers to file. Threshold information is ignored. |
| */ |
| private boolean saveToFile() { |
| Slog.i(TAG, "Saving observer state to file"); |
| synchronized (mLock) { |
| FileOutputStream stream; |
| try { |
| stream = mPolicyFile.startWrite(); |
| } catch (IOException e) { |
| Slog.w(TAG, "Cannot update monitored packages", e); |
| return false; |
| } |
| |
| try { |
| TypedXmlSerializer out = Xml.resolveSerializer(stream); |
| out.startDocument(null, true); |
| out.startTag(null, TAG_PACKAGE_WATCHDOG); |
| out.attributeInt(null, ATTR_VERSION, DB_VERSION); |
| for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { |
| mAllObservers.valueAt(oIndex).writeLocked(out); |
| } |
| out.endTag(null, TAG_PACKAGE_WATCHDOG); |
| out.endDocument(); |
| mPolicyFile.finishWrite(stream); |
| return true; |
| } catch (IOException e) { |
| Slog.w(TAG, "Failed to save monitored packages, restoring backup", e); |
| mPolicyFile.failWrite(stream); |
| return false; |
| } finally { |
| IoUtils.closeQuietly(stream); |
| } |
| } |
| } |
| |
| private void saveToFileAsync() { |
| if (!mLongTaskHandler.hasCallbacks(mSaveToFile)) { |
| mLongTaskHandler.post(mSaveToFile); |
| } |
| } |
| |
| /** Convert a {@code LongArrayQueue} to a String of comma-separated values. */ |
| public static String longArrayQueueToString(LongArrayQueue queue) { |
| if (queue.size() > 0) { |
| StringBuilder sb = new StringBuilder(); |
| sb.append(queue.get(0)); |
| for (int i = 1; i < queue.size(); i++) { |
| sb.append(","); |
| sb.append(queue.get(i)); |
| } |
| return sb.toString(); |
| } |
| return ""; |
| } |
| |
| /** Parse a comma-separated String of longs into a LongArrayQueue. */ |
| public static LongArrayQueue parseLongArrayQueue(String commaSeparatedValues) { |
| LongArrayQueue result = new LongArrayQueue(); |
| if (!TextUtils.isEmpty(commaSeparatedValues)) { |
| String[] values = commaSeparatedValues.split(","); |
| for (String value : values) { |
| result.addLast(Long.parseLong(value)); |
| } |
| } |
| return result; |
| } |
| |
| |
| /** Dump status of every observer in mAllObservers. */ |
| public void dump(IndentingPrintWriter pw) { |
| pw.println("Package Watchdog status"); |
| pw.increaseIndent(); |
| synchronized (mLock) { |
| for (String observerName : mAllObservers.keySet()) { |
| pw.println("Observer name: " + observerName); |
| pw.increaseIndent(); |
| ObserverInternal observerInternal = mAllObservers.get(observerName); |
| observerInternal.dump(pw); |
| pw.decreaseIndent(); |
| } |
| } |
| } |
| |
| @VisibleForTesting |
| @GuardedBy("mLock") |
| void registerObserverInternal(ObserverInternal observerInternal) { |
| mAllObservers.put(observerInternal.name, observerInternal); |
| } |
| |
| /** |
| * Represents an observer monitoring a set of packages along with the failure thresholds for |
| * each package. |
| * |
| * <p> Note, the PackageWatchdog#mLock must always be held when reading or writing |
| * instances of this class. |
| */ |
| static class ObserverInternal { |
| public final String name; |
| @GuardedBy("mLock") |
| private final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>(); |
| @Nullable |
| @GuardedBy("mLock") |
| public PackageHealthObserver registeredObserver; |
| private int mMitigationCount; |
| |
| ObserverInternal(String name, List<MonitoredPackage> packages) { |
| this(name, packages, /*mitigationCount=*/ 0); |
| } |
| |
| ObserverInternal(String name, List<MonitoredPackage> packages, int mitigationCount) { |
| this.name = name; |
| updatePackagesLocked(packages); |
| this.mMitigationCount = mitigationCount; |
| } |
| |
| /** |
| * Writes important {@link MonitoredPackage} details for this observer to file. |
| * Does not persist any package failure thresholds. |
| */ |
| @GuardedBy("mLock") |
| public boolean writeLocked(TypedXmlSerializer out) { |
| try { |
| out.startTag(null, TAG_OBSERVER); |
| out.attribute(null, ATTR_NAME, name); |
| if (Flags.recoverabilityDetection()) { |
| out.attributeInt(null, ATTR_MITIGATION_COUNT, mMitigationCount); |
| } |
| for (int i = 0; i < mPackages.size(); i++) { |
| MonitoredPackage p = mPackages.valueAt(i); |
| p.writeLocked(out); |
| } |
| out.endTag(null, TAG_OBSERVER); |
| return true; |
| } catch (IOException e) { |
| Slog.w(TAG, "Cannot save observer", e); |
| return false; |
| } |
| } |
| |
| public int getBootMitigationCount() { |
| return mMitigationCount; |
| } |
| |
| public void setBootMitigationCount(int mitigationCount) { |
| mMitigationCount = mitigationCount; |
| } |
| |
| @GuardedBy("mLock") |
| public void updatePackagesLocked(List<MonitoredPackage> packages) { |
| for (int pIndex = 0; pIndex < packages.size(); pIndex++) { |
| MonitoredPackage p = packages.get(pIndex); |
| MonitoredPackage existingPackage = getMonitoredPackage(p.getName()); |
| if (existingPackage != null) { |
| existingPackage.updateHealthCheckDuration(p.mDurationMs); |
| } else { |
| putMonitoredPackage(p); |
| } |
| } |
| } |
| |
| /** |
| * Reduces the monitoring durations of all packages observed by this observer by |
| * {@code elapsedMs}. If any duration is less than 0, the package is removed from |
| * observation. If any health check duration is less than 0, the health check result |
| * is evaluated. |
| * |
| * @return a {@link Set} of packages that were removed from the observer without explicit |
| * health check passing, or an empty list if no package expired for which an explicit health |
| * check was still pending |
| */ |
| @GuardedBy("mLock") |
| private Set<MonitoredPackage> prunePackagesLocked(long elapsedMs) { |
| Set<MonitoredPackage> failedPackages = new ArraySet<>(); |
| Iterator<MonitoredPackage> it = mPackages.values().iterator(); |
| while (it.hasNext()) { |
| MonitoredPackage p = it.next(); |
| int oldState = p.getHealthCheckStateLocked(); |
| int newState = p.handleElapsedTimeLocked(elapsedMs); |
| if (oldState != HealthCheckState.FAILED |
| && newState == HealthCheckState.FAILED) { |
| Slog.i(TAG, "Package " + p.getName() + " failed health check"); |
| failedPackages.add(p); |
| } |
| if (p.isExpiredLocked()) { |
| it.remove(); |
| } |
| } |
| return failedPackages; |
| } |
| |
| /** |
| * Increments failure counts of {@code packageName}. |
| * @returns {@code true} if failure threshold is exceeded, {@code false} otherwise |
| */ |
| @GuardedBy("mLock") |
| public boolean onPackageFailureLocked(String packageName) { |
| if (getMonitoredPackage(packageName) == null && registeredObserver.isPersistent() |
| && registeredObserver.mayObservePackage(packageName)) { |
| putMonitoredPackage(sPackageWatchdog.newMonitoredPackage( |
| packageName, DEFAULT_OBSERVING_DURATION_MS, false)); |
| } |
| MonitoredPackage p = getMonitoredPackage(packageName); |
| if (p != null) { |
| return p.onFailureLocked(); |
| } |
| return false; |
| } |
| |
| /** |
| * Returns the map of packages monitored by this observer. |
| * |
| * @return a mapping of package names to {@link MonitoredPackage} objects. |
| */ |
| @GuardedBy("mLock") |
| public ArrayMap<String, MonitoredPackage> getMonitoredPackages() { |
| return mPackages; |
| } |
| |
| /** |
| * Returns the {@link MonitoredPackage} associated with a given package name if the |
| * package is being monitored by this observer. |
| * |
| * @param packageName: the name of the package. |
| * @return the {@link MonitoredPackage} object associated with the package name if one |
| * exists, {@code null} otherwise. |
| */ |
| @GuardedBy("mLock") |
| @Nullable |
| public MonitoredPackage getMonitoredPackage(String packageName) { |
| return mPackages.get(packageName); |
| } |
| |
| /** |
| * Associates a {@link MonitoredPackage} with the observer. |
| * |
| * @param p: the {@link MonitoredPackage} to store. |
| */ |
| @GuardedBy("mLock") |
| public void putMonitoredPackage(MonitoredPackage p) { |
| mPackages.put(p.getName(), p); |
| } |
| |
| /** |
| * Returns one ObserverInternal from the {@code parser} and advances its state. |
| * |
| * <p>Note that this method is <b>not</b> thread safe. It should only be called from |
| * #loadFromFile which in turn is only called on construction of the |
| * singleton PackageWatchdog. |
| **/ |
| public static ObserverInternal read(TypedXmlPullParser parser, PackageWatchdog watchdog) { |
| String observerName = null; |
| int observerMitigationCount = 0; |
| if (TAG_OBSERVER.equals(parser.getName())) { |
| observerName = parser.getAttributeValue(null, ATTR_NAME); |
| if (TextUtils.isEmpty(observerName)) { |
| Slog.wtf(TAG, "Unable to read observer name"); |
| return null; |
| } |
| } |
| List<MonitoredPackage> packages = new ArrayList<>(); |
| int innerDepth = parser.getDepth(); |
| try { |
| if (Flags.recoverabilityDetection()) { |
| try { |
| observerMitigationCount = |
| parser.getAttributeInt(null, ATTR_MITIGATION_COUNT); |
| } catch (XmlPullParserException e) { |
| Slog.i( |
| TAG, |
| "ObserverInternal mitigation count was not present."); |
| } |
| } |
| while (XmlUtils.nextElementWithin(parser, innerDepth)) { |
| if (TAG_PACKAGE.equals(parser.getName())) { |
| try { |
| MonitoredPackage pkg = watchdog.parseMonitoredPackage(parser); |
| if (pkg != null) { |
| packages.add(pkg); |
| } |
| } catch (NumberFormatException e) { |
| Slog.wtf(TAG, "Skipping package for observer " + observerName, e); |
| continue; |
| } |
| } |
| } |
| } catch (XmlPullParserException | IOException e) { |
| Slog.wtf(TAG, "Unable to read observer " + observerName, e); |
| return null; |
| } |
| if (packages.isEmpty()) { |
| return null; |
| } |
| return new ObserverInternal(observerName, packages, observerMitigationCount); |
| } |
| |
| /** Dumps information about this observer and the packages it watches. */ |
| public void dump(IndentingPrintWriter pw) { |
| boolean isPersistent = registeredObserver != null && registeredObserver.isPersistent(); |
| pw.println("Persistent: " + isPersistent); |
| for (String packageName : mPackages.keySet()) { |
| MonitoredPackage p = getMonitoredPackage(packageName); |
| pw.println(packageName + ": "); |
| pw.increaseIndent(); |
| pw.println("# Failures: " + p.mFailureHistory.size()); |
| pw.println("Monitoring duration remaining: " + p.mDurationMs + "ms"); |
| pw.println("Explicit health check duration: " + p.mHealthCheckDurationMs + "ms"); |
| pw.println("Health check state: " + p.toString(p.mHealthCheckState)); |
| pw.decreaseIndent(); |
| } |
| } |
| } |
| |
| @Retention(SOURCE) |
| @IntDef(value = { |
| HealthCheckState.ACTIVE, |
| HealthCheckState.INACTIVE, |
| HealthCheckState.PASSED, |
| HealthCheckState.FAILED}) |
| public @interface HealthCheckState { |
| // The package has not passed health check but has requested a health check |
| int ACTIVE = 0; |
| // The package has not passed health check and has not requested a health check |
| int INACTIVE = 1; |
| // The package has passed health check |
| int PASSED = 2; |
| // The package has failed health check |
| int FAILED = 3; |
| } |
| |
| MonitoredPackage newMonitoredPackage( |
| String name, long durationMs, boolean hasPassedHealthCheck) { |
| return newMonitoredPackage(name, durationMs, Long.MAX_VALUE, hasPassedHealthCheck, |
| new LongArrayQueue()); |
| } |
| |
| MonitoredPackage newMonitoredPackage(String name, long durationMs, long healthCheckDurationMs, |
| boolean hasPassedHealthCheck, LongArrayQueue mitigationCalls) { |
| return new MonitoredPackage(name, durationMs, healthCheckDurationMs, |
| hasPassedHealthCheck, mitigationCalls); |
| } |
| |
| MonitoredPackage parseMonitoredPackage(TypedXmlPullParser parser) |
| throws XmlPullParserException { |
| String packageName = parser.getAttributeValue(null, ATTR_NAME); |
| long duration = parser.getAttributeLong(null, ATTR_DURATION); |
| long healthCheckDuration = parser.getAttributeLong(null, |
| ATTR_EXPLICIT_HEALTH_CHECK_DURATION); |
| boolean hasPassedHealthCheck = parser.getAttributeBoolean(null, ATTR_PASSED_HEALTH_CHECK); |
| LongArrayQueue mitigationCalls = parseLongArrayQueue( |
| parser.getAttributeValue(null, ATTR_MITIGATION_CALLS)); |
| return newMonitoredPackage(packageName, |
| duration, healthCheckDuration, hasPassedHealthCheck, mitigationCalls); |
| } |
| |
| /** |
| * Represents a package and its health check state along with the time |
| * it should be monitored for. |
| * |
| * <p> Note, the PackageWatchdog#mLock must always be held when reading or writing |
| * instances of this class. |
| */ |
| class MonitoredPackage { |
| private final String mPackageName; |
| // Times when package failures happen sorted in ascending order |
| @GuardedBy("mLock") |
| private final LongArrayQueue mFailureHistory = new LongArrayQueue(); |
| // Times when an observer was called to mitigate this package's failure. Sorted in |
| // ascending order. |
| @GuardedBy("mLock") |
| private final LongArrayQueue mMitigationCalls; |
| // One of STATE_[ACTIVE|INACTIVE|PASSED|FAILED]. Updated on construction and after |
| // methods that could change the health check state: handleElapsedTimeLocked and |
| // tryPassHealthCheckLocked |
| private int mHealthCheckState = HealthCheckState.INACTIVE; |
| // Whether an explicit health check has passed. |
| // This value in addition with mHealthCheckDurationMs determines the health check state |
| // of the package, see #getHealthCheckStateLocked |
| @GuardedBy("mLock") |
| private boolean mHasPassedHealthCheck; |
| // System uptime duration to monitor package. |
| @GuardedBy("mLock") |
| private long mDurationMs; |
| // System uptime duration to check the result of an explicit health check |
| // Initially, MAX_VALUE until we get a value from the health check service |
| // and request health checks. |
| // This value in addition with mHasPassedHealthCheck determines the health check state |
| // of the package, see #getHealthCheckStateLocked |
| @GuardedBy("mLock") |
| private long mHealthCheckDurationMs = Long.MAX_VALUE; |
| |
| MonitoredPackage(String packageName, long durationMs, |
| long healthCheckDurationMs, boolean hasPassedHealthCheck, |
| LongArrayQueue mitigationCalls) { |
| mPackageName = packageName; |
| mDurationMs = durationMs; |
| mHealthCheckDurationMs = healthCheckDurationMs; |
| mHasPassedHealthCheck = hasPassedHealthCheck; |
| mMitigationCalls = mitigationCalls; |
| updateHealthCheckStateLocked(); |
| } |
| |
| /** Writes the salient fields to disk using {@code out}. */ |
| @GuardedBy("mLock") |
| public void writeLocked(TypedXmlSerializer out) throws IOException { |
| out.startTag(null, TAG_PACKAGE); |
| out.attribute(null, ATTR_NAME, getName()); |
| out.attributeLong(null, ATTR_DURATION, mDurationMs); |
| out.attributeLong(null, ATTR_EXPLICIT_HEALTH_CHECK_DURATION, mHealthCheckDurationMs); |
| out.attributeBoolean(null, ATTR_PASSED_HEALTH_CHECK, mHasPassedHealthCheck); |
| LongArrayQueue normalizedCalls = normalizeMitigationCalls(); |
| out.attribute(null, ATTR_MITIGATION_CALLS, longArrayQueueToString(normalizedCalls)); |
| out.endTag(null, TAG_PACKAGE); |
| } |
| |
| /** |
| * Increment package failures or resets failure count depending on the last package failure. |
| * |
| * @return {@code true} if failure count exceeds a threshold, {@code false} otherwise |
| */ |
| @GuardedBy("mLock") |
| public boolean onFailureLocked() { |
| // Sliding window algorithm: find out if there exists a window containing failures >= |
| // mTriggerFailureCount. |
| final long now = mSystemClock.uptimeMillis(); |
| mFailureHistory.addLast(now); |
| while (now - mFailureHistory.peekFirst() > mTriggerFailureDurationMs) { |
| // Prune values falling out of the window |
| mFailureHistory.removeFirst(); |
| } |
| boolean failed = mFailureHistory.size() >= mTriggerFailureCount; |
| if (failed) { |
| mFailureHistory.clear(); |
| } |
| return failed; |
| } |
| |
| /** |
| * Notes the timestamp of a mitigation call into the observer. |
| */ |
| @GuardedBy("mLock") |
| public void noteMitigationCallLocked() { |
| mMitigationCalls.addLast(mSystemClock.uptimeMillis()); |
| } |
| |
| /** |
| * Prunes any mitigation calls outside of the de-escalation window, and returns the |
| * number of calls that are in the window afterwards. |
| * |
| * @return the number of mitigation calls made in the de-escalation window. |
| */ |
| @GuardedBy("mLock") |
| public int getMitigationCountLocked() { |
| try { |
| final long now = mSystemClock.uptimeMillis(); |
| while (now - mMitigationCalls.peekFirst() > DEFAULT_DEESCALATION_WINDOW_MS) { |
| mMitigationCalls.removeFirst(); |
| } |
| } catch (NoSuchElementException ignore) { |
| } |
| |
| return mMitigationCalls.size(); |
| } |
| |
| /** |
| * Before writing to disk, make the mitigation call timestamps relative to the current |
| * system uptime. This is because they need to be relative to the uptime which will reset |
| * at the next boot. |
| * |
| * @return a LongArrayQueue of the mitigation calls relative to the current system uptime. |
| */ |
| @GuardedBy("mLock") |
| public LongArrayQueue normalizeMitigationCalls() { |
| LongArrayQueue normalized = new LongArrayQueue(); |
| final long now = mSystemClock.uptimeMillis(); |
| for (int i = 0; i < mMitigationCalls.size(); i++) { |
| normalized.addLast(mMitigationCalls.get(i) - now); |
| } |
| return normalized; |
| } |
| |
| /** |
| * Sets the initial health check duration. |
| * |
| * @return the new health check state |
| */ |
| @GuardedBy("mLock") |
| public int setHealthCheckActiveLocked(long initialHealthCheckDurationMs) { |
| if (initialHealthCheckDurationMs <= 0) { |
| Slog.wtf(TAG, "Cannot set non-positive health check duration " |
| + initialHealthCheckDurationMs + "ms for package " + getName() |
| + ". Using total duration " + mDurationMs + "ms instead"); |
| initialHealthCheckDurationMs = mDurationMs; |
| } |
| if (mHealthCheckState == HealthCheckState.INACTIVE) { |
| // Transitions to ACTIVE |
| mHealthCheckDurationMs = initialHealthCheckDurationMs; |
| } |
| return updateHealthCheckStateLocked(); |
| } |
| |
| /** |
| * Updates the monitoring durations of the package. |
| * |
| * @return the new health check state |
| */ |
| @GuardedBy("mLock") |
| public int handleElapsedTimeLocked(long elapsedMs) { |
| if (elapsedMs <= 0) { |
| Slog.w(TAG, "Cannot handle non-positive elapsed time for package " + getName()); |
| return mHealthCheckState; |
| } |
| // Transitions to FAILED if now <= 0 and health check not passed |
| mDurationMs -= elapsedMs; |
| if (mHealthCheckState == HealthCheckState.ACTIVE) { |
| // We only update health check durations if we have #setHealthCheckActiveLocked |
| // This ensures we don't leave the INACTIVE state for an unexpected elapsed time |
| // Transitions to FAILED if now <= 0 and health check not passed |
| mHealthCheckDurationMs -= elapsedMs; |
| } |
| return updateHealthCheckStateLocked(); |
| } |
| |
| /** Explicitly update the monitoring duration of the package. */ |
| @GuardedBy("mLock") |
| public void updateHealthCheckDuration(long newDurationMs) { |
| mDurationMs = newDurationMs; |
| } |
| |
| /** |
| * Marks the health check as passed and transitions to {@link HealthCheckState.PASSED} |
| * if not yet {@link HealthCheckState.FAILED}. |
| * |
| * @return the new {@link HealthCheckState health check state} |
| */ |
| @GuardedBy("mLock") |
| @HealthCheckState |
| public int tryPassHealthCheckLocked() { |
| if (mHealthCheckState != HealthCheckState.FAILED) { |
| // FAILED is a final state so only pass if we haven't failed |
| // Transition to PASSED |
| mHasPassedHealthCheck = true; |
| } |
| return updateHealthCheckStateLocked(); |
| } |
| |
| /** Returns the monitored package name. */ |
| private String getName() { |
| return mPackageName; |
| } |
| |
| /** |
| * Returns the current {@link HealthCheckState health check state}. |
| */ |
| @GuardedBy("mLock") |
| @HealthCheckState |
| public int getHealthCheckStateLocked() { |
| return mHealthCheckState; |
| } |
| |
| /** |
| * Returns the shortest duration before the package should be scheduled for a prune. |
| * |
| * @return the duration or {@link Long#MAX_VALUE} if the package should not be scheduled |
| */ |
| @GuardedBy("mLock") |
| public long getShortestScheduleDurationMsLocked() { |
| // Consider health check duration only if #isPendingHealthChecksLocked is true |
| return Math.min(toPositive(mDurationMs), |
| isPendingHealthChecksLocked() |
| ? toPositive(mHealthCheckDurationMs) : Long.MAX_VALUE); |
| } |
| |
| /** |
| * Returns {@code true} if the total duration left to monitor the package is less than or |
| * equal to 0 {@code false} otherwise. |
| */ |
| @GuardedBy("mLock") |
| public boolean isExpiredLocked() { |
| return mDurationMs <= 0; |
| } |
| |
| /** |
| * Returns {@code true} if the package, {@link #getName} is expecting health check results |
| * {@code false} otherwise. |
| */ |
| @GuardedBy("mLock") |
| public boolean isPendingHealthChecksLocked() { |
| return mHealthCheckState == HealthCheckState.ACTIVE |
| || mHealthCheckState == HealthCheckState.INACTIVE; |
| } |
| |
| /** |
| * Updates the health check state based on {@link #mHasPassedHealthCheck} |
| * and {@link #mHealthCheckDurationMs}. |
| * |
| * @return the new {@link HealthCheckState health check state} |
| */ |
| @GuardedBy("mLock") |
| @HealthCheckState |
| private int updateHealthCheckStateLocked() { |
| int oldState = mHealthCheckState; |
| if (mHasPassedHealthCheck) { |
| // Set final state first to avoid ambiguity |
| mHealthCheckState = HealthCheckState.PASSED; |
| } else if (mHealthCheckDurationMs <= 0 || mDurationMs <= 0) { |
| // Set final state first to avoid ambiguity |
| mHealthCheckState = HealthCheckState.FAILED; |
| } else if (mHealthCheckDurationMs == Long.MAX_VALUE) { |
| mHealthCheckState = HealthCheckState.INACTIVE; |
| } else { |
| mHealthCheckState = HealthCheckState.ACTIVE; |
| } |
| |
| if (oldState != mHealthCheckState) { |
| Slog.i(TAG, "Updated health check state for package " + getName() + ": " |
| + toString(oldState) + " -> " + toString(mHealthCheckState)); |
| } |
| return mHealthCheckState; |
| } |
| |
| /** Returns a {@link String} representation of the current health check state. */ |
| private String toString(@HealthCheckState int state) { |
| switch (state) { |
| case HealthCheckState.ACTIVE: |
| return "ACTIVE"; |
| case HealthCheckState.INACTIVE: |
| return "INACTIVE"; |
| case HealthCheckState.PASSED: |
| return "PASSED"; |
| case HealthCheckState.FAILED: |
| return "FAILED"; |
| default: |
| return "UNKNOWN"; |
| } |
| } |
| |
| /** Returns {@code value} if it is greater than 0 or {@link Long#MAX_VALUE} otherwise. */ |
| private long toPositive(long value) { |
| return value > 0 ? value : Long.MAX_VALUE; |
| } |
| |
| /** Compares the equality of this object with another {@link MonitoredPackage}. */ |
| @VisibleForTesting |
| boolean isEqualTo(MonitoredPackage pkg) { |
| return (getName().equals(pkg.getName())) |
| && mDurationMs == pkg.mDurationMs |
| && mHasPassedHealthCheck == pkg.mHasPassedHealthCheck |
| && mHealthCheckDurationMs == pkg.mHealthCheckDurationMs |
| && (mMitigationCalls.toString()).equals(pkg.mMitigationCalls.toString()); |
| } |
| } |
| |
| @GuardedBy("mLock") |
| @SuppressWarnings("GuardedBy") |
| void saveAllObserversBootMitigationCountToMetadata(String filePath) { |
| HashMap<String, Integer> bootMitigationCounts = new HashMap<>(); |
| for (int i = 0; i < mAllObservers.size(); i++) { |
| final ObserverInternal observer = mAllObservers.valueAt(i); |
| bootMitigationCounts.put(observer.name, observer.getBootMitigationCount()); |
| } |
| |
| try { |
| FileOutputStream fileStream = new FileOutputStream(new File(filePath)); |
| ObjectOutputStream objectStream = new ObjectOutputStream(fileStream); |
| objectStream.writeObject(bootMitigationCounts); |
| objectStream.flush(); |
| objectStream.close(); |
| fileStream.close(); |
| } catch (Exception e) { |
| Slog.i(TAG, "Could not save observers metadata to file: " + e); |
| } |
| } |
| |
| /** |
| * Handles the thresholding logic for system server boots. |
| */ |
| class BootThreshold { |
| |
| private final int mBootTriggerCount; |
| private final long mTriggerWindow; |
| |
| BootThreshold(int bootTriggerCount, long triggerWindow) { |
| this.mBootTriggerCount = bootTriggerCount; |
| this.mTriggerWindow = triggerWindow; |
| } |
| |
| public void reset() { |
| setStart(0); |
| setCount(0); |
| } |
| |
| protected int getCount() { |
| return CrashRecoveryProperties.rescueBootCount().orElse(0); |
| } |
| |
| protected void setCount(int count) { |
| CrashRecoveryProperties.rescueBootCount(count); |
| } |
| |
| public long getStart() { |
| return CrashRecoveryProperties.rescueBootStart().orElse(0L); |
| } |
| |
| public int getMitigationCount() { |
| return CrashRecoveryProperties.bootMitigationCount().orElse(0); |
| } |
| |
| public void setStart(long start) { |
| CrashRecoveryProperties.rescueBootStart(getStartTime(start)); |
| } |
| |
| public void setMitigationStart(long start) { |
| CrashRecoveryProperties.bootMitigationStart(getStartTime(start)); |
| } |
| |
| public long getMitigationStart() { |
| return CrashRecoveryProperties.bootMitigationStart().orElse(0L); |
| } |
| |
| public void setMitigationCount(int count) { |
| CrashRecoveryProperties.bootMitigationCount(count); |
| } |
| |
| private static long constrain(long amount, long low, long high) { |
| return amount < low ? low : (amount > high ? high : amount); |
| } |
| |
| public long getStartTime(long start) { |
| final long now = mSystemClock.uptimeMillis(); |
| return constrain(start, 0, now); |
| } |
| |
| public void saveMitigationCountToMetadata() { |
| try (BufferedWriter writer = new BufferedWriter(new FileWriter(METADATA_FILE))) { |
| writer.write(String.valueOf(getMitigationCount())); |
| } catch (Exception e) { |
| Slog.e(TAG, "Could not save metadata to file: " + e); |
| } |
| } |
| |
| public void readMitigationCountFromMetadataIfNecessary() { |
| File bootPropsFile = new File(METADATA_FILE); |
| if (bootPropsFile.exists()) { |
| try (BufferedReader reader = new BufferedReader(new FileReader(METADATA_FILE))) { |
| String mitigationCount = reader.readLine(); |
| setMitigationCount(Integer.parseInt(mitigationCount)); |
| bootPropsFile.delete(); |
| } catch (Exception e) { |
| Slog.i(TAG, "Could not read metadata file: " + e); |
| } |
| } |
| } |
| |
| |
| /** Increments the boot counter, and returns whether the device is bootlooping. */ |
| @GuardedBy("mLock") |
| public boolean incrementAndTest() { |
| if (Flags.recoverabilityDetection()) { |
| readAllObserversBootMitigationCountIfNecessary(METADATA_FILE); |
| } else { |
| readMitigationCountFromMetadataIfNecessary(); |
| } |
| |
| final long now = mSystemClock.uptimeMillis(); |
| if (now - getStart() < 0) { |
| Slog.e(TAG, "Window was less than zero. Resetting start to current time."); |
| setStart(now); |
| setMitigationStart(now); |
| } |
| if (now - getMitigationStart() > DEFAULT_DEESCALATION_WINDOW_MS) { |
| setMitigationStart(now); |
| if (Flags.recoverabilityDetection()) { |
| resetAllObserversBootMitigationCount(); |
| } else { |
| setMitigationCount(0); |
| } |
| } |
| final long window = now - getStart(); |
| if (window >= mTriggerWindow) { |
| setCount(1); |
| setStart(now); |
| return false; |
| } else { |
| int count = getCount() + 1; |
| setCount(count); |
| EventLogTags.writeRescueNote(Process.ROOT_UID, count, window); |
| if (Flags.recoverabilityDetection()) { |
| // After a reboot (e.g. by WARM_REBOOT or mainline rollback) we apply |
| // mitigations without waiting for DEFAULT_BOOT_LOOP_TRIGGER_COUNT. |
| return (count >= mBootTriggerCount) |
| || (performedMitigationsDuringWindow() && count > 1); |
| } |
| return count >= mBootTriggerCount; |
| } |
| } |
| |
| @GuardedBy("mLock") |
| private boolean performedMitigationsDuringWindow() { |
| for (ObserverInternal observerInternal: mAllObservers.values()) { |
| if (observerInternal.getBootMitigationCount() > 0) { |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| @GuardedBy("mLock") |
| private void resetAllObserversBootMitigationCount() { |
| for (int i = 0; i < mAllObservers.size(); i++) { |
| final ObserverInternal observer = mAllObservers.valueAt(i); |
| observer.setBootMitigationCount(0); |
| } |
| saveAllObserversBootMitigationCountToMetadata(METADATA_FILE); |
| } |
| |
| @GuardedBy("mLock") |
| @SuppressWarnings("GuardedBy") |
| void readAllObserversBootMitigationCountIfNecessary(String filePath) { |
| File metadataFile = new File(filePath); |
| if (metadataFile.exists()) { |
| try { |
| FileInputStream fileStream = new FileInputStream(metadataFile); |
| ObjectInputStream objectStream = new ObjectInputStream(fileStream); |
| HashMap<String, Integer> bootMitigationCounts = |
| (HashMap<String, Integer>) objectStream.readObject(); |
| objectStream.close(); |
| fileStream.close(); |
| |
| for (int i = 0; i < mAllObservers.size(); i++) { |
| final ObserverInternal observer = mAllObservers.valueAt(i); |
| if (bootMitigationCounts.containsKey(observer.name)) { |
| observer.setBootMitigationCount( |
| bootMitigationCounts.get(observer.name)); |
| } |
| } |
| } catch (Exception e) { |
| Slog.i(TAG, "Could not read observer metadata file: " + e); |
| } |
| } |
| } |
| |
| } |
| } |
