odrefresh: add metrics support
Adds metrics to stages of odrefresh.
Bug: 169925964
Test: atest art_odrefresh_tests
Test: atest --host art_odrefresh_tests
(cherry picked from commit 3d877f082636f26ad57c92e3aae1525faacff51b)
Merged-In: I768ce5f122b0c1b839f4cdf55aa6dafb68708eb2
Change-Id: I8355fd38c28e41b04f0ea52384061b686cb1e362
diff --git a/odrefresh/Android.bp b/odrefresh/Android.bp
index 6f84e8f..8a9acd3 100644
--- a/odrefresh/Android.bp
+++ b/odrefresh/Android.bp
@@ -30,12 +30,15 @@
srcs: [
"odrefresh.cc",
"odr_fs_utils.cc",
+ "odr_metrics.cc",
+ "odr_metrics_record.cc",
],
local_include_dirs: ["include"],
header_libs: ["dexoptanalyzer_headers"],
generated_sources: [
"apex-info-list",
"art-apex-cache-info",
+ "art-odrefresh-operator-srcs",
],
shared_libs: [
"libartpalette",
@@ -81,6 +84,16 @@
visibility: ["//visibility:public"],
}
+gensrcs {
+ name: "art-odrefresh-operator-srcs",
+ cmd: "$(location generate_operator_out) art/odrefresh $(in) > $(out)",
+ tools: ["generate_operator_out"],
+ srcs: [
+ "odr_metrics.h",
+ ],
+ output_extension: "operator_out.cc",
+}
+
art_cc_binary {
name: "odrefresh",
defaults: ["odrefresh-defaults"],
@@ -126,16 +139,19 @@
defaults: [
"art_gtest_defaults",
],
+ generated_sources: ["art-odrefresh-operator-srcs"],
header_libs: ["odrefresh_headers"],
srcs: [
"odr_artifacts_test.cc",
"odr_fs_utils.cc",
"odr_fs_utils_test.cc",
+ "odr_metrics.cc",
+ "odr_metrics_test.cc",
+ "odr_metrics_record.cc",
+ "odr_metrics_record_test.cc",
"odrefresh_test.cc",
],
- shared_libs: [
- "libbase",
- ],
+ shared_libs: ["libbase"],
}
xsd_config {
diff --git a/odrefresh/odr_metrics.cc b/odrefresh/odr_metrics.cc
new file mode 100644
index 0000000..4bddb17
--- /dev/null
+++ b/odrefresh/odr_metrics.cc
@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "odr_metrics.h"
+
+#include <unistd.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <fstream>
+#include <iosfwd>
+#include <optional>
+#include <ostream>
+#include <string>
+
+#include <android-base/logging.h>
+#include <base/os.h>
+#include <base/string_view_cpp20.h>
+#include <odr_fs_utils.h>
+#include <odr_metrics_record.h>
+
+namespace art {
+namespace odrefresh {
+
+OdrMetrics::OdrMetrics(const std::string& cache_directory, const std::string& metrics_file)
+ : cache_directory_(cache_directory), metrics_file_(metrics_file), status_(Status::kOK) {
+ DCHECK(StartsWith(metrics_file_, "/"));
+
+ // Remove existing metrics file if it exists.
+ if (OS::FileExists(metrics_file.c_str())) {
+ if (unlink(metrics_file.c_str()) != 0) {
+ PLOG(ERROR) << "Failed to remove metrics file '" << metrics_file << "'";
+ }
+ }
+
+ // Create apexdata dalvik-cache directory if it does not exist. It is required before
+ // calling GetFreeSpaceMiB().
+ if (!EnsureDirectoryExists(cache_directory)) {
+ // This should never fail except for no space on device or configuration issues (e.g. SELinux).
+ LOG(WARNING) << "Cache directory '" << cache_directory << "' could not be created.";
+ }
+ cache_space_free_start_mib_ = GetFreeSpaceMiB(cache_directory);
+}
+
+OdrMetrics::~OdrMetrics() {
+ cache_space_free_end_mib_ = GetFreeSpaceMiB(cache_directory_);
+
+ // Log metrics only if odrefresh detected a reason to compile.
+ if (trigger_.has_value()) {
+ WriteToFile(metrics_file_, this);
+ }
+}
+
+void OdrMetrics::SetCompilationTime(int32_t seconds) {
+ switch (stage_) {
+ case Stage::kPrimaryBootClasspath:
+ primary_bcp_compilation_seconds_ = seconds;
+ break;
+ case Stage::kSecondaryBootClasspath:
+ secondary_bcp_compilation_seconds_ = seconds;
+ break;
+ case Stage::kSystemServerClasspath:
+ system_server_compilation_seconds_ = seconds;
+ break;
+ case Stage::kCheck:
+ case Stage::kComplete:
+ case Stage::kPreparation:
+ case Stage::kUnknown:
+ break;
+ }
+}
+
+void OdrMetrics::SetStage(Stage stage) {
+ if (status_ == Status::kOK) {
+ stage_ = stage;
+ }
+}
+
+int32_t OdrMetrics::GetFreeSpaceMiB(const std::string& path) {
+ static constexpr uint32_t kBytesPerMiB = 1024 * 1024;
+ static constexpr uint64_t kNominalMaximumCacheBytes = 1024 * kBytesPerMiB;
+
+ // Assume nominal cache space is 1GiB (much larger than expected, ~100MB).
+ uint64_t used_space_bytes;
+ if (!GetUsedSpace(path, &used_space_bytes)) {
+ used_space_bytes = 0;
+ }
+ uint64_t nominal_free_space_bytes = kNominalMaximumCacheBytes - used_space_bytes;
+
+ // Get free space on partition containing `path`.
+ uint64_t free_space_bytes;
+ if (!GetFreeSpace(path, &free_space_bytes)) {
+ free_space_bytes = kNominalMaximumCacheBytes;
+ }
+
+ // Pick the smallest free space, ie space on partition or nominal space in cache.
+ // There are two things of interest for metrics:
+ // (i) identifying failed compilations due to low space.
+ // (ii) understanding what the storage requirements are for the spectrum of boot classpaths and
+ // system_server classpaths.
+ uint64_t free_space_mib = std::min(free_space_bytes, nominal_free_space_bytes) / kBytesPerMiB;
+ return static_cast<int32_t>(free_space_mib);
+}
+
+bool OdrMetrics::ToRecord(/*out*/OdrMetricsRecord* record) const {
+ if (!trigger_.has_value()) {
+ return false;
+ }
+ record->art_apex_version = art_apex_version_;
+ record->trigger = static_cast<uint32_t>(trigger_.value());
+ record->stage_reached = static_cast<uint32_t>(stage_);
+ record->status = static_cast<uint32_t>(status_);
+ record->primary_bcp_compilation_seconds = primary_bcp_compilation_seconds_;
+ record->secondary_bcp_compilation_seconds = secondary_bcp_compilation_seconds_;
+ record->system_server_compilation_seconds = system_server_compilation_seconds_;
+ record->cache_space_free_start_mib = cache_space_free_start_mib_;
+ record->cache_space_free_end_mib = cache_space_free_end_mib_;
+ return true;
+}
+
+void OdrMetrics::WriteToFile(const std::string& path, const OdrMetrics* metrics) {
+ OdrMetricsRecord record;
+ if (!metrics->ToRecord(&record)) {
+ LOG(ERROR) << "Attempting to report metrics without a compilation trigger.";
+ return;
+ }
+
+ // Preserve order from frameworks/proto_logging/stats/atoms.proto in metrics file written.
+ std::ofstream ofs(path);
+ ofs << record;
+}
+
+} // namespace odrefresh
+} // namespace art
diff --git a/odrefresh/odr_metrics.h b/odrefresh/odr_metrics.h
new file mode 100644
index 0000000..8b8d5ff
--- /dev/null
+++ b/odrefresh/odr_metrics.h
@@ -0,0 +1,155 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_ODREFRESH_ODR_METRICS_H_
+#define ART_ODREFRESH_ODR_METRICS_H_
+
+#include <chrono>
+#include <cstdint>
+#include <iosfwd>
+#include <optional>
+#include <string>
+
+#include "base/macros.h"
+#include "odr_metrics_record.h"
+
+namespace art {
+namespace odrefresh {
+
+class OdrMetrics final {
+ public:
+ // Enumeration used to track the latest stage reached running odrefresh.
+ //
+ // These values mirror those in OdrefreshReported::Stage in frameworks/proto_logging/atoms.proto.
+ // NB There are gaps between the values in case an additional stages are introduced.
+ enum class Stage : uint8_t {
+ kUnknown = 0,
+ kCheck = 10,
+ kPreparation = 20,
+ kPrimaryBootClasspath = 30,
+ kSecondaryBootClasspath = 40,
+ kSystemServerClasspath = 50,
+ kComplete = 60,
+ };
+
+ // Enumeration describing the overall status, processing stops on the first error discovered.
+ //
+ // These values mirror those in OdrefreshReported::Status in frameworks/proto_logging/atoms.proto.
+ enum class Status : uint8_t {
+ kUnknown = 0,
+ kOK = 1,
+ kNoSpace = 2,
+ kIoError = 3,
+ kDex2OatError = 4,
+ kTimeLimitExceeded = 5,
+ kStagingFailed = 6,
+ kInstallFailed = 7,
+ };
+
+ // Enumeration describing the cause of compilation (if any) in odrefresh.
+ //
+ // These values mirror those in OdrefreshReported::Trigger in
+ // frameworks/proto_logging/atoms.proto.
+ enum class Trigger : uint8_t {
+ kUnknown = 0,
+ kApexVersionMismatch = 1,
+ kDexFilesChanged = 2,
+ kMissingArtifacts = 3,
+ };
+
+ explicit OdrMetrics(const std::string& cache_directory,
+ const std::string& metrics_file = kOdrefreshMetricsFile);
+ ~OdrMetrics();
+
+ // Sets the ART APEX that metrics are being collected on behalf of.
+ void SetArtApexVersion(int64_t version) {
+ art_apex_version_ = version;
+ }
+
+ // Sets the trigger for metrics collection. The trigger is the reason why odrefresh considers
+ // compilation necessary. Only call this method if compilation is necessary as the presence
+ // of a trigger means we will try to record and upload metrics.
+ void SetTrigger(const Trigger trigger) {
+ trigger_ = trigger;
+ }
+
+ // Sets the execution status of the current odrefresh processing stage.
+ void SetStatus(const Status status) {
+ status_ = status;
+ }
+
+ // Sets the current odrefresh processing stage.
+ void SetStage(Stage stage);
+
+ // Record metrics into an OdrMetricsRecord.
+ // returns true on success, false if instance is not valid (because the trigger value is not set).
+ bool ToRecord(/*out*/OdrMetricsRecord* record) const;
+
+ private:
+ OdrMetrics(const OdrMetrics&) = delete;
+ OdrMetrics operator=(const OdrMetrics&) = delete;
+
+ static int32_t GetFreeSpaceMiB(const std::string& path);
+ static void WriteToFile(const std::string& path, const OdrMetrics* metrics);
+
+ void SetCompilationTime(int32_t seconds);
+
+ const std::string cache_directory_;
+ const std::string metrics_file_;
+
+ int64_t art_apex_version_ = 0;
+ std::optional<Trigger> trigger_ = {}; // metrics are only logged if compilation is triggered.
+ Stage stage_ = Stage::kUnknown;
+ Status status_ = Status::kUnknown;
+
+ int32_t primary_bcp_compilation_seconds_ = 0;
+ int32_t secondary_bcp_compilation_seconds_ = 0;
+ int32_t system_server_compilation_seconds_ = 0;
+ int32_t cache_space_free_start_mib_ = 0;
+ int32_t cache_space_free_end_mib_ = 0;
+
+ friend class ScopedOdrCompilationTimer;
+};
+
+// Timer used to measure compilation time (in seconds). Automatically associates the time recorded
+// with the current stage of the metrics used.
+class ScopedOdrCompilationTimer final {
+ public:
+ explicit ScopedOdrCompilationTimer(OdrMetrics& metrics) :
+ metrics_(metrics), start_(std::chrono::steady_clock::now()) {}
+
+ ~ScopedOdrCompilationTimer() {
+ auto elapsed_time = std::chrono::steady_clock::now() - start_;
+ auto elapsed_seconds = std::chrono::duration_cast<std::chrono::seconds>(elapsed_time);
+ metrics_.SetCompilationTime(static_cast<int32_t>(elapsed_seconds.count()));
+ }
+
+ private:
+ OdrMetrics& metrics_;
+ std::chrono::time_point<std::chrono::steady_clock> start_;
+
+ DISALLOW_ALLOCATION();
+};
+
+// Generated ostream operators.
+std::ostream& operator<<(std::ostream& os, OdrMetrics::Status status);
+std::ostream& operator<<(std::ostream& os, OdrMetrics::Stage stage);
+std::ostream& operator<<(std::ostream& os, OdrMetrics::Trigger trigger);
+
+} // namespace odrefresh
+} // namespace art
+
+#endif // ART_ODREFRESH_ODR_METRICS_H_
diff --git a/odrefresh/odr_metrics_record.cc b/odrefresh/odr_metrics_record.cc
new file mode 100644
index 0000000..fc135d3
--- /dev/null
+++ b/odrefresh/odr_metrics_record.cc
@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "odr_metrics_record.h"
+
+#include <iosfwd>
+#include <istream>
+#include <ostream>
+#include <streambuf>
+#include <string>
+
+namespace art {
+namespace odrefresh {
+
+std::istream& operator>>(std::istream& is, OdrMetricsRecord& record) {
+ // Block I/O related exceptions
+ auto saved_exceptions = is.exceptions();
+ is.exceptions(std::ios_base::iostate {});
+
+ // The order here matches the field order of MetricsRecord.
+ is >> record.art_apex_version >> std::ws;
+ is >> record.trigger >> std::ws;
+ is >> record.stage_reached >> std::ws;
+ is >> record.status >> std::ws;
+ is >> record.primary_bcp_compilation_seconds >> std::ws;
+ is >> record.secondary_bcp_compilation_seconds >> std::ws;
+ is >> record.system_server_compilation_seconds >> std::ws;
+ is >> record.cache_space_free_start_mib >> std::ws;
+ is >> record.cache_space_free_end_mib >> std::ws;
+
+ // Restore I/O related exceptions
+ is.exceptions(saved_exceptions);
+ return is;
+}
+
+std::ostream& operator<<(std::ostream& os, const OdrMetricsRecord& record) {
+ static const char kSpace = ' ';
+
+ // Block I/O related exceptions
+ auto saved_exceptions = os.exceptions();
+ os.exceptions(std::ios_base::iostate {});
+
+ // The order here matches the field order of MetricsRecord.
+ os << record.art_apex_version << kSpace;
+ os << record.trigger << kSpace;
+ os << record.stage_reached << kSpace;
+ os << record.status << kSpace;
+ os << record.primary_bcp_compilation_seconds << kSpace;
+ os << record.secondary_bcp_compilation_seconds << kSpace;
+ os << record.system_server_compilation_seconds << kSpace;
+ os << record.cache_space_free_start_mib << kSpace;
+ os << record.cache_space_free_end_mib << std::endl;
+
+ // Restore I/O related exceptions
+ os.exceptions(saved_exceptions);
+ return os;
+}
+
+} // namespace odrefresh
+} // namespace art
diff --git a/odrefresh/odr_metrics_record.h b/odrefresh/odr_metrics_record.h
new file mode 100644
index 0000000..9dd51a6
--- /dev/null
+++ b/odrefresh/odr_metrics_record.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_ODREFRESH_ODR_METRICS_RECORD_H_
+#define ART_ODREFRESH_ODR_METRICS_RECORD_H_
+
+#include <cstdint>
+#include <iosfwd> // For forward-declaration of std::string.
+
+namespace art {
+namespace odrefresh {
+
+// Default location for storing metrics from odrefresh.
+constexpr const char* kOdrefreshMetricsFile = "/data/misc/odrefresh/odrefresh-metrics.txt";
+
+// MetricsRecord is a simpler container for Odrefresh metric values reported to statsd. The order
+// and types of fields here mirror definition of `OdrefreshReported` in
+// frameworks/proto_logging/stats/atoms.proto.
+struct OdrMetricsRecord {
+ int64_t art_apex_version;
+ int32_t trigger;
+ int32_t stage_reached;
+ int32_t status;
+ int32_t primary_bcp_compilation_seconds;
+ int32_t secondary_bcp_compilation_seconds;
+ int32_t system_server_compilation_seconds;
+ int32_t cache_space_free_start_mib;
+ int32_t cache_space_free_end_mib;
+};
+
+// Read a `MetricsRecord` from an `istream`.
+//
+// This method blocks istream related exceptions, the caller should check `is.fail()` is false after
+// calling.
+//
+// Returns `is`.
+std::istream& operator>>(std::istream& is, OdrMetricsRecord& record);
+
+// Write a `MetricsRecord` to an `ostream`.
+//
+// This method blocks ostream related exceptions, the caller should check `os.fail()` is false after
+// calling.
+//
+// Returns `os`
+std::ostream& operator<<(std::ostream& os, const OdrMetricsRecord& record);
+
+} // namespace odrefresh
+} // namespace art
+
+#endif // ART_ODREFRESH_ODR_METRICS_RECORD_H_
diff --git a/odrefresh/odr_metrics_record_test.cc b/odrefresh/odr_metrics_record_test.cc
new file mode 100644
index 0000000..dd739d6
--- /dev/null
+++ b/odrefresh/odr_metrics_record_test.cc
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "odr_metrics_record.h"
+
+#include <string.h>
+
+#include <fstream>
+
+#include "base/common_art_test.h"
+
+namespace art {
+namespace odrefresh {
+
+class OdrMetricsRecordTest : public CommonArtTest {};
+
+TEST_F(OdrMetricsRecordTest, HappyPath) {
+ const OdrMetricsRecord expected {
+ .art_apex_version = 0x01233456'789abcde,
+ .trigger = 0x01020304,
+ .stage_reached = 0x11121314,
+ .status = 0x21222324,
+ .primary_bcp_compilation_seconds = 0x31323334,
+ .secondary_bcp_compilation_seconds = 0x41424344,
+ .system_server_compilation_seconds = 0x51525354,
+ .cache_space_free_start_mib = 0x61626364,
+ .cache_space_free_end_mib = 0x71727374
+ };
+
+ ScratchDir dir(/*keep_files=*/false);
+ std::string file_path = dir.GetPath() + "/metrics-record.txt";
+
+ {
+ std::ofstream ofs(file_path);
+ ofs << expected;
+ ASSERT_FALSE(ofs.fail());
+ ofs.close();
+ }
+
+ OdrMetricsRecord actual {};
+ {
+ std::ifstream ifs(file_path);
+ ifs >> actual;
+ ASSERT_TRUE(ifs.eof());
+ }
+
+ ASSERT_EQ(expected.art_apex_version, actual.art_apex_version);
+ ASSERT_EQ(expected.trigger, actual.trigger);
+ ASSERT_EQ(expected.stage_reached, actual.stage_reached);
+ ASSERT_EQ(expected.status, actual.status);
+ ASSERT_EQ(expected.primary_bcp_compilation_seconds, actual.primary_bcp_compilation_seconds);
+ ASSERT_EQ(expected.secondary_bcp_compilation_seconds, actual.secondary_bcp_compilation_seconds);
+ ASSERT_EQ(expected.system_server_compilation_seconds, actual.system_server_compilation_seconds);
+ ASSERT_EQ(expected.cache_space_free_start_mib, actual.cache_space_free_start_mib);
+ ASSERT_EQ(expected.cache_space_free_end_mib, actual.cache_space_free_end_mib);
+ ASSERT_EQ(0, memcmp(&expected, &actual, sizeof(expected)));
+}
+
+TEST_F(OdrMetricsRecordTest, EmptyInput) {
+ ScratchDir dir(/*keep_files=*/false);
+ std::string file_path = dir.GetPath() + "/metrics-record.txt";
+
+ std::ifstream ifs(file_path);
+ OdrMetricsRecord record;
+ ifs >> record;
+
+ ASSERT_TRUE(ifs.fail());
+ ASSERT_TRUE(!ifs);
+}
+
+TEST_F(OdrMetricsRecordTest, ClosedInput) {
+ ScratchDir dir(/*keep_files=*/false);
+ std::string file_path = dir.GetPath() + "/metrics-record.txt";
+
+ std::ifstream ifs(file_path);
+ ifs.close();
+
+ OdrMetricsRecord record;
+ ifs >> record;
+
+ ASSERT_TRUE(ifs.fail());
+ ASSERT_TRUE(!ifs);
+}
+
+TEST_F(OdrMetricsRecordTest, ClosedOutput) {
+ ScratchDir dir(/*keep_files=*/false);
+ std::string file_path = dir.GetPath() + "/metrics-record.txt";
+
+ std::ofstream ofs(file_path);
+ ofs.close();
+
+ OdrMetricsRecord record {};
+ ofs << record;
+
+ ASSERT_TRUE(ofs.fail());
+ ASSERT_TRUE(!ofs.good());
+}
+
+} // namespace odrefresh
+} // namespace art
diff --git a/odrefresh/odr_metrics_test.cc b/odrefresh/odr_metrics_test.cc
new file mode 100644
index 0000000..4519f00
--- /dev/null
+++ b/odrefresh/odr_metrics_test.cc
@@ -0,0 +1,220 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "odr_metrics.h"
+#include "base/casts.h"
+#include "odr_metrics_record.h"
+
+#include <unistd.h>
+
+#include <fstream>
+#include <memory>
+#include <string>
+
+#include "base/common_art_test.h"
+
+namespace art {
+namespace odrefresh {
+
+class OdrMetricsTest : public CommonArtTest {
+ public:
+ void SetUp() override {
+ CommonArtTest::SetUp();
+
+ scratch_dir_ = std::make_unique<ScratchDir>();
+ metrics_file_path_ = scratch_dir_->GetPath() + "/metrics.txt";
+ cache_directory_ = scratch_dir_->GetPath() + "/dir";
+ mkdir(cache_directory_.c_str(), S_IRWXU);
+ }
+
+ void TearDown() override {
+ scratch_dir_.reset();
+ }
+
+ bool MetricsFileExists() const {
+ const char* path = metrics_file_path_.c_str();
+ return OS::FileExists(path);
+ }
+
+ bool RemoveMetricsFile() const {
+ const char* path = metrics_file_path_.c_str();
+ if (OS::FileExists(path)) {
+ return unlink(path) == 0;
+ }
+ return true;
+ }
+
+ const std::string GetCacheDirectory() const { return cache_directory_; }
+ const std::string GetMetricsFilePath() const { return metrics_file_path_; }
+
+ protected:
+ std::unique_ptr<ScratchDir> scratch_dir_;
+ std::string metrics_file_path_;
+ std::string cache_directory_;
+};
+
+TEST_F(OdrMetricsTest, ToRecordFailsIfNotTriggered) {
+ {
+ OdrMetrics metrics(GetCacheDirectory(), GetMetricsFilePath());
+ OdrMetricsRecord record {};
+ EXPECT_FALSE(metrics.ToRecord(&record));
+ }
+
+ {
+ OdrMetrics metrics(GetCacheDirectory(), GetMetricsFilePath());
+ metrics.SetArtApexVersion(99);
+ metrics.SetStage(OdrMetrics::Stage::kCheck);
+ metrics.SetStatus(OdrMetrics::Status::kNoSpace);
+ OdrMetricsRecord record {};
+ EXPECT_FALSE(metrics.ToRecord(&record));
+ }
+}
+
+TEST_F(OdrMetricsTest, ToRecordSucceedsIfTriggered) {
+ OdrMetrics metrics(GetCacheDirectory(), GetMetricsFilePath());
+ metrics.SetArtApexVersion(99);
+ metrics.SetTrigger(OdrMetrics::Trigger::kApexVersionMismatch);
+ metrics.SetStage(OdrMetrics::Stage::kCheck);
+ metrics.SetStatus(OdrMetrics::Status::kNoSpace);
+
+ OdrMetricsRecord record{};
+ EXPECT_TRUE(metrics.ToRecord(&record));
+
+ EXPECT_EQ(99, record.art_apex_version);
+ EXPECT_EQ(OdrMetrics::Trigger::kApexVersionMismatch,
+ enum_cast<OdrMetrics::Trigger>(record.trigger));
+ EXPECT_EQ(OdrMetrics::Stage::kCheck, enum_cast<OdrMetrics::Stage>(record.stage_reached));
+ EXPECT_EQ(OdrMetrics::Status::kNoSpace, enum_cast<OdrMetrics::Status>(record.status));
+}
+
+TEST_F(OdrMetricsTest, MetricsFileIsNotCreatedIfNotTriggered) {
+ EXPECT_TRUE(RemoveMetricsFile());
+
+ // Metrics file is (potentially) written in OdrMetrics destructor.
+ {
+ OdrMetrics metrics(GetCacheDirectory(), GetMetricsFilePath());
+ metrics.SetArtApexVersion(99);
+ metrics.SetStage(OdrMetrics::Stage::kCheck);
+ metrics.SetStatus(OdrMetrics::Status::kNoSpace);
+ }
+ EXPECT_FALSE(MetricsFileExists());
+}
+
+TEST_F(OdrMetricsTest, NoMetricsFileIsCreatedIfTriggered) {
+ EXPECT_TRUE(RemoveMetricsFile());
+
+ // Metrics file is (potentially) written in OdrMetrics destructor.
+ {
+ OdrMetrics metrics(GetCacheDirectory(), GetMetricsFilePath());
+ metrics.SetArtApexVersion(101);
+ metrics.SetTrigger(OdrMetrics::Trigger::kDexFilesChanged);
+ metrics.SetStage(OdrMetrics::Stage::kCheck);
+ metrics.SetStatus(OdrMetrics::Status::kNoSpace);
+ }
+ EXPECT_TRUE(MetricsFileExists());
+}
+
+TEST_F(OdrMetricsTest, StageDoesNotAdvancedAfterFailure) {
+ OdrMetrics metrics(GetCacheDirectory(), GetMetricsFilePath());
+ metrics.SetArtApexVersion(1999);
+ metrics.SetTrigger(OdrMetrics::Trigger::kMissingArtifacts);
+ metrics.SetStage(OdrMetrics::Stage::kCheck);
+ metrics.SetStatus(OdrMetrics::Status::kNoSpace);
+ metrics.SetStage(OdrMetrics::Stage::kComplete);
+
+ OdrMetricsRecord record{};
+ EXPECT_TRUE(metrics.ToRecord(&record));
+
+ EXPECT_EQ(OdrMetrics::Stage::kCheck, enum_cast<OdrMetrics::Stage>(record.stage_reached));
+}
+
+TEST_F(OdrMetricsTest, TimeValuesAreRecorded) {
+ OdrMetrics metrics(GetCacheDirectory(), GetMetricsFilePath());
+ metrics.SetArtApexVersion(1999);
+ metrics.SetTrigger(OdrMetrics::Trigger::kMissingArtifacts);
+ metrics.SetStage(OdrMetrics::Stage::kCheck);
+ metrics.SetStatus(OdrMetrics::Status::kOK);
+
+ // Primary boot classpath compilation time.
+ OdrMetricsRecord record{};
+ {
+ metrics.SetStage(OdrMetrics::Stage::kPrimaryBootClasspath);
+ ScopedOdrCompilationTimer timer(metrics);
+ sleep(2u);
+ }
+ EXPECT_TRUE(metrics.ToRecord(&record));
+ EXPECT_EQ(OdrMetrics::Stage::kPrimaryBootClasspath,
+ enum_cast<OdrMetrics::Stage>(record.stage_reached));
+ EXPECT_NE(0, record.primary_bcp_compilation_seconds);
+ EXPECT_GT(10, record.primary_bcp_compilation_seconds);
+ EXPECT_EQ(0, record.secondary_bcp_compilation_seconds);
+ EXPECT_EQ(0, record.system_server_compilation_seconds);
+
+ // Secondary boot classpath compilation time.
+ {
+ metrics.SetStage(OdrMetrics::Stage::kSecondaryBootClasspath);
+ ScopedOdrCompilationTimer timer(metrics);
+ sleep(2u);
+ }
+ EXPECT_TRUE(metrics.ToRecord(&record));
+ EXPECT_EQ(OdrMetrics::Stage::kSecondaryBootClasspath,
+ enum_cast<OdrMetrics::Stage>(record.stage_reached));
+ EXPECT_NE(0, record.primary_bcp_compilation_seconds);
+ EXPECT_NE(0, record.secondary_bcp_compilation_seconds);
+ EXPECT_GT(10, record.secondary_bcp_compilation_seconds);
+ EXPECT_EQ(0, record.system_server_compilation_seconds);
+
+ // system_server classpath compilation time.
+ {
+ metrics.SetStage(OdrMetrics::Stage::kSystemServerClasspath);
+ ScopedOdrCompilationTimer timer(metrics);
+ sleep(2u);
+ }
+ EXPECT_TRUE(metrics.ToRecord(&record));
+ EXPECT_EQ(OdrMetrics::Stage::kSystemServerClasspath,
+ enum_cast<OdrMetrics::Stage>(record.stage_reached));
+ EXPECT_NE(0, record.primary_bcp_compilation_seconds);
+ EXPECT_NE(0, record.secondary_bcp_compilation_seconds);
+ EXPECT_NE(0, record.system_server_compilation_seconds);
+ EXPECT_GT(10, record.system_server_compilation_seconds);
+}
+
+TEST_F(OdrMetricsTest, CacheSpaceValuesAreUpdated) {
+ OdrMetricsRecord snap {};
+ snap.cache_space_free_start_mib = -1;
+ snap.cache_space_free_end_mib = -1;
+ {
+ OdrMetrics metrics(GetCacheDirectory(), GetMetricsFilePath());
+ metrics.SetArtApexVersion(1999);
+ metrics.SetTrigger(OdrMetrics::Trigger::kMissingArtifacts);
+ metrics.SetStage(OdrMetrics::Stage::kCheck);
+ metrics.SetStatus(OdrMetrics::Status::kOK);
+ EXPECT_TRUE(metrics.ToRecord(&snap));
+ EXPECT_NE(0, snap.cache_space_free_start_mib);
+ EXPECT_EQ(0, snap.cache_space_free_end_mib);
+ }
+
+ OdrMetricsRecord on_disk;
+ std::ifstream ifs(GetMetricsFilePath());
+ EXPECT_TRUE(ifs);
+ ifs >> on_disk;
+ EXPECT_TRUE(ifs);
+ EXPECT_EQ(snap.cache_space_free_start_mib, on_disk.cache_space_free_start_mib);
+ EXPECT_NE(0, on_disk.cache_space_free_end_mib);
+}
+
+} // namespace odrefresh
+} // namespace art
diff --git a/odrefresh/odrefresh.cc b/odrefresh/odrefresh.cc
index 24ef7d1..5f8b072 100644
--- a/odrefresh/odrefresh.cc
+++ b/odrefresh/odrefresh.cc
@@ -64,12 +64,14 @@
#include "dex/art_dex_file_loader.h"
#include "dexoptanalyzer.h"
#include "exec_utils.h"
+#include "log/log.h"
#include "palette/palette.h"
#include "palette/palette_types.h"
#include "odr_artifacts.h"
#include "odr_config.h"
#include "odr_fs_utils.h"
+#include "odr_metrics.h"
namespace art {
namespace odrefresh {
@@ -479,7 +481,9 @@
return true;
}
- WARN_UNUSED ExitCode CheckArtifactsAreUpToDate() {
+ WARN_UNUSED ExitCode CheckArtifactsAreUpToDate(OdrMetrics& metrics) {
+ metrics.SetStage(OdrMetrics::Stage::kCheck);
+
// Clean-up helper used to simplify clean-ups and handling failures there.
auto cleanup_return = [this](ExitCode exit_code) {
return CleanApexdataDirectory() ? exit_code : ExitCode::kCleanupFailed;
@@ -487,8 +491,9 @@
const auto apex_info = GetArtApexInfo();
if (!apex_info.has_value()) {
- // This should never happen, but do not proceed if it does.
+ // This should never happen, further up-to-date checks are not possible if it does.
LOG(ERROR) << "Could not get ART APEX info.";
+ metrics.SetTrigger(OdrMetrics::Trigger::kApexVersionMismatch);
return cleanup_return(ExitCode::kCompilationRequired);
}
@@ -502,30 +507,40 @@
// If the cache info file does not exist, assume compilation is required because the
// file is missing and because the current ART APEX is not factory installed.
PLOG(ERROR) << "No prior cache-info file: " << QuotePath(cache_info_filename_);
+ metrics.SetTrigger(OdrMetrics::Trigger::kApexVersionMismatch);
return cleanup_return(ExitCode::kCompilationRequired);
}
// Get and parse the ART APEX cache info file.
std::optional<art_apex::CacheInfo> cache_info = ReadCacheInfo();
if (!cache_info.has_value()) {
+ // This should never happen, further up-to-date checks are not possible if it does.
PLOG(ERROR) << "Failed to read cache-info file: " << QuotePath(cache_info_filename_);
+ metrics.SetTrigger(OdrMetrics::Trigger::kUnknown);
return cleanup_return(ExitCode::kCompilationRequired);
}
// Generate current module info for the current ART APEX.
const auto current_info = GenerateArtModuleInfo();
if (!current_info.has_value()) {
+ // This should never happen, further up-to-date checks are not possible if it does.
LOG(ERROR) << "Failed to generate cache provenance.";
+ metrics.SetTrigger(OdrMetrics::Trigger::kUnknown);
return cleanup_return(ExitCode::kCompilationRequired);
}
+ // Record ART Apex version for metrics reporting.
+ metrics.SetArtApexVersion(current_info->getVersionCode());
+
// Check whether the current cache ART module info differs from the current ART module info.
// Always check APEX version.
const auto cached_info = cache_info->getFirstArtModuleInfo();
+
if (cached_info->getVersionCode() != current_info->getVersionCode()) {
LOG(INFO) << "ART APEX version code mismatch ("
<< cached_info->getVersionCode()
<< " != " << current_info->getVersionCode() << ").";
+ metrics.SetTrigger(OdrMetrics::Trigger::kApexVersionMismatch);
return cleanup_return(ExitCode::kCompilationRequired);
}
@@ -533,6 +548,7 @@
LOG(INFO) << "ART APEX version code mismatch ("
<< cached_info->getVersionName()
<< " != " << current_info->getVersionName() << ").";
+ metrics.SetTrigger(OdrMetrics::Trigger::kApexVersionMismatch);
return cleanup_return(ExitCode::kCompilationRequired);
}
@@ -550,6 +566,7 @@
(!cache_info->hasDex2oatBootClasspath() ||
!cache_info->getFirstDex2oatBootClasspath()->hasComponent())) {
LOG(INFO) << "Missing Dex2oatBootClasspath components.";
+ metrics.SetTrigger(OdrMetrics::Trigger::kDexFilesChanged);
return cleanup_return(ExitCode::kCompilationRequired);
}
@@ -558,6 +575,7 @@
cache_info->getFirstDex2oatBootClasspath()->getComponent();
if (!CheckComponents(expected_bcp_components, bcp_components, &error_msg)) {
LOG(INFO) << "Dex2OatClasspath components mismatch: " << error_msg;
+ metrics.SetTrigger(OdrMetrics::Trigger::kDexFilesChanged);
return cleanup_return(ExitCode::kCompilationRequired);
}
@@ -580,6 +598,7 @@
(!cache_info->hasSystemServerClasspath() ||
!cache_info->getFirstSystemServerClasspath()->hasComponent())) {
LOG(INFO) << "Missing SystemServerClasspath components.";
+ metrics.SetTrigger(OdrMetrics::Trigger::kDexFilesChanged);
return cleanup_system_server_return(ExitCode::kCompilationRequired);
}
@@ -587,6 +606,7 @@
cache_info->getFirstSystemServerClasspath()->getComponent();
if (!CheckComponents(expected_system_server_components, system_server_components, &error_msg)) {
LOG(INFO) << "SystemServerClasspath components mismatch: " << error_msg;
+ metrics.SetTrigger(OdrMetrics::Trigger::kDexFilesChanged);
return cleanup_system_server_return(ExitCode::kCompilationRequired);
}
@@ -598,6 +618,7 @@
for (const InstructionSet isa : config_.GetBootExtensionIsas()) {
if (!BootExtensionArtifactsExistOnData(isa, &error_msg)) {
LOG(INFO) << "Incomplete boot extension artifacts. " << error_msg;
+ metrics.SetTrigger(OdrMetrics::Trigger::kMissingArtifacts);
return cleanup_boot_extensions_return(ExitCode::kCompilationRequired, isa);
}
}
@@ -608,6 +629,7 @@
// `SystemServerArtifactsExistOnData()` checks in compilation order so it is possible some of
// the artifacts are here. We likely ran out of space compiling the system_server artifacts.
// Any artifacts present are usable.
+ metrics.SetTrigger(OdrMetrics::Trigger::kMissingArtifacts);
return ExitCode::kCompilationRequired;
}
@@ -980,8 +1002,10 @@
WARN_UNUSED bool CompileBootExtensionArtifacts(const InstructionSet isa,
const std::string& staging_dir,
+ OdrMetrics& metrics,
uint32_t* dex2oat_invocation_count,
std::string* error_msg) const {
+ ScopedOdrCompilationTimer compilation_timer(metrics);
std::vector<std::string> args;
args.push_back(config_.GetDex2Oat());
@@ -1032,12 +1056,14 @@
std::unique_ptr<File> staging_file(OS::CreateEmptyFile(staging_location.c_str()));
if (staging_file == nullptr) {
PLOG(ERROR) << "Failed to create " << kind << " file: " << staging_location;
+ metrics.SetStatus(OdrMetrics::Status::kIoError);
EraseFiles(staging_files);
return false;
}
if (fchmod(staging_file->Fd(), S_IRUSR | S_IWUSR) != 0) {
PLOG(ERROR) << "Could not set file mode on " << QuotePath(staging_location);
+ metrics.SetStatus(OdrMetrics::Status::kIoError);
EraseFiles(staging_files);
return false;
}
@@ -1048,6 +1074,7 @@
const std::string install_location = android::base::Dirname(image_location);
if (!EnsureDirectoryExists(install_location)) {
+ metrics.SetStatus(OdrMetrics::Status::kIoError);
return false;
}
@@ -1061,15 +1088,19 @@
}
bool timed_out = false;
- if (ExecAndReturnCode(args, timeout, &timed_out, error_msg) != 0) {
+ int dex2oat_exit_code = ExecAndReturnCode(args, timeout, &timed_out, error_msg);
+ if (dex2oat_exit_code != 0) {
if (timed_out) {
- // TODO(oth): record timeout event for compiling boot extension
+ metrics.SetStatus(OdrMetrics::Status::kTimeLimitExceeded);
+ } else {
+ metrics.SetStatus(OdrMetrics::Status::kDex2OatError);
}
EraseFiles(staging_files);
return false;
}
if (!MoveOrEraseFiles(staging_files, install_location)) {
+ metrics.SetStatus(OdrMetrics::Status::kInstallFailed);
return false;
}
@@ -1080,8 +1111,10 @@
}
WARN_UNUSED bool CompileSystemServerArtifacts(const std::string& staging_dir,
+ OdrMetrics& metrics,
uint32_t* dex2oat_invocation_count,
std::string* error_msg) const {
+ ScopedOdrCompilationTimer compilation_timer(metrics);
std::vector<std::string> classloader_context;
const std::string dex2oat = config_.GetDex2Oat();
@@ -1104,6 +1137,7 @@
if (classloader_context.empty()) {
// All images are in the same directory, we only need to check on the first iteration.
if (!EnsureDirectoryExists(install_location)) {
+ metrics.SetStatus(OdrMetrics::Status::kIoError);
return false;
}
}
@@ -1124,6 +1158,7 @@
std::unique_ptr<File> staging_file(OS::CreateEmptyFile(staging_location.c_str()));
if (staging_file == nullptr) {
PLOG(ERROR) << "Failed to create " << kind << " file: " << staging_location;
+ metrics.SetStatus(OdrMetrics::Status::kIoError);
EraseFiles(staging_files);
return false;
}
@@ -1152,15 +1187,19 @@
}
bool timed_out = false;
- if (!Exec(args, error_msg)) {
+ int dex2oat_exit_code = ExecAndReturnCode(args, timeout, &timed_out, error_msg);
+ if (dex2oat_exit_code != 0) {
if (timed_out) {
- // TODO(oth): record timeout event for compiling boot extension
+ metrics.SetStatus(OdrMetrics::Status::kTimeLimitExceeded);
+ } else {
+ metrics.SetStatus(OdrMetrics::Status::kDex2OatError);
}
EraseFiles(staging_files);
return false;
}
if (!MoveOrEraseFiles(staging_files, install_location)) {
+ metrics.SetStatus(OdrMetrics::Status::kInstallFailed);
return false;
}
@@ -1181,28 +1220,39 @@
android::base::SetProperty("service.bootanim.progress", std::to_string(value));
}
- WARN_UNUSED ExitCode Compile(bool force_compile) const {
+
+ WARN_UNUSED ExitCode Compile(OdrMetrics& metrics, bool force_compile) const {
ReportSpace(); // TODO(oth): Factor available space into compilation logic.
+ const char* staging_dir = nullptr;
+ metrics.SetStage(OdrMetrics::Stage::kPreparation);
// Clean-up existing files.
if (force_compile && !CleanApexdataDirectory()) {
+ metrics.SetStatus(OdrMetrics::Status::kIoError);
+ return ExitCode::kCleanupFailed;
+ }
+
+ // Create staging area and assign label for generating compilation artifacts.
+ if (PaletteCreateOdrefreshStagingDirectory(&staging_dir) != PALETTE_STATUS_OK) {
+ metrics.SetStatus(OdrMetrics::Status::kStagingFailed);
return ExitCode::kCleanupFailed;
}
// Emit cache info before compiling. This can be used to throttle compilation attempts later.
WriteCacheInfo();
- // Create staging area and assign label for generating compilation artifacts.
- const char* staging_dir;
- if (PaletteCreateOdrefreshStagingDirectory(&staging_dir) != PALETTE_STATUS_OK) {
- return ExitCode::kCompilationFailed;
- }
-
std::string error_msg;
uint32_t dex2oat_invocation_count = 0;
ReportNextBootAnimationProgress(dex2oat_invocation_count);
- for (const InstructionSet isa : config_.GetBootExtensionIsas()) {
+
+ const auto& bcp_instruction_sets = config_.GetBootExtensionIsas();
+ DCHECK(!bcp_instruction_sets.empty() && bcp_instruction_sets.size() <= 2);
+ for (const InstructionSet isa : bcp_instruction_sets) {
+ auto stage = (isa == bcp_instruction_sets.front()) ?
+ OdrMetrics::Stage::kPrimaryBootClasspath :
+ OdrMetrics::Stage::kSecondaryBootClasspath;
+ metrics.SetStage(stage);
if (force_compile || !BootExtensionArtifactsExistOnData(isa, &error_msg)) {
// Remove artifacts we are about to generate. Ordinarily these are removed in the checking
// step, but this is not always run (e.g. during manual testing).
@@ -1210,7 +1260,7 @@
return ExitCode::kCleanupFailed;
}
if (!CompileBootExtensionArtifacts(
- isa, staging_dir, &dex2oat_invocation_count, &error_msg)) {
+ isa, staging_dir, metrics, &dex2oat_invocation_count, &error_msg)) {
LOG(ERROR) << "Compilation of BCP failed: " << error_msg;
if (!config_.GetDryRun() && !CleanDirectory(staging_dir)) {
return ExitCode::kCleanupFailed;
@@ -1221,7 +1271,9 @@
}
if (force_compile || !SystemServerArtifactsExistOnData(&error_msg)) {
- if (!CompileSystemServerArtifacts(staging_dir, &dex2oat_invocation_count, &error_msg)) {
+ metrics.SetStage(OdrMetrics::Stage::kSystemServerClasspath);
+ if (!CompileSystemServerArtifacts(
+ staging_dir, metrics, &dex2oat_invocation_count, &error_msg)) {
LOG(ERROR) << "Compilation of system_server failed: " << error_msg;
if (!config_.GetDryRun() && !CleanDirectory(staging_dir)) {
return ExitCode::kCleanupFailed;
@@ -1230,6 +1282,7 @@
}
}
+ metrics.SetStage(OdrMetrics::Stage::kComplete);
return ExitCode::kCompilationSuccess;
}
@@ -1341,26 +1394,28 @@
static int main(int argc, const char** argv) {
OdrConfig config(argv[0]);
-
int n = InitializeConfig(argc, argv, &config);
argv += n;
argc -= n;
-
if (argc != 1) {
UsageError("Expected 1 argument, but have %d.", argc);
}
+ OdrMetrics metrics(kOdrefreshArtifactDirectory);
OnDeviceRefresh odr(config);
for (int i = 0; i < argc; ++i) {
std::string_view action(argv[i]);
if (action == "--check") {
// Fast determination of whether artifacts are up to date.
- return odr.CheckArtifactsAreUpToDate();
+ return odr.CheckArtifactsAreUpToDate(metrics);
} else if (action == "--compile") {
- const ExitCode e = odr.CheckArtifactsAreUpToDate();
- return (e == ExitCode::kCompilationRequired) ? odr.Compile(/*force_compile=*/false) : e;
+ const ExitCode exit_code = odr.CheckArtifactsAreUpToDate(metrics);
+ if (exit_code == ExitCode::kCompilationRequired) {
+ return odr.Compile(metrics, /*force_compile=*/false);
+ }
+ return exit_code;
} else if (action == "--force-compile") {
- return odr.Compile(/*force_compile=*/true);
+ return odr.Compile(metrics, /*force_compile=*/true);
} else if (action == "--verify") {
// Slow determination of whether artifacts are up to date. These are too slow for checking
// during boot (b/181689036).
