odrefresh: move file-system utilities into odr_fs_utils.{h,cc}
Preparation for metrics support which will use the same functionality.
Add tests for odrefresh file-system utilities too.
Bug: 160683548
Test: atest art_odrefresh_tests
Change-Id: I1da2f477f51b1d445bbee3b22a9c1fb268d25e80
diff --git a/odrefresh/Android.bp b/odrefresh/Android.bp
index 8e39edf..6f84e8f 100644
--- a/odrefresh/Android.bp
+++ b/odrefresh/Android.bp
@@ -29,6 +29,7 @@
defaults: ["art_defaults"],
srcs: [
"odrefresh.cc",
+ "odr_fs_utils.cc",
],
local_include_dirs: ["include"],
header_libs: ["dexoptanalyzer_headers"],
@@ -128,6 +129,8 @@
header_libs: ["odrefresh_headers"],
srcs: [
"odr_artifacts_test.cc",
+ "odr_fs_utils.cc",
+ "odr_fs_utils_test.cc",
"odrefresh_test.cc",
],
shared_libs: [
diff --git a/odrefresh/odr_config.h b/odrefresh/odr_config.h
index 6b0f212..4161944 100644
--- a/odrefresh/odr_config.h
+++ b/odrefresh/odr_config.h
@@ -36,7 +36,7 @@
kZygote32_64 = 1,
// 64-bit primary zygote, 32-bit secondary zygote.
kZygote64_32 = 2,
- // 64-bit praimry zygote, no secondary zygote.
+ // 64-bit primary zygote, no secondary zygote.
kZygote64 = 3
};
diff --git a/odrefresh/odr_fs_utils.cc b/odrefresh/odr_fs_utils.cc
new file mode 100644
index 0000000..9f3a68b
--- /dev/null
+++ b/odrefresh/odr_fs_utils.cc
@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "odr_fs_utils.h"
+
+#include <dirent.h>
+#include <ftw.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <unistd.h>
+
+#include <iosfwd>
+#include <memory>
+#include <ostream>
+#include <queue>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <vector>
+
+#include <android-base/logging.h>
+#include <android-base/macros.h>
+#include <android-base/strings.h>
+#include <base/os.h>
+
+namespace art {
+namespace odrefresh {
+
+// Callback for use with nftw(3) to assist with clearing files and sub-directories.
+// This method removes files and directories below the top-level directory passed to nftw().
+static int NftwCleanUpCallback(const char* fpath,
+ const struct stat* sb ATTRIBUTE_UNUSED,
+ int typeflag,
+ struct FTW* ftwbuf) {
+ switch (typeflag) {
+ case FTW_F:
+ return unlink(fpath);
+ case FTW_DP:
+ return (ftwbuf->level == 0) ? 0 : rmdir(fpath);
+ default:
+ return -1;
+ }
+}
+
+WARN_UNUSED bool CleanDirectory(const std::string& dir_path) {
+ if (!OS::DirectoryExists(dir_path.c_str())) {
+ return true;
+ }
+
+ static constexpr int kMaxDescriptors = 4; // Limit the need for nftw() to re-open descriptors.
+ if (nftw(dir_path.c_str(), NftwCleanUpCallback, kMaxDescriptors, FTW_DEPTH | FTW_MOUNT) != 0) {
+ LOG(ERROR) << "Failed to clean-up '" << dir_path << "'";
+ return false;
+ }
+ return true;
+}
+
+WARN_UNUSED bool EnsureDirectoryExists(const std::string& absolute_path) {
+ if (absolute_path.empty() || absolute_path[0] != '/') {
+ LOG(ERROR) << "Path not absolute '" << absolute_path << "'";
+ return false;
+ }
+ std::string path;
+ for (const std::string& directory : android::base::Split(absolute_path, "/")) {
+ path.append("/").append(directory);
+ if (!OS::DirectoryExists(path.c_str())) {
+ static constexpr mode_t kDirectoryMode = S_IRWXU | S_IRGRP | S_IXGRP| S_IROTH | S_IXOTH;
+ if (mkdir(path.c_str(), kDirectoryMode) != 0) {
+ PLOG(ERROR) << "Could not create directory: " << path;
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+bool GetFreeSpace(const std::string& path, uint64_t* bytes) {
+ struct statvfs sv;
+ if (statvfs(path.c_str(), &sv) != 0) {
+ PLOG(ERROR) << "statvfs '" << path << "'";
+ return false;
+ }
+ *bytes = sv.f_bfree * sv.f_bsize;
+ return true;
+}
+
+bool GetUsedSpace(const std::string& path, uint64_t* bytes) {
+ static constexpr std::string_view kCurrentDirectory{"."};
+ static constexpr std::string_view kParentDirectory{".."};
+ static constexpr size_t kBytesPerBlock = 512; // see manual page for stat(2).
+
+ uint64_t file_bytes = 0;
+ std::queue<std::string> unvisited;
+ unvisited.push(path);
+ while (!unvisited.empty()) {
+ std::string current = unvisited.front();
+ unvisited.pop();
+ std::unique_ptr<DIR, int (*)(DIR*)> dir(opendir(current.c_str()), closedir);
+ if (!dir) {
+ continue;
+ }
+ for (auto entity = readdir(dir.get()); entity != nullptr; entity = readdir(dir.get())) {
+ std::string_view name{entity->d_name};
+ if (name == kCurrentDirectory || name == kParentDirectory) {
+ continue;
+ }
+ std::string entity_name = current + "/" + entity->d_name;
+ if (entity->d_type == DT_DIR) {
+ unvisited.push(entity_name.c_str());
+ } else if (entity->d_type == DT_REG) {
+ struct stat sb;
+ if (stat(entity_name.c_str(), &sb) != 0) {
+ PLOG(ERROR) << "Failed to stat() file " << entity_name;
+ continue;
+ }
+ file_bytes += sb.st_blocks * kBytesPerBlock;
+ }
+ }
+ }
+ *bytes = file_bytes;
+ return true;
+}
+
+} // namespace odrefresh
+} // namespace art
diff --git a/odrefresh/odr_fs_utils.h b/odrefresh/odr_fs_utils.h
new file mode 100644
index 0000000..cefa11e
--- /dev/null
+++ b/odrefresh/odr_fs_utils.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_ODREFRESH_ODR_FS_UTILS_H_
+#define ART_ODREFRESH_ODR_FS_UTILS_H_
+
+#include <cstdint>
+#include <iosfwd>
+
+#include <android-base/macros.h>
+
+namespace art {
+namespace odrefresh {
+
+// Cleans directory by removing all files and sub-directories under `dir_path`.
+// Returns true on success, false otherwise.
+WARN_UNUSED bool CleanDirectory(const std::string& dir_path);
+
+// Create all directories on `absolute_dir_path`.
+// Returns true on success, false otherwise.
+WARN_UNUSED bool EnsureDirectoryExists(const std::string& absolute_dir_path);
+
+// Get free space for filesystem containing `path`.
+// Returns true on success, false otherwise.
+WARN_UNUSED bool GetFreeSpace(const std::string& path, uint64_t* bytes);
+
+// Gets space used under directory `dir_path`.
+// Returns true on success, false otherwise.
+WARN_UNUSED bool GetUsedSpace(const std::string& dir_path, uint64_t* bytes);
+
+} // namespace odrefresh
+} // namespace art
+
+#endif // ART_ODREFRESH_ODR_FS_UTILS_H_
diff --git a/odrefresh/odr_fs_utils_test.cc b/odrefresh/odr_fs_utils_test.cc
new file mode 100644
index 0000000..2b3cfaa
--- /dev/null
+++ b/odrefresh/odr_fs_utils_test.cc
@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <fcntl.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#include <iosfwd>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "android-base/stringprintf.h"
+#include "base/bit_utils.h"
+#include "base/common_art_test.h"
+#include "base/os.h"
+#include "base/unix_file/fd_file.h"
+#include "odr_fs_utils.h"
+
+namespace art {
+namespace odrefresh {
+
+class OdrFsUtilsTest : public CommonArtTest {};
+namespace {
+
+static bool CreateFile(const char* file_path, size_t bytes) {
+ std::unique_ptr<File> fp(OS::CreateEmptyFile(file_path));
+ if (!fp) {
+ return false;
+ }
+
+ std::vector<char> buffer(bytes, 0xa5);
+ if (!fp->WriteFully(buffer.data(), buffer.size())) {
+ fp->Erase();
+ return false;
+ }
+
+ if (fp->FlushClose() != 0) {
+ fp->Erase();
+ return false;
+ }
+
+ return true;
+}
+
+} // namespace
+
+TEST_F(OdrFsUtilsTest, CleanDirectory) {
+ ScratchDir scratch_dir(/*keep_files=*/false);
+
+ // Create some sub-directories and files
+ const std::string dir_paths[] = {
+ scratch_dir.GetPath() + "/a",
+ scratch_dir.GetPath() + "/b",
+ scratch_dir.GetPath() + "/b/c",
+ scratch_dir.GetPath() + "/d"
+ };
+ for (const auto& dir_path : dir_paths) {
+ ASSERT_EQ(0, mkdir(dir_path.c_str(), S_IRWXU));
+ }
+
+ const std::string file_paths[] = {
+ scratch_dir.GetPath() + "/zero.txt",
+ scratch_dir.GetPath() + "/a/one.txt",
+ scratch_dir.GetPath() + "/b/two.txt",
+ scratch_dir.GetPath() + "/b/c/three.txt",
+ scratch_dir.GetPath() + "/b/c/four.txt",
+ };
+ for (const auto& file_path : file_paths) {
+ ASSERT_TRUE(CreateFile(file_path.c_str(), 4096));
+ }
+
+ // Clean all files and sub-directories
+ ASSERT_TRUE(CleanDirectory(scratch_dir.GetPath()));
+
+ // Check nothing we created remains.
+ for (const auto& dir_path : dir_paths) {
+ ASSERT_FALSE(OS::DirectoryExists(dir_path.c_str()));
+ }
+
+ for (const auto& file_path : file_paths) {
+ ASSERT_FALSE(OS::FileExists(file_path.c_str(), true));
+ }
+}
+
+TEST_F(OdrFsUtilsTest, EnsureDirectoryExistsBadPath) {
+ // Pick a path where not even a root test runner can write.
+ ASSERT_FALSE(EnsureDirectoryExists("/proc/unlikely/to/be/writable"));
+}
+
+TEST_F(OdrFsUtilsTest, EnsureDirectoryExistsEmptyPath) {
+ ASSERT_FALSE(EnsureDirectoryExists(""));
+}
+
+TEST_F(OdrFsUtilsTest, EnsureDirectoryExistsRelativePath) {
+ ASSERT_FALSE(EnsureDirectoryExists("a/b/c"));
+}
+
+TEST_F(OdrFsUtilsTest, EnsureDirectoryExistsSubDirs) {
+ ScratchDir scratch_dir(/*keep_files=*/false);
+
+ const char* relative_sub_dirs[] = {"a", "b/c", "d/e/f/"};
+ for (const char* relative_sub_dir : relative_sub_dirs) {
+ ASSERT_TRUE(EnsureDirectoryExists(scratch_dir.GetPath() + "/" + relative_sub_dir));
+ }
+}
+
+TEST_F(OdrFsUtilsTest, GetUsedSpace) {
+ static constexpr size_t kFirstFileBytes = 1;
+ static constexpr size_t kSecondFileBytes = 16111;
+ static constexpr size_t kBytesPerBlock = 512;
+
+ ScratchDir scratch_dir(/*keep_files=*/false);
+
+ const std::string first_file_path = scratch_dir.GetPath() + "/1.dat";
+ ASSERT_TRUE(CreateFile(first_file_path.c_str(), kFirstFileBytes));
+
+ struct stat sb;
+ ASSERT_EQ(0, stat(first_file_path.c_str(), &sb));
+ ASSERT_EQ(kFirstFileBytes, static_cast<decltype(kFirstFileBytes)>(sb.st_size));
+
+ uint64_t bytes_used = 0;
+ ASSERT_TRUE(GetUsedSpace(scratch_dir.GetPath().c_str(), &bytes_used));
+ ASSERT_EQ(static_cast<uint64_t>(sb.st_blksize), bytes_used);
+
+ const std::string second_file_path = scratch_dir.GetPath() + "/2.dat";
+ ASSERT_TRUE(CreateFile(second_file_path.c_str(), kSecondFileBytes));
+
+ ASSERT_TRUE(GetUsedSpace(scratch_dir.GetPath().c_str(), &bytes_used));
+ uint64_t expected_bytes_used = RoundUp(kFirstFileBytes, sb.st_blocks * kBytesPerBlock) +
+ RoundUp(kSecondFileBytes, sb.st_blocks * kBytesPerBlock);
+ ASSERT_EQ(expected_bytes_used, bytes_used);
+
+ const std::string sub_dir_path = scratch_dir.GetPath() + "/sub";
+ ASSERT_TRUE(EnsureDirectoryExists(sub_dir_path));
+ for (size_t i = 1; i < 32768; i *= 17) {
+ const std::string path = android::base::StringPrintf("%s/%zu", sub_dir_path.c_str(), i);
+ ASSERT_TRUE(CreateFile(path.c_str(), i));
+ expected_bytes_used += RoundUp(i, sb.st_blocks * kBytesPerBlock);
+ ASSERT_TRUE(GetUsedSpace(scratch_dir.GetPath().c_str(), &bytes_used));
+ ASSERT_EQ(expected_bytes_used, bytes_used);
+ }
+}
+
+TEST_F(OdrFsUtilsTest, GetUsedSpaceBadPath) {
+ ScratchDir scratch_dir(/*keep_files=*/false);
+ const std::string bad_path = scratch_dir.GetPath() + "/bad_path";
+ uint64_t bytes_used = ~0ull;
+ ASSERT_TRUE(GetUsedSpace(bad_path, &bytes_used));
+ ASSERT_EQ(0ull, bytes_used);
+}
+
+} // namespace odrefresh
+} // namespace art
diff --git a/odrefresh/odrefresh.cc b/odrefresh/odrefresh.cc
index 974b3c6..24ef7d1 100644
--- a/odrefresh/odrefresh.cc
+++ b/odrefresh/odrefresh.cc
@@ -16,16 +16,13 @@
#include "odrefresh/odrefresh.h"
-#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
-#include <ftw.h>
#include <inttypes.h>
#include <limits.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
-#include <sys/statvfs.h>
#include <sysexits.h>
#include <time.h>
#include <unistd.h>
@@ -41,7 +38,6 @@
#include <memory>
#include <optional>
#include <ostream>
-#include <queue>
#include <sstream>
#include <string>
#include <string_view>
@@ -57,7 +53,6 @@
#include "android-base/strings.h"
#include "android/log.h"
#include "arch/instruction_set.h"
-#include "base/bit_utils.h"
#include "base/file_utils.h"
#include "base/globals.h"
#include "base/macros.h"
@@ -74,6 +69,7 @@
#include "odr_artifacts.h"
#include "odr_config.h"
+#include "odr_fs_utils.h"
namespace art {
namespace odrefresh {
@@ -149,23 +145,6 @@
return Concatenate({"'", path, "'"});
}
-// Create all directory and all required parents.
-static WARN_UNUSED bool EnsureDirectoryExists(const std::string& absolute_path) {
- CHECK(absolute_path.size() > 0 && absolute_path[0] == '/');
- std::string path;
- for (const std::string& directory : android::base::Split(absolute_path, "/")) {
- path.append("/").append(directory);
- if (!OS::DirectoryExists(path.c_str())) {
- static constexpr mode_t kDirectoryMode = S_IRWXU | S_IRGRP | S_IXGRP| S_IROTH | S_IXOTH;
- if (mkdir(path.c_str(), kDirectoryMode) != 0) {
- PLOG(ERROR) << "Could not create directory: " << path;
- return false;
- }
- }
- }
- return true;
-}
-
static void EraseFiles(const std::vector<std::unique_ptr<File>>& files) {
for (auto& file : files) {
file->Erase(/*unlink=*/true);
@@ -919,41 +898,6 @@
return exit_code;
}
- static bool GetFreeSpace(const char* path, uint64_t* bytes) {
- struct statvfs sv;
- if (statvfs(path, &sv) != 0) {
- PLOG(ERROR) << "statvfs '" << path << "'";
- return false;
- }
- *bytes = sv.f_bfree * sv.f_bsize;
- return true;
- }
-
- static bool GetUsedSpace(const char* path, uint64_t* bytes) {
- *bytes = 0;
-
- std::queue<std::string> unvisited;
- unvisited.push(path);
- while (!unvisited.empty()) {
- std::string current = unvisited.front();
- std::unique_ptr<DIR, int (*)(DIR*)> dir(opendir(current.c_str()), closedir);
- for (auto entity = readdir(dir.get()); entity != nullptr; entity = readdir(dir.get())) {
- if (entity->d_name[0] == '.') {
- continue;
- }
- std::string entity_name = Concatenate({current, "/", entity->d_name});
- if (entity->d_type == DT_DIR) {
- unvisited.push(entity_name.c_str());
- } else if (entity->d_type == DT_REG) {
- // RoundUp file size to number of blocks.
- *bytes += RoundUp(OS::GetFileSizeBytes(entity_name.c_str()), 512);
- }
- }
- unvisited.pop();
- }
- return true;
- }
-
static void ReportSpace() {
uint64_t bytes;
std::string data_dir = GetArtApexData();
@@ -965,44 +909,13 @@
}
}
- // Callback for use with nftw(3) to assist with clearing files and sub-directories.
- // This method removes files and directories below the top-level directory passed to nftw().
- static int NftwUnlinkRemoveCallback(const char* fpath,
- const struct stat* sb ATTRIBUTE_UNUSED,
- int typeflag,
- struct FTW* ftwbuf) {
- switch (typeflag) {
- case FTW_F:
- return unlink(fpath);
- case FTW_DP:
- return (ftwbuf->level == 0) ? 0 : rmdir(fpath);
- default:
- return -1;
- }
- }
-
- // Recursively remove files and directories under `top_dir`, but preserve `top_dir` itself.
- // Returns true on success, false otherwise.
- WARN_UNUSED bool RecursiveRemoveBelow(const char* top_dir) const {
- if (config_.GetDryRun()) {
- LOG(INFO) << "Files under " << QuotePath(top_dir) << " would be removed (dry-run).";
- return true;
- }
-
- if (!OS::DirectoryExists(top_dir)) {
- return true;
- }
-
- static constexpr int kMaxDescriptors = 4; // Limit the need for nftw() to re-open descriptors.
- if (nftw(top_dir, NftwUnlinkRemoveCallback, kMaxDescriptors, FTW_DEPTH | FTW_MOUNT) != 0) {
- LOG(ERROR) << "Failed to clean-up " << QuotePath(top_dir);
- return false;
- }
- return true;
- }
-
WARN_UNUSED bool CleanApexdataDirectory() const {
- return RecursiveRemoveBelow(GetArtApexData().c_str());
+ const std::string& apex_data_path = GetArtApexData();
+ if (config_.GetDryRun()) {
+ LOG(INFO) << "Files under `" << QuotePath(apex_data_path) << " would be removed (dry-run).";
+ return true;
+ }
+ return CleanDirectory(apex_data_path);
}
WARN_UNUSED bool RemoveArtifacts(const OdrArtifacts& artifacts) const {
@@ -1299,7 +1212,7 @@
if (!CompileBootExtensionArtifacts(
isa, staging_dir, &dex2oat_invocation_count, &error_msg)) {
LOG(ERROR) << "Compilation of BCP failed: " << error_msg;
- if (!RecursiveRemoveBelow(staging_dir)) {
+ if (!config_.GetDryRun() && !CleanDirectory(staging_dir)) {
return ExitCode::kCleanupFailed;
}
return ExitCode::kCompilationFailed;
@@ -1310,7 +1223,7 @@
if (force_compile || !SystemServerArtifactsExistOnData(&error_msg)) {
if (!CompileSystemServerArtifacts(staging_dir, &dex2oat_invocation_count, &error_msg)) {
LOG(ERROR) << "Compilation of system_server failed: " << error_msg;
- if (!RecursiveRemoveBelow(staging_dir)) {
+ if (!config_.GetDryRun() && !CleanDirectory(staging_dir)) {
return ExitCode::kCleanupFailed;
}
return ExitCode::kCompilationFailed;
