Wait for builds to complete in fetch_cvd.
Sometimes fetch_cvd is requested to run builds that have not completed
yet. Wait until the build is in a terminal state (complete, error, or
abandoned) and try to run from there.
If the state is error or abandoned, it's unlikely that the artifacts are
present or that the device works, but it falls through anyway in case it
does work.
Test: Try running with a build in the "building" state.
Bug: 137304531
Change-Id: I60230559b0d62d9eae48cf72c4d48415772f3e33
diff --git a/host/commands/fetcher/build_api.cc b/host/commands/fetcher/build_api.cc
index a82a7c6..177c565 100644
--- a/host/commands/fetcher/build_api.cc
+++ b/host/commands/fetcher/build_api.cc
@@ -15,7 +15,10 @@
#include "build_api.h"
+#include <chrono>
+#include <set>
#include <string>
+#include <thread>
#include <glog/logging.h>
@@ -24,6 +27,18 @@
const std::string BUILD_API =
"https://www.googleapis.com/android/internal/build/v3";
+bool StatusIsTerminal(const std::string& status) {
+ const static std::set<std::string> terminal_statuses = {
+ "abandoned",
+ "complete",
+ "error",
+ "ABANDONED",
+ "COMPLETE",
+ "ERROR",
+ };
+ return terminal_statuses.count(status) > 0;
+}
+
} // namespace
Artifact::Artifact(const Json::Value& json_artifact) {
@@ -60,6 +75,12 @@
return response["builds"][0]["buildId"].asString();
}
+std::string BuildApi::BuildStatus(const DeviceBuild& build) {
+ std::string url = BUILD_API + "/builds/" + build.id + "/" + build.target;
+ auto response_json = curl.DownloadToJson(url, Headers());
+ return response_json["buildAttemptStatus"].asString();
+}
+
std::vector<Artifact> BuildApi::Artifacts(const DeviceBuild& build) {
std::string url = BUILD_API + "/builds/" + build.id + "/" + build.target
+ "/attempts/latest/artifacts?maxResults=1000";
@@ -79,7 +100,9 @@
return curl.DownloadToFile(url, path, Headers());
}
-DeviceBuild ArgumentToBuild(BuildApi* build_api, const std::string& arg) {
+DeviceBuild ArgumentToBuild(BuildApi* build_api, const std::string& arg,
+ const std::string& default_build_target,
+ const std::chrono::seconds& retry_period) {
size_t slash_pos = arg.find('/');
if (slash_pos != std::string::npos
&& arg.find('/', slash_pos + 1) != std::string::npos) {
@@ -87,22 +110,33 @@
<< slash_pos << " and " << arg.find('/', slash_pos + 1);
}
std::string build_target = slash_pos == std::string::npos
- ? "aosp_cf_x86_phone-userdebug" : arg.substr(slash_pos + 1);
+ ? default_build_target : arg.substr(slash_pos + 1);
std::string branch_or_id = slash_pos == std::string::npos
? arg: arg.substr(0, slash_pos);
std::string branch_latest_build_id =
build_api->LatestBuildId(branch_or_id, build_target);
+ std::string build_id = branch_or_id;
if (branch_latest_build_id != "") {
LOG(INFO) << "The latest good build on branch \"" << branch_or_id
<< "\"with build target \"" << build_target
<< "is \"" << branch_latest_build_id << "\"";
- return DeviceBuild(branch_latest_build_id, build_target);
- } else {
- DeviceBuild proposed_build = DeviceBuild(branch_or_id, build_target);
- if (build_api->Artifacts(proposed_build).size() == 0) {
- LOG(FATAL) << '"' << branch_or_id << "\" with build target \""
- << build_target << "\" is not a valid branch or build id.";
- }
- return proposed_build;
+ build_id = branch_latest_build_id;
}
+ DeviceBuild proposed_build = DeviceBuild(build_id, build_target);
+ std::string status = build_api->BuildStatus(proposed_build);
+ if (status == "") {
+ LOG(FATAL) << '"' << build_id << "\" with build target \""
+ << build_target << "\" is not a valid branch or build id.";
+ }
+ LOG(INFO) << "Status for build " << build_id << "/" << build_target
+ << " is " << status;
+ while (retry_period != std::chrono::seconds::zero() && !StatusIsTerminal(status)) {
+ LOG(INFO) << "Status is \"" << status << "\". Waiting for " << retry_period.count()
+ << " seconds.";
+ std::this_thread::sleep_for(retry_period);
+ status = build_api->BuildStatus(proposed_build);
+ }
+ LOG(INFO) << "Status for build " << build_id << "/" << build_target
+ << " is " << status;
+ return proposed_build;
}
diff --git a/host/commands/fetcher/build_api.h b/host/commands/fetcher/build_api.h
index 128c525..36aa81f 100644
--- a/host/commands/fetcher/build_api.h
+++ b/host/commands/fetcher/build_api.h
@@ -15,6 +15,7 @@
#pragma once
+#include <chrono>
#include <functional>
#include <memory>
#include <string>
@@ -66,10 +67,14 @@
std::string LatestBuildId(const std::string& branch,
const std::string& target);
+ std::string BuildStatus(const DeviceBuild&);
+
std::vector<Artifact> Artifacts(const DeviceBuild&);
bool ArtifactToFile(const DeviceBuild& build, const std::string& artifact,
const std::string& path);
};
-DeviceBuild ArgumentToBuild(BuildApi* api, const std::string& arg);
+DeviceBuild ArgumentToBuild(BuildApi* api, const std::string& arg,
+ const std::string& default_build_target,
+ const std::chrono::seconds& retry_period);
diff --git a/host/commands/fetcher/main.cc b/host/commands/fetcher/main.cc
index 4f3e35d..cf6c4b6 100644
--- a/host/commands/fetcher/main.cc
+++ b/host/commands/fetcher/main.cc
@@ -41,6 +41,8 @@
DEFINE_string(directory, cvd::CurrentDirectory(), "Target directory to fetch "
"files into");
DEFINE_bool(run_next_stage, false, "Continue running the device through the next stage.");
+DEFINE_string(wait_retry_period, "20", "Retry period for pending builds given "
+ "in seconds. Set to 0 to not wait.");
namespace {
@@ -142,6 +144,7 @@
if (!cvd::DirectoryExists(target_dir) && mkdir(target_dir.c_str(), 0777) != 0) {
LOG(FATAL) << "Could not create " << target_dir;
}
+ std::chrono::seconds retry_period(std::stoi(FLAGS_wait_retry_period));
curl_global_init(CURL_GLOBAL_DEFAULT);
{
@@ -153,7 +156,9 @@
}
BuildApi build_api(std::move(credential_source));
- DeviceBuild default_build = ArgumentToBuild(&build_api, FLAGS_default_build);
+ DeviceBuild default_build = ArgumentToBuild(&build_api, FLAGS_default_build,
+ "aosp_cf_x86_phone-userdebug",
+ retry_period);
if (!download_host_package(&build_api, default_build, target_dir)) {
LOG(FATAL) << "Could not download host package with target "
@@ -167,7 +172,9 @@
desparse(target_dir + "/userdata.img");
if (FLAGS_system_build != "") {
- DeviceBuild system_build = ArgumentToBuild(&build_api, FLAGS_system_build);
+ DeviceBuild system_build = ArgumentToBuild(&build_api, FLAGS_system_build,
+ "aosp_cf_x86_phone-userdebug",
+ retry_period);
if (!download_images(&build_api, system_build, target_dir,
{"system.img"})) {
@@ -177,7 +184,8 @@
}
if (FLAGS_kernel_build != "") {
- DeviceBuild kernel_build = ArgumentToBuild(&build_api, FLAGS_kernel_build);
+ DeviceBuild kernel_build = ArgumentToBuild(&build_api, FLAGS_kernel_build,
+ "kernel", retry_period);
build_api.ArtifactToFile(kernel_build, "bzImage", target_dir + "/kernel");
}
