Support loading build api credentials from GCE.
This allows using credentials to get more builds on higher-privileged
GCE instances.
Based on https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances
See "Authenticating applications directly with access tokens"
Bug: 137304531
Test: ./fetch_cvd
Change-Id: Ied9813f0c9264e0b6d3c77ee30c5ecd1ccd37628
diff --git a/host/commands/fetcher/Android.bp b/host/commands/fetcher/Android.bp
index 1897cda..0c88171 100644
--- a/host/commands/fetcher/Android.bp
+++ b/host/commands/fetcher/Android.bp
@@ -17,6 +17,7 @@
name: "fetch_cvd",
srcs: [
"build_api.cc",
+ "credential_source.cc",
"curl_wrapper.cc",
"main.cc",
],
diff --git a/host/commands/fetcher/build_api.cc b/host/commands/fetcher/build_api.cc
index 231f805..7823cab 100644
--- a/host/commands/fetcher/build_api.cc
+++ b/host/commands/fetcher/build_api.cc
@@ -37,11 +37,22 @@
crc32 = json_artifact["crc32"].asUInt();
}
+BuildApi::BuildApi(std::unique_ptr<CredentialSource> credential_source)
+ : credential_source(std::move(credential_source)) {}
+
+std::vector<std::string> BuildApi::Headers() {
+ std::vector<std::string> headers;
+ if (credential_source) {
+ headers.push_back("Authorization:Bearer " + credential_source->Credential());
+ }
+ return headers;
+}
+
std::string BuildApi::LatestBuildId(const std::string& branch,
const std::string& target) {
std::string url = BUILD_API + "/builds?branch=" + branch
+ "&buildType=submitted&maxResults=1&successful=true&target=" + target;
- auto response = curl.DownloadToJson(url);
+ auto response = curl.DownloadToJson(url, Headers());
if (response["builds"].size() != 1) {
LOG(ERROR) << "invalid number of builds\n";
return "";
@@ -54,7 +65,7 @@
const std::string& attempt_id) {
std::string url = BUILD_API + "/builds/" + build_id + "/" + target
+ "/attempts/" + attempt_id + "/artifacts?maxResults=1000";
- auto artifacts_json = curl.DownloadToJson(url);
+ auto artifacts_json = curl.DownloadToJson(url, Headers());
std::vector<Artifact> artifacts;
for (const auto& artifact_json : artifacts_json["artifacts"]) {
artifacts.emplace_back(artifact_json);
@@ -69,5 +80,5 @@
const std::string& path) {
std::string url = BUILD_API + "/builds/" + build_id + "/" + target
+ "/attempts/" + attempt_id + "/artifacts/" + artifact + "?alt=media";
- return curl.DownloadToFile(url, path);
+ return curl.DownloadToFile(url, path, Headers());
}
diff --git a/host/commands/fetcher/build_api.h b/host/commands/fetcher/build_api.h
index d3fb887..be5596a 100644
--- a/host/commands/fetcher/build_api.h
+++ b/host/commands/fetcher/build_api.h
@@ -15,8 +15,11 @@
#pragma once
+#include <functional>
+#include <memory>
#include <string>
+#include "credential_source.h"
#include "curl_wrapper.h"
class Artifact {
@@ -43,9 +46,11 @@
class BuildApi {
CurlWrapper curl;
- // TODO credential fetcher
+ std::unique_ptr<CredentialSource> credential_source;
+
+ std::vector<std::string> Headers();
public:
- BuildApi() = default;
+ BuildApi(std::unique_ptr<CredentialSource> credential_source);
~BuildApi() = default;
std::string LatestBuildId(const std::string& branch,
diff --git a/host/commands/fetcher/credential_source.cc b/host/commands/fetcher/credential_source.cc
new file mode 100644
index 0000000..6133149
--- /dev/null
+++ b/host/commands/fetcher/credential_source.cc
@@ -0,0 +1,49 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "credential_source.h"
+
+namespace {
+
+std::chrono::steady_clock::duration REFRESH_WINDOW =
+ std::chrono::minutes(2);
+std::string REFRESH_URL = "http://metadata.google.internal/computeMetadata/"
+ "v1/instance/service-accounts/default/token";
+
+}
+
+GceMetadataCredentialSource::GceMetadataCredentialSource() {
+ latest_credential = "";
+ expiration = std::chrono::steady_clock::now();
+}
+
+std::string GceMetadataCredentialSource::Credential() {
+ if (expiration - std::chrono::steady_clock::now() < REFRESH_WINDOW) {
+ RefreshCredential();
+ }
+ return latest_credential;
+}
+
+void GceMetadataCredentialSource::RefreshCredential() {
+ Json::Value credential_json =
+ curl.DownloadToJson(REFRESH_URL, {"Metadata-Flavor: Google"});
+ expiration = std::chrono::steady_clock::now()
+ + std::chrono::seconds(credential_json["expires_in"].asInt());
+ latest_credential = credential_json["access_token"].asString();
+}
+
+std::unique_ptr<CredentialSource> GceMetadataCredentialSource::make() {
+ return std::unique_ptr<CredentialSource>(new GceMetadataCredentialSource());
+}
diff --git a/host/commands/fetcher/credential_source.h b/host/commands/fetcher/credential_source.h
new file mode 100644
index 0000000..f50bd12
--- /dev/null
+++ b/host/commands/fetcher/credential_source.h
@@ -0,0 +1,42 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <chrono>
+#include <memory>
+
+#include "curl_wrapper.h"
+
+class CredentialSource {
+public:
+ virtual ~CredentialSource() = default;
+ virtual std::string Credential() = 0;
+};
+
+class GceMetadataCredentialSource : public CredentialSource {
+ CurlWrapper curl;
+ std::string latest_credential;
+ std::chrono::steady_clock::time_point expiration;
+
+ void RefreshCredential();
+public:
+ GceMetadataCredentialSource();
+ GceMetadataCredentialSource(GceMetadataCredentialSource&&) = default;
+
+ virtual std::string Credential();
+
+ static std::unique_ptr<CredentialSource> make();
+};
diff --git a/host/commands/fetcher/curl_wrapper.cc b/host/commands/fetcher/curl_wrapper.cc
index 4a59c47..93dca5e 100644
--- a/host/commands/fetcher/curl_wrapper.cc
+++ b/host/commands/fetcher/curl_wrapper.cc
@@ -32,6 +32,22 @@
return nmemb;
}
+curl_slist* build_slist(const std::vector<std::string>& strings) {
+ curl_slist* curl_headers = nullptr;
+ for (const auto& str : strings) {
+ curl_slist* temp = curl_slist_append(curl_headers, str.c_str());
+ if (temp == nullptr) {
+ LOG(ERROR) << "curl_slist_append failed to add " << str;
+ if (curl_headers) {
+ curl_slist_free_all(curl_headers);
+ return nullptr;
+ }
+ }
+ curl_headers = temp;
+ }
+ return curl_headers;
+}
+
} // namespace
CurlWrapper::CurlWrapper() {
@@ -40,7 +56,6 @@
LOG(ERROR) << "failed to initialize curl";
return;
}
- curl_easy_setopt(curl, CURLOPT_CAINFO, "/etc/ssl/certs/ca-certificates.crt");
}
CurlWrapper::~CurlWrapper() {
@@ -48,12 +63,20 @@
}
bool CurlWrapper::DownloadToFile(const std::string& url, const std::string& path) {
+ return CurlWrapper::DownloadToFile(url, path, {});
+}
+
+bool CurlWrapper::DownloadToFile(const std::string& url, const std::string& path,
+ const std::vector<std::string>& headers) {
LOG(INFO) << "Attempting to save \"" << url << "\" to \"" << path << "\"";
if (!curl) {
LOG(ERROR) << "curl was not initialized\n";
return false;
}
+ curl_slist* curl_headers = build_slist(headers);
curl_easy_reset(curl);
+ curl_easy_setopt(curl, CURLOPT_CAINFO, "/etc/ssl/certs/ca-certificates.crt");
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
FILE* file = fopen(path.c_str(), "w");
if (!file) {
@@ -62,26 +85,40 @@
}
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*) file);
CURLcode res = curl_easy_perform(curl);
+ if (curl_headers) {
+ curl_slist_free_all(curl_headers);
+ }
+ fclose(file);
if(res != CURLE_OK) {
LOG(ERROR) << "curl_easy_perform() failed: " << curl_easy_strerror(res);
return false;
}
- fclose(file);
return true;
}
std::string CurlWrapper::DownloadToString(const std::string& url) {
+ return DownloadToString(url, {});
+}
+
+std::string CurlWrapper::DownloadToString(const std::string& url,
+ const std::vector<std::string>& headers) {
LOG(INFO) << "Attempting to download \"" << url << "\"";
if (!curl) {
LOG(ERROR) << "curl was not initialized\n";
return "";
}
+ curl_slist* curl_headers = build_slist(headers);
curl_easy_reset(curl);
+ curl_easy_setopt(curl, CURLOPT_CAINFO, "/etc/ssl/certs/ca-certificates.crt");
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
std::stringstream data;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, file_write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
CURLcode res = curl_easy_perform(curl);
+ if (curl_headers) {
+ curl_slist_free_all(curl_headers);
+ }
if(res != CURLE_OK) {
LOG(ERROR) << "curl_easy_perform() failed: " << curl_easy_strerror(res);
return "";
@@ -90,7 +127,12 @@
}
Json::Value CurlWrapper::DownloadToJson(const std::string& url) {
- std::string contents = DownloadToString(url);
+ return DownloadToJson(url, {});
+}
+
+Json::Value CurlWrapper::DownloadToJson(const std::string& url,
+ const std::vector<std::string>& headers) {
+ std::string contents = DownloadToString(url, headers);
Json::Reader reader;
Json::Value json;
if (!reader.parse(contents, json)) {
diff --git a/host/commands/fetcher/curl_wrapper.h b/host/commands/fetcher/curl_wrapper.h
index 40ab5a8..5a5ba9c 100644
--- a/host/commands/fetcher/curl_wrapper.h
+++ b/host/commands/fetcher/curl_wrapper.h
@@ -27,9 +27,15 @@
~CurlWrapper();
CurlWrapper(const CurlWrapper&) = delete;
CurlWrapper& operator=(const CurlWrapper*) = delete;
- CurlWrapper(const CurlWrapper&&) = delete;
+ CurlWrapper(CurlWrapper&&) = default;
bool DownloadToFile(const std::string& url, const std::string& path);
+ bool DownloadToFile(const std::string& url, const std::string& path,
+ const std::vector<std::string>& headers);
std::string DownloadToString(const std::string& url);
+ std::string DownloadToString(const std::string& url,
+ const std::vector<std::string>& headers);
Json::Value DownloadToJson(const std::string& url);
+ Json::Value DownloadToJson(const std::string& url,
+ const std::vector<std::string>& headers);
};
diff --git a/host/commands/fetcher/main.cc b/host/commands/fetcher/main.cc
index a323083..c209e01 100644
--- a/host/commands/fetcher/main.cc
+++ b/host/commands/fetcher/main.cc
@@ -24,6 +24,7 @@
#include "common/libs/utils/subprocess.h"
#include "build_api.h"
+#include "credential_source.h"
namespace {
@@ -35,6 +36,7 @@
DEFINE_string(build_id, "latest", "Build ID for all artifacts");
DEFINE_string(branch, "aosp-master", "Branch when build_id=\"latest\"");
DEFINE_string(target, "aosp_cf_x86_phone-userdebug", "Build target");
+DEFINE_string(credential_source, "", "Build API credential source");
int main(int argc, char** argv) {
::android::base::InitLogging(argv, android::base::StderrLogger);
@@ -42,7 +44,11 @@
curl_global_init(CURL_GLOBAL_DEFAULT);
{
- BuildApi build_api;
+ std::unique_ptr<CredentialSource> credential_source;
+ if (FLAGS_credential_source == "gce") {
+ credential_source = GceMetadataCredentialSource::make();
+ }
+ BuildApi build_api(std::move(credential_source));
std::string build_id = FLAGS_build_id;
if (build_id == "latest") {
build_id = build_api.LatestBuildId(FLAGS_branch, FLAGS_target);
@@ -51,7 +57,7 @@
auto artifacts = build_api.Artifacts(build_id, FLAGS_target, "latest");
bool has_host_package = false;
bool has_image_zip = false;
- const std::string img_zip_name = "aosp_cf_x86_phone-img-" + build_id + ".zip";
+ const std::string img_zip_name = FLAGS_target + "-img-" + build_id + ".zip";
for (const auto& artifact : artifacts) {
has_host_package |= artifact.Name() == HOST_TOOLS;
has_image_zip |= artifact.Name() == img_zip_name;
@@ -60,7 +66,7 @@
LOG(FATAL) << "Target build " << build_id << " did not have " << HOST_TOOLS;
}
if (!has_image_zip) {
- LOG(FATAL) << "Target build " << build_id << " did not have" << img_zip_name;
+ LOG(FATAL) << "Target build " << build_id << " did not have " << img_zip_name;
}
build_api.ArtifactToFile(build_id, FLAGS_target, "latest",
