blob: 940cf825b68c93b7325bedac21b3997a306a617c [file] [log] [blame]
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.tradefed.cluster;
import com.android.annotations.VisibleForTesting;
import com.android.tradefed.build.BuildRetrievalError;
import com.android.tradefed.build.IBuildInfo;
import com.android.tradefed.build.IBuildProvider;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionClass;
import com.android.tradefed.invoker.logger.InvocationLocal;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.FuseUtil;
import com.android.tradefed.util.TarUtil;
import com.android.tradefed.util.ZipUtil2;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/** A {@link IBuildProvider} to download TFC test resources. */
@OptionClass(alias = "cluster", global_namespace = false)
public class ClusterBuildProvider implements IBuildProvider {
private static final String DEFAULT_FILE_VERSION = "0";
@Option(name = "root-dir", description = "A root directory", mandatory = true)
private File mRootDir;
@Option(
name = "test-resource",
description = "A list of JSON-serialized test resource objects",
mandatory = true)
private List<String> mTestResources = new ArrayList<>();
@Option(name = "build-id", description = "Build ID")
private String mBuildId = IBuildInfo.UNKNOWN_BUILD_ID;
@Option(name = "build-target", description = "Build target name")
private String mBuildTarget = "stub";
// The keys are the URLs; the values are the downloaded files shared among all build providers
// in the invocation.
// TODO(b/139876060): Use dynamic download when it supports caching HTTPS and GCS files.
@VisibleForTesting
static final InvocationLocal<ConcurrentHashMap<String, File>> sDownloadCache =
new InvocationLocal<ConcurrentHashMap<String, File>>() {
@Override
protected ConcurrentHashMap<String, File> initialValue() {
return new ConcurrentHashMap<String, File>();
}
};
// The keys are the resource names; the values are the files and directories.
@VisibleForTesting
static final InvocationLocal<ConcurrentHashMap<String, File>> sCreatedResources =
new InvocationLocal<ConcurrentHashMap<String, File>>() {
@Override
protected ConcurrentHashMap<String, File> initialValue() {
return new ConcurrentHashMap<String, File>();
}
};
private List<TestResource> parseTestResources() {
final List<TestResource> objs = new ArrayList<>();
for (final String s : mTestResources) {
try {
final JSONObject json = new JSONObject(s);
final TestResource obj = TestResource.fromJson(json);
objs.add(obj);
} catch (JSONException e) {
throw new RuntimeException("Failed to parse a test resource option: " + s, e);
}
}
return objs;
}
@Override
public IBuildInfo getBuild() throws BuildRetrievalError {
mRootDir.mkdirs();
final ClusterBuildInfo buildInfo = new ClusterBuildInfo(mRootDir, mBuildId, mBuildTarget);
final TestResourceDownloader downloader = createTestResourceDownloader();
final ConcurrentHashMap<String, File> cache = sDownloadCache.get();
final ConcurrentHashMap<String, File> createdResources = sCreatedResources.get();
final List<TestResource> testResources = parseTestResources();
for (TestResource resource : testResources) {
// For backward compatibility.
if (resource.getName().endsWith(".zip") && !resource.getDecompress()) {
resource =
new TestResource(
resource.getName(),
resource.getUrl(),
true,
new File(resource.getName()).getParent(),
resource.mountZip(),
resource.getDecompressFiles());
}
// Validate the paths before the file operations.
final File resourceFile = resource.getFile(mRootDir);
validateTestResourceFile(mRootDir, resourceFile);
if (resource.getDecompress()) {
File dir = resource.getDecompressDir(mRootDir);
validateTestResourceFile(mRootDir, dir);
for (String name : resource.getDecompressFiles()) {
validateTestResourceFile(dir, new File(dir, name));
}
}
// Download and decompress.
File file;
try {
File cachedFile = retrieveFile(resource.getUrl(), cache, downloader, resourceFile);
file = prepareTestResource(resource, createdResources, cachedFile, buildInfo);
} catch (UncheckedIOException e) {
throw new BuildRetrievalError("failed to get test resources", e);
}
buildInfo.setFile(resource.getName(), file, DEFAULT_FILE_VERSION);
}
return buildInfo;
}
/** Check if a resource file is under the working directory. */
private static void validateTestResourceFile(File workDir, File file)
throws BuildRetrievalError {
if (!file.toPath().normalize().startsWith(workDir.toPath().normalize())) {
throw new BuildRetrievalError(file + " is outside of working directory.");
}
}
/**
* Retrieve a file from cache or URL.
*
* <p>If the URL is in the cache, this method returns the cached file. Otherwise, it downloads
* and adds the file to the cache. If any file operation fails, this method throws {@link
* UncheckedIOException}.
*
* @param downloadUrl the file to be retrieved.
* @param cache the cache that maps URLs to files.
* @param downloader the downloader that gets the file.
* @param downloadDest the file to be created if the URL isn't in the cache.
* @return the cached or downloaded file.
*/
private File retrieveFile(
String downloadUrl,
ConcurrentHashMap<String, File> cache,
TestResourceDownloader downloader,
File downloadDest) {
return cache.computeIfAbsent(
downloadUrl,
url -> {
CLog.i("Download %s from %s.", downloadDest, url);
try {
downloader.download(url, downloadDest);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return downloadDest;
});
}
/**
* Create a resource file from cache and decompress it if needed.
*
* <p>If any file operation fails, this method throws {@link UncheckedIOException}.
*
* @param resource the resource to be created.
* @param createdResources the map from created resource names to paths.
* @param source the local cache of the file.
* @param buildInfo the current build info.
* @return the file or directory to be added to build info.
*/
private File prepareTestResource(
TestResource resource,
ConcurrentHashMap<String, File> createdResources,
File source,
ClusterBuildInfo buildInfo) {
return createdResources.computeIfAbsent(
resource.getName(),
name -> {
// Create the file regardless of the decompress flag.
final File file = resource.getFile(mRootDir);
if (!source.equals(file)) {
if (file.exists()) {
CLog.w("Overwrite %s.", name);
file.delete();
} else {
CLog.i("Create %s.", name);
file.getParentFile().mkdirs();
}
try {
FileUtil.hardlinkFile(source, file);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
// Decompress if needed.
if (resource.getDecompress()) {
final File dir = resource.getDecompressDir(mRootDir);
try {
decompressArchive(
file,
dir,
resource.mountZip(),
resource.getDecompressFiles(),
buildInfo);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return dir;
}
return file;
});
}
@VisibleForTesting
FuseUtil getFuseUtil() {
return new FuseUtil();
}
/**
* Extracts a zip or a gzip to a directory.
*
* @param archive the archive to be extracted.
* @param destDir the directory where the archive is extracted.
* @param mountZip whether to mount the zip or extract it.
* @param fileNames the files to be extracted from the archive. If the list is empty, all files
* are extracted.
* @param buildInfo the {@link ClusterBuildInfo} that records mounted zip files.
* @throws IOException if any file operation fails or any file name is not found in the archive.
*/
private void decompressArchive(
File archive,
File destDir,
boolean mountZip,
List<String> fileNames,
ClusterBuildInfo buildInfo)
throws IOException {
if (!destDir.exists()) {
if (!destDir.mkdirs()) {
CLog.e("Cannot create %s.", destDir);
}
}
if (TarUtil.isGzip(archive)) {
decompressTarGzip(archive, destDir, fileNames);
return;
}
if (mountZip) {
FuseUtil fuseUtil = getFuseUtil();
if (fuseUtil.canMountZip()) {
File mountDir = mountZip(fuseUtil, archive, buildInfo);
// Build a shadow directory structure with symlinks to allow a test to create files
// within it. This allows xTS to write result files under its own directory
// structure (e.g. android-cts/results).
symlinkFiles(mountDir, destDir, fileNames);
return;
}
CLog.w("Mounting zip requested but not supported; falling back to extracting...");
}
decompressZip(archive, destDir, fileNames);
}
private void decompressTarGzip(File archive, File destDir, List<String> fileNames)
throws IOException {
File unGzipDir = FileUtil.createTempDir("ClusterBuildProviderUnGzip");
try {
File tar = TarUtil.unGzip(archive, unGzipDir);
if (fileNames.isEmpty()) {
TarUtil.unTar(tar, destDir);
} else {
TarUtil.unTar(tar, destDir, fileNames);
}
} finally {
FileUtil.recursiveDelete(unGzipDir);
}
}
/** Mount a zip to a temporary directory if zip mounting is supported. */
private File mountZip(FuseUtil fuseUtil, File archive, ClusterBuildInfo buildInfo)
throws IOException {
File mountDir = FileUtil.createTempDir("ClusterBuildProviderZipMount");
buildInfo.addZipMount(mountDir);
CLog.i("Mounting %s to %s...", archive, mountDir);
fuseUtil.mountZip(archive, mountDir);
return mountDir;
}
private void symlinkFiles(File origDir, File destDir, List<String> fileNames)
throws IOException {
if (fileNames.isEmpty()) {
CLog.i("Recursive symlink %s to %s...", origDir, destDir);
FileUtil.recursiveSymlink(origDir, destDir);
} else {
for (String name : fileNames) {
File origFile = new File(origDir, name);
if (!origFile.exists()) {
throw new IOException(String.format("%s does not exist.", origFile));
}
File destFile = new File(destDir, name);
CLog.i("Symlink %s to %s", origFile, destFile);
destFile.getParentFile().mkdirs();
FileUtil.symlinkFile(origFile, destFile);
}
}
}
private void decompressZip(File archive, File destDir, List<String> fileNames)
throws IOException {
try (ZipFile zip = new ZipFile(archive)) {
if (fileNames.isEmpty()) {
CLog.i("Extracting %s to %s...", archive, destDir);
ZipUtil2.extractZip(zip, destDir);
} else {
for (String name : fileNames) {
File destFile = new File(destDir, name);
CLog.i("Extracting %s from %s to %s", name, archive, destFile);
destFile.getParentFile().mkdirs();
if (!ZipUtil2.extractFileFromZip(zip, name, destFile)) {
throw new IOException(
String.format("%s is not found in %s", name, archive));
}
}
}
}
}
@Override
public void buildNotTested(IBuildInfo info) {}
@Override
public void cleanUp(IBuildInfo info) {
if (!(info instanceof ClusterBuildInfo)) {
throw new IllegalArgumentException("info is not an instance of ClusterBuildInfo");
}
FuseUtil fuseUtil = getFuseUtil();
for (File dir : ((ClusterBuildInfo) info).getZipMounts()) {
fuseUtil.unmountZip(dir);
FileUtil.recursiveDelete(dir);
}
}
@VisibleForTesting
TestResourceDownloader createTestResourceDownloader() {
return new TestResourceDownloader();
}
@VisibleForTesting
void setRootDir(File rootDir) {
mRootDir = rootDir;
}
@VisibleForTesting
void addTestResource(TestResource resource) throws JSONException {
mTestResources.add(resource.toJson().toString());
}
@VisibleForTesting
List<TestResource> getTestResources() {
return parseTestResources();
}
}