versioner: refactor to use CompilerInstance directly.
This will make it easier to switch over to a virtual filesystem,
which should drastically improve performance.
This also fixes an issue with warning/error reporting.
Bug: http://b/32748936
Test: python run_tests.py
Change-Id: I2e967acf75db29c95f3a03f4f94cccd22c367ad5
diff --git a/libc/include/android/legacy_termios_inlines.h b/libc/include/android/legacy_termios_inlines.h
index 41ea955..92cb22d 100644
--- a/libc/include/android/legacy_termios_inlines.h
+++ b/libc/include/android/legacy_termios_inlines.h
@@ -29,11 +29,12 @@
#ifndef _ANDROID_LEGACY_TERMIOS_INLINES_H_
#define _ANDROID_LEGACY_TERMIOS_INLINES_H_
-#include <linux/termios.h>
#include <sys/cdefs.h>
#include <sys/ioctl.h>
#include <sys/types.h>
+#include <linux/termios.h>
+
#if __ANDROID_API__ < 21
__BEGIN_DECLS
diff --git a/libc/include/elf.h b/libc/include/elf.h
index 1a99337..ae05d3d 100644
--- a/libc/include/elf.h
+++ b/libc/include/elf.h
@@ -29,11 +29,12 @@
#ifndef _ELF_H
#define _ELF_H
+#include <sys/cdefs.h>
+
#include <linux/auxvec.h>
#include <linux/elf.h>
#include <linux/elf-em.h>
#include <machine/elf_machdep.h>
-#include <sys/cdefs.h>
#define ELF32_R_INFO(sym, type) ((((Elf32_Word)sym) << 8) | ((type) & 0xff))
#define ELF64_R_INFO(sym, type) ((((Elf64_Xword)sym) << 32) | ((type) & 0xffffffff))
diff --git a/libc/include/link.h b/libc/include/link.h
index 92ecceb..0d07cb9 100644
--- a/libc/include/link.h
+++ b/libc/include/link.h
@@ -28,10 +28,11 @@
#ifndef _LINK_H_
#define _LINK_H_
-#include <elf.h>
#include <sys/cdefs.h>
#include <sys/types.h>
+#include <elf.h>
+
__BEGIN_DECLS
#if defined(__LP64__)
diff --git a/libc/include/signal.h b/libc/include/signal.h
index a9df04b..17bc57d 100644
--- a/libc/include/signal.h
+++ b/libc/include/signal.h
@@ -29,12 +29,13 @@
#ifndef _SIGNAL_H_
#define _SIGNAL_H_
+#include <sys/cdefs.h>
+#include <sys/types.h>
+
#include <asm/sigcontext.h>
#include <bits/pthread_types.h>
#include <bits/timespec.h>
#include <limits.h>
-#include <sys/cdefs.h>
-#include <sys/types.h>
#if defined(__LP64__) || defined(__mips__)
/* For 64-bit (and mips), the kernel's struct sigaction doesn't match the POSIX one,
diff --git a/libc/include/sys/select.h b/libc/include/sys/select.h
index 5a8a81d..df118e3 100644
--- a/libc/include/sys/select.h
+++ b/libc/include/sys/select.h
@@ -29,11 +29,12 @@
#ifndef _SYS_SELECT_H_
#define _SYS_SELECT_H_
-#include <linux/time.h>
-#include <signal.h>
#include <sys/cdefs.h>
#include <sys/types.h>
+#include <linux/time.h>
+#include <signal.h>
+
__BEGIN_DECLS
#define FD_SETSIZE 1024
diff --git a/libc/include/sys/signalfd.h b/libc/include/sys/signalfd.h
index a21c57b..79f9490 100644
--- a/libc/include/sys/signalfd.h
+++ b/libc/include/sys/signalfd.h
@@ -29,9 +29,10 @@
#ifndef _SYS_SIGNALFD_H_
#define _SYS_SIGNALFD_H_
+#include <sys/cdefs.h>
+
#include <linux/signalfd.h>
#include <signal.h>
-#include <sys/cdefs.h>
__BEGIN_DECLS
diff --git a/libc/include/sys/ucontext.h b/libc/include/sys/ucontext.h
index fdb3015..4eddf23 100644
--- a/libc/include/sys/ucontext.h
+++ b/libc/include/sys/ucontext.h
@@ -29,8 +29,9 @@
#ifndef _SYS_UCONTEXT_H_
#define _SYS_UCONTEXT_H_
-#include <signal.h>
#include <sys/cdefs.h>
+
+#include <signal.h>
#include <sys/user.h>
__BEGIN_DECLS
diff --git a/libc/include/syslog.h b/libc/include/syslog.h
index ee288e5..4b7eecb 100644
--- a/libc/include/syslog.h
+++ b/libc/include/syslog.h
@@ -89,7 +89,11 @@
void openlog(const char* _Nullable, int, int);
int setlogmask(int);
void syslog(int, const char* _Nonnull, ...) __printflike(2, 3);
+#if defined(__arm__) || defined(__aarch64__) || defined(__x86_64__)
void vsyslog(int, const char* _Nonnull, va_list) __printflike(2, 0);
+#else /* defined(__mips__) || defined(__i386__) */
+void vsyslog(int, const char* _Nonnull, va_list _Nonnull) __printflike(2, 0);
+#endif
__END_DECLS
diff --git a/tools/versioner/src/Android.mk b/tools/versioner/src/Android.mk
index c43a525..aaf5589 100644
--- a/tools/versioner/src/Android.mk
+++ b/tools/versioner/src/Android.mk
@@ -31,7 +31,7 @@
SymbolDatabase.cpp \
Utils.cpp
-LOCAL_SHARED_LIBRARIES := libclang libLLVM
+LOCAL_SHARED_LIBRARIES := libclang libLLVM libbase
include $(CLANG_HOST_BUILD_MK)
include $(CLANG_TBLGEN_RULES_MK)
diff --git a/tools/versioner/src/DeclarationDatabase.cpp b/tools/versioner/src/DeclarationDatabase.cpp
index 9c9bc83..af4e1e8 100644
--- a/tools/versioner/src/DeclarationDatabase.cpp
+++ b/tools/versioner/src/DeclarationDatabase.cpp
@@ -284,9 +284,8 @@
return false;
}
-void HeaderDatabase::parseAST(CompilationType type, ASTUnit* ast) {
+void HeaderDatabase::parseAST(CompilationType type, ASTContext& ctx) {
std::unique_lock<std::mutex> lock(this->mutex);
- ASTContext& ctx = ast->getASTContext();
Visitor visitor(*this, type, ctx);
visitor.TraverseDecl(ctx.getTranslationUnitDecl());
}
diff --git a/tools/versioner/src/DeclarationDatabase.h b/tools/versioner/src/DeclarationDatabase.h
index e957019..4ec15d9 100644
--- a/tools/versioner/src/DeclarationDatabase.h
+++ b/tools/versioner/src/DeclarationDatabase.h
@@ -30,7 +30,7 @@
#include "Utils.h"
namespace clang {
-class ASTUnit;
+class ASTContext;
class Decl;
}
@@ -220,7 +220,7 @@
public:
std::map<std::string, Symbol> symbols;
- void parseAST(CompilationType type, clang::ASTUnit* ast);
+ void parseAST(CompilationType type, clang::ASTContext& ast);
void dump(const std::string& base_path = "", FILE* out = stdout) const {
fprintf(out, "HeaderDatabase contains %zu symbols:\n", symbols.size());
diff --git a/tools/versioner/src/Preprocessor.cpp b/tools/versioner/src/Preprocessor.cpp
index da5d250..ff3ff83 100644
--- a/tools/versioner/src/Preprocessor.cpp
+++ b/tools/versioner/src/Preprocessor.cpp
@@ -121,33 +121,6 @@
return result;
}
-static std::string dirname(const std::string& path) {
- std::unique_ptr<char, decltype(&free)> path_copy(strdup(path.c_str()), free);
- return dirname(path_copy.get());
-}
-
-static bool mkdirs(const std::string& path) {
- struct stat st;
- if (stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode)) {
- return true;
- }
-
- std::string parent = dirname(path);
- if (parent == path) {
- return false;
- }
-
- if (!mkdirs(parent)) {
- return false;
- }
-
- if (mkdir(path.c_str(), 0700) != 0) {
- return false;
- }
-
- return true;
-}
-
static void writeFileLines(const std::string& path, const std::deque<std::string>& lines) {
if (!mkdirs(dirname(path))) {
err(1, "failed to create directory '%s'", dirname(path).c_str());
diff --git a/tools/versioner/src/Utils.h b/tools/versioner/src/Utils.h
index d49fc88..7876f23 100644
--- a/tools/versioner/src/Utils.h
+++ b/tools/versioner/src/Utils.h
@@ -16,6 +16,11 @@
#pragma once
+#include <errno.h>
+#include <libgen.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
#include <string>
#include <vector>
@@ -24,16 +29,53 @@
std::string getWorkingDir();
std::vector<std::string> collectFiles(const std::string& directory);
-static __attribute__((unused)) std::string to_string(const char* c) {
+static inline std::string dirname(const std::string& path) {
+ std::unique_ptr<char, decltype(&free)> path_copy(strdup(path.c_str()), free);
+ return dirname(path_copy.get());
+}
+
+static inline bool is_directory(const std::string& path) {
+ struct stat st;
+ if (stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode)) {
+ return true;
+ }
+ return false;
+}
+
+static inline bool mkdirs(const std::string& path) {
+ if (is_directory(path)) {
+ return true;
+ }
+
+ std::string parent = dirname(path);
+ if (parent == path) {
+ return false;
+ }
+
+ if (!mkdirs(parent)) {
+ return false;
+ }
+
+ if (mkdir(path.c_str(), 0700) != 0) {
+ if (errno != EEXIST) {
+ return false;
+ }
+ return is_directory(path);
+ }
+
+ return true;
+}
+
+static inline std::string to_string(const char* c) {
return c;
}
-static __attribute__((unused)) const std::string& to_string(const std::string& str) {
+static inline const std::string& to_string(const std::string& str) {
return str;
}
template <typename Collection>
-static std::string Join(Collection c, const std::string& delimiter = ", ") {
+static inline std::string Join(Collection c, const std::string& delimiter = ", ") {
std::string result;
for (const auto& item : c) {
using namespace std;
diff --git a/tools/versioner/src/versioner.cpp b/tools/versioner/src/versioner.cpp
index 74650d5..807ef16 100644
--- a/tools/versioner/src/versioner.cpp
+++ b/tools/versioner/src/versioner.cpp
@@ -23,6 +23,7 @@
#include <unistd.h>
#include <atomic>
+#include <functional>
#include <iostream>
#include <map>
#include <memory>
@@ -33,10 +34,22 @@
#include <unordered_map>
#include <vector>
+#include <clang/AST/ASTConsumer.h>
+#include <clang/Basic/TargetInfo.h>
+#include <clang/Driver/Compilation.h>
+#include <clang/Driver/Driver.h>
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/CompilerInvocation.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Frontend/FrontendActions.h>
#include <clang/Frontend/TextDiagnosticPrinter.h>
+#include <clang/Frontend/Utils.h>
+#include <clang/FrontendTool/Utils.h>
#include <clang/Tooling/Tooling.h>
#include <llvm/ADT/StringRef.h>
+#include <android-base/parseint.h>
+
#include "Arch.h"
#include "DeclarationDatabase.h"
#include "Preprocessor.h"
@@ -46,80 +59,130 @@
using namespace std::string_literals;
using namespace clang;
-using namespace clang::tooling;
+using namespace tooling;
bool verbose;
static bool add_include;
+static int max_thread_count = 48;
-class HeaderCompilationDatabase : public CompilationDatabase {
- CompilationType type;
- std::string cwd;
- std::vector<std::string> headers;
- std::vector<std::string> include_dirs;
+static std::vector<std::string> generateCompileCommand(CompilationType& type,
+ const std::string& filename,
+ const std::string& header_dir,
+ const std::vector<std::string>& include_dirs) {
+ std::vector<std::string> cmd = { "versioner" };
+ cmd.push_back("-std=c11");
+ cmd.push_back("-Wall");
+ cmd.push_back("-Wextra");
+ cmd.push_back("-Werror");
+ cmd.push_back("-Wundef");
+ cmd.push_back("-Wno-unused-macros");
+ cmd.push_back("-Wno-unused-function");
+ cmd.push_back("-Wno-unused-variable");
+ cmd.push_back("-Wno-unknown-attributes");
+ cmd.push_back("-Wno-pragma-once-outside-header");
+
+ cmd.push_back("-target");
+ cmd.push_back(arch_targets[type.arch]);
+
+ cmd.push_back("-DANDROID");
+ cmd.push_back("-D__ANDROID_API__="s + std::to_string(type.api_level));
+ cmd.push_back("-D_FORTIFY_SOURCE=2");
+ cmd.push_back("-D_GNU_SOURCE");
+ cmd.push_back("-D_FILE_OFFSET_BITS="s + std::to_string(type.file_offset_bits));
+
+ cmd.push_back("-nostdinc");
+ std::string header_path;
+ if (add_include) {
+ const char* top = getenv("ANDROID_BUILD_TOP");
+ header_path = to_string(top) + "/bionic/libc/include/android/versioning.h";
+ cmd.push_back("-include");
+ cmd.push_back(header_path);
+ }
+
+ for (const auto& dir : include_dirs) {
+ cmd.push_back("-isystem");
+ cmd.push_back(dir);
+ }
+
+ cmd.push_back(filename);
+
+ return cmd;
+}
+
+class VersionerASTConsumer : public clang::ASTConsumer {
public:
- HeaderCompilationDatabase(CompilationType type, std::string cwd, std::vector<std::string> headers,
- std::vector<std::string> include_dirs)
- : type(type),
- cwd(std::move(cwd)),
- headers(std::move(headers)),
- include_dirs(std::move(include_dirs)) {
+ HeaderDatabase* header_database;
+ CompilationType type;
+
+ VersionerASTConsumer(HeaderDatabase* header_database, CompilationType type)
+ : header_database(header_database), type(type) {
}
- CompileCommand generateCompileCommand(const std::string& filename) const {
- std::vector<std::string> command = { "clang-tool", filename, "-nostdlibinc" };
- for (const auto& dir : include_dirs) {
- command.push_back("-isystem");
- command.push_back(dir);
- }
- command.push_back("-std=c11");
- command.push_back("-DANDROID");
- command.push_back("-D__ANDROID_API__="s + std::to_string(type.api_level));
- command.push_back("-D_FORTIFY_SOURCE=2");
- command.push_back("-D_GNU_SOURCE");
- command.push_back("-Wall");
- command.push_back("-Wextra");
- command.push_back("-Werror");
- command.push_back("-Wundef");
- command.push_back("-Wno-unused-macros");
- command.push_back("-Wno-unused-function");
- command.push_back("-Wno-unused-variable");
- command.push_back("-Wno-unknown-attributes");
- command.push_back("-Wno-pragma-once-outside-header");
- command.push_back("-target");
- command.push_back(arch_targets[type.arch]);
-
- if (add_include) {
- const char* top = getenv("ANDROID_BUILD_TOP");
- std::string header_path = to_string(top) + "/bionic/libc/include/android/versioning.h";
- command.push_back("-include");
- command.push_back(std::move(header_path));
- }
-
- command.push_back("-D_FILE_OFFSET_BITS="s + std::to_string(type.file_offset_bits));
-
- return CompileCommand(cwd, filename, command);
- }
-
- std::vector<CompileCommand> getAllCompileCommands() const override {
- std::vector<CompileCommand> commands;
- for (const std::string& file : headers) {
- commands.push_back(generateCompileCommand(file));
- }
- return commands;
- }
-
- std::vector<CompileCommand> getCompileCommands(StringRef file) const override {
- std::vector<CompileCommand> commands;
- commands.push_back(generateCompileCommand(file));
- return commands;
- }
-
- std::vector<std::string> getAllFiles() const override {
- return headers;
+ virtual void HandleTranslationUnit(ASTContext& ctx) override {
+ header_database->parseAST(type, ctx);
}
};
+class VersionerASTAction : public clang::ASTFrontendAction {
+ public:
+ HeaderDatabase* header_database;
+ CompilationType type;
+
+ VersionerASTAction(HeaderDatabase* header_database, CompilationType type)
+ : header_database(header_database), type(type) {
+ }
+
+ virtual std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance& Compiler,
+ llvm::StringRef InFile) override {
+ return std::make_unique<VersionerASTConsumer>(header_database, type);
+ }
+};
+
+static void compileHeader(HeaderDatabase* header_database, CompilationType type,
+ const std::string& filename, const std::string& header_dir,
+ const std::vector<std::string>& include_dirs) {
+ DiagnosticOptions diagnostic_options;
+ auto diagnostic_printer = new TextDiagnosticPrinter(llvm::errs(), &diagnostic_options);
+
+ IntrusiveRefCntPtr<DiagnosticIDs> diagnostic_ids(new DiagnosticIDs());
+ IntrusiveRefCntPtr<DiagnosticsEngine> diags(
+ new DiagnosticsEngine(diagnostic_ids, &diagnostic_options, diagnostic_printer, false));
+ driver::Driver Driver("versioner", llvm::sys::getDefaultTargetTriple(), *diags);
+
+ std::vector<std::string> cmd = generateCompileCommand(type, filename, header_dir, include_dirs);
+ llvm::SmallVector<const char*, 32> Args;
+
+ for (const std::string& str : cmd) {
+ Args.push_back(str.c_str());
+ }
+
+ std::unique_ptr<driver::Compilation> Compilation(Driver.BuildCompilation(Args));
+
+ const driver::Command &Cmd = llvm::cast<driver::Command>(*Compilation->getJobs().begin());
+ const driver::ArgStringList &CCArgs = Cmd.getArguments();
+
+ auto invocation = std::make_unique<CompilerInvocation>();
+ if (!CompilerInvocation::CreateFromArgs(
+ *invocation.get(), const_cast<const char**>(CCArgs.data()),
+ const_cast<const char**>(CCArgs.data()) + CCArgs.size(), *diags)) {
+ errx(1, "failed to create CompilerInvocation");
+ }
+
+ clang::CompilerInstance Compiler;
+ Compiler.setInvocation(invocation.release());
+ Compiler.setDiagnostics(diags.get());
+
+ VersionerASTAction versioner_action(header_database, type);
+ if (!Compiler.ExecuteAction(versioner_action)) {
+ errx(1, "compilation generated warnings or errors");
+ }
+
+ if (diags->getNumWarnings() || diags->hasErrorOccurred()) {
+ errx(1, "compilation generated warnings or errors");
+ }
+}
+
struct CompilationRequirements {
std::vector<std::string> headers;
std::vector<std::string> dependencies;
@@ -205,20 +268,51 @@
return result;
}
+struct Job {
+ Job(CompilationType type, const std::string& header, const std::vector<std::string>& dependencies)
+ : type(type), header(header), dependencies(dependencies) {
+ }
+ CompilationType type;
+ const std::string& header;
+ const std::vector<std::string>& dependencies;
+};
+
static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<CompilationType>& types,
const std::string& header_dir,
- const std::string& dependency_dir,
- bool* failed) {
- constexpr size_t thread_count = 8;
- size_t threads_created = 0;
- std::mutex mutex;
- std::vector<std::thread> threads(thread_count);
+ const std::string& dependency_dir) {
+ if (types.empty()) {
+ errx(1, "compileHeaders received no CompilationTypes");
+ }
+
+ size_t thread_count = max_thread_count;
+ std::vector<std::thread> threads;
+ std::vector<Job> jobs;
std::map<CompilationType, HeaderDatabase> header_databases;
std::unordered_map<Arch, CompilationRequirements> requirements;
std::string cwd = getWorkingDir();
- bool errors = false;
+
+ auto result = std::make_unique<HeaderDatabase>();
+ auto spawn_threads = [&]() {
+ thread_count = std::min(thread_count, jobs.size());
+ for (size_t i = 0; i < thread_count; ++i) {
+ threads.emplace_back([&jobs, &result, &header_dir, thread_count, i]() {
+ size_t index = i;
+ while (index < jobs.size()) {
+ const auto& job = jobs[index];
+ compileHeader(result.get(), job.type, job.header, header_dir, job.dependencies);
+ index += thread_count;
+ }
+ });
+ }
+ };
+ auto reap_threads = [&]() {
+ for (auto& thread : threads) {
+ thread.join();
+ }
+ threads.clear();
+ };
for (const auto& type : types) {
if (requirements.count(type.arch) == 0) {
@@ -226,52 +320,15 @@
}
}
- auto result = std::make_unique<HeaderDatabase>();
- for (const auto& type : types) {
- size_t thread_id = threads_created++;
- if (thread_id >= thread_count) {
- thread_id = thread_id % thread_count;
- threads[thread_id].join();
+ for (CompilationType type : types) {
+ CompilationRequirements& req = requirements[type.arch];
+ for (const std::string& header : req.headers) {
+ jobs.emplace_back(type, header, req.dependencies);
}
-
- threads[thread_id] = std::thread(
- [&](CompilationType type) {
- const auto& req = requirements[type.arch];
-
- HeaderCompilationDatabase compilation_database(type, cwd, req.headers, req.dependencies);
-
- ClangTool tool(compilation_database, req.headers);
-
- clang::DiagnosticOptions diagnostic_options;
- std::vector<std::unique_ptr<ASTUnit>> asts;
- tool.buildASTs(asts);
- for (const auto& ast : asts) {
- clang::DiagnosticsEngine& diagnostics_engine = ast->getDiagnostics();
- if (diagnostics_engine.getNumWarnings() || diagnostics_engine.hasErrorOccurred()) {
- std::unique_lock<std::mutex> l(mutex);
- errors = true;
- printf("versioner: compilation failure for %s in %s\n", to_string(type).c_str(),
- ast->getOriginalSourceFileName().str().c_str());
- }
-
- result->parseAST(type, ast.get());
- }
- },
- type);
}
- if (threads_created < thread_count) {
- threads.resize(threads_created);
- }
-
- for (auto& thread : threads) {
- thread.join();
- }
-
- if (errors) {
- printf("versioner: compilation generated warnings or errors\n");
- *failed = errors;
- }
+ spawn_threads();
+ reap_threads();
return result;
}
@@ -472,6 +529,7 @@
fprintf(stderr, "\n");
fprintf(stderr, "Miscellaneous:\n");
fprintf(stderr, " -d\t\tdump function availability\n");
+ fprintf(stderr, " -j THREADS\tmaximum number of threads to use\n");
fprintf(stderr, " -h\t\tdisplay this message\n");
exit(0);
}
@@ -483,12 +541,12 @@
std::string platform_dir;
std::set<Arch> selected_architectures;
std::set<int> selected_levels;
- bool dump = false;
std::string preprocessor_output_path;
bool force = false;
+ bool dump = false;
int c;
- while ((c = getopt(argc, argv, "a:r:p:vo:fdhi")) != -1) {
+ while ((c = getopt(argc, argv, "a:r:p:vo:fdj:hi")) != -1) {
default_args = false;
switch (c) {
case 'a': {
@@ -555,6 +613,12 @@
dump = true;
break;
+ case 'j':
+ if (!android::base::ParseInt<int>(optarg, &max_thread_count, 1)) {
+ usage();
+ }
+ break;
+
case 'h':
usage(true);
break;
@@ -627,10 +691,10 @@
symbol_database = parsePlatforms(compilation_types, platform_dir);
}
- bool failed = false;
std::unique_ptr<HeaderDatabase> declaration_database =
- compileHeaders(compilation_types, header_dir, dependency_dir, &failed);
+ compileHeaders(compilation_types, header_dir, dependency_dir);
+ bool failed = false;
if (dump) {
declaration_database->dump(header_dir + "/");
} else {
diff --git a/tools/versioner/tests/compilation_error/run.sh b/tools/versioner/tests/compilation_error/run.sh
index a34fda8..7e8d489 100644
--- a/tools/versioner/tests/compilation_error/run.sh
+++ b/tools/versioner/tests/compilation_error/run.sh
@@ -1 +1 @@
-versioner headers -p platforms -r arm -a 9 -i
\ No newline at end of file
+versioner headers -p platforms -r arm -a 9 -i -j1
