| /* |
| * Copyright 2014 The Kythe Authors. All rights reserved. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.google.devtools.kythe.extractors.java; |
| |
| import static com.google.common.base.StandardSystemProperty.JAVA_HOME; |
| import static com.google.common.collect.ImmutableList.toImmutableList; |
| import static java.nio.file.LinkOption.NOFOLLOW_LINKS; |
| |
| import com.google.common.base.Joiner; |
| import com.google.common.base.Optional; |
| import com.google.common.base.Preconditions; |
| import com.google.common.base.Strings; |
| import com.google.common.collect.HashMultimap; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.collect.Iterables; |
| import com.google.common.collect.Multimap; |
| import com.google.common.io.ByteStreams; |
| import com.google.devtools.kythe.extractors.shared.CompilationDescription; |
| import com.google.devtools.kythe.extractors.shared.ExtractionException; |
| import com.google.devtools.kythe.extractors.shared.ExtractorUtils; |
| import com.google.devtools.kythe.extractors.shared.FileVNames; |
| import com.google.devtools.kythe.platform.java.JavacOptionsUtils.ModifiableOptions; |
| import com.google.devtools.kythe.proto.Analysis.CompilationUnit; |
| import com.google.devtools.kythe.proto.Analysis.CompilationUnit.FileInput; |
| import com.google.devtools.kythe.proto.Analysis.FileData; |
| import com.google.devtools.kythe.proto.Buildinfo.BuildDetails; |
| import com.google.devtools.kythe.proto.Java.JavaDetails; |
| import com.google.devtools.kythe.proto.Storage.VName; |
| import com.google.devtools.kythe.util.DeleteRecursively; |
| import com.google.protobuf.Any; |
| import com.sun.source.tree.CompilationUnitTree; |
| import com.sun.source.tree.ExpressionTree; |
| import com.sun.source.tree.ImportTree; |
| import com.sun.source.util.JavacTask; |
| import com.sun.source.util.TaskEvent; |
| import com.sun.source.util.TaskListener; |
| import com.sun.tools.javac.api.JavacTaskImpl; |
| import com.sun.tools.javac.code.Symbol.ClassSymbol; |
| import com.sun.tools.javac.code.Symtab; |
| import com.sun.tools.javac.file.CacheFSInfo; |
| import com.sun.tools.javac.file.FSInfo; |
| import com.sun.tools.javac.file.JavacFileManager; |
| import com.sun.tools.javac.main.Option; |
| import com.sun.tools.javac.util.Context; |
| import java.io.File; |
| import java.io.IOError; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.net.JarURLConnection; |
| import java.net.MalformedURLException; |
| import java.net.URI; |
| import java.net.URL; |
| import java.net.URLClassLoader; |
| import java.net.URLConnection; |
| import java.nio.charset.StandardCharsets; |
| import java.nio.file.Files; |
| import java.nio.file.Path; |
| import java.nio.file.Paths; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.EnumSet; |
| import java.util.HashMap; |
| import java.util.LinkedHashMap; |
| import java.util.LinkedHashSet; |
| import java.util.List; |
| import java.util.Locale; |
| import java.util.Map; |
| import java.util.ServiceLoader; |
| import java.util.Set; |
| import java.util.logging.Level; |
| import java.util.logging.Logger; |
| import javax.annotation.processing.Processor; |
| import javax.tools.Diagnostic; |
| import javax.tools.DiagnosticCollector; |
| import javax.tools.JavaCompiler; |
| import javax.tools.JavaCompiler.CompilationTask; |
| import javax.tools.JavaFileManager.Location; |
| import javax.tools.JavaFileObject; |
| import javax.tools.JavaFileObject.Kind; |
| import javax.tools.StandardJavaFileManager; |
| import javax.tools.StandardLocation; |
| import javax.tools.ToolProvider; |
| |
| /** |
| * Extracts all required information (set of source files, class paths, and compiler options) from a |
| * java compilation command and stores the information to replay the compilation. |
| * |
| * <p>The extractor runs the Javac compiler to get a exact description of all files required for the |
| * compilation as a whole. It then creates CompilationUnit entries for each source file. We do not |
| * do this on a per file basis as the Java Compiler takes too long to do this. |
| */ |
| public class JavaCompilationUnitExtractor { |
| public static final String JAVA_DETAILS_URL = "kythe.io/proto/kythe.proto.JavaDetails"; |
| public static final String BUILD_DETAILS_URL = "kythe.io/proto/kythe.proto.BuildDetails"; |
| |
| private static final Logger logger = |
| Logger.getLogger(JavaCompilationUnitExtractor.class.getName()); |
| |
| private static final String JDK_MODULE_PREFIX = "/modules/java."; |
| private static final String MODULE_INFO_NAME = "module-info"; |
| private static final String SOURCE_JAR_ROOT = "!SOURCE_JAR!"; |
| |
| private static String classJarRoot(Location location) { |
| return String.format("!%s_JAR!", location); |
| } |
| |
| // TODO(shahms): Use the proper methods when we can rely on JDK 9. |
| private static final ClassLoader moduleClassLoader; |
| |
| static { |
| ClassLoader loader = null; |
| try { |
| Object thisModule = |
| Class.class.getMethod("getModule").invoke(JavaCompilationUnitExtractor.class); |
| thisModule |
| .getClass() |
| .getMethod("addUses", Class.class) |
| .invoke(thisModule, JavaCompiler.class); |
| loader = (ClassLoader) thisModule.getClass().getMethod("getClassLoader").invoke(thisModule); |
| } catch (ReflectiveOperationException e) { |
| logger.info("Running on non-modular JDK, fallback compiler unavailable."); |
| } |
| moduleClassLoader = loader; |
| } |
| |
| private static final String JAR_SCHEME = "jar"; |
| private final String jdkJar; |
| private final String rootDirectory; |
| private final FileVNames fileVNames; |
| |
| /** |
| * Creates an instance of the JavaExtractor to store java compilation information in an .kindex |
| * file. |
| */ |
| public JavaCompilationUnitExtractor(String corpus) throws ExtractionException { |
| this(corpus, ExtractorUtils.getCurrentWorkingDirectory()); |
| } |
| |
| /** |
| * Creates an instance of the JavaExtractor to store java compilation information in an .kindex |
| * file. |
| */ |
| public JavaCompilationUnitExtractor(String corpus, String rootDirectory) |
| throws ExtractionException { |
| this(FileVNames.staticCorpus(corpus), rootDirectory); |
| } |
| |
| /** |
| * Creates an instance of the JavaExtractor to store java compilation information in an .kindex |
| * file. |
| */ |
| public JavaCompilationUnitExtractor(FileVNames fileVNames) throws ExtractionException { |
| this(fileVNames, ExtractorUtils.getCurrentWorkingDirectory()); |
| } |
| |
| public JavaCompilationUnitExtractor(FileVNames fileVNames, String rootDirectory) |
| throws ExtractionException { |
| this.fileVNames = fileVNames; |
| |
| Path javaHome = Paths.get(JAVA_HOME.value()).getParent(); |
| try { |
| // Remove trailing dots. Interesting trivia: in some build systems, |
| // the java.home variable is terminated with "/bin/..". |
| // However, this is not the case for the class files |
| // that we are trying to filter. |
| this.jdkJar = javaHome.toRealPath(NOFOLLOW_LINKS).toString(); |
| } catch (IOException e) { |
| throw new ExtractionException("JDK path not found: " + javaHome, e, false); |
| } |
| |
| try { |
| this.rootDirectory = Paths.get(rootDirectory).toRealPath().toString(); |
| } catch (IOException ioe) { |
| throw new ExtractionException("Root directory does not exist", ioe, false); |
| } |
| } |
| |
| private CompilationUnit buildCompilationUnit( |
| String target, |
| Iterable<String> options, |
| Iterable<FileInput> requiredInputs, |
| boolean hasErrors, |
| Set<String> newSourcePath, |
| Set<String> newClassPath, |
| Iterable<String> newBootClassPath, |
| List<String> sourceFiles, |
| String outputPath) { |
| CompilationUnit.Builder unit = CompilationUnit.newBuilder(); |
| unit.setVName(VName.newBuilder().setSignature(target).setLanguage("java")); |
| unit.addAllArgument(options); |
| unit.setHasCompileErrors(hasErrors); |
| unit.addAllRequiredInput(requiredInputs); |
| for (String sourceFile : sourceFiles) { |
| unit.addSourceFile(sourceFile); |
| } |
| unit.setOutputKey(outputPath); |
| unit.addDetails( |
| Any.newBuilder() |
| .setTypeUrl(JAVA_DETAILS_URL) |
| .setValue( |
| JavaDetails.newBuilder() |
| .addAllClasspath(newClassPath) |
| .addAllBootclasspath(newBootClassPath) |
| .addAllSourcepath(newSourcePath) |
| .build() |
| .toByteString())); |
| unit.addDetails( |
| Any.newBuilder() |
| .setTypeUrl(BUILD_DETAILS_URL) |
| .setValue(BuildDetails.newBuilder().setBuildTarget(target).build().toByteString())); |
| return unit.build(); |
| } |
| |
| /** |
| * Indexes a compilation unit to the bigtable. The extraction process will try to build a minimum |
| * set of what is needed to replay the compilation. To do this it runs the java compiler, and |
| * tracks all .class & .java files that are needed. It then builds up a new classpath & sourcepath |
| * that only contains the minimum set of paths required to replay the compilation. |
| * |
| * <p>New classpath: because we extract classes from the jars into a temp path that needs to be |
| * set. Also we only use the classpaths that are actually used, not the ones that are provided. |
| * New sourcepath: as we're doing a partial compilation, we need to set up the source path to |
| * correctly load any files that are not the main compilation but are needed to perform |
| * compilation. This is not required when doing a full compilation as all sources are |
| * automatically loaded. |
| * |
| * <p>Next we store all required files in the bigtable and writes the CompilationUnit to the |
| * bigtable. |
| * |
| * @throws ExtractionException if anything blocks the indexing to be completed. |
| */ |
| public CompilationDescription extract( |
| String target, |
| Iterable<String> sources, |
| Iterable<String> classpath, |
| Iterable<String> bootclasspath, |
| Iterable<String> sourcepath, |
| Iterable<String> processorpath, |
| Iterable<String> processors, |
| Optional<Path> genSrcDir, |
| Iterable<String> options, |
| String outputPath) |
| throws ExtractionException { |
| Preconditions.checkNotNull(target); |
| Preconditions.checkNotNull(sources); |
| Preconditions.checkNotNull(classpath); |
| Preconditions.checkNotNull(bootclasspath); |
| Preconditions.checkNotNull(sourcepath); |
| Preconditions.checkNotNull(processorpath); |
| Preconditions.checkNotNull(processors); |
| Preconditions.checkNotNull(genSrcDir); |
| Preconditions.checkNotNull(options); |
| Preconditions.checkNotNull(outputPath); |
| |
| AnalysisResults results; |
| if (sources.iterator().hasNext()) { |
| results = |
| runJavaAnalysisToExtractCompilationDetails( |
| sources, |
| classpath, |
| bootclasspath, |
| sourcepath, |
| processorpath, |
| processors, |
| genSrcDir, |
| options); |
| } else { |
| results = new AnalysisResults(); |
| } |
| |
| List<FileData> fileContents = ExtractorUtils.convertBytesToFileDatas(results.fileContents); |
| List<FileInput> compilationFileInputs = |
| ExtractorUtils.toFileInputs(fileVNames, results.relativePaths::get, fileContents).stream() |
| .map( |
| input -> { |
| String sourceBasename = results.sourceFileNames.get(input.getInfo().getPath()); |
| VName vname = input.getVName(); |
| if (sourceBasename != null |
| && vname.getPath().endsWith(".java") |
| && !vname.getPath().endsWith(sourceBasename)) { |
| Path fixedPath = Paths.get(vname.getPath()).resolveSibling(sourceBasename); |
| vname = vname.toBuilder().setPath(fixedPath.toString()).build(); |
| return input.toBuilder().setVName(vname).build(); |
| } |
| return input; |
| }) |
| .collect(toImmutableList()); |
| |
| CompilationUnit compilationUnit = |
| buildCompilationUnit( |
| target, |
| removeDestDirOptions(options), |
| compilationFileInputs, |
| results.hasErrors, |
| results.newSourcePath, |
| results.newClassPath, |
| results.newBootClassPath, |
| results.explicitSources, |
| ExtractorUtils.tryMakeRelative(rootDirectory, outputPath)); |
| return new CompilationDescription(compilationUnit, fileContents); |
| } |
| |
| /** |
| * Returns a new list with the same options except header/source destination directory options. |
| */ |
| private static ImmutableList<String> removeDestDirOptions(Iterable<String> options) { |
| // TODO(#3671): Option.D needs to remain in for module support, fix either here or in indexing. |
| return ModifiableOptions.of(options) |
| .removeOptions(EnumSet.of(Option.D, Option.S, Option.H)) |
| .build(); |
| } |
| |
| /** |
| * If the code has wildcard imports (e.g. import foo.bar.*) but doesn't actually use any of the |
| * imports, errors will happen. We don't get callbacks for file open of these files (since they |
| * aren't used) but when java runs it will report errors if it can't find any files to match the |
| * wildcard. So we add one matching file here. |
| */ |
| private void findOnDemandImportedFiles( |
| Iterable<? extends CompilationUnitTree> compilationUnits, |
| UsageAsInputReportingFileManager fileManager) |
| throws ExtractionException { |
| // Maps package names to source files that wildcard import them. |
| Multimap<String, String> pkgs = HashMultimap.create(); |
| |
| // Javac synthesizes an "import java.lang.*" for every compilation unit. |
| pkgs.put("java.lang", "*.java"); |
| |
| for (CompilationUnitTree unit : compilationUnits) { |
| for (ImportTree importTree : unit.getImports()) { |
| if (importTree.isStatic()) { |
| continue; |
| } |
| String qualifiedIdentifier = importTree.getQualifiedIdentifier().toString(); |
| if (!qualifiedIdentifier.endsWith(".*")) { |
| continue; |
| } |
| pkgs.put( |
| qualifiedIdentifier.substring(0, qualifiedIdentifier.length() - 2), |
| unit.getSourceFile().getName()); |
| } |
| } |
| |
| for (Map.Entry<String, Collection<String>> pkg : pkgs.asMap().entrySet()) { |
| try { |
| JavaFileObject firstClass = |
| Iterables.getFirst( |
| fileManager.list( |
| StandardLocation.CLASS_PATH, pkg.getKey(), EnumSet.of(Kind.CLASS), false), |
| null); |
| if (firstClass == null) { |
| firstClass = |
| Iterables.getFirst( |
| fileManager.list( |
| StandardLocation.PLATFORM_CLASS_PATH, |
| pkg.getKey(), |
| EnumSet.of(Kind.CLASS), |
| false), |
| null); |
| } |
| if (firstClass != null) { |
| firstClass.getCharContent(true); |
| } |
| JavaFileObject firstSource = |
| Iterables.getFirst( |
| fileManager.list( |
| StandardLocation.SOURCE_PATH, pkg.getKey(), EnumSet.of(Kind.SOURCE), false), |
| null); |
| if (firstSource != null) { |
| firstSource.getCharContent(true); |
| } |
| } catch (IOException e) { |
| throw new ExtractionException( |
| String.format( |
| "Unable to extract files used for on demand imports in {%s}", |
| Joiner.on(", ").join(pkg.getValue())), |
| e, |
| false); |
| } |
| } |
| } |
| |
| /** |
| * Determines -sourcepath arguments to add to the compilation unit based on the package name. This |
| * is needed as the sharded analysis will need to resolve dependent source files. Also locates |
| * sources that do not follow the package == path convention and list them as explicit sources. |
| */ |
| private void getAdditionalSourcePaths( |
| Iterable<? extends CompilationUnitTree> compilationUnits, AnalysisResults results) { |
| |
| for (CompilationUnitTree compilationUnit : compilationUnits) { |
| ExpressionTree packageExpression = compilationUnit.getPackageName(); |
| if (packageExpression != null) { |
| String packageName = packageExpression.toString(); |
| if (!Strings.isNullOrEmpty(packageName)) { |
| // If the source file specifies a package, we try to find that |
| // package name in the path to the source file and assume |
| // the correct sourcepath to add is the directory containing that |
| // package. |
| String packageSubDir = packageName.replace('.', '/'); |
| String path = compilationUnit.getSourceFile().toUri().getPath(); |
| // This needs to be lastIndexOf as there are source jars that |
| // contain the same package name in the files contained in them |
| // as the path the source jars live in. As we extract the source |
| // jars, we end up with a double named path. |
| int index = path.lastIndexOf(packageSubDir); |
| if (index >= 0) { |
| String root = ExtractorUtils.tryMakeRelative(rootDirectory, path.substring(0, index)); |
| results.newSourcePath.add(root); |
| } |
| } |
| } |
| } |
| } |
| |
| private void findRequiredFiles( |
| UsageAsInputReportingFileManager fileManager, |
| Map<URI, String> sourceFiles, |
| Optional<Path> genSrcDir, |
| AnalysisResults results) |
| throws ExtractionException { |
| for (InputUsageRecord input : fileManager.getUsages()) { |
| processRequiredInput( |
| input.fileObject(), input.location(), fileManager, sourceFiles, genSrcDir, results); |
| } |
| } |
| |
| private void processRequiredInput( |
| JavaFileObject requiredInput, |
| Location location, |
| UsageAsInputReportingFileManager fileManager, |
| Map<URI, String> sourceFiles, |
| Optional<Path> genSrcDir, |
| AnalysisResults results) |
| throws ExtractionException { |
| URI uri = requiredInput.toUri(); |
| String entryPath; |
| String jarPath = null; |
| boolean isJarPath = false; |
| |
| { |
| URLConnection conn; |
| URL url; |
| try { |
| url = uri.toURL(); |
| conn = url.openConnection(); |
| } catch (IOException e) { |
| throw new IOError(e); |
| } |
| if (conn instanceof JarURLConnection) { |
| isJarPath = true; |
| JarURLConnection jarConn = ((JarURLConnection) conn); |
| jarPath = jarConn.getJarFileURL().getFile(); |
| // jar entries don't have a leading '/', and we expect |
| // paths like "!CLASS_PATH_JAR!/com/foo/Bar.class" |
| entryPath = "/" + jarConn.getEntryName(); |
| } else { |
| entryPath = url.getFile(); |
| } |
| } |
| |
| if (uri.getScheme().equals(JAR_SCHEME)) { |
| isJarPath = true; |
| uri = URI.create(uri.getRawSchemeSpecificPart()); |
| } |
| |
| switch (requiredInput.getKind()) { |
| case CLASS: |
| case SOURCE: |
| break; |
| case OTHER: |
| if (uri.getPath().endsWith(".meta")) { |
| break; |
| } |
| throw new IllegalStateException(String.format("Unsupported OTHER file kind: '%s'", uri)); |
| default: |
| throw new IllegalStateException( |
| String.format( |
| "Unsupported java file kind: '%s' for '%s'", requiredInput.getKind().name(), uri)); |
| } |
| String path = uri.getRawSchemeSpecificPart(); |
| |
| // If the file was part of the JDK we do not store it as the JDK is tied |
| // to the analyzer we'll run on this information later on. |
| if ((isJarPath && jarPath.startsWith(jdkJar)) || path.startsWith(JDK_MODULE_PREFIX)) { |
| return; |
| } |
| |
| // Make the path relative to the indexer (e.g. a subdir of corpus/). |
| // If not possible, we store the fullpath. |
| String relativePath = ExtractorUtils.tryMakeRelative(rootDirectory, path); |
| |
| String strippedPath = relativePath; |
| if (isJarPath) { |
| // If the file came from a jar file, we strip that out as we don't care where it came from, it |
| // was not as a source file in source control. We turn it in to a fake path. This is done so |
| // we do not need to download the entire jar file for each file's compilation (e.g. instead of |
| // downloading 200MB we only download 60K for analyzing the Kythe java indexer). |
| switch (requiredInput.getKind()) { |
| case CLASS: |
| String root = classJarRoot(location); |
| strippedPath = root + entryPath; |
| (location == StandardLocation.PLATFORM_CLASS_PATH |
| ? results.newBootClassPath |
| : results.newClassPath) |
| .add(root); |
| break; |
| case SOURCE: |
| results.newSourcePath.add(SOURCE_JAR_ROOT); |
| strippedPath = SOURCE_JAR_ROOT + entryPath; |
| break; |
| default: |
| // TODO(#1845): we shouldn't need to throw an exception here because the above switch |
| // statement means that we never hit this default, but the static analysis tools can't |
| // figure that out. Try to refactor this code to remove this issue. |
| throw new IllegalStateException( |
| String.format( |
| "Unsupported java file kind: '%s' for '%s'", |
| requiredInput.getKind().name(), uri)); |
| } |
| } else { |
| // If the class file was on disk, we need to infer the correct classpath to add. |
| String binaryName = getBinaryNameForClass(fileManager, requiredInput); |
| if (binaryName != null) { |
| // Java package names map to folders on disk, so if we want to find the right directory |
| // we replace each . with a /. |
| String csubdir = binaryName.replace('.', '/'); |
| int cindex = strippedPath.indexOf(csubdir); |
| if (cindex <= 0) { |
| throw new ExtractionException( |
| String.format( |
| "unable to infer classpath for %s from %s, %s", |
| strippedPath, csubdir, binaryName), |
| false); |
| } else { |
| (location == StandardLocation.PLATFORM_CLASS_PATH |
| ? results.newBootClassPath |
| : results.newClassPath) |
| .add(strippedPath.substring(0, cindex)); |
| } |
| } |
| } |
| |
| // Identify generated sources by checking if the source file is under the genSrcDir. |
| if (genSrcDir.isPresent() |
| && !isJarPath |
| && requiredInput.getKind() == Kind.SOURCE |
| && Paths.get(relativePath).startsWith(genSrcDir.get())) { |
| results.explicitSources.add(strippedPath); |
| results.newSourcePath.add(genSrcDir.get().toString()); |
| } |
| |
| if (!results.fileContents.containsKey(strippedPath)) { |
| try { |
| // Retrieve the contents of the file. |
| InputStream stream = requiredInput.openInputStream(); |
| if (stream.markSupported()) { |
| // The stream has already been read by the compiler, we need to reset it |
| // so we can read it as well. |
| stream.reset(); |
| } |
| byte[] data = ByteStreams.toByteArray(stream); |
| if (data.length == 0) { |
| logger.warning(String.format("Empty java source file: %s", strippedPath)); |
| } |
| results.fileContents.put(strippedPath, data); |
| results.relativePaths.put(strippedPath, relativePath); |
| if (sourceFiles.containsKey(requiredInput.toUri())) { |
| results.sourceFileNames.put(strippedPath, sourceFiles.get(requiredInput.toUri())); |
| } |
| } catch (IOException e) { |
| throw new ExtractionException( |
| String.format("Unable to read file content of %s", strippedPath), false); |
| } |
| } |
| } |
| |
| /** {@link Location}s that may contain class files. */ |
| private static final ImmutableSet<Location> CLASS_LOCATIONS = |
| ImmutableSet.<Location>of( |
| StandardLocation.CLASS_OUTPUT, |
| StandardLocation.CLASS_PATH, |
| StandardLocation.PLATFORM_CLASS_PATH); |
| |
| /** |
| * Returns the location and binary name of a class file, or {@code null} if the file object is not |
| * a class. |
| */ |
| private static String getBinaryNameForClass( |
| UsageAsInputReportingFileManager fileManager, JavaFileObject fileObject) |
| throws ExtractionException { |
| if (fileObject.getKind() != Kind.CLASS) { |
| return null; |
| } |
| String binaryName; |
| for (Location location : CLASS_LOCATIONS) { |
| if ((binaryName = fileManager.inferBinaryName(location, fileObject)) != null) { |
| return binaryName; |
| } |
| } |
| if (fileObject.isNameCompatible(MODULE_INFO_NAME, Kind.CLASS)) { |
| // Ignore automatic module-info.class files |
| return null; |
| } |
| throw new ExtractionException( |
| String.format("unable to infer classpath for %s", fileObject.getName()), false); |
| } |
| |
| private static class AnalysisResults { |
| // Map from strippedPath to an input's relative path to the corpus root. |
| final Map<String, String> relativePaths = new LinkedHashMap<>(); |
| // Map from strippedPath to an input's contents. |
| final Map<String, byte[]> fileContents = new LinkedHashMap<>(); |
| // Map from strippedPath to an input's true source basename. This is usually only needed for |
| // non-public top-level classes where their filename does not match the path derived from their |
| // fully-qualified name. |
| final Map<String, String> sourceFileNames = new HashMap<>(); |
| |
| // We build a new sourcepath & classpath that contain the minimum set of paths |
| // as well as the modified set of paths that are needed to analyze the single compilation unit. |
| // This is done to speed up analysis. |
| final Set<String> newSourcePath = new LinkedHashSet<>(); |
| final Set<String> newClassPath = new LinkedHashSet<>(); |
| final Set<String> newBootClassPath = new LinkedHashSet<>(); |
| final List<String> explicitSources = new ArrayList<>(); |
| boolean hasErrors = false; |
| } |
| |
| // Install NonResolvingCacheFSInfo into Context to avoid resolving symlinks. |
| private void setupFSInfo(StandardJavaFileManager fileManager) { |
| Context context = new Context(); |
| NonResolvingCacheFSInfo.preRegister(context); |
| ((JavacFileManager) fileManager).setContext(context); |
| } |
| |
| // FSInfo class which does not resolve symlinks when canonicalizing paths. |
| private static class NonResolvingCacheFSInfo extends CacheFSInfo { |
| public static void preRegister(Context context) { |
| context.put( |
| FSInfo.class, |
| new Context.Factory<FSInfo>() { |
| @Override |
| public FSInfo make(Context c) { |
| FSInfo instance = new NonResolvingCacheFSInfo(); |
| c.put(FSInfo.class, instance); |
| return instance; |
| } |
| }); |
| } |
| |
| @Override |
| public Path getCanonicalFile(Path file) { |
| return file.toAbsolutePath(); |
| } |
| } |
| |
| private AnalysisResults runJavaAnalysisToExtractCompilationDetails( |
| Iterable<String> sources, |
| Iterable<String> classpath, |
| Iterable<String> bootclasspath, |
| Iterable<String> sourcepath, |
| Iterable<String> processorpath, |
| Iterable<String> processors, |
| Optional<Path> genSrcDir, |
| Iterable<String> options) |
| throws ExtractionException { |
| |
| AnalysisResults results = new AnalysisResults(); |
| |
| // We will initialize and run the Javac compiler to detect which dependencies |
| // the current compilation has. |
| JavaCompiler compiler = findJavaCompiler(); |
| if (compiler == null) { |
| // TODO(schroederc): provide link to further context |
| throw new IllegalStateException( |
| "Could not get system Java compiler; are you missing the JDK?"); |
| } |
| |
| DiagnosticCollector<JavaFileObject> diagnosticsCollector = new DiagnosticCollector<>(); |
| |
| StandardJavaFileManager standardFileManager = |
| compiler.getStandardFileManager(diagnosticsCollector, null, null); |
| setupFSInfo(standardFileManager); |
| |
| // We insert a filemanager that wraps the standard filemanager and records the compiler's |
| // usage of .java & .class files. |
| final UsageAsInputReportingFileManager fileManager = |
| new UsageAsInputReportingFileManager(standardFileManager); |
| |
| Iterable<? extends JavaFileObject> sourceFiles = fileManager.getJavaFileForSources(sources); |
| |
| // Generate class files in a temporary directory |
| Path tempDir; |
| try { |
| tempDir = Files.createTempDirectory("javac_extractor"); |
| } catch (IOException ioe) { |
| throw new ExtractionException( |
| "Unable to create temporary .class output directory", ioe, true); |
| } |
| List<String> completeOptions = |
| completeCompilerOptions( |
| standardFileManager, |
| options, |
| classpath, |
| bootclasspath, |
| sourcepath, |
| processorpath, |
| tempDir); |
| |
| final List<CompilationUnitTree> compilationUnits = new ArrayList<>(); |
| Symtab syms; |
| try { |
| // Launch the java compiler with our modified settings and the filemanager wrapper |
| CompilationTask task = |
| compiler.getTask( |
| null, fileManager, diagnosticsCollector, completeOptions, null, sourceFiles); |
| |
| ClassLoader loader = processingClassloader(classpath, processorpath); |
| |
| List<Processor> procs = new ArrayList<>(); |
| |
| // Add any processors passed as flags. |
| for (String processor : processors) { |
| try { |
| procs.add( |
| loader |
| .loadClass(processor) |
| .asSubclass(Processor.class) |
| .getConstructor() |
| .newInstance()); |
| } catch (Throwable e) { |
| throw new ExtractionException("Bad processor entry: " + processor, e, false); |
| } |
| } |
| |
| if (procs.isEmpty()) { |
| // If no --processors were passed, add any processors registered in the META-INF/services |
| // configuration. |
| for (Processor proc : ServiceLoader.load(Processor.class, loader)) { |
| procs.add(proc); |
| } |
| } |
| |
| procs.add(new ProcessAnnotation(fileManager)); |
| |
| JavacTask javacTask = (JavacTask) task; |
| javacTask.setProcessors(procs); |
| syms = Symtab.instance(((JavacTaskImpl) javacTask).getContext()); |
| |
| javacTask.addTaskListener( |
| new TaskListener() { |
| @Override |
| public void finished(TaskEvent e) { |
| if (e.getKind() == TaskEvent.Kind.PARSE) { |
| compilationUnits.add(e.getCompilationUnit()); |
| } |
| } |
| |
| @Override |
| public void started(TaskEvent e) {} |
| }); |
| |
| try { |
| // In order for the compiler to load all required .java & .class files we need to have it go |
| // through parsing, analysis & generate phases. Unfortunately the latter is needed to get a |
| // complete list, this was found as we were breaking on analyzing certain files. |
| // JavacTask#call() subsumes parse() and generate(), but calling those methods directly may |
| // silently ignore fatal errors. |
| results.hasErrors = !javacTask.call(); |
| } catch (com.sun.tools.javac.util.Abort e) { |
| // Type resolution issues, the diagnostics will give hints on what's going wrong. |
| for (Diagnostic<? extends JavaFileObject> diagnostic : |
| diagnosticsCollector.getDiagnostics()) { |
| if (diagnostic.getKind() == Diagnostic.Kind.ERROR) { |
| logger.severe("Fatal error in compiler: " + diagnostic.getMessage(Locale.ENGLISH)); |
| } |
| } |
| throw new ExtractionException("Fatal error while running javac compiler.", e, false); |
| } |
| |
| // If we encountered any compilation errors, we report them even though we |
| // still store the compilation information for this set of sources. |
| for (Diagnostic<? extends JavaFileObject> diag : diagnosticsCollector.getDiagnostics()) { |
| if (diag.getKind() == Diagnostic.Kind.ERROR) { |
| results.hasErrors = true; |
| if (diag.getSource() != null) { |
| logger.severe( |
| String.format( |
| "compiler error: %s(%d): %s", |
| diag.getSource().getName(), |
| diag.getLineNumber(), |
| diag.getMessage(Locale.ENGLISH))); |
| } else { |
| logger.severe("compiler error: " + diag.getMessage(Locale.ENGLISH)); |
| } |
| } |
| } |
| |
| // Ensure generated source directory is relative to root. |
| genSrcDir = |
| genSrcDir.transform( |
| p -> Paths.get(ExtractorUtils.tryMakeRelative(rootDirectory, p.toString()))); |
| |
| for (String source : sources) { |
| results.explicitSources.add(ExtractorUtils.tryMakeRelative(rootDirectory, source)); |
| } |
| |
| getAdditionalSourcePaths(compilationUnits, results); |
| |
| // Find files potentially used for resolving .* imports. |
| findOnDemandImportedFiles(compilationUnits, fileManager); |
| // We accumulate all file contents from the java compiler so we can store it in the bigtable. |
| findRequiredFiles(fileManager, mapClassesToSources(syms), genSrcDir, results); |
| } finally { |
| try { |
| DeleteRecursively.delete(tempDir); |
| } catch (IOException ioe) { |
| logger.log(Level.SEVERE, "Failed to delete temporary directory " + tempDir, ioe); |
| } |
| } |
| |
| return results; |
| } |
| |
| /** Sets the given location using command-line flags and the FileManager API. */ |
| private static void setLocation( |
| ModifiableOptions options, |
| StandardJavaFileManager fileManager, |
| Iterable<String> searchpath, |
| String flag, |
| StandardLocation location) |
| throws ExtractionException { |
| String joined = Joiner.on(":").join(searchpath); |
| if (!joined.isEmpty()) { |
| options.add(flag); |
| options.add(joined); |
| try { |
| List<File> files = new ArrayList<>(); |
| for (String elt : searchpath) { |
| files.add(new File(elt)); |
| } |
| fileManager.setLocation(location, files); |
| } catch (IOException e) { |
| throw new ExtractionException(String.format("Couldn't set %s", location), e, false); |
| } |
| } |
| } |
| |
| /** Create the ClassLoader to use for annotation processors. */ |
| private static ClassLoader processingClassloader( |
| Iterable<String> classpath, Iterable<String> processorpath) throws ExtractionException { |
| // If javac is run with -processor set and -processorpath *unset*, it will fall back to |
| // searching the regular classpath for annotation processors. |
| if (Iterables.isEmpty(processorpath)) { |
| processorpath = classpath; |
| } |
| |
| List<URL> urls = new ArrayList<>(); |
| for (String path : processorpath) { |
| try { |
| urls.add(new File(path).toURI().toURL()); |
| } catch (MalformedURLException e) { |
| throw new ExtractionException("Bad processorpath entry", e, false); |
| } |
| } |
| ClassLoader parent = new MaskedClassLoader(); |
| return new URLClassLoader(Iterables.toArray(urls, URL.class), parent); |
| } |
| |
| /** Isolated classloader for annotation processors, to avoid skew with the ambient classpath. */ |
| private static class MaskedClassLoader extends ClassLoader { |
| private MaskedClassLoader() { |
| // delegate only to the bootclasspath |
| super(getPlatformClassLoader()); |
| } |
| |
| // TODO(#2451): remove reflection and call ClassLoader.getPlatformClassLoader() directly |
| // once JDK 8 compatibility is no longer required. |
| public static ClassLoader getPlatformClassLoader() { |
| try { |
| // In JDK 9, all platform classes are visible to the platform class loader: |
| // https://docs.oracle.com/javase/9/docs/api/java/lang/ClassLoader.html#getPlatformClassLoader-- |
| return (ClassLoader) ClassLoader.class.getMethod("getPlatformClassLoader").invoke(null); |
| } catch (ReflectiveOperationException e) { |
| // In earlier releases, set 'null' as the parent to delegate to the boot class loader. |
| return null; |
| } |
| } |
| |
| @Override |
| protected Class<?> findClass(String name) throws ClassNotFoundException { |
| if (name.startsWith("com.sun.source.") || name.startsWith("com.sun.tools.")) { |
| return JavaCompilationUnitExtractor.class.getClassLoader().loadClass(name); |
| } |
| throw new ClassNotFoundException(name); |
| } |
| } |
| |
| /** |
| * Completes the given raw compiler options with the given classpath, sourcepath, and temporary |
| * destination directory. Only options supported by the Java compiler will be within the returned |
| * {@link List}. |
| */ |
| private static ImmutableList<String> completeCompilerOptions( |
| StandardJavaFileManager standardFileManager, |
| Iterable<String> rawOptions, |
| Iterable<String> classpath, |
| Iterable<String> bootclasspath, |
| Iterable<String> sourcepath, |
| Iterable<String> processorpath, |
| Path tempDestinationDir) |
| throws ExtractionException { |
| |
| ModifiableOptions completeOptions = |
| ModifiableOptions.of(rawOptions) |
| .removeUnsupportedOptions() |
| .ensureEncodingSet(StandardCharsets.UTF_8); |
| // Android always uses this option. |
| // TODO(asmundak): needs more work before contributing upstream. |
| completeOptions.add("-XDstringConcat=inline"); |
| setLocation( |
| completeOptions, standardFileManager, classpath, "-cp", StandardLocation.CLASS_PATH); |
| setLocation( |
| completeOptions, |
| standardFileManager, |
| sourcepath, |
| "-sourcepath", |
| StandardLocation.SOURCE_PATH); |
| setLocation( |
| completeOptions, |
| standardFileManager, |
| processorpath, |
| "-processorpath", |
| StandardLocation.ANNOTATION_PROCESSOR_PATH); |
| setLocation( |
| completeOptions, |
| standardFileManager, |
| bootclasspath, |
| "-bootclasspath", |
| StandardLocation.PLATFORM_CLASS_PATH); |
| |
| return completeOptions |
| .removeOptions(EnumSet.of(Option.D)) |
| .add("-d") |
| .add(tempDestinationDir.toString()) |
| .build(); |
| } |
| |
| private static JavaCompiler findJavaCompiler() { |
| JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); |
| if (compiler == null && moduleClassLoader != null) { |
| // This is all a bit of a hack to be able to extract OpenJDK itself, which |
| // uses a bootstrap compiler and a lot of JDK options to compile itself. |
| // Notably, when using modules the system compiler is inhibited and the actual compiler |
| // resides in jdk.compiler.iterim. Rather than hard-code this, just fall back to the first |
| // JavaCompiler we can find. |
| logger.warning("Unable to find system compiler, using first available."); |
| for (JavaCompiler found : ServiceLoader.load(JavaCompiler.class, moduleClassLoader)) { |
| return found; |
| } |
| } |
| return compiler; |
| } |
| |
| /** Returns a map from a classfile's {@link URI} to its sourcefile path's basename. */ |
| private static Map<URI, String> mapClassesToSources(Symtab syms) { |
| Map<URI, String> sourceBaseNames = new HashMap<>(); |
| for (ClassSymbol sym : syms.getAllClasses()) { |
| if (sym.sourcefile != null && sym.classfile != null) { |
| String path = sym.sourcefile.toUri().getPath(); |
| if (path != null) { |
| String basename = Paths.get(path).getFileName().toString(); |
| if (!basename.endsWith(".java") && !basename.endsWith(".kt")) { |
| logger.warning(String.format("Invalid sourcefile name: '%s'", basename)); |
| } |
| sourceBaseNames.put(sym.classfile.toUri(), basename); |
| } |
| } |
| } |
| return sourceBaseNames; |
| } |
| } |
