Migrate lexing to javac and complete the migration off ecj. MOE_MIGRATED_REVID=139931395
diff --git a/core/pom.xml b/core/pom.xml index 86d26d6..c4bac3f 100644 --- a/core/pom.xml +++ b/core/pom.xml
@@ -40,10 +40,6 @@ <artifactId>guava</artifactId> </dependency> <dependency> - <groupId>org.eclipse.jdt</groupId> - <artifactId>org.eclipse.jdt.core</artifactId> - </dependency> - <dependency> <groupId>com.google.errorprone</groupId> <artifactId>javac</artifactId> </dependency>
diff --git a/core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java b/core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java index d20e0ce..8c30b36 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java +++ b/core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java
@@ -14,9 +14,6 @@ package com.google.googlejavaformat.java; import static com.google.common.collect.Iterables.getLast; -import static org.eclipse.jdt.core.compiler.ITerminalSymbols.TokenNameclass; -import static org.eclipse.jdt.core.compiler.ITerminalSymbols.TokenNameenum; -import static org.eclipse.jdt.core.compiler.ITerminalSymbols.TokenNameinterface; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; @@ -24,33 +21,27 @@ import com.google.common.collect.ImmutableSortedSet; import com.google.googlejavaformat.Newlines; import com.google.googlejavaformat.java.JavaInput.Tok; -import org.eclipse.jdt.core.compiler.InvalidInputException; +import com.sun.tools.javac.parser.Tokens.TokenKind; /** Orders imports in Java source code. */ public class ImportOrderer { /** - * Reorder the inputs in {@code input}, a complete Java program. On success, another complete Java + * Reorder the inputs in {@code text}, a complete Java program. On success, another complete Java * program is returned, which is the same as the original except the imports are in order. * * @throws FormatterException if the input could not be parsed. */ public static String reorderImports(String text) throws FormatterException { - ImmutableList<Tok> toks; - try { - toks = JavaInput.buildToks(text, CLASS_START); - } catch (InvalidInputException e) { - // error handling is done during formatting - return text; - } + ImmutableList<Tok> toks = JavaInput.buildToks(text, CLASS_START); return new ImportOrderer(text, toks).reorderImports(); } /** - * Eclipse token ids that indicate the start of a type definition. We use this to avoid scanning + * {@link TokenKind}s that indicate the start of a type definition. We use this to avoid scanning * the whole file, since we know that imports must precede any type definition. */ - private static final ImmutableSet<Integer> CLASS_START = - ImmutableSet.of(TokenNameclass, TokenNameinterface, TokenNameenum); + private static final ImmutableSet<TokenKind> CLASS_START = + ImmutableSet.of(TokenKind.CLASS, TokenKind.INTERFACE, TokenKind.ENUM); /** * We use this set to find the first import, and again to check that there are no imports after
diff --git a/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java b/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java index 6f9065c..5e340e5 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java +++ b/core/src/main/java/com/google/googlejavaformat/java/JavaInput.java
@@ -16,6 +16,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Iterables.getLast; +import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.base.MoreObjects; import com.google.common.base.Verify; @@ -31,15 +32,25 @@ import com.google.common.collect.TreeRangeSet; import com.google.googlejavaformat.Input; import com.google.googlejavaformat.Newlines; +import com.google.googlejavaformat.java.JavacTokens.RawTok; +import com.sun.tools.javac.file.JavacFileManager; +import com.sun.tools.javac.parser.Tokens.TokenKind; import com.sun.tools.javac.tree.JCTree.JCCompilationUnit; +import com.sun.tools.javac.util.Context; +import com.sun.tools.javac.util.Log; +import com.sun.tools.javac.util.Log.DeferredDiagnosticHandler; +import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; -import org.eclipse.jdt.core.ToolFactory; -import org.eclipse.jdt.core.compiler.IScanner; -import org.eclipse.jdt.core.compiler.ITerminalSymbols; -import org.eclipse.jdt.core.compiler.InvalidInputException; +import javax.tools.Diagnostic; +import javax.tools.DiagnosticCollector; +import javax.tools.DiagnosticListener; +import javax.tools.JavaFileObject; +import javax.tools.JavaFileObject.Kind; +import javax.tools.SimpleJavaFileObject; /** {@code JavaInput} extends {@link Input} to represent a Java input document. */ public final class JavaInput extends Input { @@ -63,7 +74,7 @@ private final int position; private final int columnI; private final boolean isToken; - private final int id; + private final TokenKind kind; /** * The {@code Tok} constructor. @@ -74,7 +85,7 @@ * @param position its {@code 0}-origin position in the input * @param columnI its {@code 0}-origin column number in the input * @param isToken whether the {@code Tok} is a token - * @param id the token id as defined by {@link org.eclipse.jdt.core.compiler.ITerminalSymbols} + * @param kind the token kind */ Tok( int index, @@ -83,14 +94,14 @@ int position, int columnI, boolean isToken, - int id) { + TokenKind kind) { this.index = index; this.originalText = originalText; this.text = text; this.position = position; this.columnI = columnI; this.isToken = isToken; - this.id = id; + this.kind = kind; } @Override @@ -163,12 +174,8 @@ .toString(); } - /** - * The token id used by the eclipse scanner. See {@link - * org.eclipse.jdt.core.compiler.ITerminalSymbols} for possible values. - */ - public int id() { - return id; + public TokenKind kind() { + return kind; } } @@ -322,47 +329,54 @@ /** Lex the input and build the list of toks. */ private ImmutableList<Tok> buildToks(String text) throws FormatterException { - try { - ImmutableList<Tok> toks = buildToks(text, ImmutableSet.<Integer>of()); - kN = getLast(toks).getIndex(); - computeRanges(toks); - return toks; - } catch (InvalidInputException e) { - // jdt's scanner elects not to produce error messages, so we don't either - // - // problems will get caught (again!) and reported (with error messages!) - // during parsing - return ImmutableList.of(); - } + ImmutableList<Tok> toks = buildToks(text, ImmutableSet.<TokenKind>of()); + kN = getLast(toks).getIndex(); + computeRanges(toks); + return toks; } /** * Lex the input and build the list of toks. * * @param text the text to be lexed. - * @param stopIds a set of Eclipse token names which should cause lexing to stop. If one of these - * is found, the returned list will include tokens up to but not including that token. + * @param stopTokens a set of tokens which should cause lexing to stop. If one of these is found, + * the returned list will include tokens up to but not including that token. */ - static ImmutableList<Tok> buildToks(String text, ImmutableSet<Integer> stopIds) - throws InvalidInputException, FormatterException { - stopIds = - ImmutableSet.<Integer>builder().addAll(stopIds).add(ITerminalSymbols.TokenNameEOF).build(); + static ImmutableList<Tok> buildToks(String text, ImmutableSet<TokenKind> stopTokens) + throws FormatterException { + stopTokens = ImmutableSet.<TokenKind>builder().addAll(stopTokens).add(TokenKind.EOF).build(); + Context context = new Context(); + new JavacFileManager(context, true, UTF_8); + DiagnosticCollector<JavaFileObject> diagnosticCollector = new DiagnosticCollector<>(); + context.put(DiagnosticListener.class, diagnosticCollector); + Log log = Log.instance(context); + log.useSource( + new SimpleJavaFileObject(URI.create("Source.java"), Kind.SOURCE) { + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { + return text; + } + }); + DeferredDiagnosticHandler diagnostics = new DeferredDiagnosticHandler(log); + ImmutableList<RawTok> rawToks = JavacTokens.getTokens(text, context, stopTokens); + if (diagnostics.getDiagnostics().stream().anyMatch(d -> d.getKind() == Diagnostic.Kind.ERROR)) { + return ImmutableList.of(new Tok(0, "", "", 0, 0, true, null)); // EOF + } int kN = 0; - IScanner scanner = ToolFactory.createScanner(true, true, true, "1.8"); - scanner.setSource(text.toCharArray()); - int textLength = text.length(); List<Tok> toks = new ArrayList<>(); int charI = 0; int columnI = 0; - while (scanner.getCurrentTokenEndPosition() < textLength - 1) { - int tokenId = scanner.getNextToken(); - if (stopIds.contains(tokenId)) { + for (RawTok t : rawToks) { + if (stopTokens.contains(t.kind())) { break; } - int charI0 = scanner.getCurrentTokenStartPosition(); + int charI0 = t.pos(); // Get string, possibly with Unicode escapes. - String originalTokText = text.substring(charI0, scanner.getCurrentTokenEndPosition() + 1); - String tokText = new String(scanner.getCurrentTokenSource()); // Unicode escapes removed. + String originalTokText = text.substring(charI0, t.endPos()); + String tokText = + t.kind() == TokenKind.STRINGLITERAL + ? t.stringVal() // Unicode escapes removed. + : originalTokText; char tokText0 = tokText.charAt(0); // The token's first character. final boolean isToken; // Is this tok a token? final boolean isNumbered; // Is this tok numbered? (tokens and comments) @@ -427,7 +441,7 @@ charI, columnI, isToken, - tokenId)); + t.kind())); charI += originalTokText.length(); columnI = updateColumn(columnI, originalTokText); @@ -437,18 +451,18 @@ "Unicode escapes not allowed in whitespace or multi-character operators"); } for (String str : strings) { - toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, tokenId)); + toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken, null)); charI += str.length(); columnI = updateColumn(columnI, originalTokText); } } if (extraNewline != null) { - toks.add(new Tok(-1, extraNewline, extraNewline, charI, columnI, false, tokenId)); + toks.add(new Tok(-1, extraNewline, extraNewline, charI, columnI, false, null)); columnI = 0; charI += extraNewline.length(); } } - toks.add(new Tok(kN, "", "", charI, columnI, true, ITerminalSymbols.TokenNameEOF)); // EOF tok. + toks.add(new Tok(kN, "", "", charI, columnI, true, null)); // EOF tok. return ImmutableList.copyOf(toks); }
diff --git a/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java b/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java new file mode 100644 index 0000000..00efadc --- /dev/null +++ b/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java
@@ -0,0 +1,206 @@ +/* + * Copyright 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.googlejavaformat.java; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.sun.tools.javac.parser.JavaTokenizer; +import com.sun.tools.javac.parser.Scanner; +import com.sun.tools.javac.parser.ScannerFactory; +import com.sun.tools.javac.parser.Tokens.Comment; +import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle; +import com.sun.tools.javac.parser.Tokens.Token; +import com.sun.tools.javac.parser.Tokens.TokenKind; +import com.sun.tools.javac.parser.UnicodeReader; +import com.sun.tools.javac.util.Context; +import java.util.Set; + +/** A wrapper around javac's lexer. */ +public class JavacTokens { + + /** An unprocessed input token, including whitespace and comments. */ + static class RawTok { + private final String stringVal; + private final TokenKind kind; + private final int pos; + private final int endPos; + + RawTok(String stringVal, TokenKind kind, int pos, int endPos) { + this.stringVal = stringVal; + this.kind = kind; + this.pos = pos; + this.endPos = endPos; + } + + /** The token kind, or {@code null} for whitespace and comments. */ + public TokenKind kind() { + return kind; + } + + /** The start position. */ + public int pos() { + return pos; + } + + /** The end position. */ + public int endPos() { + return endPos; + } + + /** The escaped string value of a literal, or {@code null} for other tokens. */ + public String stringVal() { + return stringVal; + } + } + + /** Lex the input and return a list of {@link RawTok}s. */ + public static ImmutableList<RawTok> getTokens( + String source, Context context, Set<TokenKind> stopTokens) { + if (source == null) { + return ImmutableList.of(); + } + ScannerFactory fac = ScannerFactory.instance(context); + char[] buffer = source.toCharArray(); + Scanner scanner = + new AccessibleScanner(fac, new CommentSavingTokenizer(fac, buffer, buffer.length)); + ImmutableList.Builder<RawTok> tokens = ImmutableList.builder(); + int last = 0; + do { + scanner.nextToken(); + Token t = scanner.token(); + if (stopTokens.contains(t.kind)) { + break; + } + if (t.comments != null) { + for (Comment c : Lists.reverse(t.comments)) { + if (last < c.getSourcePos(0)) { + tokens.add(new RawTok(null, null, last, c.getSourcePos(0))); + } + tokens.add( + new RawTok(null, null, c.getSourcePos(0), c.getSourcePos(0) + c.getText().length())); + last = c.getSourcePos(0) + c.getText().length(); + } + } + if (last < t.pos) { + tokens.add(new RawTok(null, null, last, t.pos)); + } + tokens.add( + new RawTok( + t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null, + t.kind, + t.pos, + t.endPos)); + last = t.endPos; + } while (scanner.token().kind != TokenKind.EOF); + if (last < source.length()) { + tokens.add(new RawTok(null, null, last, source.length())); + } + return tokens.build(); + } + + /** A {@link JavaTokenizer} that saves comments. */ + static class CommentSavingTokenizer extends JavaTokenizer { + CommentSavingTokenizer(ScannerFactory fac, char[] buffer, int length) { + super(fac, buffer, length); + } + + @Override + protected Comment processComment(int pos, int endPos, CommentStyle style) { + char[] buf = reader.getRawCharacters(pos, endPos); + return new CommentWithTextAndPosition( + pos, endPos, new AccessibleReader(fac, buf, buf.length), style); + } + } + + /** A {@link Comment} that saves its text and start position. */ + static class CommentWithTextAndPosition implements Comment { + + private final int pos; + private final int endPos; + private final AccessibleReader reader; + private final CommentStyle style; + + private String text = null; + + public CommentWithTextAndPosition( + int pos, int endPos, AccessibleReader reader, CommentStyle style) { + this.pos = pos; + this.endPos = endPos; + this.reader = reader; + this.style = style; + } + + /** + * Returns the source position of the character at index {@code index} in the comment text. + * + * <p>The handling of javadoc comments in javac has more logic to skip over leading whitespace + * and '*' characters when indexing into doc comments, but we don't need any of that. + */ + @Override + public int getSourcePos(int index) { + checkArgument( + 0 <= index && index < (endPos - pos), + "Expected %s in the range [0, %s)", + index, + endPos - pos); + return pos + index; + } + + @Override + public CommentStyle getStyle() { + return style; + } + + @Override + public String getText() { + String text = this.text; + if (text == null) { + this.text = text = new String(reader.getRawCharacters()); + } + return text; + } + + /** + * We don't care about {@code @deprecated} javadoc tags (see the DepAnn check). + * + * @return false + */ + @Override + public boolean isDeprecated() { + return false; + } + + @Override + public String toString() { + return String.format("Comment: '%s'", getText()); + } + } + + // Scanner(ScannerFactory, JavaTokenizer) is package-private + static class AccessibleScanner extends Scanner { + protected AccessibleScanner(ScannerFactory fac, JavaTokenizer tokenizer) { + super(fac, tokenizer); + } + } + + // UnicodeReader(ScannerFactory, char[], int) is package-private + static class AccessibleReader extends UnicodeReader { + protected AccessibleReader(ScannerFactory fac, char[] buffer, int length) { + super(fac, buffer, length); + } + } +}
diff --git a/core/src/main/java/com/google/googlejavaformat/java/ModifierOrderer.java b/core/src/main/java/com/google/googlejavaformat/java/ModifierOrderer.java index 8dbf1be..d938797 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/ModifierOrderer.java +++ b/core/src/main/java/com/google/googlejavaformat/java/ModifierOrderer.java
@@ -23,6 +23,7 @@ import com.google.common.collect.TreeRangeMap; import com.google.googlejavaformat.Input.Tok; import com.google.googlejavaformat.Input.Token; +import com.sun.tools.javac.parser.Tokens.TokenKind; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -31,39 +32,42 @@ import java.util.Map; import java.util.Map.Entry; import javax.lang.model.element.Modifier; -import org.eclipse.jdt.core.compiler.ITerminalSymbols; /** Fixes sequences of modifiers to be in JLS order. */ final class ModifierOrderer { /** - * Returns the {@link javax.lang.model.element.Modifier} for the given token id, or {@code null}. + * Returns the {@link javax.lang.model.element.Modifier} for the given token kind, or {@code + * null}. */ - static Modifier getModifier(int tokenId) { - switch (tokenId) { - case ITerminalSymbols.TokenNamepublic: + private static Modifier getModifier(TokenKind kind) { + if (kind == null) { + return null; + } + switch (kind) { + case PUBLIC: return Modifier.PUBLIC; - case ITerminalSymbols.TokenNameprotected: + case PROTECTED: return Modifier.PROTECTED; - case ITerminalSymbols.TokenNameprivate: + case PRIVATE: return Modifier.PRIVATE; - case ITerminalSymbols.TokenNameabstract: + case ABSTRACT: return Modifier.ABSTRACT; - case ITerminalSymbols.TokenNamestatic: + case STATIC: return Modifier.STATIC; - case ITerminalSymbols.TokenNamedefault: + case DEFAULT: return Modifier.DEFAULT; - case ITerminalSymbols.TokenNamefinal: + case FINAL: return Modifier.FINAL; - case ITerminalSymbols.TokenNametransient: + case TRANSIENT: return Modifier.TRANSIENT; - case ITerminalSymbols.TokenNamevolatile: + case VOLATILE: return Modifier.VOLATILE; - case ITerminalSymbols.TokenNamesynchronized: + case SYNCHRONIZED: return Modifier.SYNCHRONIZED; - case ITerminalSymbols.TokenNamenative: + case NATIVE: return Modifier.NATIVE; - case ITerminalSymbols.TokenNamestrictfp: + case STRICTFP: return Modifier.STRICTFP; default: return null; @@ -148,7 +152,7 @@ * is not a modifier. */ private static Modifier asModifier(Token token) { - return getModifier(((JavaInput.Tok) token.getTok()).id()); + return getModifier(((JavaInput.Tok) token.getTok()).kind()); } /** Applies replacements to the given string. */
diff --git a/idea_plugin/test/com/google/googlejavaformat/intellij/GoogleJavaFormatCodeStyleManagerTest.java b/idea_plugin/test/com/google/googlejavaformat/intellij/GoogleJavaFormatCodeStyleManagerTest.java index dc9bddb..4fdca6b 100644 --- a/idea_plugin/test/com/google/googlejavaformat/intellij/GoogleJavaFormatCodeStyleManagerTest.java +++ b/idea_plugin/test/com/google/googlejavaformat/intellij/GoogleJavaFormatCodeStyleManagerTest.java
@@ -24,8 +24,6 @@ import com.intellij.openapi.util.TextRange; import com.intellij.psi.codeStyle.CodeStyleManager; import com.intellij.testFramework.fixtures.LightCodeInsightFixtureTestCase; -import java.net.URL; -import java.net.URLClassLoader; /** * Tests for {@link GoogleJavaFormatCodeStyleManager}. @@ -34,24 +32,6 @@ */ public class GoogleJavaFormatCodeStyleManagerTest extends LightCodeInsightFixtureTestCase { - @Override - protected void setUp() throws Exception { - if (getClass().getClassLoader() instanceof URLClassLoader) { - for (URL url : ((URLClassLoader) getClass().getClassLoader()).getURLs()) { - if (url.getPath().matches(".*/ecj-\\d(\\.\\d)*\\.jar")) { - System.err.println( - "If you see a SecurityException while running this test from within IJ,\n" - + "unfortunately you'll need to somehow remove ecj.jar from your classpath,\n" - + "as it will cause a cert conflict with org.eclipse.jdt.core.jar.\n" - + "e.g. if " - + url.getPath() - + " is in IDEA_HOME/lib, temporarily relocate it"); - } - } - } - super.setUp(); - } - public void testFormatFile() { CodeStyleManager codeStyleManager = CodeStyleManager.getInstance(getProject()); final GoogleJavaFormatCodeStyleManager googleJavaFormat =
diff --git a/pom.xml b/pom.xml index ea7a63e..dd28f2e 100644 --- a/pom.xml +++ b/pom.xml
@@ -106,11 +106,6 @@ <version>${guava.version}</version> </dependency> <dependency> - <groupId>org.eclipse.jdt</groupId> - <artifactId>org.eclipse.jdt.core</artifactId> - <version>3.10.0</version> - </dependency> - <dependency> <groupId>com.google.errorprone</groupId> <artifactId>javac</artifactId> <version>1.9.0-dev-r2973-2</version>