| /* |
| * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. Oracle designates this |
| * particular file as subject to the "Classpath" exception as provided |
| * by Oracle in the LICENSE file that accompanied this code. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| */ |
| |
| package com.sun.tools.javac.parser; |
| |
| import java.util.Locale; |
| |
| import com.sun.tools.javac.api.Formattable; |
| import com.sun.tools.javac.api.Messages; |
| import com.sun.tools.javac.parser.Tokens.Token.Tag; |
| import com.sun.tools.javac.util.List; |
| import com.sun.tools.javac.util.Name; |
| import com.sun.tools.javac.util.Context; |
| import com.sun.tools.javac.util.Filter; |
| import com.sun.tools.javac.util.ListBuffer; |
| import com.sun.tools.javac.util.Names; |
| |
| /** A class that defines codes/utilities for Java source tokens |
| * returned from lexical analysis. |
| * |
| * <p><b>This is NOT part of any supported API. |
| * If you write code that depends on this, you do so at your own risk. |
| * This code and its internal interfaces are subject to change or |
| * deletion without notice.</b> |
| */ |
| public class Tokens { |
| |
| private final Names names; |
| |
| /** |
| * Keyword array. Maps name indices to Token. |
| */ |
| private final TokenKind[] key; |
| |
| /** The number of the last entered keyword. |
| */ |
| private int maxKey = 0; |
| |
| /** The names of all tokens. |
| */ |
| private Name[] tokenName = new Name[TokenKind.values().length]; |
| |
| public static final Context.Key<Tokens> tokensKey = |
| new Context.Key<Tokens>(); |
| |
| public static Tokens instance(Context context) { |
| Tokens instance = context.get(tokensKey); |
| if (instance == null) |
| instance = new Tokens(context); |
| return instance; |
| } |
| |
| protected Tokens(Context context) { |
| context.put(tokensKey, this); |
| names = Names.instance(context); |
| for (TokenKind t : TokenKind.values()) { |
| if (t.name != null) |
| enterKeyword(t.name, t); |
| else |
| tokenName[t.ordinal()] = null; |
| } |
| |
| key = new TokenKind[maxKey+1]; |
| for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER; |
| for (TokenKind t : TokenKind.values()) { |
| if (t.name != null) |
| key[tokenName[t.ordinal()].getIndex()] = t; |
| } |
| } |
| |
| private void enterKeyword(String s, TokenKind token) { |
| Name n = names.fromString(s); |
| tokenName[token.ordinal()] = n; |
| if (n.getIndex() > maxKey) maxKey = n.getIndex(); |
| } |
| |
| /** |
| * Create a new token given a name; if the name corresponds to a token name, |
| * a new token of the corresponding kind is returned; otherwise, an |
| * identifier token is returned. |
| */ |
| TokenKind lookupKind(Name name) { |
| return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()]; |
| } |
| |
| TokenKind lookupKind(String name) { |
| return lookupKind(names.fromString(name)); |
| } |
| |
| /** |
| * This enum defines all tokens used by the javac scanner. A token is |
| * optionally associated with a name. |
| */ |
| public enum TokenKind implements Formattable, Filter<TokenKind> { |
| EOF(), |
| ERROR(), |
| IDENTIFIER(Tag.NAMED), |
| ABSTRACT("abstract"), |
| ASSERT("assert", Tag.NAMED), |
| BOOLEAN("boolean", Tag.NAMED), |
| BREAK("break"), |
| BYTE("byte", Tag.NAMED), |
| CASE("case"), |
| CATCH("catch"), |
| CHAR("char", Tag.NAMED), |
| CLASS("class"), |
| CONST("const"), |
| CONTINUE("continue"), |
| DEFAULT("default"), |
| DO("do"), |
| DOUBLE("double", Tag.NAMED), |
| ELSE("else"), |
| ENUM("enum", Tag.NAMED), |
| EXTENDS("extends"), |
| FINAL("final"), |
| FINALLY("finally"), |
| FLOAT("float", Tag.NAMED), |
| FOR("for"), |
| GOTO("goto"), |
| IF("if"), |
| IMPLEMENTS("implements"), |
| IMPORT("import"), |
| INSTANCEOF("instanceof"), |
| INT("int", Tag.NAMED), |
| INTERFACE("interface"), |
| LONG("long", Tag.NAMED), |
| NATIVE("native"), |
| NEW("new"), |
| PACKAGE("package"), |
| PRIVATE("private"), |
| PROTECTED("protected"), |
| PUBLIC("public"), |
| RETURN("return"), |
| SHORT("short", Tag.NAMED), |
| STATIC("static"), |
| STRICTFP("strictfp"), |
| SUPER("super", Tag.NAMED), |
| SWITCH("switch"), |
| SYNCHRONIZED("synchronized"), |
| THIS("this", Tag.NAMED), |
| THROW("throw"), |
| THROWS("throws"), |
| TRANSIENT("transient"), |
| TRY("try"), |
| VOID("void", Tag.NAMED), |
| VOLATILE("volatile"), |
| WHILE("while"), |
| INTLITERAL(Tag.NUMERIC), |
| LONGLITERAL(Tag.NUMERIC), |
| FLOATLITERAL(Tag.NUMERIC), |
| DOUBLELITERAL(Tag.NUMERIC), |
| CHARLITERAL(Tag.NUMERIC), |
| STRINGLITERAL(Tag.STRING), |
| TRUE("true", Tag.NAMED), |
| FALSE("false", Tag.NAMED), |
| NULL("null", Tag.NAMED), |
| UNDERSCORE("_", Tag.NAMED), |
| ARROW("->"), |
| COLCOL("::"), |
| LPAREN("("), |
| RPAREN(")"), |
| LBRACE("{"), |
| RBRACE("}"), |
| LBRACKET("["), |
| RBRACKET("]"), |
| SEMI(";"), |
| COMMA(","), |
| DOT("."), |
| ELLIPSIS("..."), |
| EQ("="), |
| GT(">"), |
| LT("<"), |
| BANG("!"), |
| TILDE("~"), |
| QUES("?"), |
| COLON(":"), |
| EQEQ("=="), |
| LTEQ("<="), |
| GTEQ(">="), |
| BANGEQ("!="), |
| AMPAMP("&&"), |
| BARBAR("||"), |
| PLUSPLUS("++"), |
| SUBSUB("--"), |
| PLUS("+"), |
| SUB("-"), |
| STAR("*"), |
| SLASH("/"), |
| AMP("&"), |
| BAR("|"), |
| CARET("^"), |
| PERCENT("%"), |
| LTLT("<<"), |
| GTGT(">>"), |
| GTGTGT(">>>"), |
| PLUSEQ("+="), |
| SUBEQ("-="), |
| STAREQ("*="), |
| SLASHEQ("/="), |
| AMPEQ("&="), |
| BAREQ("|="), |
| CARETEQ("^="), |
| PERCENTEQ("%="), |
| LTLTEQ("<<="), |
| GTGTEQ(">>="), |
| GTGTGTEQ(">>>="), |
| MONKEYS_AT("@"), |
| CUSTOM; |
| |
| public final String name; |
| public final Tag tag; |
| |
| TokenKind() { |
| this(null, Tag.DEFAULT); |
| } |
| |
| TokenKind(String name) { |
| this(name, Tag.DEFAULT); |
| } |
| |
| TokenKind(Tag tag) { |
| this(null, tag); |
| } |
| |
| TokenKind(String name, Tag tag) { |
| this.name = name; |
| this.tag = tag; |
| } |
| |
| public String toString() { |
| switch (this) { |
| case IDENTIFIER: |
| return "token.identifier"; |
| case CHARLITERAL: |
| return "token.character"; |
| case STRINGLITERAL: |
| return "token.string"; |
| case INTLITERAL: |
| return "token.integer"; |
| case LONGLITERAL: |
| return "token.long-integer"; |
| case FLOATLITERAL: |
| return "token.float"; |
| case DOUBLELITERAL: |
| return "token.double"; |
| case ERROR: |
| return "token.bad-symbol"; |
| case EOF: |
| return "token.end-of-input"; |
| case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN: |
| case LBRACKET: case RBRACKET: case LBRACE: case RBRACE: |
| return "'" + name + "'"; |
| default: |
| return name; |
| } |
| } |
| |
| public String getKind() { |
| return "Token"; |
| } |
| |
| public String toString(Locale locale, Messages messages) { |
| return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString()); |
| } |
| |
| @Override |
| public boolean accepts(TokenKind that) { |
| return this == that; |
| } |
| } |
| |
| public interface Comment { |
| |
| enum CommentStyle { |
| LINE, |
| BLOCK, |
| JAVADOC, |
| } |
| |
| String getText(); |
| int getSourcePos(int index); |
| CommentStyle getStyle(); |
| boolean isDeprecated(); |
| } |
| |
| /** |
| * This is the class representing a javac token. Each token has several fields |
| * that are set by the javac lexer (i.e. start/end position, string value, etc). |
| */ |
| public static class Token { |
| |
| /** tags constants **/ |
| enum Tag { |
| DEFAULT, |
| NAMED, |
| STRING, |
| NUMERIC; |
| } |
| |
| /** The token kind */ |
| public final TokenKind kind; |
| |
| /** The start position of this token */ |
| public final int pos; |
| |
| /** The end position of this token */ |
| public final int endPos; |
| |
| /** Comment reader associated with this token */ |
| public final List<Comment> comments; |
| |
| Token(TokenKind kind, int pos, int endPos, List<Comment> comments) { |
| this.kind = kind; |
| this.pos = pos; |
| this.endPos = endPos; |
| this.comments = comments; |
| checkKind(); |
| } |
| |
| Token[] split(Tokens tokens) { |
| if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) { |
| throw new AssertionError("Cant split" + kind); |
| } |
| |
| TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1)); |
| TokenKind t2 = tokens.lookupKind(kind.name.substring(1)); |
| |
| if (t1 == null || t2 == null) { |
| throw new AssertionError("Cant split - bad subtokens"); |
| } |
| return new Token[] { |
| new Token(t1, pos, pos + t1.name.length(), comments), |
| new Token(t2, pos + t1.name.length(), endPos, null) |
| }; |
| } |
| |
| protected void checkKind() { |
| if (kind.tag != Tag.DEFAULT) { |
| throw new AssertionError("Bad token kind - expected " + Tag.STRING); |
| } |
| } |
| |
| public Name name() { |
| throw new UnsupportedOperationException(); |
| } |
| |
| public String stringVal() { |
| throw new UnsupportedOperationException(); |
| } |
| |
| public int radix() { |
| throw new UnsupportedOperationException(); |
| } |
| |
| /** |
| * Preserve classic semantics - if multiple javadocs are found on the token |
| * the last one is returned |
| */ |
| public Comment comment(Comment.CommentStyle style) { |
| List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC); |
| return comments.isEmpty() ? |
| null : |
| comments.head; |
| } |
| |
| /** |
| * Preserve classic semantics - deprecated should be set if at least one |
| * javadoc comment attached to this token contains the '@deprecated' string |
| */ |
| public boolean deprecatedFlag() { |
| for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) { |
| if (c.isDeprecated()) { |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| private List<Comment> getComments(Comment.CommentStyle style) { |
| if (comments == null) { |
| return List.nil(); |
| } else { |
| ListBuffer<Comment> buf = new ListBuffer<>(); |
| for (Comment c : comments) { |
| if (c.getStyle() == style) { |
| buf.add(c); |
| } |
| } |
| return buf.toList(); |
| } |
| } |
| } |
| |
| final static class NamedToken extends Token { |
| /** The name of this token */ |
| public final Name name; |
| |
| public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) { |
| super(kind, pos, endPos, comments); |
| this.name = name; |
| } |
| |
| protected void checkKind() { |
| if (kind.tag != Tag.NAMED) { |
| throw new AssertionError("Bad token kind - expected " + Tag.NAMED); |
| } |
| } |
| |
| @Override |
| public Name name() { |
| return name; |
| } |
| } |
| |
| static class StringToken extends Token { |
| /** The string value of this token */ |
| public final String stringVal; |
| |
| public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) { |
| super(kind, pos, endPos, comments); |
| this.stringVal = stringVal; |
| } |
| |
| protected void checkKind() { |
| if (kind.tag != Tag.STRING) { |
| throw new AssertionError("Bad token kind - expected " + Tag.STRING); |
| } |
| } |
| |
| @Override |
| public String stringVal() { |
| return stringVal; |
| } |
| } |
| |
| final static class NumericToken extends StringToken { |
| /** The 'radix' value of this token */ |
| public final int radix; |
| |
| public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) { |
| super(kind, pos, endPos, stringVal, comments); |
| this.radix = radix; |
| } |
| |
| protected void checkKind() { |
| if (kind.tag != Tag.NUMERIC) { |
| throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC); |
| } |
| } |
| |
| @Override |
| public int radix() { |
| return radix; |
| } |
| } |
| |
| public static final Token DUMMY = |
| new Token(TokenKind.ERROR, 0, 0, null); |
| } |