| package java_cup; |
| |
| import java.util.Hashtable; |
| |
| import java_cup.runtime.str_token; |
| import java_cup.runtime.token; |
| |
| /** This class implements a small scanner (aka lexical analyzer or lexer) for |
| * the JavaCup specification. This scanner reads characters from standard |
| * input (System.in) and returns integers corresponding to the terminal |
| * number of the next token. Once end of input is reached the EOF token is |
| * returned on every subsequent call.<p> |
| * Tokens currently returned include: <pre> |
| * Symbol Constant Returned Symbol Constant Returned |
| * ------ ----------------- ------ ----------------- |
| * "package" PACKAGE "import" IMPORT |
| * "code" CODE "action" ACTION |
| * "parser" PARSER "terminal" TERMINAL |
| * "non" NON "init" INIT |
| * "scan" SCAN "with" WITH |
| * "start" START ; SEMI |
| * , COMMA * STAR |
| * . DOT : COLON |
| * ::= COLON_COLON_EQUALS | BAR |
| * identifier ID {:...:} CODE_STRING |
| * "debug" DEBUG |
| * </pre> |
| * All symbol constants are defined in sym.java which is generated by |
| * JavaCup from parser.cup.<p> |
| * |
| * In addition to the scanner proper (called first via init() then with |
| * next_token() to get each token) this class provides simple error and |
| * warning routines and keeps a count of errors and warnings that is |
| * publicly accessible.<p> |
| * |
| * This class is "static" (i.e., it has only static members and methods). |
| * |
| * @version last updated: 11/25/95 |
| * @author Scott Hudson |
| */ |
| public class lexer { |
| |
| /*-----------------------------------------------------------*/ |
| /*--- Constructor(s) ----------------------------------------*/ |
| /*-----------------------------------------------------------*/ |
| |
| /** The only constructor is private, so no instances can be created. */ |
| private lexer() { } |
| |
| /*-----------------------------------------------------------*/ |
| /*--- Static (Class) Variables ------------------------------*/ |
| /*-----------------------------------------------------------*/ |
| |
| /** First character of lookahead. */ |
| protected static int next_char; |
| |
| /** Second character of lookahead. */ |
| protected static int next_char2; |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** EOF constant. */ |
| protected static final int EOF_CHAR = -1; |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Table of keywords. Keywords are initially treated as identifiers. |
| * Just before they are returned we look them up in this table to see if |
| * they match one of the keywords. The string of the name is the key here, |
| * which indexes Integer objects holding the symbol number. |
| */ |
| protected static Hashtable keywords = new Hashtable(23); |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Table of single character symbols. For ease of implementation, we |
| * store all unambiguous single character tokens in this table of Integer |
| * objects keyed by Integer objects with the numerical value of the |
| * appropriate char (currently Character objects have a bug which precludes |
| * their use in tables). |
| */ |
| protected static Hashtable char_symbols = new Hashtable(11); |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Current line number for use in error messages. */ |
| protected static int current_line = 1; |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Character position in current line. */ |
| protected static int current_position = 1; |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Count of total errors detected so far. */ |
| public static int error_count = 0; |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Count of warnings issued so far */ |
| public static int warning_count = 0; |
| |
| /*-----------------------------------------------------------*/ |
| /*--- Static Methods ----------------------------------------*/ |
| /*-----------------------------------------------------------*/ |
| |
| /** Initialize the scanner. This sets up the keywords and char_symbols |
| * tables and reads the first two characters of lookahead. |
| */ |
| public static void init() throws java.io.IOException |
| { |
| /* set up the keyword table */ |
| keywords.put("package", new Integer(sym.PACKAGE)); |
| keywords.put("import", new Integer(sym.IMPORT)); |
| keywords.put("code", new Integer(sym.CODE)); |
| keywords.put("action", new Integer(sym.ACTION)); |
| keywords.put("parser", new Integer(sym.PARSER)); |
| keywords.put("terminal", new Integer(sym.TERMINAL)); |
| keywords.put("non", new Integer(sym.NON)); |
| keywords.put("init", new Integer(sym.INIT)); |
| keywords.put("scan", new Integer(sym.SCAN)); |
| keywords.put("with", new Integer(sym.WITH)); |
| keywords.put("start", new Integer(sym.START)); |
| keywords.put("debug", new Integer(sym.DEBUG)); |
| |
| /* set up the table of single character symbols */ |
| char_symbols.put(new Integer(';'), new Integer(sym.SEMI)); |
| char_symbols.put(new Integer(','), new Integer(sym.COMMA)); |
| char_symbols.put(new Integer('*'), new Integer(sym.STAR)); |
| char_symbols.put(new Integer('.'), new Integer(sym.DOT)); |
| char_symbols.put(new Integer('|'), new Integer(sym.BAR)); |
| |
| /* read two characters of lookahead */ |
| next_char = System.in.read(); |
| if (next_char == EOF_CHAR) |
| next_char2 = EOF_CHAR; |
| else |
| next_char2 = System.in.read(); |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Advance the scanner one character in the input stream. This moves |
| * next_char2 to next_char and then reads a new next_char2. |
| */ |
| protected static void advance() throws java.io.IOException |
| { |
| int old_char; |
| |
| old_char = next_char; |
| next_char = next_char2; |
| if (next_char == EOF_CHAR) |
| next_char2 = EOF_CHAR; |
| else |
| next_char2 = System.in.read(); |
| |
| /* count this */ |
| current_position++; |
| if (old_char == '\n') |
| { |
| current_line++; |
| current_position = 1; |
| } |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Emit an error message. The message will be marked with both the |
| * current line number and the position in the line. Error messages |
| * are printed on standard error (System.err). |
| * @param message the message to print. |
| */ |
| public static void emit_error(String message) |
| { |
| System.err.println("Error at " + current_line + "(" + current_position + |
| "): " + message); |
| error_count++; |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Emit a warning message. The message will be marked with both the |
| * current line number and the position in the line. Messages are |
| * printed on standard error (System.err). |
| * @param message the message to print. |
| */ |
| public static void emit_warn(String message) |
| { |
| System.err.println("Warning at " + current_line + "(" + current_position + |
| "): " + message); |
| warning_count++; |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Determine if a character is ok to start an id. |
| * @param ch the character in question. |
| */ |
| protected static boolean id_start_char(int ch) |
| { |
| return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || |
| (ch == '_'); |
| |
| // later need to deal with non-8-bit chars here |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Determine if a character is ok for the middle of an id. |
| * @param ch the character in question. |
| */ |
| protected static boolean id_char(int ch) |
| { |
| return id_start_char(ch) || (ch >= '0' && ch <= '9'); |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Try to look up a single character symbol, returns -1 for not found. |
| * @param ch the character in question. |
| */ |
| protected static int find_single_char(int ch) |
| { |
| Integer result; |
| |
| result = (Integer)char_symbols.get(new Integer((char)ch)); |
| if (result == null) |
| return -1; |
| else |
| return result.intValue(); |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Handle swallowing up a comment. Both old style C and new style C++ |
| * comments are handled. |
| */ |
| protected static void swallow_comment() throws java.io.IOException |
| { |
| /* next_char == '/' at this point */ |
| |
| /* is it a traditional comment */ |
| if (next_char2 == '*') |
| { |
| /* swallow the opener */ |
| advance(); advance(); |
| |
| /* swallow the comment until end of comment or EOF */ |
| for (;;) |
| { |
| /* if its EOF we have an error */ |
| if (next_char == EOF_CHAR) |
| { |
| emit_error("Specification file ends inside a comment"); |
| return; |
| } |
| |
| /* if we can see the closer we are done */ |
| if (next_char == '*' && next_char2 == '/') |
| { |
| advance(); |
| advance(); |
| return; |
| } |
| |
| /* otherwise swallow char and move on */ |
| advance(); |
| } |
| } |
| |
| /* is its a new style comment */ |
| if (next_char2 == '/') |
| { |
| /* swallow the opener */ |
| advance(); advance(); |
| |
| /* swallow to '\n', '\f', or EOF */ |
| while (next_char != '\n' && next_char != '\f' && next_char!=EOF_CHAR) |
| advance(); |
| |
| return; |
| |
| } |
| |
| /* shouldn't get here, but... if we get here we have an error */ |
| emit_error("Malformed comment in specification -- ignored"); |
| advance(); |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Swallow up a code string. Code strings begin with "{:" and include |
| all characters up to the first occurrence of ":}" (there is no way to |
| include ":}" inside a code string). The routine returns an str_token |
| object suitable for return by the scanner. |
| */ |
| protected static token do_code_string() throws java.io.IOException |
| { |
| StringBuffer result = new StringBuffer(); |
| |
| /* at this point we have lookahead of "{:" -- swallow that */ |
| advance(); advance(); |
| |
| /* save chars until we see ":}" */ |
| while (!(next_char == ':' && next_char2 == '}')) |
| { |
| /* if we have run off the end issue a message and break out of loop */ |
| if (next_char == EOF_CHAR) |
| { |
| emit_error("Specification file ends inside a code string"); |
| break; |
| } |
| |
| /* otherwise record the char and move on */ |
| result.append(new Character((char)next_char)); |
| advance(); |
| } |
| |
| /* advance past the closer and build a return token */ |
| advance(); advance(); |
| return new str_token(sym.CODE_STRING, result.toString()); |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Process an identifier. Identifiers begin with a letter, underscore, |
| * or dollar sign, which is followed by zero or more letters, numbers, |
| * underscores or dollar signs. This routine returns an str_token suitable |
| * for return by the scanner. |
| */ |
| protected static token do_id() throws java.io.IOException |
| { |
| StringBuffer result = new StringBuffer(); |
| String result_str; |
| Integer keyword_num; |
| char buffer[] = new char[1]; |
| |
| /* next_char holds first character of id */ |
| buffer[0] = (char)next_char; |
| result.append(buffer,0,1); |
| advance(); |
| |
| /* collect up characters while they fit in id */ |
| while(id_char(next_char)) |
| { |
| buffer[0] = (char)next_char; |
| result.append(buffer,0,1); |
| advance(); |
| } |
| |
| /* extract a string and try to look it up as a keyword */ |
| result_str = result.toString(); |
| keyword_num = (Integer)keywords.get(result_str); |
| |
| /* if we found something, return that keyword */ |
| if (keyword_num != null) |
| return new token(keyword_num.intValue()); |
| |
| /* otherwise build and return an id token with an attached string */ |
| return new str_token(sym.ID, result_str); |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Return one token. This is the main external interface to the scanner. |
| * It consumes sufficient characters to determine the next input token |
| * and returns it. To help with debugging, this routine actually calls |
| * real_next_token() which does the work. If you need to debug the |
| * parser, this can be changed to call debug_next_token() which prints |
| * a debugging message before returning the token. |
| */ |
| public static token next_token() throws java.io.IOException |
| { |
| return real_next_token(); |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** Debugging version of next_token(). This routine calls the real scanning |
| * routine, prints a message on System.out indicating what the token is, |
| * then returns it. |
| */ |
| public static token debug_next_token() throws java.io.IOException |
| { |
| token result = real_next_token(); |
| System.out.println("# next_token() => " + result.sym); |
| return result; |
| } |
| |
| /*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/ |
| |
| /** The actual routine to return one token. This is normally called from |
| * next_token(), but for debugging purposes can be called indirectly from |
| * debug_next_token(). |
| */ |
| protected static token real_next_token() throws java.io.IOException |
| { |
| int sym_num; |
| |
| for (;;) |
| { |
| /* look for white space */ |
| if (next_char == ' ' || next_char == '\t' || next_char == '\n' || |
| next_char == '\f' || next_char == '\r') |
| { |
| /* advance past it and try the next character */ |
| advance(); |
| continue; |
| } |
| |
| /* look for a single character symbol */ |
| sym_num = find_single_char(next_char); |
| if (sym_num != -1) |
| { |
| /* found one -- advance past it and return a token for it */ |
| advance(); |
| return new token(sym_num); |
| } |
| |
| /* look for : or ::= */ |
| if (next_char == ':') |
| { |
| /* if we don't have a second ':' return COLON */ |
| if (next_char2 != ':') |
| { |
| advance(); |
| return new token(sym.COLON); |
| } |
| |
| /* move forward and look for the '=' */ |
| advance(); |
| if (next_char2 == '=') |
| { |
| advance(); advance(); |
| return new token(sym.COLON_COLON_EQUALS); |
| } |
| else |
| { |
| /* return just the colon (already consumed) */ |
| return new token(sym.COLON); |
| } |
| } |
| |
| /* look for a comment */ |
| if (next_char == '/' && (next_char2 == '*' || next_char2 == '/')) |
| { |
| /* swallow then continue the scan */ |
| swallow_comment(); |
| continue; |
| } |
| |
| /* look for start of code string */ |
| if (next_char == '{' && next_char2 == ':') |
| return do_code_string(); |
| |
| /* look for an id or keyword */ |
| if (id_start_char(next_char)) return do_id(); |
| |
| /* look for EOF */ |
| if (next_char == EOF_CHAR) return new token(sym.EOF); |
| |
| /* if we get here, we have an unrecognized character */ |
| emit_warn("Unrecognized character '" + |
| new Character((char)next_char) + "'(" + next_char + |
| ") -- ignored"); |
| |
| /* advance past it */ |
| advance(); |
| } |
| } |
| |
| /*-----------------------------------------------------------*/ |
| }; |
| |
