antlr-3.4/tool/src/test/java/org/antlr/test/TestLexer.java

/*
 * [The "BSD license"]
 *  Copyright (c) 2010 Terence Parr
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *  1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *  2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *  3. The name of the author may not be used to endorse or promote products
 *      derived from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package org.antlr.test;

import org.antlr.Tool;
import org.antlr.codegen.CodeGenerator;
import org.stringtemplate.v4.ST;
import org.antlr.tool.Grammar;
import org.junit.Test;

public class TestLexer extends BaseTest {
	protected boolean debug = false;

	/** Public default constructor used by TestRig */
	public TestLexer() {
	}

	@Test public void testSetText() throws Exception {
		// this must return A not I to the parser; calling a nonfragment rule
		// from a nonfragment rule does not set the overall token.
		String grammar =
			"grammar P;\n"+
			"a : A {System.out.println(input);} ;\n"+
			"A : '\\\\' 't' {setText(\"\t\");} ;\n" +
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "\\t", debug);
		assertEquals("\t\n", found);
	}

	@Test public void testRefToRuleDoesNotSetTokenNorEmitAnother() throws Exception {
		// this must return A not I to the parser; calling a nonfragment rule
		// from a nonfragment rule does not set the overall token.
		String grammar =
			"grammar P;\n"+
			"a : A EOF {System.out.println(input);} ;\n"+
			"A : '-' I ;\n" +
			"I : '0'..'9'+ ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "-34", debug);
		assertEquals("-34\n", found);
	}

	@Test public void testRefToRuleDoesNotSetChannel() throws Exception {
		// this must set channel of A to HIDDEN.  $channel is local to rule
		// like $type.
		String grammar =
			"grammar P;\n"+
			"a : A EOF {System.out.println($A.text+\", channel=\"+$A.channel);} ;\n"+
			"A : '-' WS I ;\n" +
			"I : '0'..'9'+ ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "- 34", debug);
		assertEquals("- 34, channel=0\n", found);
	}

	@Test public void testWeCanSetType() throws Exception {
		String grammar =
			"grammar P;\n"+
			"tokens {X;}\n" +
			"a : X EOF {System.out.println(input);} ;\n"+
			"A : '-' I {$type = X;} ;\n" +
			"I : '0'..'9'+ ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "-34", debug);
		assertEquals("-34\n", found);
	}

	@Test public void testRefToFragment() throws Exception {
		// this must return A not I to the parser; calling a nonfragment rule
		// from a nonfragment rule does not set the overall token.
		String grammar =
			"grammar P;\n"+
			"a : A {System.out.println(input);} ;\n"+
			"A : '-' I ;\n" +
			"fragment I : '0'..'9'+ ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "-34", debug);
		assertEquals("-34\n", found);
	}

	@Test public void testMultipleRefToFragment() throws Exception {
		// this must return A not I to the parser; calling a nonfragment rule
		// from a nonfragment rule does not set the overall token.
		String grammar =
			"grammar P;\n"+
			"a : A EOF {System.out.println(input);} ;\n"+
			"A : I '.' I ;\n" +
			"fragment I : '0'..'9'+ ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "3.14159", debug);
		assertEquals("3.14159\n", found);
	}

	@Test public void testLabelInSubrule() throws Exception {
		// can we see v outside?
		String grammar =
			"grammar P;\n"+
			"a : A EOF ;\n"+
			"A : 'hi' WS (v=I)? {$channel=0; System.out.println($v.text);} ;\n" +
			"fragment I : '0'..'9'+ ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "hi 342", debug);
		assertEquals("342\n", found);
	}

	@Test public void testRefToTokenInLexer() throws Exception {
		String grammar =
			"grammar P;\n"+
			"a : A EOF ;\n"+
			"A : I {System.out.println($I.text);} ;\n" +
			"fragment I : '0'..'9'+ ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "342", debug);
		assertEquals("342\n", found);
	}

	@Test public void testListLabelInLexer() throws Exception {
		String grammar =
			"grammar P;\n"+
			"a : A ;\n"+
			"A : i+=I+ {for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n" +
			"fragment I : '0'..'9'+ ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "33 297", debug);
		assertEquals(" 33 297\n", found);
	}

	@Test public void testDupListRefInLexer() throws Exception {
		String grammar =
			"grammar P;\n"+
			"a : A ;\n"+
			"A : i+=I WS i+=I {$channel=0; for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n" +
			"fragment I : '0'..'9'+ ;\n"+
			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
		String found = execParser("P.g", grammar, "PParser", "PLexer",
				    "a", "33 297", debug);
		assertEquals(" 33 297\n", found);
	}

	@Test public void testCharLabelInLexer() {
		String grammar =
			"grammar T;\n" +
			"a : B ;\n" +
			"B : x='a' {System.out.println((char)$x);} ;\n" ;
		String found = execParser("T.g", grammar, "TParser", "TLexer",
								  "a", "a", debug);
		assertEquals("a\n", found);
	}

	@Test public void testRepeatedLabelInLexer() {
		String grammar =
			"lexer grammar T;\n" +
			"B : x='a' x='b' ;\n" ;
		boolean found =
			rawGenerateAndBuildRecognizer(
				"T.g", grammar, null, "T", false);
		boolean expecting = true; // should be ok
		assertEquals(expecting, found);
	}

	@Test public void testRepeatedRuleLabelInLexer() {
		String grammar =
			"lexer grammar T;\n" +
			"B : x=A x=A ;\n" +
			"fragment A : 'a' ;\n" ;
		boolean found =
			rawGenerateAndBuildRecognizer(
				"T.g", grammar, null, "T", false);
		boolean expecting = true; // should be ok
		assertEquals(expecting, found);
	}

	@Test public void testIsolatedEOTEdge() {
		String grammar =
			"lexer grammar T;\n" +
			"QUOTED_CONTENT \n" +
			"        : 'q' (~'q')* (('x' 'q') )* 'q' ; \n";
		boolean found =
			rawGenerateAndBuildRecognizer(
				"T.g", grammar, null, "T", false);
		boolean expecting = true; // should be ok
		assertEquals(expecting, found);
	}	

	@Test public void testEscapedLiterals() {
		/* Grammar:
			A : '\"' ;  should match a single double-quote: "
			B : '\\\"' ; should match input \"
		*/
		String grammar =
			"lexer grammar T;\n" +
			"A : '\\\"' ;\n" +
			"B : '\\\\\\\"' ;\n" ; // '\\\"'
		boolean found =
			rawGenerateAndBuildRecognizer(
				"T.g", grammar, null, "T", false);
		boolean expecting = true; // should be ok
		assertEquals(expecting, found);
	}

    @Test public void testNewlineLiterals() throws Exception {
        Grammar g = new Grammar(
            "lexer grammar T;\n" +
            "A : '\\n\\n' ;\n"  // ANTLR sees '\n\n'
        );
        String expecting = "match(\"\\n\\n\")";

        Tool antlr = newTool();
        antlr.setOutputDirectory(null); // write to /dev/null
        CodeGenerator generator = new CodeGenerator(antlr, g, "Java");
        g.setCodeGenerator(generator);
        generator.genRecognizer(); // codegen phase sets some vars we need
        ST codeST = generator.getRecognizerST();
        String code = codeST.render();
        int m = code.indexOf("match(\"");
        String found = code.substring(m,m+expecting.length());

        assertEquals(expecting, found);
    }
}