| # -*- coding: utf-8 -*- |
| |
| import os |
| import unittest |
| from StringIO import StringIO |
| import antlr3 |
| |
| |
| class TestStringStream(unittest.TestCase): |
| """Test case for the StringStream class.""" |
| |
| def testSize(self): |
| """StringStream.size()""" |
| |
| stream = antlr3.StringStream('foo') |
| |
| self.failUnlessEqual(stream.size(), 3) |
| |
| |
| def testIndex(self): |
| """StringStream.index()""" |
| |
| stream = antlr3.StringStream('foo') |
| |
| self.failUnlessEqual(stream.index(), 0) |
| |
| |
| def testConsume(self): |
| """StringStream.consume()""" |
| |
| stream = antlr3.StringStream('foo\nbar') |
| |
| stream.consume() # f |
| self.failUnlessEqual(stream.index(), 1) |
| self.failUnlessEqual(stream.charPositionInLine, 1) |
| self.failUnlessEqual(stream.line, 1) |
| |
| stream.consume() # o |
| self.failUnlessEqual(stream.index(), 2) |
| self.failUnlessEqual(stream.charPositionInLine, 2) |
| self.failUnlessEqual(stream.line, 1) |
| |
| stream.consume() # o |
| self.failUnlessEqual(stream.index(), 3) |
| self.failUnlessEqual(stream.charPositionInLine, 3) |
| self.failUnlessEqual(stream.line, 1) |
| |
| stream.consume() # \n |
| self.failUnlessEqual(stream.index(), 4) |
| self.failUnlessEqual(stream.charPositionInLine, 0) |
| self.failUnlessEqual(stream.line, 2) |
| |
| stream.consume() # b |
| self.failUnlessEqual(stream.index(), 5) |
| self.failUnlessEqual(stream.charPositionInLine, 1) |
| self.failUnlessEqual(stream.line, 2) |
| |
| stream.consume() # a |
| self.failUnlessEqual(stream.index(), 6) |
| self.failUnlessEqual(stream.charPositionInLine, 2) |
| self.failUnlessEqual(stream.line, 2) |
| |
| stream.consume() # r |
| self.failUnlessEqual(stream.index(), 7) |
| self.failUnlessEqual(stream.charPositionInLine, 3) |
| self.failUnlessEqual(stream.line, 2) |
| |
| stream.consume() # EOF |
| self.failUnlessEqual(stream.index(), 7) |
| self.failUnlessEqual(stream.charPositionInLine, 3) |
| self.failUnlessEqual(stream.line, 2) |
| |
| stream.consume() # EOF |
| self.failUnlessEqual(stream.index(), 7) |
| self.failUnlessEqual(stream.charPositionInLine, 3) |
| self.failUnlessEqual(stream.line, 2) |
| |
| |
| def testReset(self): |
| """StringStream.reset()""" |
| |
| stream = antlr3.StringStream('foo') |
| |
| stream.consume() |
| stream.consume() |
| |
| stream.reset() |
| self.failUnlessEqual(stream.index(), 0) |
| self.failUnlessEqual(stream.line, 1) |
| self.failUnlessEqual(stream.charPositionInLine, 0) |
| self.failUnlessEqual(stream.LT(1), 'f') |
| |
| |
| def testLA(self): |
| """StringStream.LA()""" |
| |
| stream = antlr3.StringStream('foo') |
| |
| self.failUnlessEqual(stream.LT(1), 'f') |
| self.failUnlessEqual(stream.LT(2), 'o') |
| self.failUnlessEqual(stream.LT(3), 'o') |
| |
| stream.consume() |
| stream.consume() |
| |
| self.failUnlessEqual(stream.LT(1), 'o') |
| self.failUnlessEqual(stream.LT(2), antlr3.EOF) |
| self.failUnlessEqual(stream.LT(3), antlr3.EOF) |
| |
| |
| def testSubstring(self): |
| """StringStream.substring()""" |
| |
| stream = antlr3.StringStream('foobar') |
| |
| self.failUnlessEqual(stream.substring(0, 0), 'f') |
| self.failUnlessEqual(stream.substring(0, 1), 'fo') |
| self.failUnlessEqual(stream.substring(0, 5), 'foobar') |
| self.failUnlessEqual(stream.substring(3, 5), 'bar') |
| |
| |
| def testSeekForward(self): |
| """StringStream.seek(): forward""" |
| |
| stream = antlr3.StringStream('foo\nbar') |
| |
| stream.seek(4) |
| |
| self.failUnlessEqual(stream.index(), 4) |
| self.failUnlessEqual(stream.line, 2) |
| self.failUnlessEqual(stream.charPositionInLine, 0) |
| self.failUnlessEqual(stream.LT(1), 'b') |
| |
| |
| ## # not yet implemented |
| ## def testSeekBackward(self): |
| ## """StringStream.seek(): backward""" |
| |
| ## stream = antlr3.StringStream('foo\nbar') |
| |
| ## stream.seek(4) |
| ## stream.seek(1) |
| |
| ## self.failUnlessEqual(stream.index(), 1) |
| ## self.failUnlessEqual(stream.line, 1) |
| ## self.failUnlessEqual(stream.charPositionInLine, 1) |
| ## self.failUnlessEqual(stream.LA(1), 'o') |
| |
| |
| def testMark(self): |
| """StringStream.mark()""" |
| |
| stream = antlr3.StringStream('foo\nbar') |
| |
| stream.seek(4) |
| |
| marker = stream.mark() |
| self.failUnlessEqual(marker, 1) |
| self.failUnlessEqual(stream.markDepth, 1) |
| |
| stream.consume() |
| marker = stream.mark() |
| self.failUnlessEqual(marker, 2) |
| self.failUnlessEqual(stream.markDepth, 2) |
| |
| |
| def testReleaseLast(self): |
| """StringStream.release(): last marker""" |
| |
| stream = antlr3.StringStream('foo\nbar') |
| |
| stream.seek(4) |
| marker1 = stream.mark() |
| |
| stream.consume() |
| marker2 = stream.mark() |
| |
| stream.release() |
| self.failUnlessEqual(stream.markDepth, 1) |
| |
| # release same marker again, nothing has changed |
| stream.release() |
| self.failUnlessEqual(stream.markDepth, 1) |
| |
| |
| def testReleaseNested(self): |
| """StringStream.release(): nested""" |
| |
| stream = antlr3.StringStream('foo\nbar') |
| |
| stream.seek(4) |
| marker1 = stream.mark() |
| |
| stream.consume() |
| marker2 = stream.mark() |
| |
| stream.consume() |
| marker3 = stream.mark() |
| |
| stream.release(marker2) |
| self.failUnlessEqual(stream.markDepth, 1) |
| |
| |
| def testRewindLast(self): |
| """StringStream.rewind(): last marker""" |
| |
| stream = antlr3.StringStream('foo\nbar') |
| |
| stream.seek(4) |
| |
| marker = stream.mark() |
| stream.consume() |
| stream.consume() |
| |
| stream.rewind() |
| self.failUnlessEqual(stream.markDepth, 0) |
| self.failUnlessEqual(stream.index(), 4) |
| self.failUnlessEqual(stream.line, 2) |
| self.failUnlessEqual(stream.charPositionInLine, 0) |
| self.failUnlessEqual(stream.LT(1), 'b') |
| |
| |
| def testRewindNested(self): |
| """StringStream.rewind(): nested""" |
| |
| stream = antlr3.StringStream('foo\nbar') |
| |
| stream.seek(4) |
| marker1 = stream.mark() |
| |
| stream.consume() |
| marker2 = stream.mark() |
| |
| stream.consume() |
| marker3 = stream.mark() |
| |
| stream.rewind(marker2) |
| self.failUnlessEqual(stream.markDepth, 1) |
| self.failUnlessEqual(stream.index(), 5) |
| self.failUnlessEqual(stream.line, 2) |
| self.failUnlessEqual(stream.charPositionInLine, 1) |
| self.failUnlessEqual(stream.LT(1), 'a') |
| |
| |
| class TestFileStream(unittest.TestCase): |
| """Test case for the FileStream class.""" |
| |
| |
| def testNoEncoding(self): |
| path = os.path.join(os.path.dirname(__file__), 'teststreams.input1') |
| |
| stream = antlr3.FileStream(path) |
| |
| stream.seek(4) |
| marker1 = stream.mark() |
| |
| stream.consume() |
| marker2 = stream.mark() |
| |
| stream.consume() |
| marker3 = stream.mark() |
| |
| stream.rewind(marker2) |
| self.failUnlessEqual(stream.markDepth, 1) |
| self.failUnlessEqual(stream.index(), 5) |
| self.failUnlessEqual(stream.line, 2) |
| self.failUnlessEqual(stream.charPositionInLine, 1) |
| self.failUnlessEqual(stream.LT(1), 'a') |
| self.failUnlessEqual(stream.LA(1), ord('a')) |
| |
| |
| def testEncoded(self): |
| path = os.path.join(os.path.dirname(__file__), 'teststreams.input2') |
| |
| stream = antlr3.FileStream(path, 'utf-8') |
| |
| stream.seek(4) |
| marker1 = stream.mark() |
| |
| stream.consume() |
| marker2 = stream.mark() |
| |
| stream.consume() |
| marker3 = stream.mark() |
| |
| stream.rewind(marker2) |
| self.failUnlessEqual(stream.markDepth, 1) |
| self.failUnlessEqual(stream.index(), 5) |
| self.failUnlessEqual(stream.line, 2) |
| self.failUnlessEqual(stream.charPositionInLine, 1) |
| self.failUnlessEqual(stream.LT(1), u'ä') |
| self.failUnlessEqual(stream.LA(1), ord(u'ä')) |
| |
| |
| |
| class TestInputStream(unittest.TestCase): |
| """Test case for the InputStream class.""" |
| |
| def testNoEncoding(self): |
| file = StringIO('foo\nbar') |
| |
| stream = antlr3.InputStream(file) |
| |
| stream.seek(4) |
| marker1 = stream.mark() |
| |
| stream.consume() |
| marker2 = stream.mark() |
| |
| stream.consume() |
| marker3 = stream.mark() |
| |
| stream.rewind(marker2) |
| self.failUnlessEqual(stream.markDepth, 1) |
| self.failUnlessEqual(stream.index(), 5) |
| self.failUnlessEqual(stream.line, 2) |
| self.failUnlessEqual(stream.charPositionInLine, 1) |
| self.failUnlessEqual(stream.LT(1), 'a') |
| self.failUnlessEqual(stream.LA(1), ord('a')) |
| |
| |
| def testEncoded(self): |
| file = StringIO(u'foo\nbär'.encode('utf-8')) |
| |
| stream = antlr3.InputStream(file, 'utf-8') |
| |
| stream.seek(4) |
| marker1 = stream.mark() |
| |
| stream.consume() |
| marker2 = stream.mark() |
| |
| stream.consume() |
| marker3 = stream.mark() |
| |
| stream.rewind(marker2) |
| self.failUnlessEqual(stream.markDepth, 1) |
| self.failUnlessEqual(stream.index(), 5) |
| self.failUnlessEqual(stream.line, 2) |
| self.failUnlessEqual(stream.charPositionInLine, 1) |
| self.failUnlessEqual(stream.LT(1), u'ä') |
| self.failUnlessEqual(stream.LA(1), ord(u'ä')) |
| |
| |
| class TestCommonTokenStream(unittest.TestCase): |
| """Test case for the StringStream class.""" |
| |
| def setUp(self): |
| """Setup test fixure |
| |
| The constructor of CommonTokenStream needs a token source. This |
| is a simple mock class providing just the nextToken() method. |
| |
| """ |
| |
| class MockSource(object): |
| def __init__(self): |
| self.tokens = [] |
| |
| def makeEOFToken(self): |
| return antlr3.CommonToken(type=antlr3.EOF) |
| |
| def nextToken(self): |
| try: |
| return self.tokens.pop(0) |
| except IndexError: |
| return None |
| |
| self.source = MockSource() |
| |
| |
| def testInit(self): |
| """CommonTokenStream.__init__()""" |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| self.failUnlessEqual(stream.index(), -1) |
| |
| |
| def testSetTokenSource(self): |
| """CommonTokenStream.setTokenSource()""" |
| |
| stream = antlr3.CommonTokenStream(None) |
| stream.setTokenSource(self.source) |
| self.failUnlessEqual(stream.index(), -1) |
| self.failUnlessEqual(stream.channel, antlr3.DEFAULT_CHANNEL) |
| |
| |
| def testLTEmptySource(self): |
| """CommonTokenStream.LT(): EOF (empty source)""" |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| |
| lt1 = stream.LT(1) |
| self.failUnlessEqual(lt1.type, antlr3.EOF) |
| |
| |
| def testLT1(self): |
| """CommonTokenStream.LT(1)""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| |
| lt1 = stream.LT(1) |
| self.failUnlessEqual(lt1.type, 12) |
| |
| |
| def testLT1WithHidden(self): |
| """CommonTokenStream.LT(1): with hidden tokens""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| |
| lt1 = stream.LT(1) |
| self.failUnlessEqual(lt1.type, 13) |
| |
| |
| def testLT2BeyondEnd(self): |
| """CommonTokenStream.LT(2): beyond end""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13, channel=antlr3.HIDDEN_CHANNEL) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| |
| lt1 = stream.LT(2) |
| self.failUnlessEqual(lt1.type, antlr3.EOF) |
| |
| |
| # not yet implemented |
| def testLTNegative(self): |
| """CommonTokenStream.LT(-1): look back""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| stream.fillBuffer() |
| stream.consume() |
| |
| lt1 = stream.LT(-1) |
| self.failUnlessEqual(lt1.type, 12) |
| |
| |
| def testLB1(self): |
| """CommonTokenStream.LB(1)""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| stream.fillBuffer() |
| stream.consume() |
| |
| self.failUnlessEqual(stream.LB(1).type, 12) |
| |
| |
| def testLTZero(self): |
| """CommonTokenStream.LT(0)""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| |
| lt1 = stream.LT(0) |
| self.failUnless(lt1 is None) |
| |
| |
| def testLBBeyondBegin(self): |
| """CommonTokenStream.LB(-1): beyond begin""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| self.failUnless(stream.LB(1) is None) |
| |
| stream.consume() |
| stream.consume() |
| self.failUnless(stream.LB(3) is None) |
| |
| |
| def testFillBuffer(self): |
| """CommonTokenStream.fillBuffer()""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=14) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=antlr3.EOF) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| stream.fillBuffer() |
| |
| self.failUnlessEqual(len(stream.tokens), 3) |
| self.failUnlessEqual(stream.tokens[0].type, 12) |
| self.failUnlessEqual(stream.tokens[1].type, 13) |
| self.failUnlessEqual(stream.tokens[2].type, 14) |
| |
| |
| def testConsume(self): |
| """CommonTokenStream.consume()""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=antlr3.EOF) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| self.failUnlessEqual(stream.LA(1), 12) |
| |
| stream.consume() |
| self.failUnlessEqual(stream.LA(1), 13) |
| |
| stream.consume() |
| self.failUnlessEqual(stream.LA(1), antlr3.EOF) |
| |
| stream.consume() |
| self.failUnlessEqual(stream.LA(1), antlr3.EOF) |
| |
| |
| def testSeek(self): |
| """CommonTokenStream.seek()""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=antlr3.EOF) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| self.failUnlessEqual(stream.LA(1), 12) |
| |
| stream.seek(2) |
| self.failUnlessEqual(stream.LA(1), antlr3.EOF) |
| |
| stream.seek(0) |
| self.failUnlessEqual(stream.LA(1), 12) |
| |
| |
| def testMarkRewind(self): |
| """CommonTokenStream.mark()/rewind()""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13) |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=antlr3.EOF) |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| stream.fillBuffer() |
| |
| stream.consume() |
| marker = stream.mark() |
| |
| stream.consume() |
| stream.rewind(marker) |
| |
| self.failUnlessEqual(stream.LA(1), 13) |
| |
| |
| def testToString(self): |
| """CommonTokenStream.toString()""" |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=12, text="foo") |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=13, text="bar") |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=14, text="gnurz") |
| ) |
| |
| self.source.tokens.append( |
| antlr3.CommonToken(type=15, text="blarz") |
| ) |
| |
| stream = antlr3.CommonTokenStream(self.source) |
| |
| assert stream.toString() == "foobargnurzblarz" |
| assert stream.toString(1, 2) == "bargnurz" |
| assert stream.toString(stream.tokens[1], stream.tokens[-2]) == "bargnurz" |
| |
| |
| if __name__ == "__main__": |
| unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) |
