Merge "ART: Improved fail reporting in Checker"
diff --git a/tools/checker.py b/tools/checker.py
index 74c6d61..5e910ec 100755
--- a/tools/checker.py
+++ b/tools/checker.py
@@ -79,6 +79,66 @@
import tempfile
from subprocess import check_call
+class Logger(object):
+ SilentMode = False
+
+ class Color(object):
+ Default, Blue, Gray, Purple, Red = range(5)
+
+ @staticmethod
+ def terminalCode(color, out=sys.stdout):
+ if not out.isatty():
+ return ''
+ elif color == Logger.Color.Blue:
+ return '\033[94m'
+ elif color == Logger.Color.Gray:
+ return '\033[37m'
+ elif color == Logger.Color.Purple:
+ return '\033[95m'
+ elif color == Logger.Color.Red:
+ return '\033[91m'
+ else:
+ return '\033[0m'
+
+ @staticmethod
+ def log(text, color=Color.Default, newLine=True, out=sys.stdout):
+ if not Logger.SilentMode:
+ text = Logger.Color.terminalCode(color, out) + text + \
+ Logger.Color.terminalCode(Logger.Color.Default, out)
+ if newLine:
+ print(text, file=out)
+ else:
+ print(text, end="", flush=True, file=out)
+
+ @staticmethod
+ def fail(msg, file=None, line=-1):
+ location = ""
+ if file:
+ location += file + ":"
+ if line > 0:
+ location += str(line) + ":"
+ if location:
+ location += " "
+
+ Logger.log(location, color=Logger.Color.Gray, newLine=False, out=sys.stderr)
+ Logger.log("error: ", color=Logger.Color.Red, newLine=False, out=sys.stderr)
+ Logger.log(msg, out=sys.stderr)
+ sys.exit(1)
+
+ @staticmethod
+ def startTest(name):
+ Logger.log("TEST ", color=Logger.Color.Purple, newLine=False)
+ Logger.log(name + "... ", newLine=False)
+
+ @staticmethod
+ def testPassed():
+ Logger.log("PASS", color=Logger.Color.Blue)
+
+ @staticmethod
+ def testFailed(msg, file=None, line=-1):
+ Logger.log("FAIL", color=Logger.Color.Red)
+ Logger.fail(msg, file, line)
+
class CommonEqualityMixin:
"""Mixin for class equality as equality of the fields."""
def __eq__(self, other):
@@ -135,14 +195,25 @@
"""Supported types of assertions."""
InOrder, DAG, Not = range(3)
- def __init__(self, content, variant=Variant.InOrder, lineNo=-1):
- self.content = content.strip()
- self.variant = variant
+ def __init__(self, content, variant=Variant.InOrder, fileName=None, lineNo=-1):
+ self.fileName = fileName
self.lineNo = lineNo
+ self.content = content.strip()
+ self.variant = variant
self.lineParts = self.__parse(self.content)
if not self.lineParts:
- raise Exception("Empty check line")
+ Logger.fail("Empty check line", self.fileName, self.lineNo)
+
+ if self.variant == CheckLine.Variant.Not:
+ for elem in self.lineParts:
+ if elem.variant == CheckElement.Variant.VarDef:
+ Logger.fail("CHECK-NOT lines cannot define variables", self.fileName, self.lineNo)
+
+ def __eq__(self, other):
+ return (isinstance(other, self.__class__) and
+ self.variant == other.variant and
+ self.lineParts == other.lineParts)
# Returns True if the given Match object was at the beginning of the line.
def __isMatchAtStart(self, match):
@@ -199,11 +270,7 @@
elif self.__isMatchAtStart(matchVariable):
var = line[0:matchVariable.end()]
line = line[matchVariable.end():]
- elem = CheckElement.parseVariable(var)
- if self.variant == CheckLine.Variant.Not and elem.variant == CheckElement.Variant.VarDef:
- raise Exception("CHECK-NOT check lines cannot define variables " +
- "(line " + str(self.lineNo) + ")")
- lineParts.append(elem)
+ lineParts.append(CheckElement.parseVariable(var))
else:
# If we're not currently looking at a special marker, this is a plain
# text match all the way until the first special marker (or the end
@@ -223,8 +290,8 @@
try:
return re.escape(varState[linePart.name])
except KeyError:
- raise Exception("Use of undefined variable '" + linePart.name + "' " +
- "(line " + str(self.lineNo))
+ Logger.testFailed("Use of undefined variable \"" + linePart.name + "\"",
+ self.fileName, self.lineNo)
else:
return linePart.pattern
@@ -262,8 +329,8 @@
matchEnd = matchStart + match.end()
if part.variant == CheckElement.Variant.VarDef:
if part.name in varState:
- raise Exception("Redefinition of variable '" + part.name + "'" +
- " (line " + str(self.lineNo) + ")")
+ Logger.testFailed("Multiple definitions of variable \"" + part.name + "\"",
+ self.fileName, self.lineNo)
varState[part.name] = outputLine[matchStart:matchEnd]
matchStart = matchEnd
@@ -277,15 +344,22 @@
"""Represents a named collection of check lines which are to be matched
against an output group of the same name."""
- def __init__(self, name, lines):
- if name:
- self.name = name
- else:
- raise Exception("Check group does not have a name")
- if lines:
- self.lines = lines
- else:
- raise Exception("Check group " + self.name + " does not have a body")
+ def __init__(self, name, lines, fileName=None, lineNo=-1):
+ self.fileName = fileName
+ self.lineNo = lineNo
+
+ if not name:
+ Logger.fail("Check group does not have a name", self.fileName, self.lineNo)
+ if not lines:
+ Logger.fail("Check group does not have a body", self.fileName, self.lineNo)
+
+ self.name = name
+ self.lines = lines
+
+ def __eq__(self, other):
+ return (isinstance(other, self.__class__) and
+ self.name == other.name and
+ self.lines == other.lines)
def __headAndTail(self, list):
return list[0], list[1:]
@@ -318,15 +392,14 @@
# check line and the updated variable state. Otherwise returns -1 and None,
# respectively. The 'lineFilter' parameter can be used to supply a list of
# line numbers (counting from 1) which should be skipped.
- def __findFirstMatch(self, checkLine, outputLines, lineFilter, varState):
- matchLineNo = 0
+ def __findFirstMatch(self, checkLine, outputLines, startLineNo, lineFilter, varState):
+ matchLineNo = startLineNo
for outputLine in outputLines:
+ if matchLineNo not in lineFilter:
+ newVarState = checkLine.match(outputLine, varState)
+ if newVarState is not None:
+ return matchLineNo, newVarState
matchLineNo += 1
- if matchLineNo in lineFilter:
- continue
- newVarState = checkLine.match(outputLine, varState)
- if newVarState is not None:
- return matchLineNo, newVarState
return -1, None
# Matches the given positive check lines against the output in order of
@@ -336,35 +409,42 @@
# together with the remaining output. The function also returns output lines
# which appear before either of the matched lines so they can be tested
# against Not checks.
- def __matchIndependentChecks(self, checkLines, outputLines, varState):
+ def __matchIndependentChecks(self, checkLines, outputLines, startLineNo, varState):
# If no checks are provided, skip over the entire output.
if not checkLines:
- return outputLines, varState, []
+ return outputLines, [], startLineNo + len(outputLines), varState
# Keep track of which lines have been matched.
matchedLines = []
# Find first unused output line which matches each check line.
for checkLine in checkLines:
- matchLineNo, varState = self.__findFirstMatch(checkLine, outputLines, matchedLines, varState)
+ matchLineNo, varState = \
+ self.__findFirstMatch(checkLine, outputLines, startLineNo, matchedLines, varState)
if varState is None:
- raise Exception("Could not match line " + str(checkLine))
+ Logger.testFailed("Could not match check line \"" + checkLine.content + "\" " +
+ "starting from output line " + str(startLineNo),
+ self.fileName, checkLine.lineNo)
matchedLines.append(matchLineNo)
# Return new variable state and the output lines which lie outside the
# match locations of this independent group.
- preceedingLines = outputLines[:min(matchedLines)-1]
- remainingLines = outputLines[max(matchedLines):]
- return preceedingLines, remainingLines, varState
+ minMatchLineNo = min(matchedLines)
+ maxMatchLineNo = max(matchedLines)
+ preceedingLines = outputLines[:minMatchLineNo - startLineNo]
+ remainingLines = outputLines[maxMatchLineNo - startLineNo + 1:]
+ return preceedingLines, remainingLines, maxMatchLineNo + 1, varState
# Makes sure that the given check lines do not match any of the given output
# lines. Variable state does not change.
- def __matchNotLines(self, checkLines, outputLines, varState):
+ def __matchNotLines(self, checkLines, outputLines, startLineNo, varState):
for checkLine in checkLines:
assert checkLine.variant == CheckLine.Variant.Not
- matchLineNo, varState = self.__findFirstMatch(checkLine, outputLines, [], varState)
+ matchLineNo, varState = \
+ self.__findFirstMatch(checkLine, outputLines, startLineNo, [], varState)
if varState is not None:
- raise Exception("CHECK-NOT line " + str(checkLine) + " matches output")
+ Logger.testFailed("CHECK-NOT line \"" + checkLine.content + "\" matches output line " + \
+ str(matchLineNo), self.fileName, checkLine.lineNo)
# Matches the check lines in this group against an output group. It is
# responsible for running the checks in the right order and scope, and
@@ -373,32 +453,42 @@
varState = {}
checkLines = self.lines
outputLines = outputGroup.body
+ startLineNo = outputGroup.lineNo
while checkLines:
# Extract the next sequence of location-independent checks to be matched.
notChecks, independentChecks, checkLines = self.__nextIndependentChecks(checkLines)
+
# Match the independent checks.
- notOutput, outputLines, newVarState = \
- self.__matchIndependentChecks(independentChecks, outputLines, varState)
+ notOutput, outputLines, newStartLineNo, newVarState = \
+ self.__matchIndependentChecks(independentChecks, outputLines, startLineNo, varState)
+
# Run the Not checks against the output lines which lie between the last
# two independent groups or the bounds of the output.
- self.__matchNotLines(notChecks, notOutput, varState)
+ self.__matchNotLines(notChecks, notOutput, startLineNo, varState)
+
# Update variable state.
+ startLineNo = newStartLineNo
varState = newVarState
class OutputGroup(CommonEqualityMixin):
"""Represents a named part of the test output against which a check group of
the same name is to be matched."""
- def __init__(self, name, body):
- if name:
- self.name = name
- else:
- raise Exception("Output group does not have a name")
- if body:
- self.body = body
- else:
- raise Exception("Output group " + self.name + " does not have a body")
+ def __init__(self, name, body, fileName=None, lineNo=-1):
+ if not name:
+ Logger.fail("Output group does not have a name", fileName, lineNo)
+ if not body:
+ Logger.fail("Output group does not have a body", fileName, lineNo)
+
+ self.name = name
+ self.body = body
+ self.lineNo = lineNo
+
+ def __eq__(self, other):
+ return (isinstance(other, self.__class__) and
+ self.name == other.name and
+ self.body == other.body)
class FileSplitMixin(object):
@@ -421,20 +511,24 @@
# entirely) and specify whether it starts a new group.
processedLine, newGroupName = self._processLine(line, lineNo)
if newGroupName is not None:
- currentGroup = (newGroupName, [])
+ currentGroup = (newGroupName, [], lineNo)
allGroups.append(currentGroup)
if processedLine is not None:
- currentGroup[1].append(processedLine)
+ if currentGroup is not None:
+ currentGroup[1].append(processedLine)
+ else:
+ self._exceptionLineOutsideGroup(line, lineNo)
# Finally, take the generated line groups and let the child class process
# each one before storing the final outcome.
- return list(map(lambda group: self._processGroup(group[0], group[1]), allGroups))
+ return list(map(lambda group: self._processGroup(group[0], group[1], group[2]), allGroups))
class CheckFile(FileSplitMixin):
"""Collection of check groups extracted from the input test file."""
- def __init__(self, prefix, checkStream):
+ def __init__(self, prefix, checkStream, fileName=None):
+ self.fileName = fileName
self.prefix = prefix
self.groups = self._parseStream(checkStream)
@@ -466,46 +560,40 @@
# Lines starting only with 'CHECK' are matched in order.
plainLine = self._extractLine(self.prefix, line)
if plainLine is not None:
- return (plainLine, CheckLine.Variant.InOrder), None
+ return (plainLine, CheckLine.Variant.InOrder, lineNo), None
# 'CHECK-DAG' lines are no-order assertions.
dagLine = self._extractLine(self.prefix + "-DAG", line)
if dagLine is not None:
- return (dagLine, CheckLine.Variant.DAG), None
+ return (dagLine, CheckLine.Variant.DAG, lineNo), None
# 'CHECK-NOT' lines are no-order negative assertions.
notLine = self._extractLine(self.prefix + "-NOT", line)
if notLine is not None:
- return (notLine, CheckLine.Variant.Not), None
+ return (notLine, CheckLine.Variant.Not, lineNo), None
# Other lines are ignored.
return None, None
def _exceptionLineOutsideGroup(self, line, lineNo):
- raise Exception("Check file line lies outside a group (line " + str(lineNo) + ")")
+ Logger.fail("Check line not inside a group", self.fileName, lineNo)
- def _processGroup(self, name, lines):
- checkLines = list(map(lambda line: CheckLine(line[0], line[1]), lines))
- return CheckGroup(name, checkLines)
+ def _processGroup(self, name, lines, lineNo):
+ checkLines = list(map(lambda line: CheckLine(line[0], line[1], self.fileName, line[2]), lines))
+ return CheckGroup(name, checkLines, self.fileName, lineNo)
- def match(self, outputFile, printInfo=False):
+ def match(self, outputFile):
for checkGroup in self.groups:
# TODO: Currently does not handle multiple occurrences of the same group
# name, e.g. when a pass is run multiple times. It will always try to
# match a check group against the first output group of the same name.
outputGroup = outputFile.findGroup(checkGroup.name)
if outputGroup is None:
- raise Exception("Group " + checkGroup.name + " not found in the output")
- if printInfo:
- print("TEST " + checkGroup.name + "... ", end="", flush=True)
- try:
- checkGroup.match(outputGroup)
- if printInfo:
- print("PASSED")
- except Exception as e:
- if printInfo:
- print("FAILED!")
- raise e
+ Logger.fail("Group \"" + checkGroup.name + "\" not found in the output",
+ self.fileName, checkGroup.lineNo)
+ Logger.startTest(checkGroup.name)
+ checkGroup.match(outputGroup)
+ Logger.testPassed()
class OutputFile(FileSplitMixin):
@@ -522,7 +610,9 @@
class ParsingState:
OutsideBlock, InsideCompilationBlock, StartingCfgBlock, InsideCfgBlock = range(4)
- def __init__(self, outputStream):
+ def __init__(self, outputStream, fileName=None):
+ self.fileName = fileName
+
# Initialize the state machine
self.lastMethodName = None
self.state = OutputFile.ParsingState.OutsideBlock
@@ -538,7 +628,7 @@
self.state = OutputFile.ParsingState.InsideCfgBlock
return (None, self.lastMethodName + " " + line.split("\"")[1])
else:
- raise Exception("Expected group name in output file (line " + str(lineNo) + ")")
+ Logger.fail("Expected output group name", self.fileName, lineNo)
elif self.state == OutputFile.ParsingState.InsideCfgBlock:
if line == "end_cfg":
@@ -549,29 +639,32 @@
elif self.state == OutputFile.ParsingState.InsideCompilationBlock:
# Search for the method's name. Format: method "<name>"
- if re.match("method\s+\"[^\"]+\"", line):
- self.lastMethodName = line.split("\"")[1]
+ if re.match("method\s+\"[^\"]*\"", line):
+ methodName = line.split("\"")[1].strip()
+ if not methodName:
+ Logger.fail("Empty method name in output", self.fileName, lineNo)
+ self.lastMethodName = methodName
elif line == "end_compilation":
self.state = OutputFile.ParsingState.OutsideBlock
return (None, None)
- else: # self.state == OutputFile.ParsingState.OutsideBlock:
+ else:
+ assert self.state == OutputFile.ParsingState.OutsideBlock
if line == "begin_cfg":
# The line starts a new group but we'll wait until the next line from
# which we can extract the name of the pass.
if self.lastMethodName is None:
- raise Exception("Output contains a pass without a method header" +
- " (line " + str(lineNo) + ")")
+ Logger.fail("Expected method header", self.fileName, lineNo)
self.state = OutputFile.ParsingState.StartingCfgBlock
return (None, None)
elif line == "begin_compilation":
self.state = OutputFile.ParsingState.InsideCompilationBlock
return (None, None)
else:
- raise Exception("Output line lies outside a group (line " + str(lineNo) + ")")
+ Logger.fail("Output line not inside a group", self.fileName, lineNo)
- def _processGroup(self, name, lines):
- return OutputGroup(name, lines)
+ def _processGroup(self, name, lines, lineNo):
+ return OutputGroup(name, lines, self.fileName, lineNo + 1)
def findGroup(self, name):
for group in self.groups:
@@ -631,22 +724,30 @@
def ListGroups(outputFilename):
outputFile = OutputFile(open(outputFilename, "r"))
for group in outputFile.groups:
- print(group.name)
+ Logger.log(group.name)
def DumpGroup(outputFilename, groupName):
outputFile = OutputFile(open(outputFilename, "r"))
group = outputFile.findGroup(groupName)
if group:
- print("\n".join(group.body))
+ lineNo = group.lineNo
+ maxLineNo = lineNo + len(group.body)
+ lenLineNo = len(str(maxLineNo)) + 2
+ for line in group.body:
+ Logger.log((str(lineNo) + ":").ljust(lenLineNo) + line)
+ lineNo += 1
else:
- raise Exception("Check group " + groupName + " not found in the output")
+ Logger.fail("Group \"" + groupName + "\" not found in the output")
def RunChecks(checkPrefix, checkFilename, outputFilename):
- checkFile = CheckFile(checkPrefix, open(checkFilename, "r"))
- outputFile = OutputFile(open(outputFilename, "r"))
- checkFile.match(outputFile, True)
+ checkBaseName = os.path.basename(checkFilename)
+ outputBaseName = os.path.splitext(checkBaseName)[0] + ".cfg"
+
+ checkFile = CheckFile(checkPrefix, open(checkFilename, "r"), checkBaseName)
+ outputFile = OutputFile(open(outputFilename, "r"), outputBaseName)
+ checkFile.match(outputFile)
if __name__ == "__main__":
diff --git a/tools/checker_test.py b/tools/checker_test.py
index 8947d8a..9b04ab0 100755
--- a/tools/checker_test.py
+++ b/tools/checker_test.py
@@ -21,6 +21,11 @@
import io
import unittest
+# The parent type of exception expected to be thrown by Checker during tests.
+# It must be specific enough to not cover exceptions thrown due to actual flaws
+# in Checker..
+CheckerException = SystemExit
+
class TestCheckFile_PrefixExtraction(unittest.TestCase):
def __tryParse(self, string):
@@ -65,7 +70,7 @@
self.assertEqual(expected, self.__getRegex(self.__tryParse(string)))
def __tryParseNot(self, string):
- return checker.CheckLine(string, checker.CheckLine.Variant.UnorderedNot)
+ return checker.CheckLine(string, checker.CheckLine.Variant.Not)
def __parsesPattern(self, string, pattern):
line = self.__tryParse(string)
@@ -167,7 +172,7 @@
self.__parsesTo("[[ABC:abc]][[DEF:def]]", "(abc)(def)")
def test_NoVarDefsInNotChecks(self):
- with self.assertRaises(Exception):
+ with self.assertRaises(CheckerException):
self.__tryParseNot("[[ABC:abc]]")
class TestCheckLine_Match(unittest.TestCase):
@@ -203,7 +208,7 @@
self.__matchSingle("foo[[X]]bar", "fooBbar", {"X": "B"})
self.__notMatchSingle("foo[[X]]bar", "foobar", {"X": "A"})
self.__notMatchSingle("foo[[X]]bar", "foo bar", {"X": "A"})
- with self.assertRaises(Exception):
+ with self.assertRaises(CheckerException):
self.__matchSingle("foo[[X]]bar", "foobar", {})
def test_VariableDefinition(self):
@@ -221,7 +226,7 @@
self.__notMatchSingle("foo[[X:A|B]]bar[[X]]baz", "fooAbarBbaz")
def test_NoVariableRedefinition(self):
- with self.assertRaises(Exception):
+ with self.assertRaises(CheckerException):
self.__matchSingle("[[X:...]][[X]][[X:...]][[X]]", "foofoobarbar")
def test_EnvNotChangedOnPartialMatch(self):
@@ -255,7 +260,7 @@
return checkGroup.match(outputGroup)
def __notMatchMulti(self, checkString, outputString):
- with self.assertRaises(Exception):
+ with self.assertRaises(CheckerException):
self.__matchMulti(checkString, outputString)
def test_TextAndPattern(self):
@@ -448,4 +453,5 @@
("def", CheckVariant.DAG) ])) ])
if __name__ == '__main__':
+ checker.Logger.SilentMode = True
unittest.main()
