blob: 8191a9d3234f49c843978a8688358673f859017f [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Self-test for skimage.
import filecmp
import os
import subprocess
import sys
import tempfile
class BinaryNotFoundException(Exception):
def __str__ (self):
return ("Could not find binary!\n"
"Did you forget to build the tools project?\n"
"Self tests failed")
# Find a path to the binary to use. Iterates through a list of possible
# locations the binary may be.
def PickBinaryPath(base_dir):
POSSIBLE_BINARY_PATHS = [
'out/Debug/skimage',
'out/Release/skimage',
'xcodebuild/Debug/skimage',
'xcodebuild/Release/skimage',
]
for binary in POSSIBLE_BINARY_PATHS:
binary_full_path = os.path.join(base_dir, binary)
if (os.path.exists(binary_full_path)):
return binary_full_path
raise BinaryNotFoundException
# Quit early if two files have different content.
def DieIfFilesMismatch(expected, actual):
if not filecmp.cmp(expected, actual):
print 'Error: file mismatch! expected=%s , actual=%s' % (
expected, actual)
exit(1)
def test_invalid_file(file_dir, skimage_binary):
""" Test the return value of skimage when an invalid file is decoded.
If there is no expectation file, or the file expects a particular
result, skimage should return nonzero indicating failure.
If the file has no expectation, or ignore-failure is set to true,
skimage should return zero indicating success. """
invalid_file = os.path.join(file_dir, "skimage", "input", "bad-images",
"invalid.png")
# No expectations file:
args = [skimage_binary, "--readPath", invalid_file]
result = subprocess.call(args)
if 0 == result:
print "'%s' should have reported failure!" % " ".join(args)
exit(1)
# Directory holding all expectations files
expectations_dir = os.path.join(file_dir, "skimage", "input", "bad-images")
# Expectations file expecting a valid decode:
incorrect_expectations = os.path.join(expectations_dir,
"incorrect-results.json")
args = [skimage_binary, "--readPath", invalid_file,
"--readExpectationsPath", incorrect_expectations]
result = subprocess.call(args)
if 0 == result:
print "'%s' should have reported failure!" % " ".join(args)
exit(1)
# Empty expectations:
empty_expectations = os.path.join(expectations_dir, "empty-results.json")
output = subprocess.check_output([skimage_binary, "--readPath", invalid_file,
"--readExpectationsPath",
empty_expectations],
stderr=subprocess.STDOUT)
if not "Missing" in output:
# Another test (in main()) tests to ensure that "Missing" does not appear
# in the output. That test could be passed if the output changed so
# "Missing" never appears. This ensures that an error is not missed if
# that happens.
print "skimage output changed! This may cause other self tests to fail!"
exit(1)
# Ignore failure:
ignore_expectations = os.path.join(expectations_dir, "ignore-results.json")
output = subprocess.check_output([skimage_binary, "--readPath", invalid_file,
"--readExpectationsPath",
ignore_expectations],
stderr=subprocess.STDOUT)
if not "failures" in output:
# Another test (in main()) tests to ensure that "failures" does not
# appear in the output. That test could be passed if the output changed
# so "failures" never appears. This ensures that an error is not missed
# if that happens.
print "skimage output changed! This may cause other self tests to fail!"
exit(1)
def test_incorrect_expectations(file_dir, skimage_binary):
""" Test that comparing to incorrect expectations fails, unless
ignore-failures is set to true. """
valid_file = os.path.join(file_dir, "skimage", "input",
"images-with-known-hashes",
"1209453360120438698.png")
expectations_dir = os.path.join(file_dir, "skimage", "input",
"images-with-known-hashes")
incorrect_results = os.path.join(expectations_dir,
"incorrect-results.json")
args = [skimage_binary, "--readPath", valid_file, "--readExpectationsPath",
incorrect_results]
result = subprocess.call(args)
if 0 == result:
print "'%s' should have reported failure!" % " ".join(args)
exit(1)
ignore_results = os.path.join(expectations_dir, "ignore-failures.json")
subprocess.check_call([skimage_binary, "--readPath", valid_file,
"--readExpectationsPath", ignore_results])
def main():
# Use the directory of this file as the out directory
file_dir = os.path.abspath(os.path.dirname(__file__))
trunk_dir = os.path.normpath(os.path.join(file_dir, os.pardir, os.pardir))
# Find the binary
skimage_binary = PickBinaryPath(trunk_dir)
print "Running " + skimage_binary
# Generate an expectations file from known images.
images_dir = os.path.join(file_dir, "skimage", "input",
"images-with-known-hashes")
expectations_path = os.path.join(file_dir, "skimage", "output-actual",
"create-expectations", "expectations.json")
subprocess.check_call([skimage_binary, "--readPath", images_dir,
"--createExpectationsPath", expectations_path])
# Make sure the expectations file was generated correctly.
golden_expectations = os.path.join(file_dir, "skimage", "output-expected",
"create-expectations",
"expectations.json")
DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path)
# Tell skimage to read back the expectations file it just wrote, and
# confirm that the images in images_dir match it.
output = subprocess.check_output([skimage_binary, "--readPath", images_dir,
"--readExpectationsPath",
expectations_path],
stderr=subprocess.STDOUT)
# Although skimage succeeded, it would have reported success if the file
# was missing from the expectations file. Consider this a failure, since
# the expectations file was created from this same image. (It will print
# "Missing" in this case before listing the missing expectations).
if "Missing" in output:
print "Expectations file was missing expectations!"
print output
exit(1)
# Again, skimage would succeed if there were known failures (and print
# "failures"), but there should be no failures, since the file just
# created did not include failures to ignore.
if "failures" in output:
print "Image failed!"
print output
exit(1)
test_incorrect_expectations(file_dir=file_dir,
skimage_binary=skimage_binary)
# Generate an expectations file from an empty directory.
empty_dir = tempfile.mkdtemp()
expectations_path = os.path.join(file_dir, "skimage", "output-actual",
"empty-dir", "expectations.json")
subprocess.check_call([skimage_binary, "--readPath", empty_dir,
"--createExpectationsPath", expectations_path])
golden_expectations = os.path.join(file_dir, "skimage", "output-expected",
"empty-dir", "expectations.json")
DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path)
os.rmdir(empty_dir)
# Generate an expectations file from a nonexistent directory.
expectations_path = os.path.join(file_dir, "skimage", "output-actual",
"nonexistent-dir", "expectations.json")
subprocess.check_call([skimage_binary, "--readPath", "/nonexistent/dir",
"--createExpectationsPath", expectations_path])
golden_expectations = os.path.join(file_dir, "skimage", "output-expected",
"nonexistent-dir", "expectations.json")
DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path)
test_invalid_file(file_dir=file_dir, skimage_binary=skimage_binary)
# Done with all tests.
print "Self tests succeeded!"
if __name__ == "__main__":
main()