Tools/GardeningServer/scripts/results.js

/*
 * Copyright (C) 2011 Google Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

var results = results || {};

(function() {

var kResultsName = 'failing_results.json';

var PASS = 'PASS';
var TIMEOUT = 'TIMEOUT';
var TEXT = 'TEXT';
var CRASH = 'CRASH';
var IMAGE = 'IMAGE';
var IMAGE_TEXT = 'IMAGE+TEXT';
var AUDIO = 'AUDIO';
var MISSING = 'MISSING';

var kFailingResults = [TEXT, IMAGE_TEXT, AUDIO];

var kExpectedImageSuffix = '-expected.png';
var kActualImageSuffix = '-actual.png';
var kImageDiffSuffix = '-diff.png';
var kExpectedAudioSuffix = '-expected.wav';
var kActualAudioSuffix = '-actual.wav';
var kExpectedTextSuffix = '-expected.txt';
var kActualTextSuffix = '-actual.txt';
var kDiffTextSuffix = '-diff.txt';
var kCrashLogSuffix = '-crash-log.txt';

var kPNGExtension = 'png';
var kTXTExtension = 'txt';
var kWAVExtension = 'wav';

var kPreferredSuffixOrder = [
    kExpectedImageSuffix,
    kActualImageSuffix,
    kImageDiffSuffix,
    kExpectedTextSuffix,
    kActualTextSuffix,
    kDiffTextSuffix,
    kCrashLogSuffix,
    kExpectedAudioSuffix,
    kActualAudioSuffix,
    // FIXME: Add support for the rest of the result types.
];

// Kinds of results.
results.kActualKind = 'actual';
results.kExpectedKind = 'expected';
results.kDiffKind = 'diff';
results.kUnknownKind = 'unknown';

// Types of tests.
results.kImageType = 'image'
results.kAudioType = 'audio'
results.kTextType = 'text'
// FIXME: There are more types of tests.

function layoutTestResultsURL(platform)
{
    return config.kPlatforms[platform].layoutTestResultsURL;
}

function possibleSuffixListFor(failureTypeList)
{
    var suffixList = [];

    function pushImageSuffixes()
    {
        suffixList.push(kExpectedImageSuffix);
        suffixList.push(kActualImageSuffix);
        suffixList.push(kImageDiffSuffix);
    }

    function pushAudioSuffixes()
    {
        suffixList.push(kExpectedAudioSuffix);
        suffixList.push(kActualAudioSuffix);
    }

    function pushTextSuffixes()
    {
        suffixList.push(kActualTextSuffix);
        suffixList.push(kExpectedTextSuffix);
        suffixList.push(kDiffTextSuffix);
        // '-wdiff.html',
        // '-pretty-diff.html',
    }

    $.each(failureTypeList, function(index, failureType) {
        switch(failureType) {
        case IMAGE:
            pushImageSuffixes();
            break;
        case TEXT:
            pushTextSuffixes();
            break;
        case AUDIO:
            pushAudioSuffixes();
            break;
        case IMAGE_TEXT:
            pushImageSuffixes();
            pushTextSuffixes();
            break;
        case CRASH:
            suffixList.push(kCrashLogSuffix);
            break;
        case MISSING:
            pushImageSuffixes();
            pushTextSuffixes();
            break;
        default:
            // FIXME: Add support for the rest of the result types.
            // '-expected.html',
            // '-expected-mismatch.html',
            // ... and possibly more.
            break;
        }
    });

    return base.uniquifyArray(suffixList);
}

results.failureTypeToExtensionList = function(failureType)
{
    switch(failureType) {
    case IMAGE:
        return [kPNGExtension];
    case AUDIO:
        return [kWAVExtension];
    case TEXT:
        return [kTXTExtension];
    case MISSING:
    case IMAGE_TEXT:
        return [kTXTExtension, kPNGExtension];
    default:
        // FIXME: Add support for the rest of the result types.
        // '-expected.html',
        // '-expected-mismatch.html',
        // ... and possibly more.
        return [];
    }
};

results.failureTypeList = function(failureBlob)
{
    return failureBlob.split(' ');
};

results.directoryForBuilder = function(builderName)
{
    return config.kPlatforms[config.currentPlatform].resultsDirectoryNameFromBuilderName(builderName);
}

function resultsDirectoryURL(platform, builderName)
{
    if (config.useLocalResults)
        return '/localresult?path=';
    return layoutTestResultsURL(platform) + '/' + results.directoryForBuilder(builderName) + '/results/layout-test-results/';
}

function resultsPrefixListingURL(platform, builderName, marker)
{
    var url =  layoutTestResultsURL(platform) + '/?prefix=' + results.directoryForBuilder(builderName) + '/&delimiter=/';
    if (marker)
        return url + '&marker=' + marker;
    return url;
}

function resultsDirectoryURLForBuildNumber(platform, builderName, buildNumber)
{
    return layoutTestResultsURL(platform) + '/' + results.directoryForBuilder(builderName) + '/' + buildNumber + '/' ;
}

function resultsSummaryURL(platform, builderName)
{
    return resultsDirectoryURL(platform, builderName) + kResultsName;
}

function resultsSummaryURLForBuildNumber(platform, builderName, buildNumber)
{
    return resultsDirectoryURLForBuildNumber(platform, builderName, buildNumber) + kResultsName;
}

var g_resultsCache = new base.AsynchronousCache(function (key, callback) {
    net.jsonp(key, callback);
});

results.ResultAnalyzer = base.extends(Object, {
    init: function(resultNode)
    {
        this._isUnexpected = resultNode.is_unexpected;
        this._actual = resultNode ? results.failureTypeList(resultNode.actual) : [];
        this._expected = resultNode ? this._addImpliedExpectations(results.failureTypeList(resultNode.expected)) : [];
    },
    _addImpliedExpectations: function(resultsList)
    {
        if (resultsList.indexOf('FAIL') == -1)
            return resultsList;
        return resultsList.concat(kFailingResults);
    },
    _hasPass: function(results)
    {
        return results.indexOf(PASS) != -1;
    },
    unexpectedResults: function()
    {
        return this._actual.filter(function(result) {
            return this._expected.indexOf(result) == -1;
        }, this);
    },
    succeeded: function()
    {
        return this._hasPass(this._actual);
    },
    flaky: function()
    {
        return this._actual.length > 1;
    },
    wontfix: function()
    {
        return this._expected.indexOf('WONTFIX') != -1;
    },
    hasUnexpectedFailures: function()
    {
        return this._isUnexpected;
    }
})

function isExpectedFailure(resultNode)
{
    var analyzer = new results.ResultAnalyzer(resultNode);
    return !analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix();
}

function isUnexpectedFailure(resultNode)
{
    var analyzer = new results.ResultAnalyzer(resultNode);
    return analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix();
}

function isResultNode(node)
{
    return !!node.actual;
}

results.expectedFailures = function(resultsTree)
{
    return base.filterTree(resultsTree.tests, isResultNode, isExpectedFailure);
};

results.unexpectedFailures = function(resultsTree)
{
    return base.filterTree(resultsTree.tests, isResultNode, isUnexpectedFailure);
};

function resultsByTest(resultsByBuilder, filter)
{
    var resultsByTest = {};

    $.each(resultsByBuilder, function(builderName, resultsTree) {
        $.each(filter(resultsTree), function(testName, resultNode) {
            resultsByTest[testName] = resultsByTest[testName] || {};
            resultsByTest[testName][builderName] = resultNode;
        });
    });

    return resultsByTest;
}

results.expectedFailuresByTest = function(resultsByBuilder)
{
    return resultsByTest(resultsByBuilder, results.expectedFailures);
};

results.unexpectedFailuresByTest = function(resultsByBuilder)
{
    return resultsByTest(resultsByBuilder, results.unexpectedFailures);
};

results.failureInfoForTestAndBuilder = function(resultsByTest, testName, builderName)
{
    var failureInfoForTest = {
        'testName': testName,
        'builderName': builderName,
        'failureTypeList': results.failureTypeList(resultsByTest[testName][builderName].actual),
    };

    return failureInfoForTest;
};

results.collectUnexpectedResults = function(dictionaryOfResultNodes)
{
    var collectedResults = [];
    $.each(dictionaryOfResultNodes, function(key, resultNode) {
        var analyzer = new results.ResultAnalyzer(resultNode);
        collectedResults = collectedResults.concat(analyzer.unexpectedResults());
    });
    return base.uniquifyArray(collectedResults);
};

// Callback data is [{ buildNumber:, url: }]
function historicalResultsLocations(platform, builderName, callback)
{
    var historicalResultsData = [];

    function parseListingDocument(prefixListingDocument) {
        $(prefixListingDocument).find("Prefix").each(function() {
            var buildString = this.textContent.replace(results.directoryForBuilder(builderName) + '/', '');
            if (buildString.match(/\d+\//)) {
                var buildNumber = parseInt(buildString);
                var resultsData = {
                    'buildNumber': buildNumber,
                    'url': resultsSummaryURLForBuildNumber(platform, builderName, buildNumber)
                };
                historicalResultsData.unshift(resultsData);
           }
        });
        var nextMarker = $(prefixListingDocument).find('NextMarker').get();
        if (nextMarker.length) {
            var nextListingURL = resultsPrefixListingURL(platform, builderName, nextMarker[0].textContent);
            net.get(nextListingURL, parseListingDocument);
        } else {
            callback(historicalResultsData);
        }
    }

    builders.mostRecentBuildForBuilder(platform, builderName, function (mostRecentBuildNumber) {
        var marker = results.directoryForBuilder(builderName) + "/" + (mostRecentBuildNumber - 100) + "/";
        var listingURL = resultsPrefixListingURL(platform, builderName, marker);
        net.get(listingURL, parseListingDocument);
    });
}

function walkHistory(platform, builderName, testName, callback)
{
    var indexOfNextKeyToFetch = 0;
    var keyList = [];

    function continueWalk()
    {
        if (indexOfNextKeyToFetch >= keyList.length) {
            processResultNode(0, null);
            return;
        }

        var resultsURL = keyList[indexOfNextKeyToFetch].url;
        ++indexOfNextKeyToFetch;
        g_resultsCache.get(resultsURL, function(resultsTree) {
            if ($.isEmptyObject(resultsTree)) {
                continueWalk();
                return;
            }
            var resultNode = results.resultNodeForTest(resultsTree, testName);
            var revision = parseInt(resultsTree['blink_revision'])
            if (isNaN(revision))
                revision = 0;
            processResultNode(revision, resultNode);
        });
    }

    function processResultNode(revision, resultNode)
    {
        var shouldContinue = callback(revision, resultNode);
        if (!shouldContinue)
            return;
        continueWalk();
    }

    historicalResultsLocations(platform, builderName, function(resultsLocations) {
        keyList = resultsLocations;
        continueWalk();
    });
}

results.regressionRangeForFailure = function(builderName, testName, callback)
{
    var oldestFailingRevision = 0;
    var newestPassingRevision = 0;

    // FIXME: should treat {platform, builderName} as a tuple
    walkHistory(config.currentPlatform, builderName, testName, function(revision, resultNode) {
        if (!revision) {
            callback(oldestFailingRevision, newestPassingRevision);
            return false;
        }
        if (!resultNode) {
            newestPassingRevision = revision;
            callback(oldestFailingRevision, newestPassingRevision);
            return false;
        }
        if (isUnexpectedFailure(resultNode)) {
            oldestFailingRevision = revision;
            return true;
        }
        if (!oldestFailingRevision)
            return true;  // We need to keep looking for a failing revision.
        newestPassingRevision = revision;
        callback(oldestFailingRevision, newestPassingRevision);
        return false;
    });
};

function mergeRegressionRanges(regressionRanges)
{
    var mergedRange = {};

    mergedRange.oldestFailingRevision = 0;
    mergedRange.newestPassingRevision = 0;

    $.each(regressionRanges, function(builderName, range) {
        if (!range.oldestFailingRevision && !range.newestPassingRevision)
            return

        if (!mergedRange.oldestFailingRevision)
            mergedRange.oldestFailingRevision = range.oldestFailingRevision;
        if (!mergedRange.newestPassingRevision)
            mergedRange.newestPassingRevision = range.newestPassingRevision;

        if (range.oldestFailingRevision && range.oldestFailingRevision < mergedRange.oldestFailingRevision)
            mergedRange.oldestFailingRevision = range.oldestFailingRevision;
        if (range.newestPassingRevision > mergedRange.newestPassingRevision)
            mergedRange.newestPassingRevision = range.newestPassingRevision;
    });

    return mergedRange;
}

results.unifyRegressionRanges = function(builderNameList, testName, callback)
{
    var regressionRanges = {};

    var tracker = new base.RequestTracker(builderNameList.length, function() {
        var mergedRange = mergeRegressionRanges(regressionRanges);
        callback(mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision);
    });

    $.each(builderNameList, function(index, builderName) {
        results.regressionRangeForFailure(builderName, testName, function(oldestFailingRevision, newestPassingRevision) {
            var range = {};
            range.oldestFailingRevision = oldestFailingRevision;
            range.newestPassingRevision = newestPassingRevision;
            regressionRanges[builderName] = range;
            tracker.requestComplete();
        });
    });
};

results.resultNodeForTest = function(resultsTree, testName)
{
    var testNamePath = testName.split('/');
    var currentNode = resultsTree['tests'];
    $.each(testNamePath, function(index, segmentName) {
        if (!currentNode)
            return;
        currentNode = (segmentName in currentNode) ? currentNode[segmentName] : null;
    });
    return currentNode;
};

results.resultKind = function(url)
{
    if (/-actual\.[a-z]+$/.test(url))
        return results.kActualKind;
    else if (/-expected\.[a-z]+$/.test(url))
        return results.kExpectedKind;
    else if (/diff\.[a-z]+$/.test(url))
        return results.kDiffKind;
    return results.kUnknownKind;
}

results.resultType = function(url)
{
    if (/\.png$/.test(url))
        return results.kImageType;
    if (/\.wav$/.test(url))
        return results.kAudioType;
    return results.kTextType;
}

function sortResultURLsBySuffix(urls)
{
    var sortedURLs = [];
    $.each(kPreferredSuffixOrder, function(i, suffix) {
        $.each(urls, function(j, url) {
            if (!base.endsWith(url, suffix))
                return;
            sortedURLs.push(url);
        });
    });
    if (sortedURLs.length != urls.length)
        throw "sortResultURLsBySuffix failed to return the same number of URLs."
    return sortedURLs;
}

results.fetchResultsURLs = function(failureInfo, callback)
{
    var testNameStem = base.trimExtension(failureInfo.testName);
    var urlStem = resultsDirectoryURL(config.currentPlatform, failureInfo.builderName);

    var suffixList = possibleSuffixListFor(failureInfo.failureTypeList);
    var resultURLs = [];
    var tracker = new base.RequestTracker(suffixList.length, function() {
        callback(sortResultURLsBySuffix(resultURLs));
    });
    $.each(suffixList, function(index, suffix) {
        var url = urlStem + testNameStem + suffix;
        net.probe(url, {
            success: function() {
                resultURLs.push(url);
                tracker.requestComplete();
            },
            error: function() {
                tracker.requestComplete();
            },
        });
    });
};

results.fetchResultsByBuilder = function(builderNameList, callback)
{
    var resultsByBuilder = {};
    var tracker = new base.RequestTracker(builderNameList.length, function() {
        callback(resultsByBuilder);
    });
    $.each(builderNameList, function(index, builderName) {
        var resultsURL = resultsSummaryURL(config.currentPlatform, builderName);
        net.jsonp(resultsURL, function(resultsTree) {
            resultsByBuilder[builderName] = resultsTree;
            tracker.requestComplete();
        });
    });
};

})();