hostsidetests/media/bitstreams/src/android/media/cts/bitstreams/MediaBitstreamsTest.java

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package android.media.cts.bitstreams;

import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
import com.android.compatibility.common.tradefed.targetprep.MediaPreparer;
import com.android.compatibility.common.util.MetricsReportLog;
import com.android.compatibility.common.util.ResultType;
import com.android.compatibility.common.util.ResultUnit;
import com.android.tradefed.build.IBuildInfo;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionClass;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.invoker.TestInformation;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.testtype.IAbi;
import com.android.tradefed.testtype.IAbiReceiver;
import com.android.tradefed.testtype.IBuildReceiver;
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.ITestInformationReceiver;
import com.android.tradefed.util.FileUtil;

import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import org.xmlpull.v1.XmlPullParserFactory;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.nio.file.Files;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

/**
 * Test that verifies video bitstreams decode pixel perfectly
 */
@OptionClass(alias="media-bitstreams-test")
public abstract class MediaBitstreamsTest implements IDeviceTest, IBuildReceiver, IAbiReceiver, ITestInformationReceiver {

    @Option(name = MediaBitstreams.OPT_HOST_BITSTREAMS_PATH,
            description = "Absolute path of Ittiam bitstreams (host)",
            mandatory = true)
    private File mHostBitstreamsPath = getDefaultBitstreamsDir();

    @Option(name = MediaBitstreams.OPT_DEVICE_BITSTREAMS_PATH,
            description = "Absolute path of Ittiam bitstreams (device)")
    private String mDeviceBitstreamsPath = MediaBitstreams.DEFAULT_DEVICE_BITSTEAMS_PATH;

    @Option(name = MediaBitstreams.OPT_DOWNLOAD_BITSTREAMS,
            description = "Whether to download the bitstreams files")
    private boolean mDownloadBitstreams = false;

    @Option(name = MediaBitstreams.OPT_UTILIZATION_RATE,
            description = "Percentage of external storage space used for test")
    private int mUtilizationRate = 80;

    @Option(name = MediaBitstreams.OPT_NUM_BATCHES,
            description = "Number of batches to test;"
                    + " each batch uses external storage up to utilization rate")
    private int mNumBatches = Integer.MAX_VALUE;

    @Option(name = MediaBitstreams.OPT_DEBUG_TARGET_DEVICE,
            description = "Whether to debug target device under test")
    private boolean mDebugTargetDevice = false;

    @Option(name = MediaBitstreams.OPT_BITSTREAMS_PREFIX,
            description = "Only test bitstreams in this sub-directory")
    private String mPrefix = "";

    private String mPath = "";

    private static ConcurrentMap<String, List<ConformanceEntry>> mResults = new ConcurrentHashMap<>();

    /**
     * Which subset of bitstreams to test
     */
    enum BitstreamPackage {
        STANDARD,
        FULL,
    }

    private BitstreamPackage mPackage = BitstreamPackage.FULL;
    private BitstreamPackage mPackageToRun = BitstreamPackage.STANDARD;
    private boolean mEnforce = false;

    static class ConformanceEntry {
        final String mPath, mCodecName, mStatus;
        ConformanceEntry(String path, String codecName, String status) {
            mPath = path;
            mCodecName = codecName;
            mStatus = status;
        }
        @Override
        public String toString() {
            return String.format("%s,%s,%s", mPath, mCodecName, mStatus);
        }
    }

    /**
     * A helper to access resources in the build.
     */
    private CompatibilityBuildHelper mBuildHelper;

    private IAbi mAbi;
    private ITestDevice mDevice;
    private TestInformation mTestInfo;

    static File getDefaultBitstreamsDir() {
        File mediaDir = MediaPreparer.getDefaultMediaDir();
        File[] subDirs = mediaDir.listFiles(new FileFilter() {
            @Override
            public boolean accept(File child) {
                return child.isDirectory();
            }
        });
        if (subDirs != null && subDirs.length == 1) {
            File parent = new File(mediaDir, subDirs[0].getName());
            return new File(parent, MediaBitstreams.DEFAULT_HOST_BITSTREAMS_PATH);
        } else {
            return new File(MediaBitstreams.DEFAULT_HOST_BITSTREAMS_PATH);
        }
    }

    static Collection<Object[]> bitstreams(String prefix, BitstreamPackage packageToRun) {
        final String dynConfXml = new File("/", MediaBitstreams.DYNAMIC_CONFIG_XML).toString();
        try (InputStream is = MediaBitstreamsTest.class.getResourceAsStream(dynConfXml)) {
            List<Object[]> entries = new ArrayList<>();
            XmlPullParser parser = XmlPullParserFactory.newInstance().newPullParser();
            parser.setInput(is, null);
            parser.nextTag();
            parser.require(XmlPullParser.START_TAG, null, MediaBitstreams.DYNAMIC_CONFIG);
            while (parser.next() != XmlPullParser.END_DOCUMENT) {
                if (parser.getEventType() != XmlPullParser.START_TAG
                        || !MediaBitstreams.DYNAMIC_CONFIG_ENTRY.equals(parser.getName())) {
                    continue;
                }
                final String key = MediaBitstreams.DYNAMIC_CONFIG_KEY;
                String bitstream = parser.getAttributeValue(null, key);
                if (!bitstream.startsWith(prefix)) {
                    continue;
                }
                while (parser.next() != XmlPullParser.END_DOCUMENT) {
                    if (parser.getEventType() != XmlPullParser.START_TAG) {
                        continue;
                    }
                    if (MediaBitstreams.DYNAMIC_CONFIG_VALUE.equals(parser.getName())) {
                        parser.next();
                        break;
                    }
                }
                String format = parser.getText();
                String[] kvPairs = format.split(",");
                BitstreamPackage curPackage = BitstreamPackage.FULL;
                boolean enforce = false;
                for (String kvPair : kvPairs) {
                    String[] kv = kvPair.split("=");
                    if (MediaBitstreams.DYNAMIC_CONFIG_PACKAGE.equals(kv[0])) {
                        String packageName = kv[1];
                        try {
                            curPackage = BitstreamPackage.valueOf(packageName.toUpperCase());
                        } catch (Exception e) {
                            CLog.w(e);
                        }
                    } else if (MediaBitstreams.DYNAMIC_CONFIG_ENFORCE.equals(kv[0])) {
                        enforce = "true".equals(kv[1]);
                    }
                }
                if (curPackage.compareTo(packageToRun) <= 0) {
                    entries.add(new Object[] {prefix, bitstream, curPackage, packageToRun, enforce});
                }
            }
            return entries;
        } catch (XmlPullParserException | IOException e) {
            CLog.e(e);
            return Collections.emptyList();
        }
    }

    public MediaBitstreamsTest(String prefix, String path, BitstreamPackage pkg, BitstreamPackage packageToRun
            ) {
        this(prefix, path, pkg, packageToRun, false);
    }

    public MediaBitstreamsTest(String prefix, String path, BitstreamPackage pkg, BitstreamPackage packageToRun,
            boolean enforce) {
        mPrefix = prefix;
        mPath = path;
        mPackage = pkg;
        mPackageToRun = packageToRun;
        mEnforce = enforce;
    }

    @Override
    public void setBuild(IBuildInfo buildInfo) {
        // Get the build, this is used to access the APK.
        mBuildHelper = new CompatibilityBuildHelper(buildInfo);
    }

    @Override
    public void setAbi(IAbi abi) {
        mAbi = abi;
    }

    @Override
    public void setDevice(ITestDevice device) {
        mDevice = device;
    }

    @Override
    public ITestDevice getDevice() {
        return mDevice;
    }

    @Override
    public void setTestInformation(TestInformation testInformation) {
        mTestInfo = testInformation;
    }

    @Override
    public TestInformation getTestInformation() {
        return mTestInfo;
    }

    /*
     * Returns true if all necessary media files exist on the device, and false otherwise.
     *
     * This method is exposed for unit testing.
     */
    private boolean bitstreamsExistOnDevice(ITestDevice device)
            throws DeviceNotAvailableException {
        return device.doesFileExist(mDeviceBitstreamsPath)
                && device.isDirectory(mDeviceBitstreamsPath);
    }

    private String getCurrentMethod() {
        return Thread.currentThread().getStackTrace()[2].getMethodName();
    }

    private MetricsReportLog createReport(String methodName) {
        String className = MediaBitstreamsTest.class.getCanonicalName();
        MetricsReportLog report = new MetricsReportLog(
                mBuildHelper.getBuildInfo(), mAbi.getName(),
                String.format("%s#%s", className, methodName),
                MediaBitstreams.K_MODULE + "." + this.getClass().getSimpleName(),
                "media_bitstreams_conformance", true);
        return report;
    }

    /**
     * @param method test method name in the form class#method
     * @param p path to bitstream
     * @param d decoder name
     * @param s test status: unsupported, true, false, crash, or timeout.
     */
    private void addConformanceEntry(String method, String p, String d, String s) {
        MetricsReportLog report = createReport(method);
        report.addValue(MediaBitstreams.KEY_PATH, p, ResultType.NEUTRAL, ResultUnit.NONE);
        report.addValue(MediaBitstreams.KEY_CODEC_NAME, d, ResultType.NEUTRAL, ResultUnit.NONE);
        report.addValue(MediaBitstreams.KEY_STATUS, s, ResultType.NEUTRAL, ResultUnit.NONE);
        report.submit();

        ConformanceEntry ce = new ConformanceEntry(p, d, s);
        mResults.putIfAbsent(p, new ArrayList<>());
        mResults.get(p).add(ce);
    }

    Map<String, String> getArgs() {
        Map<String, String> args = new HashMap<>();
        args.put(MediaBitstreams.OPT_DEBUG_TARGET_DEVICE, Boolean.toString(mDebugTargetDevice));
        args.put(MediaBitstreams.OPT_DEVICE_BITSTREAMS_PATH, mDeviceBitstreamsPath);
        return args;
    }

    private class ProcessBitstreamsFormats extends ReportProcessor {

        @Override
        void setUp(ITestDevice device) throws DeviceNotAvailableException {
            if (mDownloadBitstreams || !bitstreamsExistOnDevice(device)) {
                device.pushDir(mHostBitstreamsPath, mDeviceBitstreamsPath);
            }
        }

        @Override
        Map<String, String> getArgs() {
            return MediaBitstreamsTest.this.getArgs();
        }

        @Override
        void process(ITestDevice device, String reportPath)
                throws DeviceNotAvailableException, IOException {
            File dynamicConfigFile = mBuildHelper.getTestFile(MediaBitstreams.K_MODULE + ".dynamic");
            device.pullFile(reportPath, dynamicConfigFile);
            CLog.i("Pulled bitstreams formats to %s", dynamicConfigFile.getPath());
        }

    }

    private class ProcessBitstreamsValidation extends ReportProcessor {

        Set<String> mBitstreams;
        Deque<String> mProcessedBitstreams = new ArrayDeque<>();
        private final String mMethodName;
        private final String mBitstreamsListTxt = new File(
                mDeviceBitstreamsPath,
                MediaBitstreams.K_BITSTREAMS_LIST_TXT).toString();
        private String mLastCrash;

        ProcessBitstreamsValidation(Set<String> bitstreams, String methodName) {
            mBitstreams = bitstreams;
            mMethodName = methodName;
        }

        private String getBitstreamsListString() {
            OutputStream baos = new ByteArrayOutputStream();
            PrintStream ps = new PrintStream(baos, true);
            try {
                for (String b : mBitstreams) {
                    ps.println(b);
                }
                return baos.toString();
            } finally {
                ps.close();
            }
        }

        private void pushBitstreams(ITestDevice device)
                throws IOException, DeviceNotAvailableException {
            File tmp = null;
            try {
                CLog.i("Pushing %d bitstream(s) from %s to %s",
                        mBitstreams.size(),
                        mHostBitstreamsPath,
                        mDeviceBitstreamsPath);
                tmp = Files.createTempDirectory(null).toFile();
                for (String b : mBitstreams) {
                    String m = MediaBitstreams.getMd5Path(b);
                    for (String f : new String[] {m, b}) {
                        File tmpf = new File(tmp, f);
                        new File(tmpf.getParent()).mkdirs();
                        FileUtil.copyFile(new File(mHostBitstreamsPath, f), tmpf);
                    }
                }
                device.executeShellCommand(String.format("rm -rf %s", mDeviceBitstreamsPath));
                device.pushDir(tmp, mDeviceBitstreamsPath);
                device.pushString(getBitstreamsListString(), mBitstreamsListTxt);
            } finally {
                FileUtil.recursiveDelete(tmp);
            }
        }

        @Override
        void setUp(ITestDevice device) throws DeviceNotAvailableException, IOException {
            pushBitstreams(device);
        }

        @Override
        Map<String, String> getArgs() {
            Map<String, String> args = MediaBitstreamsTest.this.getArgs();
            if (mLastCrash != null) {
                args.put(MediaBitstreams.OPT_LAST_CRASH, mLastCrash);
            }
            return args;
        }

        private void parse(ITestDevice device, String reportPath)
                throws DeviceNotAvailableException {
            String[] lines = getReportLines(device, reportPath);
            mProcessedBitstreams.clear();
            for (int i = 0; i < lines.length;) {

                String path = lines[i++];
                mProcessedBitstreams.add(path);
                String errMsg;

                boolean failedEarly;
                if (i < lines.length) {
                    failedEarly = Boolean.parseBoolean(lines[i++]);
                    errMsg = failedEarly ? lines[i++] : "";
                } else {
                    failedEarly = true;
                    errMsg = MediaBitstreams.K_NATIVE_CRASH;
                    mLastCrash = MediaBitstreams.generateCrashSignature(path, "");
                    mProcessedBitstreams.removeLast();
                }

                if (failedEarly) {
                    addConformanceEntry(mMethodName, path, null, errMsg);
                    continue;
                }

                int n = Integer.parseInt(lines[i++]);
                for (int j = 0; j < n && i < lines.length; j++) {
                    String decoderName = lines[i++];
                    String result;
                    if (i < lines.length) {
                        result = lines[i++];
                    } else {
                        result = MediaBitstreams.K_NATIVE_CRASH;
                        mLastCrash = MediaBitstreams.generateCrashSignature(path, decoderName);
                        mProcessedBitstreams.removeLast();
                    }
                    addConformanceEntry(mMethodName, path, decoderName, result);
                }


            }
        }

        @Override
        void process(ITestDevice device, String reportPath)
                throws DeviceNotAvailableException, IOException {
            parse(device, reportPath);
        }

        @Override
        boolean recover(ITestDevice device, String reportPath)
                throws DeviceNotAvailableException, IOException {
            try {
                parse(device, reportPath);
                mBitstreams.removeAll(mProcessedBitstreams);
                device.pushString(getBitstreamsListString(), mBitstreamsListTxt);
                return true;
            } catch (RuntimeException e) {
                File hostFile = reportPath == null ? null : device.pullFile(reportPath);
                CLog.e("Error parsing report; saving report to %s", hostFile);
                CLog.e(e);
                return false;
            }
        }

    }

    @Ignore
    @Test
    public void testGetBitstreamsFormats() throws DeviceNotAvailableException, IOException {
        ReportProcessor processor = new ProcessBitstreamsFormats();
        processor.processDeviceReport(
                getTestInformation(),
                getDevice(),
                getCurrentMethod(),
                MediaBitstreams.KEY_BITSTREAMS_FORMATS_XML);
    }

    @Test
    public void testBitstreamsConformance() {
        File bitstreamFile = new File(mHostBitstreamsPath, mPath);
        if (!bitstreamFile.exists()) {
            // todo(b/65165250): throw Exception once MediaPreparer can auto-download
            CLog.w(bitstreamFile + " not found; skipping");
            return;
        }

        if (!mResults.containsKey(mPath)) {
            try {
                testBitstreamsConformance(mPrefix);
            } catch (DeviceNotAvailableException | IOException e) {
                String curMethod = getCurrentMethod();
                addConformanceEntry(curMethod, mPath, MediaBitstreams.K_UNAVAILABLE, e.toString());
            }
        }

        if (mEnforce) {
            if (!mResults.containsKey(mPath)) {
                Assert.fail("no results captured for " + mPath);
            }
            List<ConformanceEntry> entries = mResults.get(mPath);
            for (ConformanceEntry ce : entries) {
                if (!"true".equals(ce.mStatus) && !"unsupported".equals(ce.mStatus)) {
                    Assert.fail(ce.toString());
                }
            }
        }

    }

    private void testBitstreamsConformance(String prefix)
            throws DeviceNotAvailableException, IOException {

        ITestDevice device = getDevice();
        SupportedBitstreamsProcessor preparer;
        preparer = new SupportedBitstreamsProcessor(prefix, mDebugTargetDevice);
        preparer.processDeviceReport(
                getTestInformation(),
                device,
                MediaBitstreams.K_TEST_GET_SUPPORTED_BITSTREAMS,
                MediaBitstreams.KEY_SUPPORTED_BITSTREAMS_TXT);
        Collection<Object[]> bitstreams = bitstreams(mPrefix, mPackageToRun);
        Set<String> supportedBitstreams = preparer.getSupportedBitstreams();
        CLog.i("%d supported bitstreams under %s", supportedBitstreams.size(), prefix);

        int n = 0;
        long size = 0;
        long limit = device.getExternalStoreFreeSpace() * mUtilizationRate * 1024 / 100;

        String curMethod = getCurrentMethod();
        Set<String> toPush = new LinkedHashSet<>();
        Iterator<Object[]> iter = bitstreams.iterator();

        for (int i = 0; i < bitstreams.size(); i++) {

            if (n >= mNumBatches) {
                break;
            }

            String p = (String) iter.next()[1];
            Map<String, Boolean> decoderCapabilities;
            decoderCapabilities = preparer.getDecoderCapabilitiesForPath(p);
            if (decoderCapabilities.isEmpty()) {
                addConformanceEntry(
                        curMethod, p,
                        MediaBitstreams.K_UNAVAILABLE,
                        MediaBitstreams.K_UNSUPPORTED);
            }
            for (Entry<String, Boolean> entry : decoderCapabilities.entrySet()) {
                Boolean supported = entry.getValue();
                if (supported) {
                    File bitstreamFile = new File(mHostBitstreamsPath, p);
                    String md5Path = MediaBitstreams.getMd5Path(p);
                    File md5File = new File(mHostBitstreamsPath, md5Path);
                    if (md5File.exists() && bitstreamFile.exists() && toPush.add(p)) {
                        size += md5File.length();
                        size += bitstreamFile.length();
                    }
                } else {
                    String d = entry.getKey();
                    addConformanceEntry(curMethod, p, d, MediaBitstreams.K_UNSUPPORTED);
                }
            }

            if (size > limit || i + 1 == bitstreams.size()) {
                ReportProcessor processor;
                processor = new ProcessBitstreamsValidation(toPush, curMethod);
                processor.processDeviceReport(
                        getTestInformation(),
                        device,
                        curMethod,
                        MediaBitstreams.KEY_BITSTREAMS_VALIDATION_TXT);
                toPush.clear();
                size = 0;
                n++;
            }

        }

    }


}