| /* |
| * Copyright (C) 2021 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package android.videoencodingquality.cts; |
| |
| import static com.android.media.videoquality.bdrate.BdRateMain.verifyBdRate; |
| |
| import android.cts.host.utils.DeviceJUnit4ClassRunnerWithParameters; |
| import android.cts.host.utils.DeviceJUnit4Parameterized; |
| import android.platform.test.annotations.AppModeFull; |
| |
| import android.media.cts.codecdb.CodecDbResultReporter; |
| import android.media.cts.codecdb.VideoEncoderConfig; |
| import android.media.cts.codecdb.VideoEncoderResult; |
| import android.videoencoding.app.proto.VideoEncodingAppResultProto.VideoEncodingAppResult; |
| |
| import com.android.compatibility.common.util.CddTest; |
| import com.android.ddmlib.IDevice; |
| import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; |
| import com.android.ddmlib.testrunner.TestResult.TestStatus; |
| import com.android.tradefed.build.IBuildInfo; |
| import com.android.tradefed.config.Option; |
| import com.android.tradefed.config.OptionClass; |
| import com.android.tradefed.device.DeviceNotAvailableException; |
| import com.android.tradefed.device.ITestDevice; |
| import com.android.tradefed.log.LogUtil; |
| import com.android.tradefed.result.CollectingTestListener; |
| import com.android.tradefed.result.TestDescription; |
| import com.android.tradefed.result.TestResult; |
| import com.android.tradefed.result.TestRunResult; |
| import com.android.tradefed.testtype.IAbi; |
| import com.android.tradefed.testtype.IAbiReceiver; |
| import com.android.tradefed.testtype.IBuildReceiver; |
| import com.android.tradefed.testtype.IDeviceTest; |
| |
| import com.google.common.collect.ImmutableList; |
| |
| import org.json.JSONArray; |
| import org.json.JSONObject; |
| import org.junit.Assert; |
| import org.junit.Assume; |
| import org.junit.Rule; |
| import org.junit.Test; |
| import org.junit.rules.TestName; |
| import org.junit.runner.RunWith; |
| import org.junit.runners.Parameterized; |
| import org.junit.runners.Parameterized.UseParametersRunnerFactory; |
| |
| import java.io.BufferedReader; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileReader; |
| import java.io.FileWriter; |
| import java.io.IOException; |
| import java.nio.charset.StandardCharsets; |
| import java.nio.file.Files; |
| import java.nio.file.Paths; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.concurrent.TimeUnit; |
| import java.util.concurrent.locks.Condition; |
| import java.util.concurrent.locks.Lock; |
| import java.util.concurrent.locks.ReentrantLock; |
| |
| import javax.annotation.Nullable; |
| |
| /** |
| * This class constitutes host-part of video encoding quality test (go/pc14-veq). This test is aimed |
| * towards benchmarking encoders on the target device. |
| * |
| * <p>Video encoding quality test quantifies encoders on the test device by encoding a set of clips |
| * at various configurations. The encoded output is analysed for vmaf and compared against |
| * reference. This entire process is not carried on the device. The host side of the test prepares |
| * the test environment by installing a VideoEncodingApp on the device. It also pushes the test |
| * vectors and test configurations on to the device. The VideoEncodingApp transcodes the input clips |
| * basing on the configurations shared. The host side of the test then pulls output files from the |
| * device and analyses for vmaf. These values are compared against reference using Bjontegaard |
| * metric. |
| */ |
| @AppModeFull(reason = "Instant apps cannot access the SD card") |
| @RunWith(DeviceJUnit4Parameterized.class) |
| @UseParametersRunnerFactory(DeviceJUnit4ClassRunnerWithParameters.RunnerFactory.class) |
| @OptionClass(alias = "pc-veq-test") |
| public class CtsVideoEncodingQualityHostTest implements IDeviceTest, IAbiReceiver, IBuildReceiver { |
| private static final String RES_VER = "2.2"; |
| private static final String RES_URL = |
| "https://dl.google.com/android/xts/cts/hostsidetests/mediapc/videoencodingquality/CtsVideoEncodingQualityHostTestCases-" |
| + RES_VER |
| + ".tar.gz"; |
| |
| // variables related to host-side of the test |
| private static final int MEDIA_PERFORMANCE_CLASS_14 = 34; |
| private static final int MINIMUM_VALID_SDK = 31; |
| // test is not valid before sdk 31, aka Android 12, aka Android S |
| |
| private static final Lock sLock = new ReentrantLock(); |
| private static final Condition sCondition = sLock.newCondition(); |
| private static boolean sIsTestSetUpDone = false; |
| private static boolean sSkipAll = false; |
| private static int sResult = 0; |
| private static String sReason = ""; |
| // install apk, push necessary resources to device to run the test. lock/condition |
| // pair is to keep setupTestEnv() thread safe |
| private static File sHostWorkDir; |
| private static int sMpc; |
| |
| // Variables related to device-side of the test. These need to kept in sync with definitions of |
| // VideoEncodingApp.apk |
| private static final String DEVICE_SIDE_TEST_PACKAGE = "android.videoencoding.app"; |
| private static final String DEVICE_SIDE_TEST_CLASS = |
| "android.videoencoding.app.VideoTranscoderTest"; |
| private static final String VIDEO_ENCODING_APP_RESULT_FILE = "VideoEncodingAppResult.pb"; |
| private static final String RUNNER = "androidx.test.runner.AndroidJUnitRunner"; |
| private static final String TEST_CONFIG_INST_ARGS_KEY = "conf-json"; |
| private static final long DEFAULT_SHELL_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); |
| private static final String TEST_TIMEOUT_INST_ARGS_KEY = "timeout_msec"; |
| private static final long DEFAULT_TEST_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(3); |
| |
| // local variables related to host-side of the test |
| private final String mJsonName; |
| private ITestDevice mDevice; |
| private IBuildInfo mBuildInfo; |
| private IAbi mAbi; |
| |
| @Rule |
| public TestName mTestName = new TestName(); |
| |
| @Override |
| public void setBuild(IBuildInfo buildInfo) { |
| mBuildInfo = buildInfo; |
| } |
| |
| @Override |
| public void setAbi(IAbi abi) { |
| mAbi = abi; |
| } |
| |
| @Option( |
| name = "force-to-run", |
| description = |
| "Force to run the test even if the device is not" |
| + " a right performance class device.") |
| private boolean mForceToRun = true; |
| |
| @Option(name = "skip-avc", description = "Skip avc encoder testing") |
| private boolean mSkipAvc = false; |
| |
| @Option(name = "skip-hevc", description = "Skip hevc encoder testing") |
| private boolean mSkipHevc = false; |
| |
| @Option(name = "skip-p", description = "Skip P only testing") |
| private boolean mSkipP = false; |
| |
| @Option(name = "skip-b", description = "Skip B frame testing") |
| private boolean mSkipB = false; |
| |
| @Option(name = "reset", description = "Start with a fresh directory.") |
| private boolean mReset = false; |
| |
| @Option(name = "quick-check", description = "Run a quick check.") |
| private boolean mQuickCheck = false; |
| |
| public CtsVideoEncodingQualityHostTest( |
| String jsonName, @SuppressWarnings("unused") String testLabel) { |
| mJsonName = jsonName; |
| } |
| |
| private static final List<Object[]> AVC_VBR_B0_PARAMS = |
| Arrays.asList( |
| new Object[][] { |
| { |
| "AVICON-MOBILE-HikeNear-SO01-CRW02-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", |
| "HikeNear_SO01_CRW02_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-RestaurantFar-SI24-CRUW01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", |
| "RestaurantFar_SI24_CRUW01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", |
| "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-SelfieCoupleGarden-SF12-CF01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", |
| "SelfieCoupleGarden_SF12_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-SelfieManStreet-SF07-CF01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", |
| "SelfieManStreet_SF07_CF01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0" |
| } |
| }); |
| |
| private static final List<Object[]> AVC_VBR_B1_PARAMS = |
| Arrays.asList( |
| new Object[][] { |
| { |
| "AVICON-MOBILE-HikeNear-SO01-CRW02-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b1.json", |
| "HikeNear_SO01_CRW02_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b1" |
| }, |
| { |
| "AVICON-MOBILE-RestaurantFar-SI24-CRUW01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b1.json", |
| "RestaurantFar_SI24_CRUW01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b1" |
| }, |
| { |
| "AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b1.json", |
| "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b1" |
| }, |
| { |
| "AVICON-MOBILE-SelfieCoupleGarden-SF12-CF01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b1.json", |
| "SelfieCoupleGarden_SF12_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b1" |
| }, |
| { |
| "AVICON-MOBILE-SelfieManStreet-SF07-CF01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b1.json", |
| "SelfieManStreet_SF07_CF01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b1" |
| } |
| }); |
| |
| private static final List<Object[]> HEVC_VBR_B0_PARAMS = |
| Arrays.asList( |
| new Object[][] { |
| { |
| "AVICON-MOBILE-HikeNear-SO01-CRW02-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", |
| "HikeNear_SO01_CRW02_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-RestaurantFar-SI24-CRUW01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", |
| "RestaurantFar_SI24_CRUW01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", |
| "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-SelfieCoupleGarden-SF12-CF01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", |
| "SelfieCoupleGarden_SF12_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-SelfieManStreet-SF07-CF01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", |
| "SelfieManStreet_SF07_CF01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0" |
| } |
| }); |
| |
| private static final List<Object[]> HEVC_VBR_B1_PARAMS = |
| Arrays.asList( |
| new Object[][] { |
| { |
| "AVICON-MOBILE-HikeNear-SO01-CRW02-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b1.json", |
| "HikeNear_SO01_CRW02_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b1" |
| }, |
| { |
| "AVICON-MOBILE-RestaurantFar-SI24-CRUW01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b1.json", |
| "RestaurantFar_SI24_CRUW01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b1" |
| }, |
| { |
| "AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b1.json", |
| "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b1" |
| }, |
| { |
| "AVICON-MOBILE-SelfieCoupleGarden-SF12-CF01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b1.json", |
| "SelfieCoupleGarden_SF12_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b1" |
| }, |
| { |
| "AVICON-MOBILE-SelfieManStreet-SF07-CF01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b1.json", |
| "SelfieManStreet_SF07_CF01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b1" |
| } |
| }); |
| |
| private static final List<Object[]> QUICK_RUN_PARAMS = |
| Arrays.asList( |
| new Object[][] { |
| { |
| "AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", |
| "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", |
| "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-SelfieCoupleGarden-SF12-CF01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", |
| "SelfieCoupleGarden_SF12_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0" |
| }, |
| { |
| "AVICON-MOBILE-SelfieCoupleGarden-SF12-CF01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", |
| "SelfieCoupleGarden_SF12_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0" |
| } |
| }); |
| |
| @Parameterized.Parameters(name = "{index}_{1}") |
| public static List<Object[]> input() { |
| final List<Object[]> args = new ArrayList<>(); |
| args.addAll(AVC_VBR_B0_PARAMS); |
| args.addAll(getRescaledParams(AVC_VBR_B0_PARAMS, "_rescaled_360p")); |
| args.addAll(getRescaledParams(AVC_VBR_B0_PARAMS, "_rescaled_540p")); |
| args.addAll(getRescaledParams(AVC_VBR_B0_PARAMS, "_rescaled_720p")); |
| |
| args.addAll(AVC_VBR_B1_PARAMS); |
| args.addAll(getRescaledParams(AVC_VBR_B1_PARAMS, "_rescaled_360p")); |
| args.addAll(getRescaledParams(AVC_VBR_B1_PARAMS, "_rescaled_540p")); |
| args.addAll(getRescaledParams(AVC_VBR_B1_PARAMS, "_rescaled_720p")); |
| |
| args.addAll(HEVC_VBR_B0_PARAMS); |
| args.addAll(getRescaledParams(HEVC_VBR_B0_PARAMS, "_rescaled_360p")); |
| args.addAll(getRescaledParams(HEVC_VBR_B0_PARAMS, "_rescaled_540p")); |
| args.addAll(getRescaledParams(HEVC_VBR_B0_PARAMS, "_rescaled_720p")); |
| |
| args.addAll(HEVC_VBR_B1_PARAMS); |
| args.addAll(getRescaledParams(HEVC_VBR_B1_PARAMS, "_rescaled_360p")); |
| args.addAll(getRescaledParams(HEVC_VBR_B1_PARAMS, "_rescaled_540p")); |
| args.addAll(getRescaledParams(HEVC_VBR_B1_PARAMS, "_rescaled_720p")); |
| return args; |
| } |
| |
| private static List<Object[]> getRescaledParams(List<Object[]> params, String suffix) { |
| List<Object[]> rescaledParams = new ArrayList<>(); |
| for (Object[] param : params) { |
| String jsonName = (String) param[0]; |
| String testName = (String) param[1]; |
| String rescaledJsonName = jsonName.replace(".json", suffix + ".json"); |
| String rescaledTestName = testName + suffix; |
| rescaledParams.add(new Object[] {rescaledJsonName, rescaledTestName}); |
| } |
| return rescaledParams; |
| } |
| |
| @Override |
| public void setDevice(ITestDevice device) { |
| mDevice = device; |
| } |
| |
| @Override |
| public ITestDevice getDevice() { |
| return mDevice; |
| } |
| |
| /** Sets up the necessary environment for the video encoding quality test. */ |
| public void setupTestEnv() throws Exception { |
| String sdkAsString = getDevice().getProperty("ro.build.version.sdk"); |
| int sdk = sdkAsString == null ? 0 : Integer.parseInt(sdkAsString); |
| if (sdk < MINIMUM_VALID_SDK) { |
| sSkipAll = true; |
| sReason = "Test requires sdk >= " + MINIMUM_VALID_SDK + " test device has sdk = " + sdk; |
| sIsTestSetUpDone = true; |
| return; |
| } |
| |
| String pcAsString = getDevice().getProperty("ro.odm.build.media_performance_class"); |
| sMpc = pcAsString == null ? 0 : Integer.parseInt(pcAsString); |
| if (!(mForceToRun |
| || sMpc >= MEDIA_PERFORMANCE_CLASS_14 |
| || (sMpc == 0 && sdk >= 34 /* Build.VERSION_CODES.UPSIDE_DOWN_CAKE */))) { |
| sSkipAll = true; |
| sReason = |
| "Performance class advertised by the test device is less than " |
| + MEDIA_PERFORMANCE_CLASS_14; |
| sIsTestSetUpDone = true; |
| return; |
| } |
| |
| if (!getDevice().isPackageInstalled(DEVICE_SIDE_TEST_PACKAGE)) { |
| sResult = 1; |
| sReason = "Failed to install package on device : " + DEVICE_SIDE_TEST_PACKAGE; |
| sIsTestSetUpDone = true; |
| return; |
| } |
| |
| // set up host-side working directory |
| String tmpBase = System.getProperty("java.io.tmpdir"); |
| String dirName = "CtsVideoEncodingQualityHostTest_" + RES_VER; |
| String tmpDir = tmpBase + "/" + dirName; |
| LogUtil.CLog.i("tmpBase= " + tmpBase + " tmpDir =" + tmpDir); |
| sHostWorkDir = new File(tmpDir); |
| if (mReset || !sHostWorkDir.isDirectory()) { |
| File cwd = new File("."); |
| sResult = runCommand("rm -rf " + tmpDir, cwd); |
| if (0 != sResult) { |
| sReason = "Failed to remove file / dir " + tmpDir; |
| sIsTestSetUpDone = true; |
| return; |
| } |
| } |
| if (!sHostWorkDir.exists() && !sHostWorkDir.mkdirs()) { |
| sResult = 1; |
| sReason = "Failed to create directory : " + sHostWorkDir.getAbsolutePath(); |
| sIsTestSetUpDone = true; |
| return; |
| } |
| |
| // Clean up output folders before starting the test |
| sResult = runCommand("rm -rf " + "output_*", sHostWorkDir); |
| if (0 != sResult) { |
| sReason = "Failed to remove output_* dirs from " + tmpDir; |
| sIsTestSetUpDone = true; |
| return; |
| } |
| |
| // Download the test suite tar file. |
| downloadFile(RES_URL, sHostWorkDir); |
| |
| // Unpack the test suite tar file. |
| String fileName = RES_URL.substring(RES_URL.lastIndexOf('/') + 1); |
| sResult = runCommand("tar xvzf " + fileName, sHostWorkDir); |
| if (0 != sResult) { |
| sReason = "Failed to untar " + fileName; |
| sIsTestSetUpDone = true; |
| return; |
| } |
| |
| // Run download_ffmpeg script |
| sResult = runCommand("./bin/download_ffmpeg.sh", sHostWorkDir); |
| if (0 != sResult) { |
| sReason = "Failed to run " + sHostWorkDir.getPath() + "/bin/download_ffmpeg.sh"; |
| sIsTestSetUpDone = true; |
| return; |
| } |
| |
| sIsTestSetUpDone = true; |
| } |
| |
| public static boolean containsJson(String jsonName, List<Object[]> params) { |
| for (Object[] param : params) { |
| if (param[0].equals(jsonName)) { |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| /** Verify the video encoding quality requirements for the performance class 14 devices. */ |
| @CddTest(requirements = {"2.2.7.1/5.8/H-1-1"}) |
| @Test |
| public void testEncoding() throws Exception { |
| Assume.assumeFalse( |
| "Skipping due to quick run mode", |
| mQuickCheck && !containsJson(mJsonName, QUICK_RUN_PARAMS)); |
| Assume.assumeFalse( |
| "Skipping avc encoder tests", |
| mSkipAvc |
| && (containsJson(mJsonName, AVC_VBR_B0_PARAMS) |
| || containsJson(mJsonName, AVC_VBR_B1_PARAMS))); |
| Assume.assumeFalse( |
| "Skipping hevc encoder tests", |
| mSkipHevc |
| && (containsJson(mJsonName, HEVC_VBR_B0_PARAMS) |
| || containsJson(mJsonName, HEVC_VBR_B1_PARAMS))); |
| Assume.assumeFalse( |
| "Skipping b-frame tests", |
| mSkipB |
| && (containsJson(mJsonName, AVC_VBR_B1_PARAMS) |
| || containsJson(mJsonName, HEVC_VBR_B1_PARAMS))); |
| Assume.assumeFalse( |
| "Skipping non b-frame tests", |
| mSkipP |
| && (containsJson(mJsonName, AVC_VBR_B0_PARAMS) |
| || containsJson(mJsonName, HEVC_VBR_B0_PARAMS))); |
| |
| // set up test environment |
| sLock.lock(); |
| try { |
| if (!sIsTestSetUpDone) setupTestEnv(); |
| sCondition.signalAll(); |
| } finally { |
| sLock.unlock(); |
| } |
| Assume.assumeFalse(sReason, sSkipAll); |
| Assert.assertEquals(sReason, 0, sResult); |
| |
| // Push input files to device |
| String mntPoint = getDevice().getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); |
| String deviceInDir = mntPoint + "/veq/input/"; |
| String deviceJsonDir = deviceInDir + "json/"; |
| String deviceSamplesDir = deviceInDir + "samples/"; |
| Assert.assertNotNull( |
| "Failed to create directory " + deviceJsonDir + " on device ", |
| getDevice().executeAdbCommand("shell", "mkdir", "-p", deviceJsonDir)); |
| Assert.assertNotNull( |
| "Failed to create directory " + deviceSamplesDir + " on device ", |
| getDevice().executeAdbCommand("shell", "mkdir", "-p", deviceSamplesDir)); |
| String jsonPath = sHostWorkDir.getPath() + "/json/" + mJsonName; |
| String devJsonPath = deviceJsonDir + mJsonName; |
| if (!getDevice().doesFileExist(devJsonPath)) { |
| Assert.assertTrue( |
| "Failed to push json file to device ", |
| getDevice().pushFile(new File(jsonPath), devJsonPath)); |
| } |
| |
| // Parse json file |
| String jsonString = |
| new String(Files.readAllBytes(Paths.get(jsonPath)), StandardCharsets.UTF_8); |
| JSONArray jsonArray = new JSONArray(jsonString); |
| JSONObject obj = jsonArray.getJSONObject(0); |
| String refFileName = obj.getString("RefFileName"); |
| int fps = obj.getInt("FrameRate"); |
| int frameCount = obj.getInt("FrameCount"); |
| int clipDuration = frameCount / fps; |
| boolean mandatory = obj.optBoolean("Mandatory", true); |
| boolean rescale = obj.optBoolean("Rescale", false); |
| |
| String refFilePath = sHostWorkDir.getPath() + "/samples/" + refFileName; |
| String devRefFilePath = deviceSamplesDir + refFileName; |
| if (!getDevice().doesFileExist(devRefFilePath)) { |
| Assert.assertTrue( |
| "Failed to push mp4 file to device ", |
| getDevice().pushFile(new File(refFilePath), devRefFilePath)); |
| } |
| |
| // transcode input |
| try { |
| runDeviceTests(DEVICE_SIDE_TEST_PACKAGE, DEVICE_SIDE_TEST_CLASS, "testTranscode"); |
| } catch (AssertionError e) { |
| if (!mandatory) { |
| Assume.assumeNoException("Optional Test: failed to transcode", e); |
| } else if (sMpc >= MEDIA_PERFORMANCE_CLASS_14) { |
| Assert.fail(e.getMessage()); |
| } else { |
| Assume.assumeTrue(e.getMessage(), false); |
| } |
| } |
| |
| // copy the encoded output from the device to the host. |
| String outDir = "output_" + mJsonName.substring(0, mJsonName.indexOf('.')); |
| File outHostPath = new File(sHostWorkDir, outDir); |
| if (!outHostPath.isDirectory()) { |
| Assert.assertTrue( |
| "Failed to create directory : " + outHostPath.getAbsolutePath(), |
| outHostPath.mkdirs()); |
| } |
| String outDevPath = mntPoint + "/veq/output/" + outDir; |
| Assert.assertTrue( |
| "Failed to pull mp4 files from " + outDevPath + " to " + outHostPath.getPath(), |
| getDevice().pullDir(outDevPath, outHostPath)); |
| getDevice().deleteFile(outDevPath); |
| |
| // Compute Vmaf |
| try { |
| JSONArray codecConfigs = obj.getJSONArray("CodecConfigs"); |
| int th = Runtime.getRuntime().availableProcessors() / 2; |
| th = Math.min(Math.max(1, th), 8); |
| String filter; |
| if (rescale) { |
| filter = |
| "[0:v]setpts=PTS-STARTPTS[ref_pts];[1:v]setpts=PTS-STARTPTS[dist_pts];" |
| + "[dist_pts][ref_pts]scale2ref=flags=bicubic[dist_scaled][ref_out];" |
| + "[dist_scaled][ref_out]libvmaf=feature=name=psnr:model=version=vmaf_v0.6.1:n_threads=" |
| + th; |
| } else { |
| filter = |
| "[0:v]setpts=PTS-STARTPTS[reference];[1:v]setpts=PTS-STARTPTS[distorted];" |
| + "[distorted][reference]libvmaf=feature=name=psnr:model=version" |
| + "=vmaf_v0.6.1:n_threads=" |
| + th; |
| } |
| |
| // First loop to kick off VMAF calculations in parallel |
| List<Process> vmafProcesses = new ArrayList<>(); |
| for (int i = 0; i < codecConfigs.length(); i++) { |
| JSONObject codecConfig = codecConfigs.getJSONObject(i); |
| String outputName = codecConfig.getString("EncodedFileName"); |
| outputName = outputName.substring(0, outputName.lastIndexOf(".")); |
| String outputVmafPath = outDir + "/" + outputName + ".txt"; |
| String cmd = "ffmpeg"; |
| cmd += " -hide_banner"; |
| cmd += " -r " + fps; |
| cmd += " -i " + "samples/" + refFileName + " -an"; // reference video |
| cmd += " -r " + fps; |
| cmd += " -i " + outDir + "/" + outputName + ".mp4" + " -an"; // distorted video |
| cmd += " -filter_complex " + "\"" + filter + "\""; |
| cmd += " -f null -"; |
| cmd += " > " + outputVmafPath + " 2>&1"; |
| LogUtil.CLog.i("ffmpeg command : " + cmd); |
| ProcessBuilder pb = |
| new ProcessBuilder("/bin/sh", "-c", cmd).directory(sHostWorkDir); |
| updateEnv(pb); |
| vmafProcesses.add(pb.start()); |
| } |
| |
| // Wait for all VMAF computations to complete and check results |
| for (Process p : vmafProcesses) { |
| try { |
| p.waitFor(); |
| } catch (InterruptedException e) { |
| throw new AssertionError("VMAF computation interrupted", e); |
| } |
| int result = p.exitValue(); |
| if (!mandatory && result != 0) { |
| Assume.assumeTrue( |
| "!mandatory test encountered error during vmaf computation.", false); |
| } else if (sMpc >= MEDIA_PERFORMANCE_CLASS_14) { |
| Assert.assertEquals("Encountered error during vmaf computation.", 0, result); |
| } else { |
| Assume.assumeTrue( |
| "Encountered error during vmaf computation but the " |
| + "test device does not advertise performance class", |
| result == 0); |
| } |
| } |
| |
| // Second loop to read results and write to all_vmafs.txt |
| ImmutableList.Builder<Long> targetBitratesBuilder = ImmutableList.builder(); |
| ImmutableList.Builder<Double> actualBitratesBuilder = ImmutableList.builder(); |
| ImmutableList.Builder<Double> vmafScoresBuilder = ImmutableList.builder(); |
| ImmutableList.Builder<Double> transcodingFpsBuilder = ImmutableList.builder(); |
| ImmutableList.Builder<Double> encodeFpsBuilder = ImmutableList.builder(); |
| |
| Map<String, Double> transcodingFpsMap = new HashMap<>(); |
| Map<String, Double> encodeFpsMap = new HashMap<>(); |
| File fpsFile = new File(outHostPath, VIDEO_ENCODING_APP_RESULT_FILE); |
| if (fpsFile.exists()) { |
| try (FileInputStream fis = new FileInputStream(fpsFile)) { |
| VideoEncodingAppResult result = VideoEncodingAppResult.parseFrom(fis); |
| transcodingFpsMap = result.getTranscodingFpsMapMap(); |
| encodeFpsMap = result.getEncodeFpsMapMap(); |
| } |
| } |
| |
| try (FileWriter writer = |
| new FileWriter(outHostPath.getPath() + "/" + "all_vmafs.txt")) { |
| for (int i = 0; i < codecConfigs.length(); i++) { |
| JSONObject codecConfig = codecConfigs.getJSONObject(i); |
| String outputName = codecConfig.getString("EncodedFileName"); |
| double fpsVal = transcodingFpsMap.getOrDefault(outputName, 0.0); |
| double encodeFpsVal = encodeFpsMap.getOrDefault(outputName, 0.0); |
| outputName = outputName.substring(0, outputName.lastIndexOf(".")); |
| String outputVmafPath = outDir + "/" + outputName + ".txt"; |
| |
| double vmafValue = 0; |
| String vmafLine = ""; |
| try (BufferedReader reader = |
| new BufferedReader( |
| new FileReader( |
| sHostWorkDir.getPath() + "/" + outputVmafPath))) { |
| String token = "VMAF score: "; |
| String line; |
| while ((line = reader.readLine()) != null) { |
| if (line.contains(token)) { |
| line = line.substring(line.indexOf(token)); |
| String vmafStr = line.substring(token.length()).trim(); |
| vmafValue = Double.parseDouble(vmafStr); |
| vmafLine = "VMAF score = " + vmafValue; |
| LogUtil.CLog.i(vmafLine); |
| break; |
| } |
| } |
| } |
| |
| writer.write(vmafLine + "\n"); |
| writer.write("Y4M file = " + refFileName + "\n"); |
| writer.write("MP4 file = " + refFileName + "\n"); |
| File file = new File(outHostPath + "/" + outputName + ".mp4"); |
| Assert.assertTrue("output file from device missing", file.exists()); |
| long fileSize = file.length(); |
| writer.write("Filesize = " + fileSize + "\n"); |
| writer.write("FPS = " + fps + "\n"); |
| writer.write("FRAME_COUNT = " + frameCount + "\n"); |
| writer.write("CLIP_DURATION = " + clipDuration + "\n"); |
| long totalBits = fileSize * 8; |
| long totalBits_kbps = totalBits / 1000; |
| long bitrate_kbps = totalBits_kbps / clipDuration; |
| writer.write("Bitrate kbps = " + bitrate_kbps + "\n"); |
| writer.write("Transcoding FPS = " + fpsVal + "\n"); |
| writer.write("Encode FPS = " + encodeFpsVal + "\n"); |
| |
| targetBitratesBuilder.add(codecConfig.optLong("BitRate", 0)); |
| actualBitratesBuilder.add((double) bitrate_kbps); |
| vmafScoresBuilder.add(vmafValue); |
| transcodingFpsBuilder.add(fpsVal); |
| encodeFpsBuilder.add(encodeFpsVal); |
| } |
| } |
| |
| // Report to CodecDB |
| if (mBuildInfo != null && mAbi != null) { |
| String strippedRefName = refFileName.replaceFirst("^AVICON-MOBILE-", "") |
| .replaceFirst("\\.[^.]+$", ""); |
| String codec = obj.optString("TestMediaType", "video/avc"); |
| int width = obj.optInt("Width", 0); |
| int height = obj.optInt("Height", 0); |
| if (rescale) { |
| width = obj.optInt("RescaleWidth", width); |
| height = obj.optInt("RescaleHeight", height); |
| } |
| |
| JSONObject firstConfig = codecConfigs.getJSONObject(0); |
| int maxBFrames = firstConfig.optInt("MaxBFrames", 0); |
| int profile = firstConfig.optInt("Profile", 0); |
| int level = firstConfig.optInt("Level", 0); |
| |
| VideoEncoderConfig config = VideoEncoderConfig.builder() |
| .setReference(strippedRefName) |
| .addBitrates(targetBitratesBuilder.build()) |
| .setCodec(codec) |
| .setMaxBFrames(maxBFrames) |
| .setProfile(profile) |
| .setLevel(level) |
| .setOutputWidth(width) |
| .setOutputHeight(height) |
| .build(); |
| |
| VideoEncoderResult resultData = VideoEncoderResult.builder() |
| .addBitrates(actualBitratesBuilder.build()) |
| .addVmafs(vmafScoresBuilder.build()) |
| .addTranscodingFpsList(transcodingFpsBuilder.build()) |
| .addEncodeFpsList(encodeFpsBuilder.build()) |
| .build(); |
| |
| CodecDbResultReporter reporter = CodecDbResultReporter.create( |
| mBuildInfo, mAbi.getName(), |
| this.getClass().getName() + "#" + mTestName.getMethodName(), RES_VER); |
| reporter.reportResult(config, resultData); |
| } |
| } catch (IOException e) { |
| throw new AssertionError("Unexpected IOException", e); |
| } |
| |
| if (!obj.has("RefRate") || !obj.has("RefVmaf")) { |
| String msg = "RefRate or RefVmaf missing, skipping bd rate verification."; |
| LogUtil.CLog.i(msg); |
| if (mandatory && sMpc >= MEDIA_PERFORMANCE_CLASS_14) { |
| Assert.fail(msg); |
| } |
| return; |
| } |
| |
| // bd rate verification |
| String refJsonFilePath = sHostWorkDir.getPath() + "/json/" + mJsonName; |
| String testVmafFilePath = sHostWorkDir.getPath() + "/" + outDir + "/" + "all_vmafs.txt"; |
| String resultFilePath = sHostWorkDir.getPath() + "/" + outDir + "/result.txt"; |
| int result = verifyBdRate(refJsonFilePath, testVmafFilePath, resultFilePath); |
| if (!mandatory && result != 0) { |
| Assume.assumeTrue("Optional Test: BD-RATE validation failed.", false); |
| } else if (sMpc >= MEDIA_PERFORMANCE_CLASS_14) { |
| Assert.assertEquals("bd rate validation failed.", 0, result); |
| } else { |
| Assume.assumeTrue( |
| "bd rate validation failed but the test device does not " |
| + "advertise performance class", |
| result == 0); |
| } |
| LogUtil.CLog.i("Finished executing the process."); |
| } |
| |
| private int runCommand(String command, File dir) throws IOException, InterruptedException { |
| ProcessBuilder pb = |
| new ProcessBuilder("/bin/sh", "-c", command) |
| .directory(dir) |
| .redirectErrorStream(true) |
| .redirectOutput(ProcessBuilder.Redirect.INHERIT); |
| |
| updateEnv(pb); |
| |
| Process p = pb.start(); |
| |
| return p.waitFor(); |
| } |
| |
| private void updateEnv(ProcessBuilder pb) { |
| if (sHostWorkDir != null) { |
| Map<String, String> env = pb.environment(); |
| String binPath = new File(sHostWorkDir, "bin").getAbsolutePath(); |
| String path = env.get("PATH"); |
| if (path != null && !path.isEmpty()) { |
| env.put("PATH", binPath + File.pathSeparator + path); |
| } else { |
| env.put("PATH", binPath); |
| } |
| } |
| } |
| |
| // Download the indicated file (within the base_url folder) to our desired destination |
| // simple caching -- if file exists, we do not re-download |
| private void downloadFile(String url, File destDir) { |
| String fileName = url.substring(RES_URL.lastIndexOf('/') + 1); |
| File destination = new File(destDir, fileName); |
| |
| // save bandwidth, also allows a user to manually preload files |
| LogUtil.CLog.i("Do we already have a copy of file " + destination.getPath()); |
| if (destination.isFile()) { |
| LogUtil.CLog.i("Skipping re-download of file " + destination.getPath()); |
| return; |
| } |
| |
| String cmd = "wget -O " + destination.getPath() + " " + url; |
| LogUtil.CLog.i("wget_cmd = " + cmd); |
| |
| int result = 0; |
| try { |
| result = runCommand(cmd, destDir); |
| } catch (IOException e) { |
| result = -2; |
| } catch (InterruptedException e) { |
| result = -3; |
| } |
| Assert.assertEquals("download file failed.\n", 0, result); |
| } |
| |
| private void runDeviceTests( |
| String pkgName, @Nullable String testClassName, @Nullable String testMethodName) |
| throws DeviceNotAvailableException { |
| RemoteAndroidTestRunner testRunner = getTestRunner(pkgName, testClassName, testMethodName); |
| CollectingTestListener listener = new CollectingTestListener(); |
| Assert.assertTrue(getDevice().runInstrumentationTests(testRunner, listener)); |
| assertTestsPassed(listener.getCurrentRunResults()); |
| } |
| |
| private RemoteAndroidTestRunner getTestRunner( |
| String pkgName, String testClassName, String testMethodName) { |
| if (testClassName != null && testClassName.startsWith(".")) { |
| testClassName = pkgName + testClassName; |
| } |
| RemoteAndroidTestRunner testRunner = |
| new RemoteAndroidTestRunner(pkgName, RUNNER, getDevice().getIDevice()); |
| testRunner.setMaxTimeToOutputResponse(DEFAULT_SHELL_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); |
| testRunner.addInstrumentationArg( |
| TEST_TIMEOUT_INST_ARGS_KEY, Long.toString(DEFAULT_TEST_TIMEOUT_MILLIS)); |
| testRunner.addInstrumentationArg(TEST_CONFIG_INST_ARGS_KEY, mJsonName); |
| if (testClassName != null && testMethodName != null) { |
| testRunner.setMethodName(testClassName, testMethodName); |
| } else if (testClassName != null) { |
| testRunner.setClassName(testClassName); |
| } |
| return testRunner; |
| } |
| |
| private void assertTestsPassed(TestRunResult testRunResult) { |
| if (testRunResult.isRunFailure()) { |
| throw new AssertionError( |
| "Failed to successfully run device tests for " |
| + testRunResult.getName() |
| + ": " |
| + testRunResult.getRunFailureMessage()); |
| } |
| if (testRunResult.getNumTests() != testRunResult.getPassedTests().size()) { |
| for (Map.Entry<TestDescription, TestResult> resultEntry : |
| testRunResult.getTestResults().entrySet()) { |
| if (resultEntry.getValue().getStatus().equals(TestStatus.FAILURE)) { |
| StringBuilder errorBuilder = new StringBuilder("On-device tests failed:\n"); |
| errorBuilder.append(resultEntry.getKey().toString()); |
| errorBuilder.append(":\n"); |
| errorBuilder.append(resultEntry.getValue().getStackTrace()); |
| throw new AssertionError(errorBuilder.toString()); |
| } |
| if (resultEntry.getValue().getStatus().equals(TestStatus.ASSUMPTION_FAILURE)) { |
| StringBuilder errorBuilder = |
| new StringBuilder("On-device tests assumption failed:\n"); |
| errorBuilder.append(resultEntry.getKey().toString()); |
| errorBuilder.append(":\n"); |
| errorBuilder.append(resultEntry.getValue().getStackTrace()); |
| throw new AssertionError(errorBuilder.toString()); |
| } |
| } |
| } |
| } |
| } |
