| // Copyright (C) 2022 The Android Open Source Project |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "android/emulation/MediaVideoToolBoxHevcVideoHelper.h" |
| #include "host-common/MediaVideoToolBoxUtils.h" |
| #include "host-common/YuvConverter.h" |
| #include "android/utils/debug.h" |
| |
| #define MEDIA_VTB_DEBUG 0 |
| |
| #if MEDIA_VTB_DEBUG |
| #define VTB_DPRINT(fmt, ...) \ |
| fprintf(stderr, "media-vtb-hevc-video-helper: %s:%d " fmt "\n", __func__, \ |
| __LINE__, ##__VA_ARGS__); |
| #else |
| #define VTB_DPRINT(fmt, ...) |
| #endif |
| |
| namespace android { |
| namespace emulation { |
| |
| using TextureFrame = MediaTexturePool::TextureFrame; |
| using FrameInfo = MediaSnapshotState::FrameInfo; |
| using ColorAspects = MediaSnapshotState::ColorAspects; |
| using HevcNaluType = HevcNaluParser::HevcNaluType; |
| |
| MediaVideoToolBoxHevcVideoHelper::MediaVideoToolBoxHevcVideoHelper( |
| int w, |
| int h, |
| OutputTreatmentMode oMode, |
| FrameStorageMode fMode) |
| : mOutputWidth(w), |
| mOutputHeight(h), |
| mUseGpuTexture(fMode == FrameStorageMode::USE_GPU_TEXTURE) { |
| mIgnoreDecoderOutput = (oMode == OutputTreatmentMode::IGNORE_RESULT); |
| } |
| |
| MediaVideoToolBoxHevcVideoHelper::~MediaVideoToolBoxHevcVideoHelper() { |
| deInit(); |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::deInit() { |
| VTB_DPRINT("deInit calling"); |
| |
| resetDecoderSession(); |
| |
| resetFormatDesc(); |
| |
| mVtbReady = false; |
| |
| mFfmpegVideoHelper.reset(); |
| |
| mSavedDecodedFrames.clear(); |
| } |
| |
| bool MediaVideoToolBoxHevcVideoHelper::init() { |
| VTB_DPRINT("init calling"); |
| mVtbReady = false; |
| return true; |
| } |
| |
| static void dumpBytes(const uint8_t* img, size_t szBytes, bool all = false) { |
| printf("data="); |
| size_t numBytes = szBytes; |
| if (!all) { |
| numBytes = 32; |
| } |
| |
| for (size_t i = 0; i < (numBytes > szBytes ? szBytes : numBytes); ++i) { |
| if (i % 8 == 0) { |
| printf("\n"); |
| } |
| printf("0x%02x ", img[i]); |
| } |
| printf("\n"); |
| |
| fflush(stdout); |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::extractFrameInfo() { |
| // at this moment, ffmpeg should have decoded the first frame to obtain |
| // w/h/colorspace info and number of b frame buffers |
| |
| mFfmpegVideoHelper->flush(); |
| MediaSnapshotState::FrameInfo frame; |
| bool success = mFfmpegVideoHelper->receiveFrame(&frame); |
| if (success) { |
| mColorAspects = frame.color; |
| mVtbBufferSize = mFfmpegVideoHelper->frameReorderBufferSize(); |
| VTB_DPRINT("vtb buffer size is %d", mVtbBufferSize); |
| } |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::decode(const uint8_t* frame, |
| size_t szBytes, |
| uint64_t inputPts) { |
| VTB_DPRINT("%s(frame=%p, sz=%zu)", __func__, frame, szBytes); |
| |
| ++mTotalFrames; |
| const bool parseOk = parseInputFrames(frame, szBytes); |
| if (!parseOk) { |
| // cannot parse, probably cannot decode either |
| // just fail |
| VTB_DPRINT("Failed to parse frame=%p, sz=%zu, give up.", frame, |
| szBytes); |
| mIsGood = false; |
| return; |
| } |
| |
| // has to go in the FIFO order |
| for (int i = 0; i < mInputFrames.size(); ++i) { |
| InputFrame& f = mInputFrames[i]; |
| switch (f.type) { |
| case HevcNaluType::SPS: |
| mSPS.assign(f.data, f.data + f.size); |
| VTB_DPRINT("this is an SPS frame"); |
| dumpBytes(mSPS.data(), mSPS.size()); |
| mVtbReady = false; |
| { // create ffmpeg decoder with only 1 thread to avoid frame |
| // delay |
| constexpr int numthreads = 1; |
| mFfmpegVideoHelper.reset( |
| new MediaFfmpegVideoHelper(265, numthreads)); |
| mFfmpegVideoHelper->init(); |
| } |
| break; |
| case HevcNaluType::VPS: |
| VTB_DPRINT("this is an VPS frame"); |
| mVPS.assign(f.data, f.data + f.size); |
| dumpBytes(mVPS.data(), mVPS.size()); |
| mVtbReady = false; |
| break; |
| case HevcNaluType::PPS: |
| VTB_DPRINT("this is an PPS frame"); |
| mPPS.assign(f.data, f.data + f.size); |
| dumpBytes(mPPS.data(), mPPS.size()); |
| mVtbReady = false; |
| break; |
| case HevcNaluType::SEI_PREFIX: |
| case HevcNaluType::SEI_SUFFIX: |
| VTB_DPRINT("this is SEI frame"); |
| break; |
| case HevcNaluType::IDR_W_RADL: |
| case HevcNaluType::CRA_NUT: |
| case HevcNaluType::IDR_N_LP: |
| VTB_DPRINT("this is an IDR frame"); |
| if (mFfmpegVideoHelper) { |
| mFfmpegVideoHelper->decode(frame, szBytes, inputPts); |
| extractFrameInfo(); |
| mFfmpegVideoHelper.reset(); |
| } |
| if (!mVtbReady) { |
| createCMFormatDescription(); |
| Boolean needNewSession = |
| (VTDecompressionSessionCanAcceptFormatDescription( |
| mDecoderSession, mCmFmtDesc) == false); |
| if (needNewSession) { |
| resetDecoderSession(); |
| } |
| if (!mDecoderSession) { |
| // all the previously decoded frames need to be |
| // retrieved |
| flush(); |
| recreateDecompressionSession(); |
| } |
| if (mIsGood) { |
| mVtbReady = true; |
| } |
| } |
| handleIDRFrame(f.data, f.size, inputPts); |
| break; |
| default: |
| VTB_DPRINT("warning Support for nalu_type=%u not implemented", |
| (uint8_t)f.type); |
| // cannot handle such video, give up so we can fall |
| // back to ffmpeg |
| handleIDRFrame(f.data, f.size, inputPts); |
| break; |
| // mIsGood = false; |
| } |
| } |
| |
| if (mFfmpegVideoHelper) { |
| // we have not reached idr frame yet, keep decoding |
| mFfmpegVideoHelper->decode(frame, szBytes, inputPts); |
| } |
| |
| mInputFrames.clear(); |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::flush() { |
| VTB_DPRINT("started flushing"); |
| for (auto& iter : mVtbBufferMap) { |
| mSavedDecodedFrames.push_back(std::move(iter.second)); |
| } |
| mVtbBufferMap.clear(); |
| VTB_DPRINT("done one flushing"); |
| } |
| |
| //------------------------ private helper functions |
| //----------------------------------------- |
| |
| void MediaVideoToolBoxHevcVideoHelper::handleIDRFrame(const uint8_t* ptr, |
| size_t szBytes, |
| uint64_t pts) { |
| uint8_t* fptr = const_cast<uint8_t*>(ptr); |
| |
| // We can assume fptr has a valid start code header because it has already |
| // gone through validation in HevcNaluParser. |
| uint8_t startHeaderSz = fptr[2] == 1 ? 3 : 4; |
| uint32_t dataSz = szBytes - startHeaderSz; |
| std::unique_ptr<uint8_t> idr(new uint8_t[dataSz + 4]); |
| uint32_t dataSzNl = htonl(dataSz); |
| |
| // AVCC format requires us to replace the start code header on this NALU |
| // with the size of the data. Start code is either 0x000001 or 0x00000001. |
| // The size needs to be the first four bytes in network byte order. |
| memcpy(idr.get(), &dataSzNl, 4); |
| memcpy(idr.get() + 4, ptr + startHeaderSz, dataSz); |
| |
| CMSampleBufferRef sampleBuf = nullptr; |
| sampleBuf = createSampleBuffer(mCmFmtDesc, (void*)idr.get(), dataSz + 4); |
| if (!sampleBuf) { |
| VTB_DPRINT("%s: Failed to create CMSampleBufferRef", __func__); |
| return; |
| } |
| |
| CMSampleBufferSetOutputPresentationTimeStamp(sampleBuf, CMTimeMake(pts, 1)); |
| |
| OSStatus status; |
| status = VTDecompressionSessionDecodeFrame(mDecoderSession, sampleBuf, |
| 0, // decodeFlags |
| NULL, // sourceFrameRefCon |
| 0); // infoFlagsOut |
| |
| if (status == noErr) { |
| // TODO: this call blocks until the frame has been decoded. Perhaps it |
| // will be more efficient to signal the guest when the frame is ready to |
| // be read instead. |
| status = VTDecompressionSessionWaitForAsynchronousFrames( |
| mDecoderSession); |
| VTB_DPRINT("Success decoding IDR frame"); |
| } else { |
| VTB_DPRINT("%s: Failed to decompress frame (err=%d)", __func__, status); |
| } |
| |
| CFRelease(sampleBuf); |
| } |
| |
| // chop the frame into sub frames that video tool box will consume |
| // one by one |
| bool MediaVideoToolBoxHevcVideoHelper::parseInputFrames(const uint8_t* frame, |
| size_t sz) { |
| mInputFrames.clear(); |
| VTB_DPRINT("input frame %d bytes", (int)sz); |
| while (1) { |
| const uint8_t* remainingFrame = parseOneFrame(frame, sz); |
| if (remainingFrame == nullptr) |
| break; |
| int consumed = (remainingFrame - frame); |
| VTB_DPRINT("consumed %d bytes", consumed); |
| frame = remainingFrame; |
| sz = sz - consumed; |
| } |
| return mInputFrames.size() > 0; |
| } |
| |
| const uint8_t* MediaVideoToolBoxHevcVideoHelper::parseOneFrame( |
| const uint8_t* frame, |
| size_t szBytes) { |
| if (frame == nullptr || szBytes <= 0) { |
| return nullptr; |
| } |
| |
| const uint8_t* currentNalu = |
| HevcNaluParser::getNextStartCodeHeader(frame, szBytes); |
| if (currentNalu == nullptr) { |
| VTB_DPRINT("No start code header found in this frame"); |
| return nullptr; |
| } |
| |
| const uint8_t* nextNalu = nullptr; |
| size_t remaining = szBytes - (currentNalu - frame); |
| |
| // Figure out the size of |currentNalu|. |
| size_t currentNaluSize = remaining; |
| // 3 is the minimum size of the start code header (3 or 4 bytes). |
| // dumpBytes(currentNalu, currentNaluSize); |
| |
| // usually guest sends one frame a time, but sometime, SEI prefixes IDR |
| // frame as one frame from guest, we need to separate SEI out from IDR: |
| // video tool box cannot handle SEI+IDR combo-frame; for other cases, there |
| // is no need to check. |
| if (HevcNaluType::SEI_PREFIX == |
| HevcNaluParser::getFrameNaluType(frame, szBytes, nullptr) || |
| HevcNaluType::SEI_SUFFIX == |
| HevcNaluParser::getFrameNaluType(frame, szBytes, nullptr) || |
| HevcNaluType::SPS == |
| HevcNaluParser::getFrameNaluType(frame, szBytes, nullptr) || |
| HevcNaluType::PPS == |
| HevcNaluParser::getFrameNaluType(frame, szBytes, nullptr) || |
| HevcNaluType::VPS == |
| HevcNaluParser::getFrameNaluType(frame, szBytes, nullptr) |
| |
| ) { |
| nextNalu = HevcNaluParser::getNextStartCodeHeader(currentNalu + 3, |
| remaining - 3); |
| if (nextNalu != nullptr) { |
| currentNaluSize = nextNalu - currentNalu; |
| } |
| } |
| |
| const uint8_t* remainingFrame = currentNalu + currentNaluSize; |
| |
| // |data| is currentNalu, but with the start code header discarded. |
| uint8_t* data = nullptr; |
| HevcNaluType naluType = HevcNaluParser::getFrameNaluType( |
| currentNalu, currentNaluSize, &data); |
| |
| if (naluType == HevcNaluType::Undefined) |
| return nullptr; |
| |
| size_t dataSize = currentNaluSize - (data - currentNalu); |
| const std::string naluTypeStr = HevcNaluParser::naluTypeToString(naluType); |
| VTB_DPRINT("Got frame type=%u (%s)", (uint8_t)naluType, |
| naluTypeStr.c_str()); |
| |
| if (naluType == HevcNaluType::SPS || naluType == HevcNaluType::PPS || |
| naluType == HevcNaluType::VPS) { |
| mInputFrames.push_back(InputFrame(naluType, data, dataSize)); |
| } else { |
| mInputFrames.push_back( |
| InputFrame(naluType, currentNalu, currentNaluSize)); |
| } |
| return remainingFrame; |
| } |
| |
| // static |
| void MediaVideoToolBoxHevcVideoHelper::videoToolboxDecompressCallback( |
| void* opaque, |
| void* sourceFrameRefCon, |
| OSStatus status, |
| VTDecodeInfoFlags flags, |
| CVPixelBufferRef image_buffer, |
| CMTime pts, |
| CMTime duration) { |
| VTB_DPRINT("%s", __func__); |
| auto ptr = static_cast<MediaVideoToolBoxHevcVideoHelper*>(opaque); |
| |
| // if (ptr->mDecodedFrame) { |
| // CVPixelBufferRelease(ptr->mDecodedFrame); |
| // ptr->mDecodedFrame = nullptr; |
| // } |
| |
| if (!image_buffer) { |
| VTB_DPRINT("%s: output image buffer is null", __func__); |
| ptr->mIsGood = false; |
| return; |
| } |
| |
| ptr->mOutputPts = pts.value; |
| ptr->mDecodedFrame = CVPixelBufferRetain(image_buffer); |
| CVOpenGLTextureCacheRef _CVGLTextureCache; |
| CVOpenGLTextureRef _CVGLTexture; |
| CGLPixelFormatObj _CGLPixelFormat; |
| |
| // Image is ready to be comsumed |
| ptr->copyFrame(); |
| ptr->mImageReady = true; |
| VTB_DPRINT("Got decoded frame"); |
| CVPixelBufferRelease(ptr->mDecodedFrame); |
| ptr->mDecodedFrame = nullptr; |
| } |
| |
| // static |
| CFDictionaryRef MediaVideoToolBoxHevcVideoHelper::createOutputBufferAttributes( |
| int width, |
| int height, |
| OSType pix_fmt) { |
| CFMutableDictionaryRef buffer_attributes; |
| CFMutableDictionaryRef io_surface_properties; |
| CFNumberRef cv_pix_fmt; |
| CFNumberRef w; |
| CFNumberRef h; |
| |
| w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width); |
| h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height); |
| cv_pix_fmt = |
| CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt); |
| |
| buffer_attributes = CFDictionaryCreateMutable( |
| kCFAllocatorDefault, 4, &kCFTypeDictionaryKeyCallBacks, |
| &kCFTypeDictionaryValueCallBacks); |
| io_surface_properties = CFDictionaryCreateMutable( |
| kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, |
| &kCFTypeDictionaryValueCallBacks); |
| |
| if (pix_fmt) { |
| CFDictionarySetValue(buffer_attributes, |
| kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt); |
| } |
| CFDictionarySetValue(buffer_attributes, |
| kCVPixelBufferIOSurfacePropertiesKey, |
| io_surface_properties); |
| CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w); |
| CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h); |
| // Not sure if this will work becuase we are passing the pixel buffer back |
| // into the guest |
| CFDictionarySetValue(buffer_attributes, |
| kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, |
| kCFBooleanTrue); |
| |
| CFRelease(io_surface_properties); |
| CFRelease(cv_pix_fmt); |
| CFRelease(w); |
| CFRelease(h); |
| |
| return buffer_attributes; |
| } |
| |
| // static |
| CMSampleBufferRef MediaVideoToolBoxHevcVideoHelper::createSampleBuffer( |
| CMFormatDescriptionRef fmtDesc, |
| void* buffer, |
| size_t sz) { |
| OSStatus status; |
| CMBlockBufferRef blockBuf = nullptr; |
| CMSampleBufferRef sampleBuf = nullptr; |
| |
| status = CMBlockBufferCreateWithMemoryBlock( |
| kCFAllocatorDefault, // structureAllocator |
| buffer, // memoryBlock |
| sz, // blockLength |
| kCFAllocatorNull, // blockAllocator |
| NULL, // customBlockSource |
| 0, // offsetToData |
| sz, // dataLength |
| 0, // flags |
| &blockBuf); |
| |
| if (!status) { |
| status = CMSampleBufferCreate(kCFAllocatorDefault, // allocator |
| blockBuf, // dataBuffer |
| TRUE, // dataReady |
| 0, // makeDataReadyCallback |
| 0, // makeDataReadyRefCon |
| fmtDesc, // formatDescription |
| 1, // numSamples |
| 0, // numSampleTimingEntries |
| NULL, // sampleTimingArray |
| 0, // numSampleSizeEntries |
| NULL, // sampleSizeArray |
| &sampleBuf); |
| } |
| |
| if (blockBuf) { |
| CFRelease(blockBuf); |
| } |
| |
| return sampleBuf; |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::resetDecoderSession() { |
| if (mDecoderSession) { |
| VTDecompressionSessionInvalidate(mDecoderSession); |
| CFRelease(mDecoderSession); |
| mDecoderSession = nullptr; |
| } |
| if (mDecodedFrame) { |
| CVPixelBufferRelease(mDecodedFrame); |
| mDecodedFrame = nullptr; |
| } |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::getOutputWH() { |
| if (!mCmFmtDesc) |
| return; |
| |
| CMVideoDimensions vsize = CMVideoFormatDescriptionGetDimensions(mCmFmtDesc); |
| |
| if (mOutputWidth == vsize.width && mOutputHeight == vsize.height) { |
| VTB_DPRINT("initial w/h is the same as vtb w/h"); |
| } else { |
| VTB_DPRINT("initial w/h are not the same as vtb w/h"); |
| } |
| mOutputWidth = vsize.width; |
| mOutputHeight = vsize.height; |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::resetFormatDesc() { |
| if (mCmFmtDesc) { |
| CFRelease(mCmFmtDesc); |
| mCmFmtDesc = nullptr; |
| } |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::createCMFormatDescription() { |
| uint8_t* parameterSets[3] = {mVPS.data(), mSPS.data(), mPPS.data()}; |
| size_t parameterSetSizes[3] = {mVPS.size(), mSPS.size(), mPPS.size()}; |
| |
| resetFormatDesc(); |
| |
| OSStatus status = CMVideoFormatDescriptionCreateFromHEVCParameterSets( |
| kCFAllocatorDefault, 3, (const uint8_t* const*)parameterSets, |
| parameterSetSizes, 4, nullptr, &mCmFmtDesc); |
| |
| if (status == noErr) { |
| getOutputWH(); |
| VTB_DPRINT("Created CMFormatDescription from SPS/PPS sets w %d h %d", |
| mOutputWidth, mOutputHeight); |
| } else { |
| VTB_DPRINT("Unable to create CMFormatDescription (%d)\n", (int)status); |
| } |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::recreateDecompressionSession() { |
| if (mCmFmtDesc == nullptr) { |
| VTB_DPRINT("CMFormatDescription not created. Need sps and pps NALUs."); |
| return; |
| } |
| |
| CMVideoCodecType codecType = kCMVideoCodecType_HEVC; |
| CFMutableDictionaryRef decoder_spec = CFDictionaryCreateMutable( |
| kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, |
| &kCFTypeDictionaryValueCallBacks); |
| // enable hardware acceleration, but allows vtb to fallback to |
| // its software implementation. |
| CFDictionarySetValue( |
| decoder_spec, |
| kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder, |
| kCFBooleanTrue); |
| |
| CFDictionaryRef bufAttr; |
| |
| if (mUseGpuTexture) { |
| // use NV12 format |
| bufAttr = createOutputBufferAttributes( |
| mOutputWidth, mOutputHeight, |
| kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); |
| } else { |
| bufAttr = createOutputBufferAttributes( |
| mOutputWidth, mOutputHeight, |
| kCVPixelFormatType_420YpCbCr8Planar); |
| } |
| |
| VTDecompressionOutputCallbackRecord decoderCb; |
| decoderCb.decompressionOutputCallback = videoToolboxDecompressCallback; |
| decoderCb.decompressionOutputRefCon = this; |
| |
| OSStatus status; |
| status = VTDecompressionSessionCreate( |
| NULL, // allocator |
| mCmFmtDesc, // videoFormatDescription |
| decoder_spec, // videoDecoderSpecification |
| bufAttr, // destinationImageBufferAttributes |
| &decoderCb, // outputCallback |
| &mDecoderSession); // decompressionSessionOut |
| |
| if (decoder_spec) { |
| CFRelease(decoder_spec); |
| } |
| if (bufAttr) { |
| CFRelease(bufAttr); |
| } |
| |
| mIsGood = false; |
| switch (status) { |
| case kVTVideoDecoderNotAvailableNowErr: |
| VTB_DPRINT("VideoToolbox session not available"); |
| return; |
| case kVTVideoDecoderUnsupportedDataFormatErr: |
| VTB_DPRINT("VideoToolbox does not support this format"); |
| return; |
| case kVTVideoDecoderMalfunctionErr: |
| VTB_DPRINT("VideoToolbox malfunction"); |
| return; |
| case kVTVideoDecoderBadDataErr: |
| VTB_DPRINT("VideoToolbox reported invalid data"); |
| return; |
| case 0: |
| dprint("VideoToolBox decoding session is created successfully"); |
| mIsGood = true; |
| return; |
| default: |
| VTB_DPRINT("Unknown VideoToolbox session creation error %d", |
| status); |
| return; |
| } |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::copyFrameToTextures() { |
| // TODO |
| auto startTime = std::chrono::steady_clock::now(); |
| TextureFrame texFrame; |
| if (mUseGpuTexture && mTexturePool != nullptr) { |
| VTB_DPRINT("decoded surface is %p w %d h %d", |
| CVPixelBufferGetIOSurface(mDecodedFrame), mOutputWidth, |
| mOutputHeight); |
| auto my_copy_context = media_vtb_utils_copy_context{ |
| CVPixelBufferGetIOSurface(mDecodedFrame), mOutputWidth, |
| mOutputHeight}; |
| texFrame = mTexturePool->getTextureFrame(mOutputWidth, mOutputHeight); |
| VTB_DPRINT("ask videocore to copy to %d and %d", texFrame.Ytex, |
| texFrame.UVtex); |
| mTexturePool->saveDecodedFrameToTexture( |
| texFrame, &my_copy_context, |
| (void*)media_vtb_utils_nv12_updater); |
| } |
| |
| auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>( |
| std::chrono::steady_clock::now() - startTime); |
| VTB_DPRINT("used %d ms", (int)elapsed.count()); |
| |
| auto ptspair = std::make_pair(mOutputPts, mTotalFrames); |
| mVtbBufferMap[ptspair] = MediaSnapshotState::FrameInfo{ |
| std::vector<uint8_t>(), |
| std::vector<uint32_t>{texFrame.Ytex, texFrame.UVtex}, |
| (int)mOutputWidth, |
| (int)mOutputHeight, |
| (uint64_t)(mOutputPts), |
| ColorAspects{mColorAspects}}; |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::copyFrameToCPU() { |
| int imageSize = CVPixelBufferGetDataSize(mDecodedFrame); |
| int stride = CVPixelBufferGetBytesPerRow(mDecodedFrame); |
| |
| mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2; |
| |
| VTB_DPRINT("copying size=%d dimension=[%dx%d] stride=%d", imageSize, |
| mOutputWidth, mOutputHeight, stride); |
| |
| // Copies the image data to the guest. |
| mSavedDecodedFrame.resize(mOutputWidth * mOutputHeight * 3 / 2); |
| uint8_t* dst = mSavedDecodedFrame.data(); |
| |
| CVPixelBufferLockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly); |
| if (CVPixelBufferIsPlanar(mDecodedFrame)) { |
| imageSize = 0; // add up the size from the planes |
| int planes = CVPixelBufferGetPlaneCount(mDecodedFrame); |
| for (int i = 0; i < planes; ++i) { |
| void* planeData = |
| CVPixelBufferGetBaseAddressOfPlane(mDecodedFrame, i); |
| int linesize = CVPixelBufferGetBytesPerRowOfPlane(mDecodedFrame, i); |
| int planeWidth = CVPixelBufferGetWidthOfPlane(mDecodedFrame, i); |
| int planeHeight = CVPixelBufferGetHeightOfPlane(mDecodedFrame, i); |
| VTB_DPRINT("plane=%d data=%p linesize=%d pwidth=%d pheight=%d", i, |
| planeData, linesize, planeWidth, planeHeight); |
| // For kCVPixelFormatType_420YpCbCr8Planar, plane 0 is Y, UV planes |
| // are 1 and 2 |
| if (planeWidth != mOutputWidth && planeWidth != mOutputWidth / 2) { |
| VTB_DPRINT("ERROR: Unable to determine YUV420 plane type"); |
| continue; |
| } |
| |
| // Sometimes the buffer stride can be longer than the actual data |
| // width. This means that the extra bytes are just padding and need |
| // to be discarded. |
| if (linesize <= planeWidth) { |
| int sz = planeHeight * planeWidth; |
| imageSize += sz; |
| memcpy(dst, planeData, sz); |
| dst += sz; |
| } else { |
| // Need to copy line by line |
| int sz = planeWidth; |
| for (int j = 0; j < planeHeight; ++j) { |
| uint8_t* ptr = (uint8_t*)planeData; |
| ptr += linesize * j; |
| memcpy(dst, ptr, sz); |
| imageSize += sz; |
| dst += sz; |
| } |
| } |
| } |
| if (imageSize != mOutBufferSize) { |
| VTB_DPRINT( |
| "ERROR: Total size of planes not same as guestSz " |
| "(guestSz=%u, imageSize=%d)", |
| mOutBufferSize, imageSize); |
| } |
| } else { |
| if (imageSize > mOutBufferSize) { |
| VTB_DPRINT( |
| "Buffer size mismatch (guestSz=%u, imageSize=%d). Using " |
| "guestSz instead.", |
| mOutBufferSize, imageSize); |
| imageSize = mOutBufferSize; |
| } |
| |
| // IMPORTANT: mDecodedFrame must be locked before accessing the contents |
| // with CPU |
| void* data = CVPixelBufferGetBaseAddress(mDecodedFrame); |
| memcpy(dst, data, imageSize); |
| } |
| CVPixelBufferUnlockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly); |
| |
| auto ptspair = std::make_pair(mOutputPts, mTotalFrames); |
| mVtbBufferMap[ptspair] = MediaSnapshotState::FrameInfo{ |
| std::vector<uint8_t>(), std::vector<uint32_t>{}, |
| (int)mOutputWidth, (int)mOutputHeight, |
| (uint64_t)(mOutputPts), ColorAspects{mColorAspects}}; |
| mVtbBufferMap[ptspair].data.swap(mSavedDecodedFrame); |
| } |
| |
| void MediaVideoToolBoxHevcVideoHelper::copyFrame() { |
| mOutputWidth = CVPixelBufferGetWidth(mDecodedFrame); |
| mOutputHeight = CVPixelBufferGetHeight(mDecodedFrame); |
| |
| if (mUseGpuTexture) { |
| copyFrameToTextures(); |
| } else { |
| copyFrameToCPU(); |
| } |
| |
| if (mVtbBufferMap.size() > mVtbBufferSize) { |
| mSavedDecodedFrames.push_back(std::move(mVtbBufferMap.begin()->second)); |
| mVtbBufferMap.erase(mVtbBufferMap.begin()); |
| } |
| } |
| |
| } // namespace emulation |
| } // namespace android |
