goldfish-codecs: add support for UseAndroidNativeBuffers
1. guest will send the host color buffer id in the
getImage call to host
2. host should send decoded Frame to the specified
host color buffer id
3. return other metadata about the decoded frame;
except the frame data.
BUG: 124388359
Test: Build
ANDROID_EMU_CODEC_USE_CUVID_DECODER=1 emulator -qemu -append qemu.hwcodec.avcdec=2
adb shell stagefright -S /data/local/tmp/buck1080p_h264.mp4
Change-Id: I5c684f3e0d438b38f66f9a9f41597f20832c9654
diff --git a/system/codecs/omx/avcdec/Android.mk b/system/codecs/omx/avcdec/Android.mk
index ea107db..798255f 100644
--- a/system/codecs/omx/avcdec/Android.mk
+++ b/system/codecs/omx/avcdec/Android.mk
@@ -33,13 +33,20 @@
libbinder_headers \
libhidlbase_impl_internal \
libbase
-
+LOCAL_HEADER_LIBRARIES += libui_headers \
+ libnativewindow_headers \
+ libhardware_headers \
+ libarect_headers \
+ libarect_headers_for_ndk
LOCAL_SHARED_LIBRARIES := \
libbinder \
libutils \
liblog \
libcutils \
+ libui \
android.hardware.media.omx@1.0 \
+ android.hardware.graphics.allocator@3.0 \
+ android.hardware.graphics.mapper@3.0 \
libstagefright_foundation
$(call emugl-export,C_INCLUDES,$(LOCAL_PATH))
diff --git a/system/codecs/omx/avcdec/GoldfishAVCDec.cpp b/system/codecs/omx/avcdec/GoldfishAVCDec.cpp
index 2cde213..662fe4e 100644
--- a/system/codecs/omx/avcdec/GoldfishAVCDec.cpp
+++ b/system/codecs/omx/avcdec/GoldfishAVCDec.cpp
@@ -24,6 +24,15 @@
#include <OMX_VideoExt.h>
#include <inttypes.h>
+#include <nativebase/nativebase.h>
+
+#include <android/hardware/graphics/allocator/3.0/IAllocator.h>
+#include <android/hardware/graphics/mapper/3.0/IMapper.h>
+#include <hidl/LegacySupport.h>
+
+using ::android::hardware::graphics::common::V1_2::PixelFormat;
+using ::android::hardware::graphics::common::V1_0::BufferUsage;
+
namespace android {
#define componentName "video_decoder.avc"
@@ -58,7 +67,7 @@
const char *name,
const OMX_CALLBACKTYPE *callbacks,
OMX_PTR appData,
- OMX_COMPONENTTYPE **component)
+ OMX_COMPONENTTYPE **component, RenderMode renderMode)
: GoldfishVideoDecoderOMXComponent(
name, componentName, codingType,
kProfileLevels, ARRAY_SIZE(kProfileLevels),
@@ -67,7 +76,10 @@
mOmxColorFormat(OMX_COLOR_FormatYUV420Planar),
mChangingResolution(false),
mSignalledError(false),
- mInputOffset(0){
+ mInputOffset(0), mRenderMode(renderMode){
+ if (renderMode == RenderMode::RENDER_BY_HOST_GPU) {
+ mOutputFormat = (OMX_COLOR_FORMATTYPE)HAL_PIXEL_FORMAT_YCBCR_420_888;
+ }
initPorts(
1 /* numMinInputBuffers */, kNumBuffers, INPUT_BUF_SIZE,
1 /* numMinOutputBuffers */, kNumBuffers, CODEC_MIME_TYPE);
@@ -116,7 +128,7 @@
status_t GoldfishAVCDec::initDecoder() {
/* Initialize the decoder */
- mContext.reset(new MediaH264Decoder());
+ mContext.reset(new MediaH264Decoder(mRenderMode));
mContext->initH264Context(mWidth,
mHeight,
mWidth,
@@ -252,6 +264,15 @@
}
}
+int GoldfishAVCDec::getHostColorBufferId(void* header) {
+ if (mNWBuffers.find(header) == mNWBuffers.end()) {
+ return -1;
+ }
+ sp<ANativeWindowBuffer> nBuf = mNWBuffers[header];
+ cb_handle_t *handle = (cb_handle_t*)nBuf->handle;
+ return handle->hostHandle;
+}
+
void GoldfishAVCDec::onQueueFilled(OMX_U32 portIndex) {
static int count1=0;
ALOGD("calling %s count %d", __func__, ++count1);
@@ -363,7 +384,13 @@
} else {
ALOGD("No more input data. Attempting to get a decoded frame, if any.");
}
- h264_image_t img = mContext->getImage();
+ h264_image_t img = {};
+
+ if (mRenderMode == RenderMode::RENDER_BY_GUEST_CPU) {
+ img = mContext->getImage();
+ } else {
+ img = mContext->renderOnHostAndReturnImageMetadata(getHostColorBufferId(outHeader));
+ }
if (img.data != nullptr) {
@@ -419,10 +446,12 @@
}
}
outHeader->nFilledLen = (outputBufferWidth() * outputBufferHeight() * 3) / 2;
- if (outputBufferWidth() == mWidth && outputBufferHeight() == mHeight) {
+ if (mRenderMode == RenderMode::RENDER_BY_GUEST_CPU) {
+ if (outputBufferWidth() == mWidth && outputBufferHeight() == mHeight) {
memcpy(outHeader->pBuffer, img.data, outHeader->nFilledLen);
- } else {
+ } else {
copyImageData(outHeader, img);
+ }
}
outHeader->nTimeStamp = img.pts;
@@ -488,6 +517,89 @@
}
}
+OMX_ERRORTYPE GoldfishAVCDec::internalGetParameter(
+ OMX_INDEXTYPE index, OMX_PTR params) {
+ const int32_t indexFull = index;
+ switch (indexFull) {
+ case kGetAndroidNativeBufferUsageIndex:
+ {
+ ALOGD("calling kGetAndroidNativeBufferUsageIndex");
+ GetAndroidNativeBufferUsageParams* nativeBuffersUsage = (GetAndroidNativeBufferUsageParams *) params;
+ nativeBuffersUsage->nUsage = (unsigned int)(BufferUsage::GPU_DATA_BUFFER);
+ return OMX_ErrorNone;
+ }
+
+ default:
+ return GoldfishVideoDecoderOMXComponent::internalGetParameter(index, params);
+ }
+}
+
+OMX_ERRORTYPE GoldfishAVCDec::internalSetParameter(
+ OMX_INDEXTYPE index, const OMX_PTR params) {
+ // Include extension index OMX_INDEXEXTTYPE.
+ const int32_t indexFull = index;
+
+ switch (indexFull) {
+ case kEnableAndroidNativeBuffersIndex:
+ {
+ ALOGD("calling kEnableAndroidNativeBuffersIndex");
+ EnableAndroidNativeBuffersParams* enableNativeBuffers = (EnableAndroidNativeBuffersParams *) params;
+ if (enableNativeBuffers) {
+ mEnableAndroidNativeBuffers = enableNativeBuffers->enable;
+ if (mEnableAndroidNativeBuffers == false) {
+ mNWBuffers.clear();
+ ALOGD("disabled kEnableAndroidNativeBuffersIndex");
+ } else {
+ ALOGD("enabled kEnableAndroidNativeBuffersIndex");
+ }
+ }
+ return OMX_ErrorNone;
+ }
+
+ case kUseAndroidNativeBufferIndex:
+ {
+ if (mEnableAndroidNativeBuffers == false) {
+ ALOGE("Error: not enabled Android Native Buffers");
+ return OMX_ErrorBadParameter;
+ }
+ UseAndroidNativeBufferParams *use_buffer_params = (UseAndroidNativeBufferParams *)params;
+ if (use_buffer_params) {
+ sp<ANativeWindowBuffer> nBuf = use_buffer_params->nativeBuffer;
+ cb_handle_t *handle = (cb_handle_t*)nBuf->handle;
+ void* dst = NULL;
+ ALOGD("kUseAndroidNativeBufferIndex with handle %p host color handle %d calling usebuffer", handle,
+ handle->hostHandle);
+ useBufferCallerLockedAlready(use_buffer_params->bufferHeader,use_buffer_params->nPortIndex,
+ use_buffer_params->pAppPrivate,handle->allocatedSize(), (OMX_U8*)dst);
+ mNWBuffers[*(use_buffer_params->bufferHeader)] = use_buffer_params->nativeBuffer;;
+ }
+ return OMX_ErrorNone;
+ }
+
+ default:
+ return GoldfishVideoDecoderOMXComponent::internalSetParameter(index, params);
+ }
+}
+
+OMX_ERRORTYPE GoldfishAVCDec::getExtensionIndex(
+ const char *name, OMX_INDEXTYPE *index) {
+
+ if (mRenderMode == RenderMode::RENDER_BY_HOST_GPU) {
+ if (!strcmp(name, "OMX.google.android.index.enableAndroidNativeBuffers")) {
+ ALOGD("calling getExtensionIndex for enable ANB");
+ *(int32_t*)index = kEnableAndroidNativeBuffersIndex;
+ return OMX_ErrorNone;
+ } else if (!strcmp(name, "OMX.google.android.index.useAndroidNativeBuffer")) {
+ *(int32_t*)index = kUseAndroidNativeBufferIndex;
+ return OMX_ErrorNone;
+ } else if (!strcmp(name, "OMX.google.android.index.getAndroidNativeBufferUsage")) {
+ *(int32_t*)index = kGetAndroidNativeBufferUsageIndex;
+ return OMX_ErrorNone;
+ }
+ }
+ return GoldfishVideoDecoderOMXComponent::getExtensionIndex(name, index);
+}
+
int GoldfishAVCDec::getColorAspectPreference() {
return kPreferBitstream;
}
@@ -497,6 +609,10 @@
android::GoldfishOMXComponent *createGoldfishOMXComponent(
const char *name, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData,
OMX_COMPONENTTYPE **component) {
- return new android::GoldfishAVCDec(name, callbacks, appData, component);
+ if (!strncmp("OMX.android.goldfish", name, 20)) {
+ return new android::GoldfishAVCDec(name, callbacks, appData, component, RenderMode::RENDER_BY_HOST_GPU);
+ } else {
+ return new android::GoldfishAVCDec(name, callbacks, appData, component, RenderMode::RENDER_BY_GUEST_CPU);
+ }
}
diff --git a/system/codecs/omx/avcdec/GoldfishAVCDec.h b/system/codecs/omx/avcdec/GoldfishAVCDec.h
index f6e68c9..dc9dc96 100644
--- a/system/codecs/omx/avcdec/GoldfishAVCDec.h
+++ b/system/codecs/omx/avcdec/GoldfishAVCDec.h
@@ -22,6 +22,17 @@
#include "MediaH264Decoder.h"
#include <sys/time.h>
+#include <vector>
+#include <map>
+
+#include "gralloc_cb.h"
+#include <utils/RefBase.h>
+#include <utils/threads.h>
+#include <utils/Vector.h>
+#include <utils/List.h>
+#include <ui/GraphicBuffer.h>
+
+
namespace android {
/** Number of entries in the time-stamp array */
@@ -44,7 +55,7 @@
struct GoldfishAVCDec : public GoldfishVideoDecoderOMXComponent {
GoldfishAVCDec(const char *name, const OMX_CALLBACKTYPE *callbacks,
- OMX_PTR appData, OMX_COMPONENTTYPE **component);
+ OMX_PTR appData, OMX_COMPONENTTYPE **component, RenderMode renderMode);
protected:
virtual ~GoldfishAVCDec();
@@ -53,12 +64,25 @@
virtual void onPortFlushCompleted(OMX_U32 portIndex);
virtual void onReset();
virtual int getColorAspectPreference();
+
+ virtual OMX_ERRORTYPE internalGetParameter(OMX_INDEXTYPE index, OMX_PTR params);
+
+ virtual OMX_ERRORTYPE internalSetParameter(OMX_INDEXTYPE index, const OMX_PTR params);
+
+ virtual OMX_ERRORTYPE getExtensionIndex(const char *name, OMX_INDEXTYPE *index);
+
private:
// Number of input and output buffers
enum {
kNumBuffers = 8
};
+ RenderMode mRenderMode = RenderMode::RENDER_BY_GUEST_CPU;
+ bool mEnableAndroidNativeBuffers = false;
+ std::map<void*, sp<ANativeWindowBuffer>> mNWBuffers;
+
+ int getHostColorBufferId(void* header);
+
size_t mNumCores; // Number of cores to be uesd by the codec
nsecs_t mTimeStart; // Time at the start of decode()
diff --git a/system/codecs/omx/avcdec/MediaH264Decoder.cpp b/system/codecs/omx/avcdec/MediaH264Decoder.cpp
index 3ecc489..e7aaf75 100644
--- a/system/codecs/omx/avcdec/MediaH264Decoder.cpp
+++ b/system/codecs/omx/avcdec/MediaH264Decoder.cpp
@@ -20,6 +20,14 @@
#include "goldfish_media_utils.h"
#include <string.h>
+MediaH264Decoder::MediaH264Decoder(RenderMode renderMode) :mRenderMode(renderMode) {
+ if (renderMode == RenderMode::RENDER_BY_HOST_GPU) {
+ mVersion = 200;
+ } else if (renderMode == RenderMode::RENDER_BY_GUEST_CPU) {
+ mVersion = 100;
+ }
+}
+
void MediaH264Decoder::initH264Context(unsigned int width,
unsigned int height,
unsigned int outWidth,
@@ -152,3 +160,41 @@
}
return res;
}
+
+
+h264_image_t MediaH264Decoder::renderOnHostAndReturnImageMetadata(int hostColorBufferId) {
+ ALOGD("%s: use handle to host %lld", __func__, mHostHandle);
+ h264_image_t res { };
+ if (hostColorBufferId < 0) {
+ ALOGE("%s negative color buffer id %d", __func__, hostColorBufferId);
+ return res;
+ }
+ ALOGD("%s send color buffer id %d", __func__, hostColorBufferId);
+ if (!mHasAddressSpaceMemory) {
+ ALOGE("%s no address space memory", __func__);
+ return res;
+ }
+ auto transport = GoldfishMediaTransport::getInstance();
+ uint8_t* dst = transport->getInputAddr(mAddressOffSet); // Note: reuse the same addr for input and output
+ transport->writeParam((uint64_t)mHostHandle, 0, mAddressOffSet);
+ transport->writeParam(transport->offsetOf((uint64_t)(dst)) - mAddressOffSet, 1, mAddressOffSet);
+ transport->writeParam((uint64_t)hostColorBufferId, 2, mAddressOffSet);
+ transport->sendOperation(MediaCodecType::H264Codec,
+ MediaOperation::GetImage, mAddressOffSet);
+ auto* retptr = transport->getReturnAddr(mAddressOffSet);
+ res.ret = *(int*)(retptr);
+ if (res.ret >= 0) {
+ res.data = dst; // note: the data could be junk
+ res.width = *(uint32_t*)(retptr + 8);
+ res.height = *(uint32_t*)(retptr + 16);
+ res.pts = *(uint32_t*)(retptr + 24);
+ res.color_primaries = *(uint32_t*)(retptr + 32);
+ res.color_range = *(uint32_t*)(retptr + 40);
+ res.color_trc = *(uint32_t*)(retptr + 48);
+ res.colorspace = *(uint32_t*)(retptr + 56);
+ } else if (res.ret == (int)(Err::DecoderRestarted)) {
+ res.width = *(uint32_t*)(retptr + 8);
+ res.height = *(uint32_t*)(retptr + 16);
+ }
+ return res;
+}
diff --git a/system/codecs/omx/avcdec/MediaH264Decoder.h b/system/codecs/omx/avcdec/MediaH264Decoder.h
index c0f1105..8264d3c 100644
--- a/system/codecs/omx/avcdec/MediaH264Decoder.h
+++ b/system/codecs/omx/avcdec/MediaH264Decoder.h
@@ -41,15 +41,21 @@
int ret;
};
+enum class RenderMode {
+ RENDER_BY_HOST_GPU = 1,
+ RENDER_BY_GUEST_CPU = 2,
+};
+
class MediaH264Decoder {
uint64_t mHostHandle = 0;
- const uint32_t mVersion = 100;
+ uint32_t mVersion = 100;
+ RenderMode mRenderMode = RenderMode::RENDER_BY_GUEST_CPU;
bool mHasAddressSpaceMemory = false;
uint64_t mAddressOffSet = 0;
public:
- MediaH264Decoder() = default;
+ MediaH264Decoder(RenderMode renderMode);
virtual ~MediaH264Decoder() = default;
enum class PixelFormat : uint8_t {
@@ -80,6 +86,11 @@
void destroyH264Context();
h264_result_t decodeFrame(uint8_t* img, size_t szBytes, uint64_t pts);
void flush();
+ // ask host to copy image data back to guest, with image metadata
+ // to guest as well
h264_image_t getImage();
+ // ask host to render to hostColorBufferId, return only image metadata back to
+ // guest
+ h264_image_t renderOnHostAndReturnImageMetadata(int hostColorBufferId);
};
#endif
diff --git a/system/codecs/omx/plugin/GoldfishOMXPlugin.cpp b/system/codecs/omx/plugin/GoldfishOMXPlugin.cpp
index f7eaacf..47c22a3 100644
--- a/system/codecs/omx/plugin/GoldfishOMXPlugin.cpp
+++ b/system/codecs/omx/plugin/GoldfishOMXPlugin.cpp
@@ -33,7 +33,7 @@
namespace android {
OMXPluginBase *createOMXPlugin() {
- ALOGE("called createOMXPlugin for Goldfish");
+ ALOGD("called createOMXPlugin for Goldfish");
return new GoldfishOMXPlugin;
}
@@ -45,21 +45,41 @@
const char *mRole;
};
-static bool useGoldfishComponentInstance(const char* libname) {
+static bool useGoogleGoldfishComponentInstance(const char* libname) {
// We have a property set indicating whether to use the host side codec
// or not (ro.kernel.qemu.hwcodec.<mLibNameSuffix>).
char propValue[PROP_VALUE_MAX];
AString prop = "ro.kernel.qemu.hwcodec.";
prop.append(libname);
- return property_get(prop.c_str(), propValue, "") > 0 &&
+ bool myret = property_get(prop.c_str(), propValue, "") > 0 &&
strcmp("1", propValue) == 0;
+ if (myret) {
+ ALOGD("%s %d found prop %s val %s", __func__, __LINE__, prop.c_str(), propValue);
+ }
+ return myret;
+}
+
+static bool useAndroidGoldfishComponentInstance(const char* libname) {
+ // We have a property set indicating whether to use the host side codec
+ // or not (ro.kernel.qemu.hwcodec.<mLibNameSuffix>).
+ char propValue[PROP_VALUE_MAX];
+ AString prop = "ro.kernel.qemu.hwcodec.";
+ prop.append(libname);
+
+ bool myret = property_get(prop.c_str(), propValue, "") > 0 &&
+ strcmp("2", propValue) == 0;
+ if (myret) {
+ ALOGD("%s %d found prop %s val %s", __func__, __LINE__, prop.c_str(), propValue);
+ }
+ return myret;
}
static const GoldfishComponent kComponents[] = {
{ "OMX.google.goldfish.vp8.decoder", "vpxdec", "video_decoder.vp8" },
{ "OMX.google.goldfish.vp9.decoder", "vpxdec", "video_decoder.vp9" },
{ "OMX.google.goldfish.h264.decoder", "avcdec", "video_decoder.avc" },
+ { "OMX.android.goldfish.h264.decoder", "avcdec", "video_decoder.avc" },
};
static std::vector<GoldfishComponent> kActiveComponents;
@@ -69,7 +89,13 @@
GoldfishOMXPlugin::GoldfishOMXPlugin() {
for (int i = 0; i < kNumComponents; ++i) {
- if (useGoldfishComponentInstance(kComponents[i].mLibNameSuffix)) {
+ if ( !strncmp("OMX.google", kComponents[i].mName, 10) &&
+ useGoogleGoldfishComponentInstance(kComponents[i].mLibNameSuffix)) {
+ ALOGD("found and use kComponents[i].name %s", kComponents[i].mName);
+ kActiveComponents.push_back(kComponents[i]);
+ } else if (!strncmp("OMX.android", kComponents[i].mName, 11) &&
+ useAndroidGoldfishComponentInstance(kComponents[i].mLibNameSuffix)) {
+ ALOGD("found and use kComponents[i].name %s", kComponents[i].mName);
kActiveComponents.push_back(kComponents[i]);
}
}
@@ -172,7 +198,7 @@
return OMX_ErrorNoMore;
}
- ALOGE("enumerate %s component", kActiveComponents[index].mName);
+ ALOGD("enumerate %s component", kActiveComponents[index].mName);
strcpy(name, kActiveComponents[index].mName);
return OMX_ErrorNone;
diff --git a/system/codecs/omx/plugin/GoldfishVideoDecoderOMXComponent.h b/system/codecs/omx/plugin/GoldfishVideoDecoderOMXComponent.h
index 5921510..bf60f61 100644
--- a/system/codecs/omx/plugin/GoldfishVideoDecoderOMXComponent.h
+++ b/system/codecs/omx/plugin/GoldfishVideoDecoderOMXComponent.h
@@ -51,6 +51,9 @@
kDescribeColorAspectsIndex = kPrepareForAdaptivePlaybackIndex + 1,
kDescribeHdrStaticInfoIndex = kPrepareForAdaptivePlaybackIndex + 2,
kDescribeHdr10PlusInfoIndex = kPrepareForAdaptivePlaybackIndex + 3,
+ kEnableAndroidNativeBuffersIndex = kPrepareForAdaptivePlaybackIndex + 4,
+ kUseAndroidNativeBufferIndex = kPrepareForAdaptivePlaybackIndex + 5,
+ kGetAndroidNativeBufferUsageIndex = kPrepareForAdaptivePlaybackIndex + 6,
};
enum {
diff --git a/system/codecs/omx/plugin/SimpleGoldfishOMXComponent.cpp b/system/codecs/omx/plugin/SimpleGoldfishOMXComponent.cpp
index a8fae39..8c8e694 100644
--- a/system/codecs/omx/plugin/SimpleGoldfishOMXComponent.cpp
+++ b/system/codecs/omx/plugin/SimpleGoldfishOMXComponent.cpp
@@ -227,6 +227,16 @@
OMX_U32 size,
OMX_U8 *ptr) {
Mutex::Autolock autoLock(mLock);
+ return useBufferCallerLockedAlready(header, portIndex, appPrivate, size, ptr);
+}
+
+OMX_ERRORTYPE SimpleGoldfishOMXComponent::useBufferCallerLockedAlready(
+ OMX_BUFFERHEADERTYPE **header,
+ OMX_U32 portIndex,
+ OMX_PTR appPrivate,
+ OMX_U32 size,
+ OMX_U8 *ptr) {
+ CHECK_LT(portIndex, mPorts.size());
CHECK_LT(portIndex, mPorts.size());
PortInfo *port = &mPorts.editItemAt(portIndex);
diff --git a/system/codecs/omx/plugin/SimpleGoldfishOMXComponent.h b/system/codecs/omx/plugin/SimpleGoldfishOMXComponent.h
index 127fb71..d2a3bf5 100644
--- a/system/codecs/omx/plugin/SimpleGoldfishOMXComponent.h
+++ b/system/codecs/omx/plugin/SimpleGoldfishOMXComponent.h
@@ -91,7 +91,7 @@
PortInfo *editPortInfo(OMX_U32 portIndex);
-private:
+protected:
enum {
kWhatSendCommand,
kWhatEmptyThisBuffer,
@@ -131,6 +131,13 @@
OMX_U32 size,
OMX_U8 *ptr);
+ OMX_ERRORTYPE useBufferCallerLockedAlready(
+ OMX_BUFFERHEADERTYPE **buffer,
+ OMX_U32 portIndex,
+ OMX_PTR appPrivate,
+ OMX_U32 size,
+ OMX_U8 *ptr);
+
virtual OMX_ERRORTYPE allocateBuffer(
OMX_BUFFERHEADERTYPE **buffer,
OMX_U32 portIndex,
