Codec2: Properly support color aspects
- implement configure and reading color aspects
decoder: configure default aspects, read back final aspects
encoder: configure source aspects, read back source and coded aspects
- directly set final aspects for components that do not support color
aspects (via local param)
- set default color aspects for surface mode. Do this in getSdkParams
so that it can be shared by decoder and encoder. Note: this also
must update dataspace which again can be shared by decoder and encoder.
Bug: 110225406
Change-Id: I486600e4b6ca02def89c3e5b65a9b861362035c7
diff --git a/media/sfplugin/CCodec.cpp b/media/sfplugin/CCodec.cpp
index a24be2f..e1081e7 100644
--- a/media/sfplugin/CCodec.cpp
+++ b/media/sfplugin/CCodec.cpp
@@ -664,6 +664,7 @@
}
Mutexed<Config>::Locked config(mConfig);
+ config->mUsingSurface = surface != nullptr;
/*
* Handle input surface configuration
@@ -954,6 +955,7 @@
Mutexed<Config>::Locked config(mConfig);
config->mInputSurface = surface;
+ config->mUsingSurface = true;
if (config->mISConfig) {
surface->configure(*config->mISConfig);
} else {
@@ -1495,6 +1497,7 @@
const static std::vector<C2Param::Index> stdGfxInfos = {
C2StreamRotationInfo::output::PARAM_TYPE,
C2StreamColorAspectsInfo::output::PARAM_TYPE,
+ C2StreamDataSpaceInfo::output::PARAM_TYPE,
C2StreamHdrStaticInfo::output::PARAM_TYPE,
C2StreamPixelAspectRatioInfo::output::PARAM_TYPE,
C2StreamSurfaceScalingInfo::output::PARAM_TYPE
diff --git a/media/sfplugin/CCodecBufferChannel.cpp b/media/sfplugin/CCodecBufferChannel.cpp
index fd3e87d..781f7d3 100644
--- a/media/sfplugin/CCodecBufferChannel.cpp
+++ b/media/sfplugin/CCodecBufferChannel.cpp
@@ -1539,84 +1539,9 @@
videoScalingMode = surfaceScaling->value;
}
- // Use dataspace if component provides it. Otherwise, compose dataspace from color aspects
- std::shared_ptr<const C2StreamDataSpaceInfo::output> dataSpaceInfo =
- std::static_pointer_cast<const C2StreamDataSpaceInfo::output>(
- c2Buffer->getInfo(C2StreamDataSpaceInfo::output::PARAM_TYPE));
- uint32_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
- if (dataSpaceInfo) {
- dataSpace = dataSpaceInfo->value;
- } else {
- std::shared_ptr<const C2StreamColorAspectsInfo::output> colorAspects =
- std::static_pointer_cast<const C2StreamColorAspectsInfo::output>(
- c2Buffer->getInfo(C2StreamColorAspectsInfo::output::PARAM_TYPE));
- C2Color::range_t range =
- colorAspects == nullptr ? C2Color::RANGE_UNSPECIFIED : colorAspects->range;
- C2Color::primaries_t primaries =
- colorAspects == nullptr ? C2Color::PRIMARIES_UNSPECIFIED : colorAspects->primaries;
- C2Color::transfer_t transfer =
- colorAspects == nullptr ? C2Color::TRANSFER_UNSPECIFIED : colorAspects->transfer;
- C2Color::matrix_t matrix =
- colorAspects == nullptr ? C2Color::MATRIX_UNSPECIFIED : colorAspects->matrix;
-
- switch (range) {
- case C2Color::RANGE_FULL: dataSpace |= HAL_DATASPACE_RANGE_FULL; break;
- case C2Color::RANGE_LIMITED: dataSpace |= HAL_DATASPACE_RANGE_LIMITED; break;
- default: break;
- }
-
- switch (transfer) {
- case C2Color::TRANSFER_LINEAR: dataSpace |= HAL_DATASPACE_TRANSFER_LINEAR; break;
- case C2Color::TRANSFER_SRGB: dataSpace |= HAL_DATASPACE_TRANSFER_SRGB; break;
- case C2Color::TRANSFER_170M: dataSpace |= HAL_DATASPACE_TRANSFER_SMPTE_170M; break;
- case C2Color::TRANSFER_GAMMA22: dataSpace |= HAL_DATASPACE_TRANSFER_GAMMA2_2; break;
- case C2Color::TRANSFER_GAMMA28: dataSpace |= HAL_DATASPACE_TRANSFER_GAMMA2_8; break;
- case C2Color::TRANSFER_ST2084: dataSpace |= HAL_DATASPACE_TRANSFER_ST2084; break;
- case C2Color::TRANSFER_HLG: dataSpace |= HAL_DATASPACE_TRANSFER_HLG; break;
- default: break;
- }
-
- switch (primaries) {
- case C2Color::PRIMARIES_BT601_525:
- dataSpace |= (matrix == C2Color::MATRIX_SMPTE240M
- || matrix == C2Color::MATRIX_BT709)
- ? HAL_DATASPACE_STANDARD_BT601_525_UNADJUSTED
- : HAL_DATASPACE_STANDARD_BT601_525;
- break;
- case C2Color::PRIMARIES_BT601_625:
- dataSpace |= (matrix == C2Color::MATRIX_SMPTE240M
- || matrix == C2Color::MATRIX_BT709)
- ? HAL_DATASPACE_STANDARD_BT601_625_UNADJUSTED
- : HAL_DATASPACE_STANDARD_BT601_625;
- break;
- case C2Color::PRIMARIES_BT2020:
- dataSpace |= (matrix == C2Color::MATRIX_BT2020CONSTANT
- ? HAL_DATASPACE_STANDARD_BT2020_CONSTANT_LUMINANCE
- : HAL_DATASPACE_STANDARD_BT2020);
- break;
- case C2Color::PRIMARIES_BT470_M:
- dataSpace |= HAL_DATASPACE_STANDARD_BT470M;
- break;
- case C2Color::PRIMARIES_BT709:
- dataSpace |= HAL_DATASPACE_STANDARD_BT709;
- break;
- default: break;
- }
- }
-
- // convert legacy dataspace values to v0 values
- const static
- ALookup<android_dataspace, android_dataspace> sLegacyDataSpaceToV0 {
- {
- { HAL_DATASPACE_SRGB, HAL_DATASPACE_V0_SRGB },
- { HAL_DATASPACE_BT709, HAL_DATASPACE_V0_BT709 },
- { HAL_DATASPACE_SRGB_LINEAR, HAL_DATASPACE_V0_SRGB_LINEAR },
- { HAL_DATASPACE_BT601_525, HAL_DATASPACE_V0_BT601_525 },
- { HAL_DATASPACE_BT601_625, HAL_DATASPACE_V0_BT601_625 },
- { HAL_DATASPACE_JFIF, HAL_DATASPACE_V0_JFIF },
- }
- };
- sLegacyDataSpaceToV0.lookup((android_dataspace_t)dataSpace, (android_dataspace_t*)&dataSpace);
+ // Use dataspace from format as it has the default aspects already applied
+ android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
+ (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
// HDR static info
std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
@@ -1641,8 +1566,8 @@
// TODO: revisit this after C2Fence implementation.
android::IGraphicBufferProducer::QueueBufferInput qbi(
timestampNs,
- false,
- (android_dataspace_t)dataSpace,
+ false, // droppable
+ dataSpace,
Rect(blocks.front().crop().left,
blocks.front().crop().top,
blocks.front().crop().right(),
diff --git a/media/sfplugin/CCodecBufferChannel.h b/media/sfplugin/CCodecBufferChannel.h
index 92150f8..c043686 100644
--- a/media/sfplugin/CCodecBufferChannel.h
+++ b/media/sfplugin/CCodecBufferChannel.h
@@ -25,6 +25,7 @@
#include <C2Buffer.h>
#include <C2Component.h>
+#include <Codec2Mapper.h>
#include <codec2/hidl/client.h>
#include <media/stagefright/bqhelper/GraphicBufferSource.h>
diff --git a/media/sfplugin/CCodecConfig.cpp b/media/sfplugin/CCodecConfig.cpp
index e5d5fbb..51af6bc 100644
--- a/media/sfplugin/CCodecConfig.cpp
+++ b/media/sfplugin/CCodecConfig.cpp
@@ -22,6 +22,7 @@
#include <C2Component.h>
#include <C2Debug.h>
#include <C2Param.h>
+#include <util/C2InterfaceHelper.h>
#include <media/stagefright/MediaCodecConstants.h>
@@ -87,6 +88,31 @@
return *this;
}
+ /// Adds SDK <=> Codec 2.0 value mappers based on C2Mapper
+ template<typename C2Type, typename SdkType=int32_t>
+ ConfigMapper &withC2Mappers() {
+ C2_CHECK(!mMapper);
+ C2_CHECK(!mReverse);
+ mMapper = [](C2Value v) -> C2Value {
+ SdkType sdkValue;
+ C2Type c2Value;
+ if (v.get(&sdkValue) && C2Mapper::map(sdkValue, &c2Value)) {
+ return c2Value;
+ }
+ return C2Value();
+ };
+ mReverse = [](C2Value v) -> C2Value {
+ SdkType sdkValue;
+ C2Type c2Value;
+ using C2ValueType=typename _c2_reduce_enum_to_underlying_type<C2Type>::type;
+ if (v.get((C2ValueType*)&c2Value) && C2Mapper::map(c2Value, &sdkValue)) {
+ return sdkValue;
+ }
+ return C2Value();
+ };
+ return *this;
+ }
+
/// Maps from SDK values in an AMessage to a suitable C2Value.
C2Value mapFromMessage(const AMessage::ItemData &item) const {
C2Value value;
@@ -285,7 +311,8 @@
CCodecConfig::CCodecConfig()
: mInputFormat(new AMessage),
- mOutputFormat(new AMessage) { }
+ mOutputFormat(new AMessage),
+ mUsingSurface(false) { }
void CCodecConfig::initializeStandardParams() {
typedef Domain D;
@@ -366,6 +393,67 @@
.limitTo(D::VIDEO & D::RAW)
.withMappers(negate, negate));
+ // android 'video-scaling'
+ add(ConfigMapper("android._video-scaling", C2_PARAMKEY_SURFACE_SCALING_MODE, "value")
+ .limitTo(D::VIDEO & D::DECODER & D::RAW));
+
+ // Color Aspects
+ //
+ // configure default for decoders
+ add(ConfigMapper(KEY_COLOR_RANGE, C2_PARAMKEY_DEFAULT_COLOR_ASPECTS, "range")
+ .limitTo((D::VIDEO | D::IMAGE) & D::DECODER & D::CODED & (D::CONFIG | D::PARAM))
+ .withC2Mappers<C2Color::range_t>());
+ add(ConfigMapper(KEY_COLOR_TRANSFER, C2_PARAMKEY_DEFAULT_COLOR_ASPECTS, "transfer")
+ .limitTo((D::VIDEO | D::IMAGE) & D::DECODER & D::CODED & (D::CONFIG | D::PARAM))
+ .withC2Mappers<C2Color::transfer_t>());
+ add(ConfigMapper("color-primaries", C2_PARAMKEY_DEFAULT_COLOR_ASPECTS, "primaries")
+ .limitTo((D::VIDEO | D::IMAGE) & D::DECODER & D::CODED & (D::CONFIG | D::PARAM)));
+ add(ConfigMapper("color-matrix", C2_PARAMKEY_DEFAULT_COLOR_ASPECTS, "matrix")
+ .limitTo((D::VIDEO | D::IMAGE) & D::DECODER & D::CODED & (D::CONFIG | D::PARAM)));
+
+ // read back final for decoder output (also, configure final aspects as well. This should be
+ // overwritten based on coded/default values if component supports color aspects, but is used
+ // as final values if component does not support aspects at all)
+ add(ConfigMapper(KEY_COLOR_RANGE, C2_PARAMKEY_COLOR_ASPECTS, "range")
+ .limitTo((D::VIDEO | D::IMAGE) & D::DECODER & D::RAW)
+ .withC2Mappers<C2Color::range_t>());
+ add(ConfigMapper(KEY_COLOR_TRANSFER, C2_PARAMKEY_COLOR_ASPECTS, "transfer")
+ .limitTo((D::VIDEO | D::IMAGE) & D::DECODER & D::RAW)
+ .withC2Mappers<C2Color::transfer_t>());
+ add(ConfigMapper("color-primaries", C2_PARAMKEY_COLOR_ASPECTS, "primaries")
+ .limitTo((D::VIDEO | D::IMAGE) & D::DECODER & D::RAW));
+ add(ConfigMapper("color-matrix", C2_PARAMKEY_COLOR_ASPECTS, "matrix")
+ .limitTo((D::VIDEO | D::IMAGE) & D::DECODER & D::RAW));
+
+ // configure source aspects for encoders and read them back on the coded(!) port.
+ // This is to ensure muxing the desired aspects into the container.
+ add(ConfigMapper(KEY_COLOR_RANGE, C2_PARAMKEY_COLOR_ASPECTS, "range")
+ .limitTo((D::VIDEO | D::IMAGE) & D::ENCODER & D::CODED)
+ .withC2Mappers<C2Color::range_t>());
+ add(ConfigMapper(KEY_COLOR_TRANSFER, C2_PARAMKEY_COLOR_ASPECTS, "transfer")
+ .limitTo((D::VIDEO | D::IMAGE) & D::ENCODER & D::CODED)
+ .withC2Mappers<C2Color::transfer_t>());
+ add(ConfigMapper("color-primaries", C2_PARAMKEY_COLOR_ASPECTS, "primaries")
+ .limitTo((D::VIDEO | D::IMAGE) & D::ENCODER & D::CODED));
+ add(ConfigMapper("color-matrix", C2_PARAMKEY_COLOR_ASPECTS, "matrix")
+ .limitTo((D::VIDEO | D::IMAGE) & D::ENCODER & D::CODED));
+
+ // read back coded aspects for encoders (on the raw port)
+ add(ConfigMapper(KEY_COLOR_RANGE, C2_PARAMKEY_VUI_COLOR_ASPECTS, "range")
+ .limitTo((D::VIDEO | D::IMAGE) & D::ENCODER & D::RAW & D::READ)
+ .withC2Mappers<C2Color::range_t>());
+ add(ConfigMapper(KEY_COLOR_TRANSFER, C2_PARAMKEY_VUI_COLOR_ASPECTS, "transfer")
+ .limitTo((D::VIDEO | D::IMAGE) & D::ENCODER & D::RAW & D::READ)
+ .withC2Mappers<C2Color::transfer_t>());
+ add(ConfigMapper("color-primaries", C2_PARAMKEY_VUI_COLOR_ASPECTS, "primaries")
+ .limitTo((D::VIDEO | D::IMAGE) & D::ENCODER & D::RAW & D::READ));
+ add(ConfigMapper("color-matrix", C2_PARAMKEY_VUI_COLOR_ASPECTS, "matrix")
+ .limitTo((D::VIDEO | D::IMAGE) & D::ENCODER & D::RAW & D::READ));
+
+ // Dataspace
+ add(ConfigMapper("android._dataspace", C2_PARAMKEY_DATA_SPACE, "value")
+ .limitTo((D::VIDEO | D::IMAGE) & D::RAW));
+
add(ConfigMapper(std::string(KEY_FEATURE_) + FEATURE_SecurePlayback,
C2_PARAMKEY_SECURE_MODE, "value"));
@@ -751,19 +839,22 @@
if (kind.value == C2Component::KIND_ENCODER) {
mParamUpdater->addStandardParam<C2StreamInitDataInfo::output>(C2_PARAMKEY_INIT_DATA);
}
- if (kind.value != C2Component::KIND_ENCODER
- && (domain.value == C2Component::DOMAIN_IMAGE
- || domain.value == C2Component::DOMAIN_VIDEO)) {
- addLocalParam<C2StreamPictureSizeInfo::output>(C2_PARAMKEY_PICTURE_SIZE);
- addLocalParam<C2StreamCropRectInfo::output>(C2_PARAMKEY_CROP_RECT);
- addLocalParam(
- new C2StreamPixelAspectRatioInfo::output(0u, 1u, 1u),
- C2_PARAMKEY_PIXEL_ASPECT_RATIO);
- addLocalParam(new C2StreamRotationInfo::output(0u, 0), C2_PARAMKEY_ROTATION);
- addLocalParam(new C2StreamColorAspectsInfo::output(0u), C2_PARAMKEY_COLOR_ASPECTS);
- addLocalParam<C2StreamHdrStaticInfo::output>(C2_PARAMKEY_HDR_STATIC_INFO);
- addLocalParam<C2StreamDataSpaceInfo::output>(C2_PARAMKEY_DATA_SPACE);
- addLocalParam<C2StreamSurfaceScalingInfo::output>(C2_PARAMKEY_SURFACE_SCALING_MODE);
+ if (domain.value == C2Component::DOMAIN_IMAGE || domain.value == C2Component::DOMAIN_VIDEO) {
+ if (kind.value != C2Component::KIND_ENCODER) {
+ addLocalParam<C2StreamPictureSizeInfo::output>(C2_PARAMKEY_PICTURE_SIZE);
+ addLocalParam<C2StreamCropRectInfo::output>(C2_PARAMKEY_CROP_RECT);
+ addLocalParam(
+ new C2StreamPixelAspectRatioInfo::output(0u, 1u, 1u),
+ C2_PARAMKEY_PIXEL_ASPECT_RATIO);
+ addLocalParam(new C2StreamRotationInfo::output(0u, 0), C2_PARAMKEY_ROTATION);
+ addLocalParam(new C2StreamColorAspectsInfo::output(0u), C2_PARAMKEY_COLOR_ASPECTS);
+ addLocalParam<C2StreamDataSpaceInfo::output>(C2_PARAMKEY_DATA_SPACE);
+ addLocalParam<C2StreamHdrStaticInfo::output>(C2_PARAMKEY_HDR_STATIC_INFO);
+ addLocalParam(new C2StreamSurfaceScalingInfo::output(0u, VIDEO_SCALING_MODE_SCALE_TO_FIT),
+ C2_PARAMKEY_SURFACE_SCALING_MODE);
+ } else {
+ addLocalParam(new C2StreamColorAspectsInfo::input(0u), C2_PARAMKEY_COLOR_ASPECTS);
+ }
}
initializeStandardParams();
@@ -975,6 +1066,62 @@
}
}
+ { // convert color info
+ // TODO: apply platform default to decoder output
+
+ C2Color::primaries_t primaries;
+ C2Color::matrix_t matrix;
+ if (msg->findInt32("color-primaries", (int32_t*)&primaries)
+ && msg->findInt32("color-matrix", (int32_t*)&matrix)) {
+ int32_t standard;
+
+ if (C2Mapper::map(primaries, matrix, &standard)) {
+ msg->setInt32(KEY_COLOR_STANDARD, standard);
+ }
+
+ msg->removeEntryAt(msg->findEntryByName("color-primaries"));
+ msg->removeEntryAt(msg->findEntryByName("color-matrix"));
+ }
+
+
+ // calculate dataspace for raw graphic buffers if not specified by component, or if
+ // using surface with unspecified aspects (as those must be defaulted which may change
+ // the dataspace)
+ if ((portDomain & IS_RAW) && (mDomain & (IS_IMAGE | IS_VIDEO))) {
+ android_dataspace dataspace;
+ ColorAspects aspects = {
+ ColorAspects::RangeUnspecified, ColorAspects::PrimariesUnspecified,
+ ColorAspects::TransferUnspecified, ColorAspects::MatrixUnspecified
+ };
+ ColorUtils::getColorAspectsFromFormat(msg, aspects);
+ ColorAspects origAspects = aspects;
+ if (mUsingSurface) {
+ // get image size (default to HD)
+ int32_t width = 1280;
+ int32_t height = 720;
+ int32_t left, top, right, bottom;
+ if (msg->findRect("crop", &left, &top, &right, &bottom)) {
+ width = right - left + 1;
+ height = bottom - top + 1;
+ } else {
+ (void)msg->findInt32(KEY_WIDTH, &width);
+ (void)msg->findInt32(KEY_HEIGHT, &height);
+ }
+ ColorUtils::setDefaultCodecColorAspectsIfNeeded(aspects, width, height);
+ ColorUtils::setColorAspectsIntoFormat(aspects, msg);
+ }
+
+ if (!msg->findInt32("android._dataspace", (int32_t*)&dataspace)
+ || aspects.mRange != origAspects.mRange
+ || aspects.mPrimaries != origAspects.mPrimaries
+ || aspects.mTransfer != origAspects.mTransfer
+ || aspects.mMatrixCoeffs != origAspects.mMatrixCoeffs) {
+ dataspace = ColorUtils::getDataSpaceForColorAspects(aspects, true /* mayExpand */);
+ msg->setInt32("android._dataspace", dataspace);
+ }
+ }
+ }
+
ALOGV("converted to SDK values as %s", msg->debugString().c_str());
return msg;
}
@@ -1108,6 +1255,19 @@
}
}
+ { // convert color info
+ int32_t standard;
+ if (params->findInt32(KEY_COLOR_STANDARD, &standard)) {
+ C2Color::primaries_t primaries;
+ C2Color::matrix_t matrix;
+
+ if (C2Mapper::map(standard, &primaries, &matrix)) {
+ params->setInt32("color-primaries", primaries);
+ params->setInt32("color-matrix", matrix);
+ }
+ }
+ }
+
// this is to verify that we set proper signedness for standard parameters
bool beVeryStrict = property_get_bool("debug.stagefright.ccodec_strict_type", false);
// this is to allow vendors to use the wrong signedness for standard parameters
diff --git a/media/sfplugin/CCodecConfig.h b/media/sfplugin/CCodecConfig.h
index 12d6176..22a31f5 100644
--- a/media/sfplugin/CCodecConfig.h
+++ b/media/sfplugin/CCodecConfig.h
@@ -117,6 +117,8 @@
sp<AMessage> mInputFormat;
sp<AMessage> mOutputFormat;
+ bool mUsingSurface; ///< using input or output surface
+
std::shared_ptr<InputSurfaceWrapper> mInputSurface;
std::unique_ptr<InputSurfaceWrapper::Config> mISConfig;
