SkJpegCodec: Add Multi-Picture Format (MPF) Support
Add support for the CIPA DC-x007-2009 Multi-Picture Format. Jpeg
images taken on iPhones make extensive use of this standard to include
multiple images (e.g, depth and gain maps), in addition to the base
image.
This information is stored in the APP2 segment in a structure that
lists the offsets and sizes of all images. Add a the function
SkJpegParseMultiPicture that will parse this segment and retrieve this
information.
Note that the offsets that are specified are relative to the APP2
segment in which they are specified. The location of the APP2 segment
inside the Jpeg's SkStream is not available using libjpeg (libjpeg
will extract the APP2 segment if requested, but does not compute or
save where it found it). To fix this, we use SkJpegSegmentScan to
find the location of the APP2 segment in the Jpeg's SkStream.
Add the function SkJpegMultiPictureStreams to extract an SkStream
for all of the separate images. Add the function SkJpegSegmentScan::
getSubsetStream to retrieve an SkStream for a subset of the Jpeg's
SkStream. This is inefficiently implemented and could be made much
more efficient.
Add tests, and a test image that has two auxiliary images.
Bug: skia: 14031
Change-Id: I411a6984d85e0e759fac5a3d48590f0d2a1f159e
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/625921
Commit-Queue: Christopher Cameron <ccameron@google.com>
Reviewed-by: Brian Osman <brianosman@google.com>
diff --git a/BUILD.gn b/BUILD.gn
index 5b6b618..f58729d 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -1203,6 +1203,7 @@
sources = [
"src/codec/SkJpegCodec.cpp",
"src/codec/SkJpegDecoderMgr.cpp",
+ "src/codec/SkJpegMultiPicture.cpp",
"src/codec/SkJpegSegmentScan.cpp",
"src/codec/SkJpegUtility.cpp",
]
diff --git a/resources/images/iphone_13_pro.jpeg b/resources/images/iphone_13_pro.jpeg
new file mode 100644
index 0000000..9d0bf69
--- /dev/null
+++ b/resources/images/iphone_13_pro.jpeg
Binary files differ
diff --git a/src/codec/BUILD.bazel b/src/codec/BUILD.bazel
index 619888f..d4fe17c 100644
--- a/src/codec/BUILD.bazel
+++ b/src/codec/BUILD.bazel
@@ -88,6 +88,8 @@
"SkJpegDecoderMgr.h",
"SkJpegUtility.cpp",
"SkJpegUtility.h",
+ "SkJpegMultiPicture.cpp",
+ "SkJpegMultiPicture.h",
"SkJpegSegmentScan.cpp",
"SkJpegSegmentScan.h",
"SkParseEncodedOrigin.cpp",
diff --git a/src/codec/SkJpegMultiPicture.cpp b/src/codec/SkJpegMultiPicture.cpp
new file mode 100644
index 0000000..7b704b5
--- /dev/null
+++ b/src/codec/SkJpegMultiPicture.cpp
@@ -0,0 +1,218 @@
+/*
+ * Copyright 2023 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "src/codec/SkJpegMultiPicture.h"
+
+#include "include/core/SkData.h"
+#include "include/core/SkStream.h"
+#include "src/codec/SkCodecPriv.h"
+#include "src/codec/SkJpegPriv.h"
+#include "src/codec/SkJpegSegmentScan.h"
+
+#include <cstring>
+
+// Helper macro for SkJpegParseMultiPicture. Define the indicated variable VAR of type TYPE, and
+// read it from the stream, performing any endian-ness conversions as needed. If any errors are
+// encountered, then return nullptr. The last void line is present to suppress unused variable
+// warnings for parameters that we don't use.
+#define DEFINE_AND_READ_UINT(TYPE, VAR) \
+ TYPE VAR = 0; \
+ { \
+ uint8_t VAR##Data[sizeof(TYPE)] = {0}; \
+ if (!stream->read(VAR##Data, sizeof(TYPE))) { \
+ return nullptr; \
+ } \
+ for (size_t VAR##i = 0; VAR##i < sizeof(TYPE); ++VAR##i) { \
+ VAR *= 256; \
+ VAR += VAR##Data[streamIsBigEndian ? VAR##i : (sizeof(TYPE) - VAR##i - 1)]; \
+ } \
+ } \
+ (void)VAR
+
+std::unique_ptr<SkJpegMultiPictureParameters> SkJpegParseMultiPicture(
+ const sk_sp<const SkData>& data) {
+ std::unique_ptr<SkMemoryStream> stream = SkMemoryStream::MakeDirect(data->data(), data->size());
+ // This function reads the structure described in Figure 6 of CIPA DC-x007-2009.
+
+ // Determine the endianness of the values in the structure. See Figure 5 (MP endian tag
+ // structure).
+ bool streamIsBigEndian = false;
+ {
+ constexpr uint8_t kMpLittleEndian[] = {0x49, 0x49, 0x2A, 0x00};
+ constexpr uint8_t kMpBigEndian[] = {0x4D, 0x4D, 0x00, 0x2A};
+ uint8_t endianTag[4] = {0};
+ if (!stream->read(endianTag, sizeof(endianTag))) {
+ SkCodecPrintf("Failed to read MP endian tag.\n");
+ return nullptr;
+ }
+ if (!memcmp(endianTag, kMpBigEndian, 4)) {
+ streamIsBigEndian = true;
+ } else if (!memcmp(endianTag, kMpLittleEndian, 4)) {
+ streamIsBigEndian = false;
+ } else {
+ SkCodecPrintf("MP endian tag was invalid.\n");
+ return nullptr;
+ }
+ }
+
+ // Seek to the Index Image File Directory (Index IFD).
+ DEFINE_AND_READ_UINT(uint32_t, indexIfdOffset);
+ if (stream->getPosition() < indexIfdOffset) {
+ SkCodecPrintf("MP Index IFD offset moves backwards.\n");
+ return nullptr;
+ }
+ if (!stream->seek(indexIfdOffset)) {
+ SkCodecPrintf("Failed to seek to MPF IFD.\n");
+ return nullptr;
+ }
+
+ // Read the number of tags in the Index IFD. See Table 3 (MP Index IFD Tags) for a description
+ // of all possible tags.
+ DEFINE_AND_READ_UINT(uint16_t, tagCount);
+
+ // We will extract the number of images from the tags.
+ uint32_t numberOfImages = 0;
+
+ // We will need to MP Entries in order to determine the image offsets.
+ bool hasMpEntryTag = false;
+
+ // The MP Index IFD tags shall be specified in the order of their tag IDs (text from
+ // section 5.2.3), so keep track of the previous tag id read.
+ uint16_t previousTagId = 0;
+ for (uint16_t tagIndex = 0; tagIndex < tagCount; ++tagIndex) {
+ DEFINE_AND_READ_UINT(uint16_t, tagId);
+ DEFINE_AND_READ_UINT(uint16_t, type);
+ DEFINE_AND_READ_UINT(uint32_t, count);
+ DEFINE_AND_READ_UINT(uint32_t, value);
+
+ if (previousTagId >= tagId) {
+ SkCodecPrintf("MPF tags not in order.\n");
+ return nullptr;
+ }
+ previousTagId = tagId;
+
+ switch (tagId) {
+ case 0xB000:
+ // Version. We ignore this.
+ break;
+ case 0xB001:
+ // Number of images.
+ numberOfImages = value;
+ if (numberOfImages < 1) {
+ SkCodecPrintf("Invalid number of images.\n");
+ return nullptr;
+ }
+ break;
+ case 0xB002:
+ // MP Entry.
+ hasMpEntryTag = true;
+ if (count != 16 * numberOfImages) {
+ SkCodecPrintf("Invalid MPEntry count.\n");
+ return nullptr;
+ }
+ break;
+ case 0xB003:
+ // Individual Image Unique ID list. Validate it, but otherwise ignore it.
+ if (count != 33 * numberOfImages) {
+ SkCodecPrintf("Invalid Image Unique ID count.\n");
+ return nullptr;
+ }
+ break;
+ case 0xB004:
+ // Total number of captured frames. We ignore this.
+ break;
+ default:
+ return nullptr;
+ }
+ }
+ if (!numberOfImages) {
+ SkCodecPrintf("Number of images must be greater than zero.\n");
+ return nullptr;
+ }
+ if (!hasMpEntryTag) {
+ SkCodecPrintf("MP Entry tag was not present.\n");
+ return nullptr;
+ }
+
+ // Start to prepare the result that we will return.
+ auto result = std::make_unique<SkJpegMultiPictureParameters>();
+ result->images.resize(numberOfImages);
+
+ // Read the Attribute IFD offset, and verify that it is zero (absent) or greater than our
+ // current offset. We will not read or validate the Attribute IFD.
+ DEFINE_AND_READ_UINT(uint32_t, attributeIfdOffset);
+ if (attributeIfdOffset > 0) {
+ if (stream->getPosition() < attributeIfdOffset) {
+ SkCodecPrintf("MP Attribute IFD offset moves backwards.\n");
+ return nullptr;
+ }
+ }
+
+ // Read the MP Entries (which we verified to be present with hasMpEntryTag).
+ for (uint32_t i = 0; i < numberOfImages; ++i) {
+ DEFINE_AND_READ_UINT(uint32_t, attribute);
+ constexpr uint32_t kAttributeTypeMask = 0x7000000;
+ if ((attribute & kAttributeTypeMask) != 0) {
+ SkCodecPrintf("Image type must be 0 (JPEG).\n");
+ }
+
+ DEFINE_AND_READ_UINT(uint32_t, size);
+ DEFINE_AND_READ_UINT(uint32_t, dataOffset);
+ if (i == 0 && dataOffset != 0) {
+ SkCodecPrintf("First individual Image offset must be NULL.\n");
+ }
+
+ DEFINE_AND_READ_UINT(uint16_t, dependentImage1EntryNumber);
+ DEFINE_AND_READ_UINT(uint16_t, dependentImage2EntryNumber);
+ result->images[i].dataOffset = dataOffset;
+ result->images[i].size = size;
+ }
+
+ return result;
+}
+
+std::unique_ptr<SkJpegMultiPictureStreams> SkJpegExtractMultiPictureStreams(
+ SkJpegSegmentScan* scan) {
+ // Look through the scanned segments until we arrive at a MultiPicture segment that we can
+ // parse.
+ size_t mpSegmentOffset = 0;
+ std::unique_ptr<SkJpegMultiPictureParameters> mpParams;
+ for (const auto& segment : scan->segments()) {
+ if (segment.marker != kMpfMarker) {
+ continue;
+ }
+ auto parameterData = scan->copyParameters(segment, kMpfSig, sizeof(kMpfSig));
+ if (!parameterData) {
+ continue;
+ }
+ mpParams = SkJpegParseMultiPicture(parameterData);
+ if (mpParams) {
+ mpSegmentOffset = segment.offset;
+ break;
+ }
+ }
+ if (!mpParams) {
+ return nullptr;
+ }
+
+ // Create streams for each of the specified segments.
+ auto result = std::make_unique<SkJpegMultiPictureStreams>();
+ size_t numberOfImages = mpParams->images.size();
+ result->images.resize(numberOfImages);
+ for (size_t i = 0; i < numberOfImages; ++i) {
+ const auto& imageParams = mpParams->images[i];
+ if (imageParams.dataOffset == 0) {
+ continue;
+ }
+ size_t imageStreamOffset = mpSegmentOffset + SkJpegSegmentScan::kMarkerCodeSize +
+ SkJpegSegmentScan::kParameterLengthSize + sizeof(kMpfSig) +
+ imageParams.dataOffset;
+ size_t imageStreamSize = imageParams.size;
+ result->images[i].stream = scan->getSubsetStream(imageStreamOffset, imageStreamSize);
+ }
+ return result;
+}
diff --git a/src/codec/SkJpegMultiPicture.h b/src/codec/SkJpegMultiPicture.h
new file mode 100644
index 0000000..417f5ca
--- /dev/null
+++ b/src/codec/SkJpegMultiPicture.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2023 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#ifndef SkJpegMultiPicture_codec_DEFINED
+#define SkJpegMultiPicture_codec_DEFINED
+
+#include "include/core/SkRefCnt.h"
+#include "include/core/SkStream.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+class SkData;
+class SkJpegSegmentScan;
+
+/*
+ * Parsed Jpeg Multi-Picture Format structure as specified in CIPA DC-x007-2009. An introduction to
+ * the format can be found in Figure 1 (Basic MP File format data structure) and Figure 6 (Internal
+ * Structure of the MP Index IFD) in that document. This parsing will extract only the size and
+ * offset parameters from the images in the Index Image File Directory.
+ */
+struct SkJpegMultiPictureParameters {
+ // An individual image.
+ struct Image {
+ // The size of the image in bytes.
+ uint32_t size;
+ // The offset of the image in bytes. This offset is specified relative to the address of
+ // the MP Endian field in the MP Header, unless the image is a First Individual Image, in
+ // which case the value of the offest [sic] shall be NULL (from section 5.2.3.3).
+ uint32_t dataOffset;
+ };
+
+ // The images listed in the Index Image File Directory.
+ std::vector<Image> images;
+};
+
+/*
+ * Parse Jpeg Multi-Picture Format parameters. The specified data should start with the MP Header.
+ * Returns nullptr on error.
+ */
+std::unique_ptr<SkJpegMultiPictureParameters> SkJpegParseMultiPicture(
+ const sk_sp<const SkData>& data);
+
+/*
+ * Create SkStreams for all MultiPicture images, given a SkJpegSegmentScan of the image. This will
+ * return nullptr if there is not MultiPicture segment, or if the MultiPicture parameters fail to
+ * parse.
+ */
+struct SkJpegMultiPictureStreams {
+ // An individual image.
+ struct Image {
+ // An SkStream from which the image's data may be read. This is nullptr for the First
+ // Individual Image and for any images which encounter errors (e.g, they are outside of
+ // the range of the stream).
+ std::unique_ptr<SkStream> stream;
+ };
+
+ // The images as listed in the Index Image File Directory.
+ std::vector<Image> images;
+};
+std::unique_ptr<SkJpegMultiPictureStreams> SkJpegExtractMultiPictureStreams(
+ SkJpegSegmentScan* scan);
+
+#endif
diff --git a/src/codec/SkJpegPriv.h b/src/codec/SkJpegPriv.h
index ad88736..98067b3 100644
--- a/src/codec/SkJpegPriv.h
+++ b/src/codec/SkJpegPriv.h
@@ -38,6 +38,9 @@
static constexpr uint32_t kExifHeaderSize = 14;
constexpr uint8_t kExifSig[] = {'E', 'x', 'i', 'f', '\0'};
+static constexpr uint32_t kMpfMarker = JPEG_APP0 + 2;
+static constexpr uint8_t kMpfSig[] = {'M', 'P', 'F', '\0'};
+
/*
* Error handling struct
*/
diff --git a/src/codec/SkJpegSegmentScan.cpp b/src/codec/SkJpegSegmentScan.cpp
index 2130f53..a379d74 100644
--- a/src/codec/SkJpegSegmentScan.cpp
+++ b/src/codec/SkJpegSegmentScan.cpp
@@ -229,6 +229,7 @@
auto segmentSignature = SkData::MakeUninitialized(signatureLength);
if (fStream->read(segmentSignature->writable_data(), segmentSignature->size()) !=
segmentSignature->size()) {
+ SkCodecPrintf("Failed to read parameters\n");
return nullptr;
}
if (memcmp(segmentSignature->data(), signature, signatureLength) != 0) {
@@ -243,3 +244,19 @@
return result;
}
+
+std::unique_ptr<SkStream> SkJpegSegmentScan::getSubsetStream(size_t offset, size_t size) {
+ // Read the image's data. It would be better to fork `stream` and limit its position and size.
+ if (!fStream->seek(fInitialPosition + offset)) {
+ SkCodecPrintf("Failed to seek to subset stream position.\n");
+ return nullptr;
+ }
+
+ sk_sp<SkData> data = SkData::MakeUninitialized(size);
+ if (fStream->read(data->writable_data(), size) != size) {
+ SkCodecPrintf("Failed to read subset stream data.\n");
+ return nullptr;
+ }
+
+ return SkMemoryStream::Make(data);
+}
diff --git a/src/codec/SkJpegSegmentScan.h b/src/codec/SkJpegSegmentScan.h
index dd6f47d..c55dd89 100644
--- a/src/codec/SkJpegSegmentScan.h
+++ b/src/codec/SkJpegSegmentScan.h
@@ -58,6 +58,10 @@
const void* signature,
const size_t signatureLength);
+ // Return a stream for a subset of the original stream, starting at the specified offset, and
+ // with the specified length.
+ std::unique_ptr<SkStream> getSubsetStream(size_t offset, size_t size);
+
// The number of bytes in a marker code is two.
static constexpr size_t kMarkerCodeSize = 2;
diff --git a/tests/CodecTest.cpp b/tests/CodecTest.cpp
index a1b44f4..a7979ff 100644
--- a/tests/CodecTest.cpp
+++ b/tests/CodecTest.cpp
@@ -46,6 +46,7 @@
#include "tools/ToolUtils.h"
#ifdef SK_CODEC_DECODES_JPEG
+#include "src/codec/SkJpegMultiPicture.h"
#include "src/codec/SkJpegSegmentScan.h"
#endif
@@ -1999,4 +2000,46 @@
REPORTER_ASSERT(r, rec.testSegmentParameterLength == segment.parameterLength);
}
}
+
+DEF_TEST(Codec_jpegMultiPicture, r) {
+ const char* path = "images/iphone_13_pro.jpeg";
+ auto stream = GetResourceAsStream(path);
+ REPORTER_ASSERT(r, stream);
+
+ auto segmentScan = SkJpegSegmentScan::Create(stream.get(), SkJpegSegmentScan::Options());
+ REPORTER_ASSERT(r, segmentScan);
+
+ // Extract the streams for the MultiPicture images.
+ auto mpStreams = SkJpegExtractMultiPictureStreams(segmentScan.get());
+ REPORTER_ASSERT(r, mpStreams);
+ size_t numberOfImages = mpStreams->images.size();
+
+ // Decode them into bitmaps.
+ std::vector<SkBitmap> bitmaps(numberOfImages);
+ for (size_t i = 0; i < numberOfImages; ++i) {
+ auto imageStream = std::move(mpStreams->images[i].stream);
+ if (i == 0) {
+ REPORTER_ASSERT(r, !imageStream);
+ continue;
+ }
+ REPORTER_ASSERT(r, imageStream);
+
+ std::unique_ptr<SkCodec> codec = SkCodec::MakeFromStream(std::move(imageStream));
+ REPORTER_ASSERT(r, codec);
+
+ SkBitmap bm;
+ bm.allocPixels(codec->getInfo());
+ REPORTER_ASSERT(
+ r, SkCodec::kSuccess == codec->getPixels(bm.info(), bm.getPixels(), bm.rowBytes()));
+ bitmaps[i] = bm;
+ }
+
+ // Spot-check the image size and pixels.
+ REPORTER_ASSERT(r, bitmaps[1].dimensions() == SkISize::Make(1512, 2016));
+ REPORTER_ASSERT(r, bitmaps[1].getColor(0, 0) == 0xFF3B3B3B);
+ REPORTER_ASSERT(r, bitmaps[1].getColor(1511, 2015) == 0xFF101010);
+ REPORTER_ASSERT(r, bitmaps[2].dimensions() == SkISize::Make(576, 768));
+ REPORTER_ASSERT(r, bitmaps[2].getColor(0, 0) == 0xFF010101);
+ REPORTER_ASSERT(r, bitmaps[2].getColor(575, 767) == 0xFFB5B5B5);
+}
#endif
