blob: 8e4c40819c9d688ca1b6fbd4f343b222c5da1f69 [file] [log] [blame]
// Copyright 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <CoreVideo/CoreVideo.h>
#include <OpenGL/CGLIOSurface.h>
#include "base/bind.h"
#include "base/command_line.h"
#include "base/sys_byteorder.h"
#include "base/thread_task_runner_handle.h"
#include "content/common/gpu/media/vt_video_decode_accelerator.h"
#include "content/public/common/content_switches.h"
#include "media/filters/h264_parser.h"
#include "ui/gl/scoped_binders.h"
#include "ui/gl/scoped_cgl.h"
using content_common_gpu_media::kModuleVt;
using content_common_gpu_media::InitializeStubs;
using content_common_gpu_media::IsVtInitialized;
using content_common_gpu_media::StubPathMap;
namespace content {
// Size of NALU length headers in AVCC/MPEG-4 format (can be 1, 2, or 4).
static const int kNALUHeaderLength = 4;
// We only request 5 picture buffers from the client which are used to hold the
// decoded samples. These buffers are then reused when the client tells us that
// it is done with the buffer.
static const int kNumPictureBuffers = 5;
// Route decoded frame callbacks back into the VTVideoDecodeAccelerator.
static void OutputThunk(
void* decompression_output_refcon,
void* source_frame_refcon,
OSStatus status,
VTDecodeInfoFlags info_flags,
CVImageBufferRef image_buffer,
CMTime presentation_time_stamp,
CMTime presentation_duration) {
// TODO(sandersd): Implement flush-before-delete to guarantee validity.
VTVideoDecodeAccelerator* vda =
reinterpret_cast<VTVideoDecodeAccelerator*>(decompression_output_refcon);
int32_t bitstream_id = reinterpret_cast<intptr_t>(source_frame_refcon);
vda->Output(bitstream_id, status, image_buffer);
}
VTVideoDecodeAccelerator::DecodedFrame::DecodedFrame(
int32_t bitstream_id,
CVImageBufferRef image_buffer)
: bitstream_id(bitstream_id),
image_buffer(image_buffer) {
}
VTVideoDecodeAccelerator::DecodedFrame::~DecodedFrame() {
}
VTVideoDecodeAccelerator::VTVideoDecodeAccelerator(CGLContextObj cgl_context)
: cgl_context_(cgl_context),
client_(NULL),
format_(NULL),
session_(NULL),
gpu_task_runner_(base::ThreadTaskRunnerHandle::Get()),
weak_this_factory_(this),
decoder_thread_("VTDecoderThread") {
callback_.decompressionOutputCallback = OutputThunk;
callback_.decompressionOutputRefCon = this;
}
VTVideoDecodeAccelerator::~VTVideoDecodeAccelerator() {
}
bool VTVideoDecodeAccelerator::Initialize(
media::VideoCodecProfile profile,
Client* client) {
DCHECK(CalledOnValidThread());
client_ = client;
// Only H.264 is supported.
if (profile < media::H264PROFILE_MIN || profile > media::H264PROFILE_MAX)
return false;
// Require --no-sandbox until VideoToolbox library loading is part of sandbox
// startup (and this VDA is ready for regular users).
if (!base::CommandLine::ForCurrentProcess()->HasSwitch(switches::kNoSandbox))
return false;
if (!IsVtInitialized()) {
// CoreVideo is also required, but the loader stops after the first
// path is loaded. Instead we rely on the transitive dependency from
// VideoToolbox to CoreVideo.
// TODO(sandersd): Fallback to PrivateFrameworks for VideoToolbox.
StubPathMap paths;
paths[kModuleVt].push_back(FILE_PATH_LITERAL(
"/System/Library/Frameworks/VideoToolbox.framework/VideoToolbox"));
if (!InitializeStubs(paths))
return false;
}
// Spawn a thread to handle parsing and calling VideoToolbox.
if (!decoder_thread_.Start())
return false;
return true;
}
// TODO(sandersd): Proper error reporting instead of CHECKs.
void VTVideoDecodeAccelerator::ConfigureDecoder(
const std::vector<const uint8_t*>& nalu_data_ptrs,
const std::vector<size_t>& nalu_data_sizes) {
DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
// Construct a new format description from the parameter sets.
// TODO(sandersd): Replace this with custom code to support OS X < 10.9.
format_.reset();
CHECK(!CMVideoFormatDescriptionCreateFromH264ParameterSets(
kCFAllocatorDefault,
nalu_data_ptrs.size(), // parameter_set_count
&nalu_data_ptrs.front(), // &parameter_set_pointers
&nalu_data_sizes.front(), // &parameter_set_sizes
kNALUHeaderLength, // nal_unit_header_length
format_.InitializeInto()));
CMVideoDimensions coded_dimensions =
CMVideoFormatDescriptionGetDimensions(format_);
// Prepare VideoToolbox configuration dictionaries.
base::ScopedCFTypeRef<CFMutableDictionaryRef> decoder_config(
CFDictionaryCreateMutable(
kCFAllocatorDefault,
1, // capacity
&kCFTypeDictionaryKeyCallBacks,
&kCFTypeDictionaryValueCallBacks));
CFDictionarySetValue(
decoder_config,
// kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
CFSTR("EnableHardwareAcceleratedVideoDecoder"),
kCFBooleanTrue);
base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config(
CFDictionaryCreateMutable(
kCFAllocatorDefault,
4, // capacity
&kCFTypeDictionaryKeyCallBacks,
&kCFTypeDictionaryValueCallBacks));
#define CFINT(i) CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &i)
// TODO(sandersd): RGBA option for 4:4:4 video.
int32_t pixel_format = kCVPixelFormatType_422YpCbCr8;
base::ScopedCFTypeRef<CFNumberRef> cf_pixel_format(CFINT(pixel_format));
base::ScopedCFTypeRef<CFNumberRef> cf_width(CFINT(coded_dimensions.width));
base::ScopedCFTypeRef<CFNumberRef> cf_height(CFINT(coded_dimensions.height));
#undef CFINT
CFDictionarySetValue(
image_config, kCVPixelBufferPixelFormatTypeKey, cf_pixel_format);
CFDictionarySetValue(image_config, kCVPixelBufferWidthKey, cf_width);
CFDictionarySetValue(image_config, kCVPixelBufferHeightKey, cf_height);
CFDictionarySetValue(
image_config, kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
// TODO(sandersd): Check if the session is already compatible.
// TODO(sandersd): Flush.
session_.reset();
CHECK(!VTDecompressionSessionCreate(
kCFAllocatorDefault,
format_, // video_format_description
decoder_config, // video_decoder_specification
image_config, // destination_image_buffer_attributes
&callback_, // output_callback
session_.InitializeInto()));
// If the size has changed, trigger a request for new picture buffers.
gfx::Size new_coded_size(coded_dimensions.width, coded_dimensions.height);
if (coded_size_ != new_coded_size) {
coded_size_ = new_coded_size;
gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
&VTVideoDecodeAccelerator::SizeChangedTask,
weak_this_factory_.GetWeakPtr(),
coded_size_));;
}
}
void VTVideoDecodeAccelerator::Decode(const media::BitstreamBuffer& bitstream) {
DCHECK(CalledOnValidThread());
// TODO(sandersd): Test what happens if bitstream buffers are passed to VT out
// of order.
decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
&VTVideoDecodeAccelerator::DecodeTask, base::Unretained(this),
bitstream));
}
// TODO(sandersd): Proper error reporting instead of CHECKs.
void VTVideoDecodeAccelerator::DecodeTask(
const media::BitstreamBuffer bitstream) {
DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
// Map the bitstream buffer.
base::SharedMemory memory(bitstream.handle(), true);
size_t size = bitstream.size();
CHECK(memory.Map(size));
const uint8_t* buf = static_cast<uint8_t*>(memory.memory());
// NALUs are stored with Annex B format in the bitstream buffer (start codes),
// but VideoToolbox expects AVCC/MPEG-4 format (length headers), so we must
// rewrite the data.
//
// 1. Locate relevant NALUs and compute the size of the translated data.
// Also record any parameter sets for VideoToolbox initialization.
size_t data_size = 0;
std::vector<media::H264NALU> nalus;
std::vector<const uint8_t*> config_nalu_data_ptrs;
std::vector<size_t> config_nalu_data_sizes;
parser_.SetStream(buf, size);
media::H264NALU nalu;
while (true) {
media::H264Parser::Result result = parser_.AdvanceToNextNALU(&nalu);
if (result == media::H264Parser::kEOStream)
break;
CHECK_EQ(result, media::H264Parser::kOk);
// TODO(sandersd): Check that these are only at the start.
if (nalu.nal_unit_type == media::H264NALU::kSPS ||
nalu.nal_unit_type == media::H264NALU::kPPS ||
nalu.nal_unit_type == media::H264NALU::kSPSExt) {
DVLOG(2) << "Parameter set " << nalu.nal_unit_type;
config_nalu_data_ptrs.push_back(nalu.data);
config_nalu_data_sizes.push_back(nalu.size);
} else {
nalus.push_back(nalu);
data_size += kNALUHeaderLength + nalu.size;
}
}
// 2. Initialize VideoToolbox.
// TODO(sandersd): Reinitialize when there are new parameter sets.
if (!session_)
ConfigureDecoder(config_nalu_data_ptrs, config_nalu_data_sizes);
// 3. Allocate a memory-backed CMBlockBuffer for the translated data.
base::ScopedCFTypeRef<CMBlockBufferRef> data;
CHECK(!CMBlockBufferCreateWithMemoryBlock(
kCFAllocatorDefault,
NULL, // &memory_block
data_size, // block_length
kCFAllocatorDefault, // block_allocator
NULL, // &custom_block_source
0, // offset_to_data
data_size, // data_length
0, // flags
data.InitializeInto()));
// 4. Copy NALU data, inserting length headers.
size_t offset = 0;
for (size_t i = 0; i < nalus.size(); i++) {
media::H264NALU& nalu = nalus[i];
uint32_t header = base::HostToNet32(static_cast<uint32_t>(nalu.size));
CHECK(!CMBlockBufferReplaceDataBytes(
&header, data, offset, kNALUHeaderLength));
offset += kNALUHeaderLength;
CHECK(!CMBlockBufferReplaceDataBytes(nalu.data, data, offset, nalu.size));
offset += nalu.size;
}
// 5. Package the data for VideoToolbox and request decoding.
base::ScopedCFTypeRef<CMSampleBufferRef> frame;
CHECK(!CMSampleBufferCreate(
kCFAllocatorDefault,
data, // data_buffer
true, // data_ready
NULL, // make_data_ready_callback
NULL, // make_data_ready_refcon
format_, // format_description
1, // num_samples
0, // num_sample_timing_entries
NULL, // &sample_timing_array
0, // num_sample_size_entries
NULL, // &sample_size_array
frame.InitializeInto()));
// Asynchronous Decompression allows for parallel submission of frames
// (without it, DecodeFrame() does not return until the frame has been
// decoded). We don't enable Temporal Processing so that frames are always
// returned in decode order; this makes it easier to avoid deadlock.
VTDecodeFrameFlags decode_flags =
kVTDecodeFrame_EnableAsynchronousDecompression;
intptr_t bitstream_id = bitstream.id();
CHECK(!VTDecompressionSessionDecodeFrame(
session_,
frame, // sample_buffer
decode_flags, // decode_flags
reinterpret_cast<void*>(bitstream_id), // source_frame_refcon
NULL)); // &info_flags_out
}
// This method may be called on any VideoToolbox thread.
// TODO(sandersd): Proper error reporting instead of CHECKs.
void VTVideoDecodeAccelerator::Output(
int32_t bitstream_id,
OSStatus status,
CVImageBufferRef image_buffer) {
CHECK(!status);
CHECK_EQ(CFGetTypeID(image_buffer), CVPixelBufferGetTypeID());
CFRetain(image_buffer);
gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
&VTVideoDecodeAccelerator::OutputTask,
weak_this_factory_.GetWeakPtr(),
DecodedFrame(bitstream_id, image_buffer)));
}
void VTVideoDecodeAccelerator::OutputTask(DecodedFrame frame) {
DCHECK(CalledOnValidThread());
decoded_frames_.push(frame);
SendPictures();
}
void VTVideoDecodeAccelerator::SizeChangedTask(gfx::Size coded_size) {
DCHECK(CalledOnValidThread());
texture_size_ = coded_size;
// TODO(sandersd): Dismiss existing picture buffers.
client_->ProvidePictureBuffers(
kNumPictureBuffers, texture_size_, GL_TEXTURE_RECTANGLE_ARB);
}
void VTVideoDecodeAccelerator::AssignPictureBuffers(
const std::vector<media::PictureBuffer>& pictures) {
DCHECK(CalledOnValidThread());
for (size_t i = 0; i < pictures.size(); i++) {
CHECK(!texture_ids_.count(pictures[i].id()));
available_picture_ids_.push(pictures[i].id());
texture_ids_[pictures[i].id()] = pictures[i].texture_id();
}
// Pictures are not marked as uncleared until this method returns. They will
// become broken if they are used before that happens.
gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
&VTVideoDecodeAccelerator::SendPictures,
weak_this_factory_.GetWeakPtr()));
}
void VTVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_id) {
DCHECK(CalledOnValidThread());
DCHECK_EQ(CFGetRetainCount(picture_bindings_[picture_id]), 1);
picture_bindings_.erase(picture_id);
available_picture_ids_.push(picture_id);
SendPictures();
}
// TODO(sandersd): Proper error reporting instead of CHECKs.
void VTVideoDecodeAccelerator::SendPictures() {
DCHECK(CalledOnValidThread());
if (available_picture_ids_.empty() || decoded_frames_.empty())
return;
gfx::ScopedCGLSetCurrentContext scoped_set_current_context(cgl_context_);
glEnable(GL_TEXTURE_RECTANGLE_ARB);
while (!available_picture_ids_.empty() && !decoded_frames_.empty()) {
int32_t picture_id = available_picture_ids_.front();
available_picture_ids_.pop();
DecodedFrame frame = decoded_frames_.front();
decoded_frames_.pop();
IOSurfaceRef surface = CVPixelBufferGetIOSurface(frame.image_buffer);
gfx::ScopedTextureBinder
texture_binder(GL_TEXTURE_RECTANGLE_ARB, texture_ids_[picture_id]);
CHECK(!CGLTexImageIOSurface2D(
cgl_context_, // ctx
GL_TEXTURE_RECTANGLE_ARB, // target
GL_RGB, // internal_format
texture_size_.width(), // width
texture_size_.height(), // height
GL_YCBCR_422_APPLE, // format
GL_UNSIGNED_SHORT_8_8_APPLE, // type
surface, // io_surface
0)); // plane
picture_bindings_[picture_id] = frame.image_buffer;
client_->PictureReady(media::Picture(
picture_id, frame.bitstream_id, gfx::Rect(texture_size_)));
client_->NotifyEndOfBitstreamBuffer(frame.bitstream_id);
}
glDisable(GL_TEXTURE_RECTANGLE_ARB);
}
void VTVideoDecodeAccelerator::Flush() {
DCHECK(CalledOnValidThread());
// TODO(sandersd): Trigger flush, sending frames.
}
void VTVideoDecodeAccelerator::Reset() {
DCHECK(CalledOnValidThread());
// TODO(sandersd): Trigger flush, discarding frames.
}
void VTVideoDecodeAccelerator::Destroy() {
DCHECK(CalledOnValidThread());
// TODO(sandersd): Trigger flush, discarding frames, and wait for them.
delete this;
}
bool VTVideoDecodeAccelerator::CanDecodeOnIOThread() {
return false;
}
} // namespace content