blob: 5066162a30190d3054d4d6acdb754cea32ec99bc [file] [log] [blame]
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.car.cluster.sample;
import android.annotation.NonNull;
import android.content.Context;
import android.hardware.display.DisplayManager;
import android.hardware.display.DisplayManager.DisplayListener;
import android.hardware.display.VirtualDisplay;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaCodec.CodecException;
import android.media.MediaCodecInfo;
import android.media.MediaCodecInfo.CodecProfileLevel;
import android.media.MediaFormat;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Looper;
import android.os.Message;
import android.util.Log;
import android.view.Display;
import android.view.Surface;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;
import java.nio.ByteBuffer;
import java.util.UUID;
/**
* This class encapsulates all work related to managing networked virtual display.
* <p>
* It opens server socket and listens on port {@code PORT} for incoming connections. Once connection
* is established it creates virtual display and media encoder and starts streaming video to that
* socket. If the receiving part is disconnected, it will keep port open and virtual display won't
* be destroyed.
*/
public class NetworkedVirtualDisplay {
private static final String TAG = "Cluster." + NetworkedVirtualDisplay.class.getSimpleName();
private final String mUniqueId = UUID.randomUUID().toString();
private final DisplayManager mDisplayManager;
private final int mWidth;
private final int mHeight;
private final int mDpi;
private static final int PORT = 5151;
private static final int FPS = 25;
private static final int BITRATE = 6144000;
private static final String MEDIA_FORMAT_MIMETYPE = MediaFormat.MIMETYPE_VIDEO_AVC;
private static final int MSG_START = 0;
private static final int MSG_STOP = 1;
private static final int MSG_RESUBMIT_FRAME = 2;
private VirtualDisplay mVirtualDisplay;
private MediaCodec mVideoEncoder;
private HandlerThread mThread = new HandlerThread("NetworkThread");
private Handler mHandler;
private ServerSocket mServerSocket;
private OutputStream mOutputStream;
private byte[] mBuffer = null;
private int mLastFrameLength = 0;
private final DebugCounter mCounter = new DebugCounter();
NetworkedVirtualDisplay(Context context, int width, int height, int dpi) {
mDisplayManager = context.getSystemService(DisplayManager.class);
mWidth = width;
mHeight = height;
mDpi = dpi;
DisplayListener displayListener = new DisplayListener() {
@Override
public void onDisplayAdded(int i) {
final Display display = mDisplayManager.getDisplay(i);
if (display != null && getDisplayName().equals(display.getName())) {
onVirtualDisplayReady(display);
}
}
@Override
public void onDisplayRemoved(int i) {}
@Override
public void onDisplayChanged(int i) {}
};
mDisplayManager.registerDisplayListener(displayListener, new Handler());
}
/**
* Opens socket and creates virtual display asynchronously once connection established. Clients
* of this class may subscribe to
* {@link android.hardware.display.DisplayManager#registerDisplayListener(
* DisplayListener, Handler)} to be notified when virtual display is created.
* Note, that this method should be called only once.
*
* @return Unique display name associated with the instance of this class.
*
* @see {@link Display#getName()}
*
* @throws IllegalStateException thrown if networked display already started
*/
public String start() {
if (mThread.isAlive()) {
throw new IllegalStateException("Already started");
}
mThread.start();
mHandler = new NetworkThreadHandler(mThread.getLooper());
mHandler.sendMessage(Message.obtain(mHandler, MSG_START));
return getDisplayName();
}
public void release() {
stopCasting();
if (mVirtualDisplay != null) {
mVirtualDisplay.release();
mVirtualDisplay = null;
}
mThread.quit();
}
private String getDisplayName() {
return "Cluster-" + mUniqueId;
}
private VirtualDisplay createVirtualDisplay() {
Log.i(TAG, "createVirtualDisplay " + mWidth + "x" + mHeight +"@" + mDpi);
return mDisplayManager.createVirtualDisplay(getDisplayName(), mWidth, mHeight, mDpi,
null, 0 /* flags */, null, null );
}
private void onVirtualDisplayReady(Display display) {
Log.i(TAG, "onVirtualDisplayReady, display: " + display);
}
private void startCasting(Handler handler) {
Log.i(TAG, "Start casting...");
mVideoEncoder = createVideoStream(handler);
if (mVirtualDisplay == null) {
mVirtualDisplay = createVirtualDisplay();
}
mVirtualDisplay.setSurface(mVideoEncoder.createInputSurface());
mVideoEncoder.start();
Log.i(TAG, "Video encoder started");
}
private MediaCodec createVideoStream(Handler handler) {
MediaCodec encoder;
try {
encoder = MediaCodec.createEncoderByType(MEDIA_FORMAT_MIMETYPE);
} catch (IOException e) {
Log.e(TAG, "Failed to create video encoder for " + MEDIA_FORMAT_MIMETYPE, e);
return null;
}
encoder.setCallback(new MediaCodec.Callback() {
@Override
public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) {
Log.i(TAG, "onInputBufferAvailable, index: " + index);
}
@Override
public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index,
@NonNull BufferInfo info) {
Log.i(TAG, "onOutputBufferAvailable, index: " + index);
mCounter.outputBuffers++;
doOutputBufferAvailable(index, info);
}
@Override
public void onError(@NonNull MediaCodec codec, @NonNull CodecException e) {
Log.e(TAG, "onError, codec: " + codec, e);
mCounter.bufferErrors++;
}
@Override
public void onOutputFormatChanged(@NonNull MediaCodec codec,
@NonNull MediaFormat format) {
Log.i(TAG, "onOutputFormatChanged, codec: " + codec + ", format: " + format);
}
}, handler);
configureVideoEncoder(encoder, mWidth, mHeight);
return encoder;
}
private void doOutputBufferAvailable(int index, @NonNull BufferInfo info) {
mHandler.removeMessages(MSG_RESUBMIT_FRAME);
ByteBuffer encodedData = mVideoEncoder.getOutputBuffer(index);
if (encodedData == null) {
throw new RuntimeException("couldn't fetch buffer at index " + index);
}
if (info.size != 0) {
encodedData.position(info.offset);
encodedData.limit(info.offset + info.size);
mLastFrameLength = encodedData.remaining();
if (mBuffer == null || mBuffer.length < mLastFrameLength) {
Log.i(TAG, "Allocating new buffer: " + mLastFrameLength);
mBuffer = new byte[mLastFrameLength];
}
encodedData.get(mBuffer, 0, mLastFrameLength);
mVideoEncoder.releaseOutputBuffer(index, false);
sendFrame(mBuffer, mLastFrameLength);
// If nothing happens in Virtual Display we won't receive new frames. If we won't keep
// sending frames it could be a problem for the receiver because it needs certain
// number of frames in order to start decoding.
scheduleResendingLastFrame(1000 / FPS);
} else {
Log.e(TAG, "Skipping empty buffer");
mVideoEncoder.releaseOutputBuffer(index, false);
}
}
private void scheduleResendingLastFrame(long delayMs) {
Message msg = mHandler.obtainMessage(MSG_RESUBMIT_FRAME);
mHandler.sendMessageDelayed(msg, delayMs);
}
private void sendFrame(byte[] buf, int len) {
try {
mOutputStream.write(buf, 0, len);
Log.i(TAG, "Bytes written: " + len);
} catch (IOException e) {
mCounter.clientsDisconnected++;
mOutputStream = null;
Log.e(TAG, "Failed to write data to socket, restart casting", e);
restart();
}
}
private void stopCasting() {
Log.i(TAG, "Stopping casting...");
if (mServerSocket != null) {
try {
mServerSocket.close();
} catch (IOException e) {
Log.w(TAG, "Failed to close server socket, ignoring", e);
}
mServerSocket = null;
}
if (mVirtualDisplay != null) {
// We do not want to destroy virtual display (as it will also destroy all the
// activities on that display, instead we will turn off the display by setting
// a null surface.
Surface surface = mVirtualDisplay.getSurface();
if (surface != null) surface.release();
mVirtualDisplay.setSurface(null);
}
if (mVideoEncoder != null) {
// Releasing encoder as stop/start didn't work well (couldn't create or reuse input
// surface).
mVideoEncoder.stop();
mVideoEncoder.release();
mVideoEncoder = null;
}
Log.i(TAG, "Casting stopped");
}
private synchronized void restart() {
// This method could be called from different threads when receiver has disconnected.
if (mHandler.hasMessages(MSG_START)) return;
mHandler.sendMessage(Message.obtain(mHandler, MSG_STOP));
mHandler.sendMessage(Message.obtain(mHandler, MSG_START));
}
private class NetworkThreadHandler extends Handler {
NetworkThreadHandler(Looper looper) {
super(looper);
}
@Override
public void handleMessage(Message msg) {
switch (msg.what) {
case MSG_START:
if (mServerSocket == null) {
mServerSocket = openServerSocket();
}
Log.i(TAG, "Server socket opened");
mOutputStream = waitForReceiver(mServerSocket);
if (mOutputStream == null) {
sendMessage(Message.obtain(this, MSG_START));
break;
}
mCounter.clientsConnected++;
startCasting(this);
break;
case MSG_STOP:
stopCasting();
break;
case MSG_RESUBMIT_FRAME:
if (mServerSocket != null && mOutputStream != null) {
Log.i(TAG, "Resending the last frame again. Buffer: " + mLastFrameLength);
sendFrame(mBuffer, mLastFrameLength);
}
// We will keep sending last frame every second as a heartbeat.
scheduleResendingLastFrame(1000L);
break;
}
}
}
private static void configureVideoEncoder(MediaCodec codec, int width, int height) {
MediaFormat format = MediaFormat.createVideoFormat(MEDIA_FORMAT_MIMETYPE, width, height);
format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_BIT_RATE, BITRATE);
format.setInteger(MediaFormat.KEY_FRAME_RATE, FPS);
format.setInteger(MediaFormat.KEY_CAPTURE_RATE, FPS);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
format.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, 1); // 1 second between I-frames
format.setInteger(MediaFormat.KEY_LEVEL, CodecProfileLevel.AVCLevel31);
format.setInteger(MediaFormat.KEY_PROFILE,
MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline);
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
}
private OutputStream waitForReceiver(ServerSocket serverSocket) {
try {
Log.i(TAG, "Listening for incoming connections on port: " + PORT);
Socket socket = serverSocket.accept();
Log.i(TAG, "Receiver connected: " + socket);
listenReceiverDisconnected(socket.getInputStream());
return socket.getOutputStream();
} catch (IOException e) {
Log.e(TAG, "Failed to accept connection");
return null;
}
}
private void listenReceiverDisconnected(InputStream inputStream) {
new Thread(() -> {
try {
if (inputStream.read() == -1) throw new IOException();
} catch (IOException e) {
Log.w(TAG, "Receiver has disconnected", e);
}
restart();
}).start();
}
private static ServerSocket openServerSocket() {
try {
return new ServerSocket(PORT);
} catch (IOException e) {
Log.e(TAG, "Failed to create server socket", e);
throw new RuntimeException(e);
}
}
@Override
public String toString() {
return getClass() + "{"
+ mServerSocket
+", receiver connected: " + (mOutputStream != null)
+", encoder: " + mVideoEncoder
+", virtualDisplay" + mVirtualDisplay
+ "}";
}
private static class DebugCounter {
long outputBuffers;
long bufferErrors;
long clientsConnected;
long clientsDisconnected;
@Override
public String toString() {
return getClass().getSimpleName() + "{"
+ "outputBuffers=" + outputBuffers
+ ", bufferErrors=" + bufferErrors
+ ", clientsConnected=" + clientsConnected
+ ", clientsDisconnected= " + clientsDisconnected
+ "}";
}
}
}