Merge "Camera: Add postview to advanced extensions" into main
diff --git a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/AutoAdvancedExtenderImpl.java b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/AutoAdvancedExtenderImpl.java
index 06c8c44..c818386 100644
--- a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/AutoAdvancedExtenderImpl.java
+++ b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/AutoAdvancedExtenderImpl.java
@@ -33,6 +33,8 @@
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -62,6 +64,39 @@
                 AWB_MODE_DAYLIGHT);
     }
 
+    @Override
+    public boolean isPostviewAvailable() {
+        return true;
+    }
+
+    @Override
+    public Map<Integer, List<Size>> getSupportedPostviewResolutions(Size captureSize) {
+        Map<Integer, List<Size>> res =  new HashMap<>();
+        Map<Integer, List<Size>> captureSupportedResolutions =
+                getSupportedCaptureOutputResolutions(mCameraId);
+        float targetAr = ((float) captureSize.getWidth()) / captureSize.getHeight();
+
+        List<Size> currFormatSizes = captureSupportedResolutions.get(ImageFormat.YUV_420_888);
+        if (currFormatSizes != null) {
+            List<Size> postviewSizes = new ArrayList<>();
+
+            for (Size s : currFormatSizes) {
+                if ((s.equals(captureSize)) || (s.getWidth() > captureSize.getWidth())
+                        || (s.getHeight() > captureSize.getHeight())) continue;
+                float currentAr = ((float) s.getWidth()) / s.getHeight();
+                if (Math.abs(targetAr - currentAr) < 0.01) {
+                    postviewSizes.add(s);
+                }
+            }
+
+            if (!postviewSizes.isEmpty()) {
+                res.put(ImageFormat.YUV_420_888, postviewSizes);
+            }
+        }
+
+        return res;
+    }
+
     public class AutoAdvancedSessionProcessor extends BaseAdvancedSessionProcessor {
 
         public AutoAdvancedSessionProcessor() {
@@ -78,6 +113,11 @@
             RequestBuilder build = new RequestBuilder(mCaptureOutputConfig.getId(),
                     CameraDevice.TEMPLATE_STILL_CAPTURE, DEFAULT_CAPTURE_ID);
             build.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_DAYLIGHT);
+
+            if (mPostviewOutputSurfaceConfig.getSurface() != null) {
+                build.addTargetOutputConfigIds(mPostviewOutputConfig.getId());
+            }
+
             applyParameters(build);
 
             requestList.add(build.build());
diff --git a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BaseAdvancedExtenderImpl.java b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BaseAdvancedExtenderImpl.java
index b6604ae..1b61d93 100644
--- a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BaseAdvancedExtenderImpl.java
+++ b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BaseAdvancedExtenderImpl.java
@@ -75,6 +75,7 @@
     }
 
     protected CameraCharacteristics mCameraCharacteristics;
+    protected String mCameraId;
 
     protected static final Key REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP =
             new Key<long[]>("android.request.availableDynamicRangeProfilesMap", long[].class);
@@ -92,6 +93,7 @@
     public void init(String cameraId,
             Map<String, CameraCharacteristics> characteristicsMap) {
         mCameraCharacteristics = characteristicsMap.get(cameraId);
+        mCameraId = cameraId;
     }
 
     @Override
@@ -130,7 +132,7 @@
 
     @Override
     public Map<Integer, List<Size>> getSupportedPostviewResolutions(Size captureSize) {
-        return new HashMap<>();
+        return new HashMap<Integer, List<Size>>();
     }
 
     @Override
@@ -148,22 +150,31 @@
 
         protected Camera2OutputConfigImpl mPreviewOutputConfig;
         protected Camera2OutputConfigImpl mCaptureOutputConfig;
+        protected Camera2OutputConfigImpl mPostviewOutputConfig;
 
         protected OutputSurfaceImpl mPreviewOutputSurfaceConfig;
         protected OutputSurfaceImpl mCaptureOutputSurfaceConfig;
+        protected OutputSurfaceImpl mPostviewOutputSurfaceConfig;
 
         protected final Object mLock = new Object();
         @GuardedBy("mLock")
         protected Map<CaptureRequest.Key<?>, Object> mParameters = new LinkedHashMap<>();
 
-        protected final Object mLockCaptureSurfaceImageWriter = new Object();
-        @GuardedBy("mLockCaptureSurfaceImageWriter")
+        protected final Object mLockImageWriter = new Object();
+        @GuardedBy("mLockImageWriter")
         protected ImageWriter mCaptureSurfaceImageWriter;
 
+        @GuardedBy("mLockImageWriter")
+        protected ImageWriter mPostviewSurfaceImageWriter;
+
         protected CaptureResultImageMatcher mImageCaptureCaptureResultImageMatcher =
                 new CaptureResultImageMatcher();
+        protected CaptureResultImageMatcher mPostviewCaptureCaptureResultImageMatcher =
+                new CaptureResultImageMatcher();
         protected HashMap<Integer, Pair<ImageReferenceImpl, TotalCaptureResult>> mCaptureResults =
                 new HashMap<>();
+        protected HashMap<Integer, Pair<ImageReferenceImpl, TotalCaptureResult>> mPostviewResults =
+                new HashMap<>();
         protected RequestProcessorImpl mRequestProcessor;
 
         protected List<Integer> mCaptureIdList = List.of(DEFAULT_CAPTURE_ID);
@@ -187,6 +198,7 @@
 
             mPreviewOutputSurfaceConfig = surfaceConfigs.getPreviewOutputSurface();
             mCaptureOutputSurfaceConfig = surfaceConfigs.getImageCaptureOutputSurface();
+            mPostviewOutputSurfaceConfig = surfaceConfigs.getPostviewOutputSurface();
 
             Camera2SessionConfigImplBuilder builder =
                     new Camera2SessionConfigImplBuilder()
@@ -253,6 +265,22 @@
                 }
             }
 
+            // Postview
+            if (mPostviewOutputSurfaceConfig.getSurface() != null) {
+                Camera2OutputConfigImplBuilder postviewOutputConfigBuilder;
+
+                postviewOutputConfigBuilder =
+                        Camera2OutputConfigImplBuilder.newImageReaderConfig(
+                                mPostviewOutputSurfaceConfig.getSize(),
+                                ImageFormat.YUV_420_888,
+                                BASIC_CAPTURE_PROCESS_MAX_IMAGES,
+                                mPostviewOutputSurfaceConfig.getUsage());
+
+                mPostviewOutputConfig = postviewOutputConfigBuilder.build();
+
+                builder.addOutputConfig(mPostviewOutputConfig);
+            }
+
             builder.setColorSpace(surfaceConfigs.getColorSpace());
             addSessionParameter(builder);
 
@@ -283,11 +311,16 @@
 
         @Override
         public void deInitSession() {
-            synchronized (mLockCaptureSurfaceImageWriter) {
+            synchronized (mLockImageWriter) {
                 if (mCaptureSurfaceImageWriter != null) {
                     mCaptureSurfaceImageWriter.close();
                     mCaptureSurfaceImageWriter = null;
                 }
+
+                if (mPostviewSurfaceImageWriter != null) {
+                    mPostviewSurfaceImageWriter.close();
+                    mPostviewSurfaceImageWriter = null;
+                }
             }
         }
 
@@ -401,30 +434,39 @@
             mRequestProcessor = requestProcessor;
 
             if (mCaptureOutputSurfaceConfig.getSurface() != null) {
-                synchronized (mLockCaptureSurfaceImageWriter) {
-                    if (!mProcessCapture) {
-                        return;
+                synchronized (mLockImageWriter) {
+                    if (mProcessCapture) {
+                        if (isJpeg(mCaptureOutputSurfaceConfig)) {
+                            mCaptureSurfaceImageWriter = new ImageWriter
+                                    .Builder(mCaptureOutputSurfaceConfig.getSurface())
+                                    .setImageFormat(ImageFormat.JPEG)
+                                    .setMaxImages(MAX_NUM_IMAGES)
+                                    // For JPEG format, width x height should be set to (w*h) x 1
+                                    // since the JPEG image is returned as a 1D byte array
+                                    .setWidthAndHeight(
+                                            mCaptureOutputSurfaceConfig.getSize().getWidth()
+                                            * mCaptureOutputSurfaceConfig.getSize().getHeight(),
+                                            1)
+                                    .build();
+                        } else {
+                            mCaptureSurfaceImageWriter = new ImageWriter
+                                    .Builder(mCaptureOutputSurfaceConfig.getSurface())
+                                    .setImageFormat(mCaptureOutputSurfaceConfig.getImageFormat())
+                                    .setMaxImages(MAX_NUM_IMAGES)
+                                    .build();
+                        }
                     }
+                }
+            }
 
-                    if (isJpeg(mCaptureOutputSurfaceConfig)) {
-                        mCaptureSurfaceImageWriter = new ImageWriter
-                                .Builder(mCaptureOutputSurfaceConfig.getSurface())
-                                .setImageFormat(ImageFormat.JPEG)
-                                .setMaxImages(MAX_NUM_IMAGES)
-                                // For JPEG format, width x height should be set to (w*h) x 1
-                                // since the JPEG image is returned as a 1D byte array
-                                .setWidthAndHeight(
-                                        mCaptureOutputSurfaceConfig.getSize().getWidth()
-                                        * mCaptureOutputSurfaceConfig.getSize().getHeight(),
-                                        1)
-                                .build();
-                    } else {
-                        mCaptureSurfaceImageWriter = new ImageWriter
-                                .Builder(mCaptureOutputSurfaceConfig.getSurface())
-                                .setImageFormat(mCaptureOutputSurfaceConfig.getImageFormat())
-                                .setMaxImages(MAX_NUM_IMAGES)
-                                .build();
-                    }
+            if (mPostviewOutputSurfaceConfig != null
+                    && mPostviewOutputSurfaceConfig.getSurface() != null) {
+                synchronized (mLockImageWriter) {
+                    mPostviewSurfaceImageWriter = new ImageWriter
+                            .Builder(mPostviewOutputSurfaceConfig.getSurface())
+                            .setImageFormat(mPostviewOutputSurfaceConfig.getImageFormat())
+                            .setMaxImages(MAX_NUM_IMAGES)
+                            .build();
                 }
             }
         }
@@ -433,6 +475,7 @@
         public void onCaptureSessionEnd() {
             synchronized (this) {
                 mImageCaptureCaptureResultImageMatcher.clear();
+                mPostviewCaptureCaptureResultImageMatcher.clear();
             }
 
             mRequestProcessor = null;
@@ -526,6 +569,7 @@
 
         @Override
         public int startCaptureWithPostview(@NonNull CaptureCallback captureCallback) {
+            Log.d(TAG, "startCaptureWithPostview");
             return startCapture(captureCallback);
         }
 
@@ -557,6 +601,12 @@
 
                     addCaptureResultKeys(seqId, totalCaptureResult, captureCallback);
 
+                    if (mPostviewOutputSurfaceConfig != null) {
+                        mPostviewCaptureCaptureResultImageMatcher.setCameraCaptureCallback(
+                            totalCaptureResult,
+                            requestProcessorRequest.getCaptureStageId());
+                    }
+
                     if (!mProcessCapture) {
                         captureCallback.onCaptureProcessStarted(seqId);
                     } else {
@@ -594,6 +644,33 @@
 
             mRequestProcessor.submit(requestList, callback);
 
+            if (mPostviewOutputSurfaceConfig != null &&
+                    mPostviewOutputSurfaceConfig.getSurface() != null) {
+                mRequestProcessor.setImageProcessor(mPostviewOutputConfig.getId(),
+                        new ImageProcessorImpl() {
+                                @Override
+                                public void onNextImageAvailable(int outputStreamId,
+                                        long timestampNs,
+                                        @NonNull ImageReferenceImpl imgReferenceImpl,
+                                        @Nullable String physicalCameraId) {
+                                    mPostviewCaptureCaptureResultImageMatcher
+                                            .setInputImage(imgReferenceImpl);
+                                }
+                });
+
+                mPostviewCaptureCaptureResultImageMatcher.setImageReferenceListener(
+                        new CaptureResultImageMatcher.ImageReferenceListener() {
+                                    @Override
+                                    public void onImageReferenceIncoming(
+                                            @NonNull ImageReferenceImpl imageReferenceImpl,
+                                            @NonNull TotalCaptureResult totalCaptureResult,
+                                            int captureId) {
+                                        processImageCapture(imageReferenceImpl, totalCaptureResult,
+                                                captureId, true /*isPostview*/);
+                                    }
+                });
+            }
+
             if (mCaptureOutputSurfaceConfig.getSurface() != null && mProcessCapture) {
                 mRequestProcessor.setImageProcessor(mCaptureOutputConfig.getId(),
                         new ImageProcessorImpl() {
@@ -616,7 +693,7 @@
                                             int captureId) {
                                         captureCallback.onCaptureProcessStarted(seqId);
                                         processImageCapture(imageReferenceImpl, totalCaptureResult,
-                                                captureId);
+                                                captureId, false /*isPostview*/);
                                     }
                 });
             }
@@ -625,20 +702,25 @@
         }
 
         protected void processImageCapture(@NonNull ImageReferenceImpl imageReferenceImpl,
-                @NonNull TotalCaptureResult totalCaptureResult,
-                int captureId) {
+                @NonNull TotalCaptureResult totalCaptureResult, int captureId,
+                boolean isPostview) {
 
-            mCaptureResults.put(captureId, new Pair<>(imageReferenceImpl, totalCaptureResult));
+            HashMap<Integer, Pair<ImageReferenceImpl, TotalCaptureResult>> captureResults =
+                    isPostview ? mPostviewResults : mCaptureResults;
+            ImageWriter imageWriter = isPostview ? mPostviewSurfaceImageWriter :
+                    mCaptureSurfaceImageWriter;
 
-            if (mCaptureResults.keySet().containsAll(mCaptureIdList)) {
+            captureResults.put(captureId, new Pair<>(imageReferenceImpl, totalCaptureResult));
+
+            if (captureResults.keySet().containsAll(mCaptureIdList)) {
                 List<Pair<ImageReferenceImpl, TotalCaptureResult>> imageDataPairs =
-                        new ArrayList<>(mCaptureResults.values());
+                        new ArrayList<>(captureResults.values());
 
                 Image resultImage = null;
                 int captureSurfaceWriterImageFormat = ImageFormat.UNKNOWN;
-                synchronized (mLockCaptureSurfaceImageWriter) {
-                    resultImage = mCaptureSurfaceImageWriter.dequeueInputImage();
-                    captureSurfaceWriterImageFormat = mCaptureSurfaceImageWriter.getFormat();
+                synchronized (mLockImageWriter) {
+                    resultImage = imageWriter.dequeueInputImage();
+                    captureSurfaceWriterImageFormat = imageWriter.getFormat();
                 }
 
                 if (captureSurfaceWriterImageFormat == ImageFormat.JPEG) {
@@ -676,11 +758,11 @@
                                 DEFAULT_CAPTURE_ID).first.get().getTimestamp());
                 }
 
-                synchronized (mLockCaptureSurfaceImageWriter) {
-                    mCaptureSurfaceImageWriter.queueInputImage(resultImage);
+                synchronized (mLockImageWriter) {
+                    imageWriter.queueInputImage(resultImage);
                 }
 
-                for (Pair<ImageReferenceImpl, TotalCaptureResult> val : mCaptureResults.values()) {
+                for (Pair<ImageReferenceImpl, TotalCaptureResult> val : captureResults.values()) {
                     val.first.decrement();
                 }
             } else {
diff --git a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/HdrAdvancedExtenderImpl.java b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/HdrAdvancedExtenderImpl.java
index 68a84b7..1a3a099 100644
--- a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/HdrAdvancedExtenderImpl.java
+++ b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/HdrAdvancedExtenderImpl.java
@@ -247,7 +247,7 @@
 
                 Image resultImage = null;
                 int captureSurfaceWriterImageFormat = ImageFormat.UNKNOWN;
-                synchronized (mLockCaptureSurfaceImageWriter) {
+                synchronized (mLockImageWriter) {
                     resultImage = mCaptureSurfaceImageWriter.dequeueInputImage();
                     captureSurfaceWriterImageFormat = mCaptureSurfaceImageWriter.getFormat();
                 }
@@ -290,7 +290,7 @@
                         UNDER_EXPOSED_CAPTURE_ID).first.get().getTimestamp());
                 }
 
-                synchronized (mLockCaptureSurfaceImageWriter) {
+                synchronized (mLockImageWriter) {
                     mCaptureSurfaceImageWriter.queueInputImage(resultImage);
                 }