Add optimized rendering feedback loop tracking.

This can be used both for WebGL and the Vulkan back-end workaround for
Manhattan. Uses the recently added tracking for Textures being bound as
samplers. Then caches this information in the Framebuffer using the
Subject/Observer pattern.

Bug: angleproject:4490
Change-Id: I08bef0a1b95c4333da19c2dae1f02a993e5835e1
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2109335
Commit-Queue: Jamie Madill <jmadill@chromium.org>
Reviewed-by: Cody Northrop <cnorthrop@google.com>
diff --git a/src/libANGLE/Context.cpp b/src/libANGLE/Context.cpp
index bc331fd..3be9404 100644
--- a/src/libANGLE/Context.cpp
+++ b/src/libANGLE/Context.cpp
@@ -8493,7 +8493,8 @@
         default:
             if (index < kTextureMaxSubjectIndex)
             {
-                if (message != angle::SubjectMessage::ContentsChanged)
+                if (message != angle::SubjectMessage::ContentsChanged &&
+                    message != angle::SubjectMessage::BindingChanged)
                 {
                     mState.onActiveTextureStateChange(this, index);
                     mStateCache.onActiveTextureChange(this);
diff --git a/src/libANGLE/Framebuffer.cpp b/src/libANGLE/Framebuffer.cpp
index 1404d9c..1e81dc8 100644
--- a/src/libANGLE/Framebuffer.cpp
+++ b/src/libANGLE/Framebuffer.cpp
@@ -316,6 +316,8 @@
       mDefaultFixedSampleLocations(GL_FALSE),
       mDefaultLayers(0),
       mWebGLDepthStencilConsistent(true),
+      mDepthBufferFeedbackLoop(false),
+      mStencilBufferFeedbackLoop(false),
       mDefaultFramebufferReadAttachmentInitialized(false)
 {
     ASSERT(mDrawBufferStates.size() > 0);
@@ -335,6 +337,8 @@
       mDefaultFixedSampleLocations(GL_FALSE),
       mDefaultLayers(0),
       mWebGLDepthStencilConsistent(true),
+      mDepthBufferFeedbackLoop(false),
+      mStencilBufferFeedbackLoop(false),
       mDefaultFramebufferReadAttachmentInitialized(false)
 {
     ASSERT(mId != Framebuffer::kDefaultDrawFramebufferHandle);
@@ -673,6 +677,26 @@
     return mId == Framebuffer::kDefaultDrawFramebufferHandle;
 }
 
+bool FramebufferState::updateAttachmentFeedbackLoop(size_t dirtyBit)
+{
+    switch (dirtyBit)
+    {
+        case Framebuffer::DIRTY_BIT_DEPTH_ATTACHMENT:
+            mDepthBufferFeedbackLoop = mDepthAttachment.isBoundAsSamplerOrImage();
+            return mDepthBufferFeedbackLoop;
+
+        case Framebuffer::DIRTY_BIT_STENCIL_ATTACHMENT:
+            mStencilBufferFeedbackLoop = mStencilAttachment.isBoundAsSamplerOrImage();
+            return mStencilBufferFeedbackLoop;
+
+        default:
+            ASSERT(dirtyBit <= Framebuffer::DIRTY_BIT_COLOR_ATTACHMENT_MAX);
+            mDrawBufferFeedbackLoops[dirtyBit] =
+                mColorAttachments[dirtyBit].isBoundAsSamplerOrImage();
+            return mDrawBufferFeedbackLoops.test(dirtyBit);
+    }
+}
+
 const FramebufferID Framebuffer::kDefaultDrawFramebufferHandle = {0};
 
 Framebuffer::Framebuffer(const Caps &caps, rx::GLImplFactory *factory, FramebufferID id)
@@ -1859,18 +1883,14 @@
 
             if (!resource)
             {
-                mColorAttachmentBits.reset(colorIndex);
                 mFloat32ColorAttachmentBits.reset(colorIndex);
             }
             else
             {
-                mColorAttachmentBits.set(colorIndex);
                 updateFloat32ColorAttachmentBits(
                     colorIndex, resource->getAttachmentFormat(binding, textureIndex).info);
             }
 
-            // TODO(jmadill): ASSERT instead of checking the attachment exists in
-            // formsRenderingFeedbackLoopWith
             bool enabled = (type != GL_NONE && getDrawBufferState(colorIndex) != GL_NONE);
             mState.mEnabledDrawBuffers.set(colorIndex, enabled);
             SetComponentTypeMask(getDrawbufferWriteType(colorIndex), colorIndex,
@@ -1899,6 +1919,7 @@
     mState.mResourceNeedsInit.set(dirtyBit, attachment->initState() == InitState::MayNeedInit);
     onDirtyBinding->bind(resource);
 
+    mState.updateAttachmentFeedbackLoop(dirtyBit);
     invalidateCompletenessCache();
 }
 
@@ -1931,6 +1952,17 @@
             return;
         }
 
+        // Triggered by changes to Texture feedback loops.
+        if (message == angle::SubjectMessage::BindingChanged)
+        {
+            if (mState.updateAttachmentFeedbackLoop(index))
+            {
+                mDirtyBits.set(index);
+                onStateChange(angle::SubjectMessage::DirtyBitsFlagged);
+            }
+            return;
+        }
+
         // This can be triggered by the GL back-end TextureGL class.
         ASSERT(message == angle::SubjectMessage::DirtyBitsFlagged);
         return;
@@ -1973,68 +2005,16 @@
 
 bool Framebuffer::formsRenderingFeedbackLoopWith(const Context *context) const
 {
-    const State &state     = context->getState();
-    const Program *program = state.getProgram();
-
-    // TODO(jmadill): Default framebuffer feedback loops.
+    // We don't handle tricky cases where the default FBO is bound as a sampler.
+    // We also don't handle tricky cases with EGLImages and mipmap selection.
+    // TODO(http://anglebug.com/4500): Tricky rendering feedback loop cases.
     if (mState.isDefault())
     {
         return false;
     }
 
-    const FramebufferAttachment *depth   = getDepthAttachment();
-    const FramebufferAttachment *stencil = getStencilAttachment();
-
-    const bool checkDepth = depth && depth->type() == GL_TEXTURE;
-    // Skip the feedback loop check for stencil if depth/stencil point to the same resource.
-    const bool checkStencil =
-        (stencil && stencil->type() == GL_TEXTURE) && (!depth || *stencil != *depth);
-
-    const gl::ActiveTextureMask &activeTextures   = program->getActiveSamplersMask();
-    const gl::ActiveTexturePointerArray &textures = state.getActiveTexturesCache();
-
-    for (size_t textureUnit : activeTextures)
-    {
-        Texture *texture = textures[textureUnit];
-
-        if (texture == nullptr)
-        {
-            continue;
-        }
-
-        // Depth and stencil attachment form feedback loops
-        // Regardless of if enabled or masked.
-        if (checkDepth)
-        {
-            if (texture->getId() == depth->id())
-            {
-                return true;
-            }
-        }
-
-        if (checkStencil)
-        {
-            if (texture->getId() == stencil->id())
-            {
-                return true;
-            }
-        }
-
-        // Check if any color attachment forms a feedback loop.
-        for (size_t drawIndex : mColorAttachmentBits)
-        {
-            const FramebufferAttachment &attachment = mState.mColorAttachments[drawIndex];
-            ASSERT(attachment.isAttached());
-
-            if (attachment.isTextureWithId(texture->id()))
-            {
-                // TODO(jmadill): Check for appropriate overlap.
-                return true;
-            }
-        }
-    }
-
-    return false;
+    return mState.mDrawBufferFeedbackLoops.any() || mState.mDepthBufferFeedbackLoop ||
+           mState.mStencilBufferFeedbackLoop;
 }
 
 bool Framebuffer::formsCopyingFeedbackLoopWith(TextureID copyTextureID,
diff --git a/src/libANGLE/Framebuffer.h b/src/libANGLE/Framebuffer.h
index 0ba9400..4c550f2 100644
--- a/src/libANGLE/Framebuffer.h
+++ b/src/libANGLE/Framebuffer.h
@@ -127,6 +127,7 @@
     const FramebufferAttachment *getWebGLDepthStencilAttachment() const;
     const FramebufferAttachment *getWebGLDepthAttachment() const;
     const FramebufferAttachment *getWebGLStencilAttachment() const;
+    bool updateAttachmentFeedbackLoop(size_t dirtyBit);
 
     friend class Framebuffer;
 
@@ -155,6 +156,11 @@
     FramebufferAttachment mWebGLStencilAttachment;
     bool mWebGLDepthStencilConsistent;
 
+    // Tracks rendering feedback loops.
+    DrawBufferMask mDrawBufferFeedbackLoops;
+    bool mDepthBufferFeedbackLoop;
+    bool mStencilBufferFeedbackLoop;
+
     // Tracks if we need to initialize the resources for each attachment.
     angle::BitSet<IMPLEMENTATION_MAX_FRAMEBUFFER_ATTACHMENTS + 2> mResourceNeedsInit;
 
@@ -470,7 +476,6 @@
 
     mutable DirtyBits mDirtyBits;
     DrawBufferMask mFloat32ColorAttachmentBits;
-    DrawBufferMask mColorAttachmentBits;
 
     // The dirty bits guard is checked when we get a dependent state change message. We verify that
     // we don't set a dirty bit that isn't already set, when inside the dirty bits syncState.
diff --git a/src/libANGLE/FramebufferAttachment.cpp b/src/libANGLE/FramebufferAttachment.cpp
index 58f07f7..2848019 100644
--- a/src/libANGLE/FramebufferAttachment.cpp
+++ b/src/libANGLE/FramebufferAttachment.cpp
@@ -281,6 +281,17 @@
     mResource->setInitState(mTarget.textureIndex(), initState);
 }
 
+bool FramebufferAttachment::isBoundAsSamplerOrImage() const
+{
+    if (mType != GL_TEXTURE)
+    {
+        return false;
+    }
+
+    const gl::TextureState &textureState = getTexture()->getTextureState();
+    return textureState.isBoundAsImageTexture() || textureState.isBoundAsSamplerTexture();
+}
+
 ////// FramebufferAttachmentObject Implementation //////
 
 FramebufferAttachmentObject::FramebufferAttachmentObject() {}
diff --git a/src/libANGLE/FramebufferAttachment.h b/src/libANGLE/FramebufferAttachment.h
index 1370d1b..2887305 100644
--- a/src/libANGLE/FramebufferAttachment.h
+++ b/src/libANGLE/FramebufferAttachment.h
@@ -92,6 +92,8 @@
     GLenum getComponentType() const;
     GLenum getColorEncoding() const;
 
+    bool isBoundAsSamplerOrImage() const;
+
     bool isTextureWithId(TextureID textureId) const
     {
         return mType == GL_TEXTURE && id() == textureId.value;
diff --git a/src/libANGLE/Observer.h b/src/libANGLE/Observer.h
index b06e705..51c8203 100644
--- a/src/libANGLE/Observer.h
+++ b/src/libANGLE/Observer.h
@@ -33,7 +33,7 @@
 enum class SubjectMessage
 {
     // Used by gl::VertexArray to notify gl::Context of a gl::Buffer binding count change. Triggers
-    // a validation cache update.
+    // a validation cache update. Also used by gl::Texture to notify gl::Framebuffer of loops.
     BindingChanged,
 
     // Only the contents (pixels, bytes, etc) changed in this Subject. Distinct from the object
diff --git a/src/libANGLE/Texture.h b/src/libANGLE/Texture.h
index 8f90581..e69532a 100644
--- a/src/libANGLE/Texture.h
+++ b/src/libANGLE/Texture.h
@@ -426,12 +426,23 @@
         mState.mImageBindingCount--;
     }
 
-    ANGLE_INLINE void onBindAsSamplerTexture() { mState.mSamplerBindingCount++; }
+    ANGLE_INLINE void onBindAsSamplerTexture()
+    {
+        mState.mSamplerBindingCount++;
+        if (mState.mSamplerBindingCount == 1)
+        {
+            onStateChange(angle::SubjectMessage::BindingChanged);
+        }
+    }
 
     ANGLE_INLINE void onUnbindAsSamplerTexture()
     {
         ASSERT(mState.isBoundAsSamplerTexture());
         mState.mSamplerBindingCount--;
+        if (mState.mSamplerBindingCount == 0)
+        {
+            onStateChange(angle::SubjectMessage::BindingChanged);
+        }
     }
 
     egl::Surface *getBoundSurface() const;