Revert r183721. It caused cleanups to be delayed too long in some cases.
Testcase to follow.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@183776 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/CodeGen/CGCleanup.cpp b/lib/CodeGen/CGCleanup.cpp
index 65de4d4..9f693ca 100644
--- a/lib/CodeGen/CGCleanup.cpp
+++ b/lib/CodeGen/CGCleanup.cpp
@@ -387,33 +387,6 @@
   }
 }
 
-/// Pops cleanup blocks until the given savepoint is reached, then add the
-/// cleanups from the given savepoint in the lifetime-extended cleanups stack.
-void
-CodeGenFunction::PopCleanupBlocks(EHScopeStack::stable_iterator Old,
-                                  size_t OldLifetimeExtendedSize) {
-  PopCleanupBlocks(Old);
-
-  // Move our deferred cleanups onto the EH stack.
-  for (size_t I = OldLifetimeExtendedSize,
-              E = LifetimeExtendedCleanupStack.size(); I != E; /**/) {
-    // Alignment should be guaranteed by the vptrs in the individual cleanups.
-    assert((I % llvm::alignOf<LifetimeExtendedCleanupHeader>() == 0) &&
-           "misaligned cleanup stack entry");
-
-    LifetimeExtendedCleanupHeader &Header =
-        reinterpret_cast<LifetimeExtendedCleanupHeader&>(
-            LifetimeExtendedCleanupStack[I]);
-    I += sizeof(Header);
-
-    EHStack.pushCopyOfCleanup(Header.getKind(),
-                              &LifetimeExtendedCleanupStack[I],
-                              Header.getSize());
-    I += Header.getSize();
-  }
-  LifetimeExtendedCleanupStack.resize(OldLifetimeExtendedSize);
-}
-
 static llvm::BasicBlock *CreateNormalEntry(CodeGenFunction &CGF,
                                            EHCleanupScope &Scope) {
   assert(Scope.isNormalCleanup());
diff --git a/lib/CodeGen/CGCleanup.h b/lib/CodeGen/CGCleanup.h
index 61d9f02..40a7502 100644
--- a/lib/CodeGen/CGCleanup.h
+++ b/lib/CodeGen/CGCleanup.h
@@ -374,11 +374,6 @@
     return new (Buffer) T(N, a0, a1, a2);
   }
 
-  void pushCopyOfCleanup(CleanupKind Kind, const void *Cleanup, size_t Size) {
-    void *Buffer = pushCleanup(Kind, Size);
-    std::memcpy(Buffer, Cleanup, Size);
-  }
-
   /// Pops a cleanup scope off the stack.  This is private to CGCleanup.cpp.
   void popCleanup();
 
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index 7fc79e0..4b19b54 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -1340,26 +1340,6 @@
                                      destroyer, useEHCleanupForArray);
 }
 
-void CodeGenFunction::pushLifetimeExtendedDestroy(
-    CleanupKind cleanupKind, llvm::Value *addr, QualType type,
-    Destroyer *destroyer, bool useEHCleanupForArray) {
-  assert(!isInConditionalBranch() &&
-         "performing lifetime extension from within conditional");
-
-  // Push an EH-only cleanup for the object now.
-  // FIXME: When popping normal cleanups, we need to keep this EH cleanup
-  // around in case a temporary's destructor throws an exception.
-  if (cleanupKind & EHCleanup)
-    EHStack.pushCleanup<DestroyObject>(
-        static_cast<CleanupKind>(cleanupKind & ~NormalCleanup), addr, type,
-        destroyer, useEHCleanupForArray);
-
-  // Remember that we need to push a full cleanup for the object at the
-  // end of the full-expression.
-  pushCleanupAfterFullExpr<DestroyObject>(
-      cleanupKind, addr, type, destroyer, useEHCleanupForArray);
-}
-
 /// emitDestroy - Immediately perform the destruction of the given
 /// object.
 ///
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index d534ff4..501d7ec 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -171,164 +171,66 @@
   llvm_unreachable("bad evaluation kind");
 }
 
-static void
-pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
-                     const Expr *E, llvm::Value *ReferenceTemporary) {
+static llvm::Value *
+CreateReferenceTemporary(CodeGenFunction &CGF, QualType Type,
+                         const NamedDecl *InitializedDecl) {
+  if (const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
+    if (VD->hasGlobalStorage()) {
+      SmallString<256> Name;
+      llvm::raw_svector_ostream Out(Name);
+      CGF.CGM.getCXXABI().getMangleContext().mangleReferenceTemporary(VD, Out);
+      Out.flush();
+
+      llvm::Type *RefTempTy = CGF.ConvertTypeForMem(Type);
+  
+      // Create the reference temporary.
+      llvm::GlobalVariable *RefTemp =
+        new llvm::GlobalVariable(CGF.CGM.getModule(), 
+                                 RefTempTy, /*isConstant=*/false,
+                                 llvm::GlobalValue::InternalLinkage,
+                                 llvm::Constant::getNullValue(RefTempTy),
+                                 Name.str());
+      // If we're binding to a thread_local variable, the temporary is also
+      // thread local.
+      if (VD->getTLSKind())
+        CGF.CGM.setTLSMode(RefTemp, *VD);
+      return RefTemp;
+    }
+  }
+
+  return CGF.CreateMemTemp(Type, "ref.tmp");
+}
+
+static llvm::Value *
+EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
+                            llvm::Value *&ReferenceTemporary,
+                            const CXXDestructorDecl *&ReferenceTemporaryDtor,
+                            const InitListExpr *&ReferenceInitializerList,
+                            QualType &ObjCARCReferenceLifetimeType,
+                            const NamedDecl *InitializedDecl) {
+  const MaterializeTemporaryExpr *M = NULL;
+  E = E->findMaterializedTemporary(M);
   // Objective-C++ ARC:
   //   If we are binding a reference to a temporary that has ownership, we
   //   need to perform retain/release operations on the temporary.
-  //
-  // FIXME: This should be looking at E, not M.
-  if (CGF.getLangOpts().ObjCAutoRefCount &&
-      M->getType()->isObjCLifetimeType()) {
-    QualType ObjCARCReferenceLifetimeType = M->getType();
-    switch (Qualifiers::ObjCLifetime Lifetime =
-                ObjCARCReferenceLifetimeType.getObjCLifetime()) {
-    case Qualifiers::OCL_None:
-    case Qualifiers::OCL_ExplicitNone:
-      // Carry on to normal cleanup handling.
-      break;
+  if (M && CGF.getLangOpts().ObjCAutoRefCount &&
+      M->getType()->isObjCLifetimeType() &&
+      (M->getType().getObjCLifetime() == Qualifiers::OCL_Strong ||
+       M->getType().getObjCLifetime() == Qualifiers::OCL_Weak ||
+       M->getType().getObjCLifetime() == Qualifiers::OCL_Autoreleasing))
+    ObjCARCReferenceLifetimeType = M->getType();
 
-    case Qualifiers::OCL_Autoreleasing:
-      // Nothing to do; cleaned up by an autorelease pool.
-      return;
-
-    case Qualifiers::OCL_Strong:
-    case Qualifiers::OCL_Weak:
-      switch (StorageDuration Duration = M->getStorageDuration()) {
-      case SD_Static:
-        // Note: we intentionally do not register a cleanup to release
-        // the object on program termination.
-        return;
-
-      case SD_Thread:
-        // FIXME: We should probably register a cleanup in this case.
-        return;
-
-      case SD_Automatic:
-      case SD_FullExpression:
-        assert(!ObjCARCReferenceLifetimeType->isArrayType());
-        CodeGenFunction::Destroyer *Destroy;
-        CleanupKind CleanupKind;
-        if (Lifetime == Qualifiers::OCL_Strong) {
-          const ValueDecl *VD = M->getExtendingDecl();
-          bool Precise =
-              VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>();
-          CleanupKind = CGF.getARCCleanupKind();
-          Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
-                            : &CodeGenFunction::destroyARCStrongImprecise;
-        } else {
-          // __weak objects always get EH cleanups; otherwise, exceptions
-          // could cause really nasty crashes instead of mere leaks.
-          CleanupKind = NormalAndEHCleanup;
-          Destroy = &CodeGenFunction::destroyARCWeak;
-        }
-        if (Duration == SD_FullExpression)
-          CGF.pushDestroy(CleanupKind, ReferenceTemporary,
-                          ObjCARCReferenceLifetimeType, *Destroy,
-                          CleanupKind & EHCleanup);
-        else
-          CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary,
-                                          ObjCARCReferenceLifetimeType,
-                                          *Destroy, CleanupKind & EHCleanup);
-        return;
-
-      case SD_Dynamic:
-        llvm_unreachable("temporary cannot have dynamic storage duration");
-      }
-      llvm_unreachable("unknown storage duration");
-    }
-  }
-
-  if (const InitListExpr *ILE = dyn_cast<InitListExpr>(E)) {
-    if (ILE->initializesStdInitializerList()) {
-      // FIXME: This is wrong if the temporary has static or thread storage
-      // duration.
-      CGF.EmitStdInitializerListCleanup(ReferenceTemporary, ILE);
-      return;
-    }
-  }
-
-  CXXDestructorDecl *ReferenceTemporaryDtor = 0;
-  if (const RecordType *RT =
-          E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
-    // Get the destructor for the reference temporary.
-    CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
-    if (!ClassDecl->hasTrivialDestructor())
-      ReferenceTemporaryDtor = ClassDecl->getDestructor();
-  }
-
-  if (!ReferenceTemporaryDtor)
-    return;
-
-  // Call the destructor for the temporary.
-  switch (M->getStorageDuration()) {
-  case SD_Static:
-  case SD_Thread: {
-    llvm::Constant *CleanupFn;
-    llvm::Constant *CleanupArg;
-    if (E->getType()->isArrayType()) {
-      CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
-          cast<llvm::Constant>(ReferenceTemporary), E->getType(),
-          CodeGenFunction::destroyCXXObject, CGF.getLangOpts().Exceptions);
-      CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy);
-    } else {
-      CleanupFn =
-        CGF.CGM.GetAddrOfCXXDestructor(ReferenceTemporaryDtor, Dtor_Complete);
-      CleanupArg = cast<llvm::Constant>(ReferenceTemporary);
-    }
-    CGF.CGM.getCXXABI().registerGlobalDtor(
-        CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
-    break;
-  }
-
-  case SD_FullExpression:
-    CGF.pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(),
-                    CodeGenFunction::destroyCXXObject,
-                    CGF.getLangOpts().Exceptions);
-    break;
-
-  case SD_Automatic:
-    CGF.pushLifetimeExtendedDestroy(NormalAndEHCleanup,
-                                    ReferenceTemporary, E->getType(),
-                                    CodeGenFunction::destroyCXXObject,
-                                    CGF.getLangOpts().Exceptions);
-    break;
-
-  case SD_Dynamic:
-    llvm_unreachable("temporary cannot have dynamic storage duration");
-  }
-}
-
-static llvm::Value *
-createReferenceTemporary(CodeGenFunction &CGF,
-                         const MaterializeTemporaryExpr *M, const Expr *Inner) {
-  switch (M->getStorageDuration()) {
-  case SD_FullExpression:
-  case SD_Automatic:
-    return CGF.CreateMemTemp(Inner->getType(), "ref.tmp");
-
-  case SD_Thread:
-  case SD_Static:
-    return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner);
-
-  case SD_Dynamic:
-    llvm_unreachable("temporary can't have dynamic storage duration");
-  }
-  llvm_unreachable("unknown storage duration");
-}
-
-static llvm::Value *
-emitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
-                            const NamedDecl *InitializedDecl) {
   if (const ExprWithCleanups *EWC = dyn_cast<ExprWithCleanups>(E)) {
     CGF.enterFullExpression(EWC);
     CodeGenFunction::RunCleanupsScope Scope(CGF);
-    return emitExprForReferenceBinding(CGF, EWC->getSubExpr(), InitializedDecl);
-  }
 
-  const MaterializeTemporaryExpr *M = 0;
-  E = E->findMaterializedTemporary(M);
+    return EmitExprForReferenceBinding(CGF, EWC->getSubExpr(), 
+                                       ReferenceTemporary, 
+                                       ReferenceTemporaryDtor,
+                                       ReferenceInitializerList,
+                                       ObjCARCReferenceLifetimeType,
+                                       InitializedDecl);
+  }
 
   if (E->isGLValue()) {
     // Emit the expression as an lvalue.
@@ -336,22 +238,61 @@
     assert(LV.isSimple());
     return LV.getAddress();
   }
-
-  assert(M && "prvalue reference initializer but not a materialized temporary");
-
-  if (CGF.getLangOpts().ObjCAutoRefCount &&
-      M->getType()->isObjCLifetimeType() &&
-      M->getType().getObjCLifetime() != Qualifiers::OCL_None &&
-      M->getType().getObjCLifetime() != Qualifiers::OCL_ExplicitNone) {
-    // FIXME: Fold this into the general case below.
-    llvm::Value *Object = createReferenceTemporary(CGF, M, E);
-    LValue RefTempDst = CGF.MakeAddrLValue(Object, M->getType());
+  
+  if (!ObjCARCReferenceLifetimeType.isNull()) {
+    ReferenceTemporary = CreateReferenceTemporary(CGF, 
+                                                ObjCARCReferenceLifetimeType, 
+                                                  InitializedDecl);
+    
+    
+    LValue RefTempDst = CGF.MakeAddrLValue(ReferenceTemporary, 
+                                           ObjCARCReferenceLifetimeType);
 
     CGF.EmitScalarInit(E, dyn_cast_or_null<ValueDecl>(InitializedDecl),
                        RefTempDst, false);
-
-    pushTemporaryCleanup(CGF, M, E, Object);
-    return Object;
+    
+    bool ExtendsLifeOfTemporary = false;
+    if (const VarDecl *Var = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
+      if (Var->extendsLifetimeOfTemporary())
+        ExtendsLifeOfTemporary = true;
+    } else if (InitializedDecl && isa<FieldDecl>(InitializedDecl)) {
+      ExtendsLifeOfTemporary = true;
+    }
+    
+    if (!ExtendsLifeOfTemporary) {
+      // Since the lifetime of this temporary isn't going to be extended,
+      // we need to clean it up ourselves at the end of the full expression.
+      switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
+      case Qualifiers::OCL_None:
+      case Qualifiers::OCL_ExplicitNone:
+      case Qualifiers::OCL_Autoreleasing:
+        break;
+          
+      case Qualifiers::OCL_Strong: {
+        assert(!ObjCARCReferenceLifetimeType->isArrayType());
+        CleanupKind cleanupKind = CGF.getARCCleanupKind();
+        CGF.pushDestroy(cleanupKind, 
+                        ReferenceTemporary,
+                        ObjCARCReferenceLifetimeType,
+                        CodeGenFunction::destroyARCStrongImprecise,
+                        cleanupKind & EHCleanup);
+        break;
+      }
+        
+      case Qualifiers::OCL_Weak:
+        assert(!ObjCARCReferenceLifetimeType->isArrayType());
+        CGF.pushDestroy(NormalAndEHCleanup, 
+                        ReferenceTemporary,
+                        ObjCARCReferenceLifetimeType,
+                        CodeGenFunction::destroyARCWeak,
+                        /*useEHCleanupForArray*/ true);
+        break;
+      }
+      
+      ObjCARCReferenceLifetimeType = QualType();
+    }
+    
+    return ReferenceTemporary;
   }
 
   SmallVector<const Expr *, 2> CommaLHSs;
@@ -361,59 +302,112 @@
   for (unsigned I = 0, N = CommaLHSs.size(); I != N; ++I)
     CGF.EmitIgnoredExpr(CommaLHSs[I]);
 
-  if (const OpaqueValueExpr *opaque = dyn_cast<OpaqueValueExpr>(E)) {
-    if (opaque->getType()->isRecordType()) {
-      assert(Adjustments.empty());
+  if (const OpaqueValueExpr *opaque = dyn_cast<OpaqueValueExpr>(E))
+    if (opaque->getType()->isRecordType())
       return CGF.EmitOpaqueValueLValue(opaque).getAddress();
+
+  // Create a reference temporary if necessary.
+  AggValueSlot AggSlot = AggValueSlot::ignored();
+  if (CGF.hasAggregateEvaluationKind(E->getType())) {
+    ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(), 
+                                                  InitializedDecl);
+    CharUnits Alignment = CGF.getContext().getTypeAlignInChars(E->getType());
+    AggValueSlot::IsDestructed_t isDestructed
+      = AggValueSlot::IsDestructed_t(InitializedDecl != 0);
+    AggSlot = AggValueSlot::forAddr(ReferenceTemporary, Alignment,
+                                    Qualifiers(), isDestructed,
+                                    AggValueSlot::DoesNotNeedGCBarriers,
+                                    AggValueSlot::IsNotAliased);
+  }
+  
+  if (InitializedDecl) {
+    if (const InitListExpr *ILE = dyn_cast<InitListExpr>(E)) {
+      if (ILE->initializesStdInitializerList()) {
+        ReferenceInitializerList = ILE;
+      }
+    }
+    else if (const RecordType *RT =
+               E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()){
+      // Get the destructor for the reference temporary.
+      CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
+      if (!ClassDecl->hasTrivialDestructor())
+        ReferenceTemporaryDtor = ClassDecl->getDestructor();
     }
   }
 
-  // Create and initialize the reference temporary.
-  llvm::Value *Object = createReferenceTemporary(CGF, M, E);
-  CGF.EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
-  pushTemporaryCleanup(CGF, M, E, Object);
+  RValue RV = CGF.EmitAnyExpr(E, AggSlot);
 
-  // Perform derived-to-base casts and/or field accesses, to get from the
-  // temporary object we created (and, potentially, for which we extended
-  // the lifetime) to the subobject we're binding the reference to.
-  for (unsigned I = Adjustments.size(); I != 0; --I) {
-    SubobjectAdjustment &Adjustment = Adjustments[I-1];
-    switch (Adjustment.Kind) {
-    case SubobjectAdjustment::DerivedToBaseAdjustment:
-      Object =
-          CGF.GetAddressOfBaseClass(Object,
-                                    Adjustment.DerivedToBase.DerivedClass,
-                          Adjustment.DerivedToBase.BasePath->path_begin(),
-                          Adjustment.DerivedToBase.BasePath->path_end(),
-                                    /*NullCheckValue=*/false);
-      break;
+  // FIXME: This is wrong. We need to register the destructor for the temporary
+  // now, *before* we perform the adjustments, because in the case of a
+  // pointer-to-member adjustment, the adjustment might throw.
 
-    case SubobjectAdjustment::FieldAdjustment: {
-      LValue LV = CGF.MakeAddrLValue(Object, E->getType());
-      LV = CGF.EmitLValueForField(LV, Adjustment.Field);
-      assert(LV.isSimple() &&
-             "materialized temporary field is not a simple lvalue");
-      Object = LV.getAddress();
-      break;
+  // Check if need to perform derived-to-base casts and/or field accesses, to
+  // get from the temporary object we created (and, potentially, for which we
+  // extended the lifetime) to the subobject we're binding the reference to.
+  if (!Adjustments.empty()) {
+    llvm::Value *Object = RV.getAggregateAddr();
+    for (unsigned I = Adjustments.size(); I != 0; --I) {
+      SubobjectAdjustment &Adjustment = Adjustments[I-1];
+      switch (Adjustment.Kind) {
+      case SubobjectAdjustment::DerivedToBaseAdjustment:
+        Object = 
+            CGF.GetAddressOfBaseClass(Object, 
+                                      Adjustment.DerivedToBase.DerivedClass, 
+                            Adjustment.DerivedToBase.BasePath->path_begin(),
+                            Adjustment.DerivedToBase.BasePath->path_end(),
+                                      /*NullCheckValue=*/false);
+        break;
+          
+      case SubobjectAdjustment::FieldAdjustment: {
+        LValue LV = CGF.MakeAddrLValue(Object, E->getType());
+        LV = CGF.EmitLValueForField(LV, Adjustment.Field);
+        assert(LV.isSimple() &&
+               "materialized temporary field is not a simple lvalue");
+        Object = LV.getAddress();
+        break;
+      }
+
+      case SubobjectAdjustment::MemberPointerAdjustment: {
+        llvm::Value *Ptr = CGF.EmitScalarExpr(Adjustment.Ptr.RHS);
+        Object = CGF.CGM.getCXXABI().EmitMemberDataPointerAddress(
+                      CGF, Object, Ptr, Adjustment.Ptr.MPT);
+        break;
+      }
+      }
     }
 
-    case SubobjectAdjustment::MemberPointerAdjustment: {
-      llvm::Value *Ptr = CGF.EmitScalarExpr(Adjustment.Ptr.RHS);
-      Object = CGF.CGM.getCXXABI().EmitMemberDataPointerAddress(
-                    CGF, Object, Ptr, Adjustment.Ptr.MPT);
-      break;
-    }
-    }
+    return Object;
   }
 
-  return Object;
+  if (RV.isAggregate())
+    return RV.getAggregateAddr();
+
+  // Create a temporary variable that we can bind the reference to.
+  ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(), 
+                                                InitializedDecl);
+
+
+  LValue tempLV = CGF.MakeNaturalAlignAddrLValue(ReferenceTemporary,
+                                                 E->getType());
+  if (RV.isScalar())
+    CGF.EmitStoreOfScalar(RV.getScalarVal(), tempLV, /*init*/ true);
+  else
+    CGF.EmitStoreOfComplex(RV.getComplexVal(), tempLV, /*init*/ true);
+  return ReferenceTemporary;
 }
 
 RValue
 CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E,
                                             const NamedDecl *InitializedDecl) {
-  llvm::Value *Value = emitExprForReferenceBinding(*this, E, InitializedDecl);
-
+  llvm::Value *ReferenceTemporary = 0;
+  const CXXDestructorDecl *ReferenceTemporaryDtor = 0;
+  const InitListExpr *ReferenceInitializerList = 0;
+  QualType ObjCARCReferenceLifetimeType;
+  llvm::Value *Value = EmitExprForReferenceBinding(*this, E, ReferenceTemporary,
+                                                   ReferenceTemporaryDtor,
+                                                   ReferenceInitializerList,
+                                                   ObjCARCReferenceLifetimeType,
+                                                   InitializedDecl);
   if (SanitizePerformTypeCheck && !E->getType()->isFunctionType()) {
     // C++11 [dcl.ref]p5 (as amended by core issue 453):
     //   If a glvalue to which a reference is directly bound designates neither
@@ -423,7 +417,80 @@
     QualType Ty = E->getType();
     EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty);
   }
+  if (!ReferenceTemporaryDtor && !ReferenceInitializerList &&
+      ObjCARCReferenceLifetimeType.isNull())
+    return RValue::get(Value);
+  
+  // Make sure to call the destructor for the reference temporary.
+  const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl);
+  if (VD && VD->hasGlobalStorage()) {
+    if (ReferenceTemporaryDtor) {
+      llvm::Constant *CleanupFn;
+      llvm::Constant *CleanupArg;
+      if (E->getType()->isArrayType()) {
+        CleanupFn = CodeGenFunction(CGM).generateDestroyHelper(
+            cast<llvm::Constant>(ReferenceTemporary), E->getType(),
+            destroyCXXObject, getLangOpts().Exceptions);
+        CleanupArg = llvm::Constant::getNullValue(Int8PtrTy);
+      } else {
+        CleanupFn =
+          CGM.GetAddrOfCXXDestructor(ReferenceTemporaryDtor, Dtor_Complete);
+        CleanupArg = cast<llvm::Constant>(ReferenceTemporary);
+      }
+      CGM.getCXXABI().registerGlobalDtor(*this, *VD, CleanupFn, CleanupArg);
+    } else if (ReferenceInitializerList) {
+      // FIXME: This is wrong. We need to register a global destructor to clean
+      // up the initializer_list object, rather than adding it as a local
+      // cleanup.
+      EmitStdInitializerListCleanup(ReferenceTemporary,
+                                    ReferenceInitializerList);
+    } else {
+      assert(!ObjCARCReferenceLifetimeType.isNull() && !VD->getTLSKind());
+      // Note: We intentionally do not register a global "destructor" to
+      // release the object.
+    }
+    
+    return RValue::get(Value);
+  }
 
+  if (ReferenceTemporaryDtor) {
+    if (E->getType()->isArrayType())
+      pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(),
+                  destroyCXXObject, getLangOpts().Exceptions);
+    else
+      PushDestructorCleanup(ReferenceTemporaryDtor, ReferenceTemporary);
+  } else if (ReferenceInitializerList) {
+    EmitStdInitializerListCleanup(ReferenceTemporary,
+                                  ReferenceInitializerList);
+  } else {
+    switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
+    case Qualifiers::OCL_None:
+      llvm_unreachable(
+                      "Not a reference temporary that needs to be deallocated");
+    case Qualifiers::OCL_ExplicitNone:
+    case Qualifiers::OCL_Autoreleasing:
+      // Nothing to do.
+      break;        
+        
+    case Qualifiers::OCL_Strong: {
+      bool precise = VD && VD->hasAttr<ObjCPreciseLifetimeAttr>();
+      CleanupKind cleanupKind = getARCCleanupKind();
+      pushDestroy(cleanupKind, ReferenceTemporary, ObjCARCReferenceLifetimeType,
+                  precise ? destroyARCStrongPrecise : destroyARCStrongImprecise,
+                  cleanupKind & EHCleanup);
+      break;
+    }
+        
+    case Qualifiers::OCL_Weak: {
+      // __weak objects always get EH cleanups; otherwise, exceptions
+      // could cause really nasty crashes instead of mere leaks.
+      pushDestroy(NormalAndEHCleanup, ReferenceTemporary,
+                  ObjCARCReferenceLifetimeType, destroyARCWeak, true);
+      break;        
+    }
+    }
+  }
+  
   return RValue::get(Value);
 }
 
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index f87f203..81b5d05 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -241,18 +241,6 @@
   llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
 
   EHScopeStack EHStack;
-  llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
-
-  /// Header for data within LifetimeExtendedCleanupStack.
-  struct LifetimeExtendedCleanupHeader {
-    /// The size of the following cleanup object.
-    size_t Size : 29;
-    /// The kind of cleanup to push: a value from the CleanupKind enumeration.
-    unsigned Kind : 3;
-
-    size_t getSize() const { return Size; }
-    CleanupKind getKind() const { return static_cast<CleanupKind>(Kind); }
-  };
 
   /// i32s containing the indexes of the cleanup destinations.
   llvm::AllocaInst *NormalCleanupDest;
@@ -388,23 +376,6 @@
     initFullExprCleanup();
   }
 
-  /// \brief Queue a cleanup to be pushed after finishing the current
-  /// full-expression.
-  template <class T, class A0, class A1, class A2, class A3>
-  void pushCleanupAfterFullExpr(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3) {
-    assert(!isInConditionalBranch() && "can't defer conditional cleanup");
-
-    LifetimeExtendedCleanupHeader Header = { sizeof(T), Kind };
-
-    size_t OldSize = LifetimeExtendedCleanupStack.size();
-    LifetimeExtendedCleanupStack.resize(
-        LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size);
-
-    char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
-    new (Buffer) LifetimeExtendedCleanupHeader(Header);
-    new (Buffer + sizeof(Header)) T(a0, a1, a2, a3);
-  }
-
   /// Set up the last cleaup that was pushed as a conditional
   /// full-expression cleanup.
   void initFullExprCleanup();
@@ -450,7 +421,6 @@
   /// will be executed once the scope is exited.
   class RunCleanupsScope {
     EHScopeStack::stable_iterator CleanupStackDepth;
-    size_t LifetimeExtendedCleanupStackSize;
     bool OldDidCallStackSave;
   protected:
     bool PerformCleanup;
@@ -468,8 +438,6 @@
       : PerformCleanup(true), CGF(CGF)
     {
       CleanupStackDepth = CGF.EHStack.stable_begin();
-      LifetimeExtendedCleanupStackSize =
-          CGF.LifetimeExtendedCleanupStack.size();
       OldDidCallStackSave = CGF.DidCallStackSave;
       CGF.DidCallStackSave = false;
     }
@@ -479,8 +447,7 @@
     ~RunCleanupsScope() {
       if (PerformCleanup) {
         CGF.DidCallStackSave = OldDidCallStackSave;
-        CGF.PopCleanupBlocks(CleanupStackDepth,
-                             LifetimeExtendedCleanupStackSize);
+        CGF.PopCleanupBlocks(CleanupStackDepth);
       }
     }
 
@@ -494,8 +461,7 @@
     void ForceCleanup() {
       assert(PerformCleanup && "Already forced cleanup");
       CGF.DidCallStackSave = OldDidCallStackSave;
-      CGF.PopCleanupBlocks(CleanupStackDepth,
-                           LifetimeExtendedCleanupStackSize);
+      CGF.PopCleanupBlocks(CleanupStackDepth);
       PerformCleanup = false;
     }
   };
@@ -547,16 +513,10 @@
   };
 
 
-  /// \brief Takes the old cleanup stack size and emits the cleanup blocks
-  /// that have been added.
+  /// PopCleanupBlocks - Takes the old cleanup stack size and emits
+  /// the cleanup blocks that have been added.
   void PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize);
 
-  /// \brief Takes the old cleanup stack size and emits the cleanup blocks
-  /// that have been added, then adds all lifetime-extended cleanups from
-  /// the given position to the stack.
-  void PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
-                        size_t OldLifetimeExtendedStackSize);
-
   void ResolveBranchFixups(llvm::BasicBlock *Target);
 
   /// The given basic block lies in the current EH scope, but may be a
@@ -1028,9 +988,6 @@
                      llvm::Value *addr, QualType type);
   void pushDestroy(CleanupKind kind, llvm::Value *addr, QualType type,
                    Destroyer *destroyer, bool useEHCleanupForArray);
-  void pushLifetimeExtendedDestroy(CleanupKind kind, llvm::Value *addr,
-                                   QualType type, Destroyer *destroyer,
-                                   bool useEHCleanupForArray);
   void emitDestroy(llvm::Value *addr, QualType type, Destroyer *destroyer,
                    bool useEHCleanupForArray);
   llvm::Function *generateDestroyHelper(llvm::Constant *addr,
diff --git a/lib/Sema/SemaInit.cpp b/lib/Sema/SemaInit.cpp
index 847b935..7d73e08 100644
--- a/lib/Sema/SemaInit.cpp
+++ b/lib/Sema/SemaInit.cpp
@@ -5214,9 +5214,6 @@
   Init = const_cast<Expr *>(
       Init->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments));
 
-  if (CXXBindTemporaryExpr *BTE = dyn_cast<CXXBindTemporaryExpr>(Init))
-    Init = BTE->getSubExpr();
-
   if (InitListExpr *ILE = dyn_cast<InitListExpr>(Init)) {
     if (ILE->initializesStdInitializerList() || ILE->getType()->isArrayType()) {
       // FIXME: If this is an InitListExpr which creates a std::initializer_list
diff --git a/test/CodeGenCXX/cxx11-thread-local.cpp b/test/CodeGenCXX/cxx11-thread-local.cpp
index 04ca089..2987703 100644
--- a/test/CodeGenCXX/cxx11-thread-local.cpp
+++ b/test/CodeGenCXX/cxx11-thread-local.cpp
@@ -35,7 +35,7 @@
 
 // CHECK: @_ZZ8tls_dtorvE1u = internal thread_local global
 // CHECK: @_ZGVZ8tls_dtorvE1u = internal thread_local global i8 0
-// CHECK: @_ZGRZ8tls_dtorvE1u = private thread_local global
+// CHECK: @_ZGRZ8tls_dtorvE1u = internal thread_local global
 
 // CHECK: @_ZGVN1VIiE1mE = weak_odr thread_local global i64 0
 
diff --git a/test/CodeGenCXX/temporaries.cpp b/test/CodeGenCXX/temporaries.cpp
index f8f7f9f..61b517a 100644
--- a/test/CodeGenCXX/temporaries.cpp
+++ b/test/CodeGenCXX/temporaries.cpp
@@ -584,9 +584,11 @@
   // CHECK: call void @_ZN15BindToSubobject1fEv()
   // CHECK: call void @_ZN15BindToSubobject1gEv()
   // CHECK: call void @_ZN15BindToSubobject1AC1Ev({{.*}} @_ZGRN15BindToSubobject1cE)
-  // CHECK: call i32 @__cxa_atexit({{.*}} bitcast ({{.*}} @_ZN15BindToSubobject1AD1Ev to void (i8*)*), i8* bitcast ({{.*}} @_ZGRN15BindToSubobject1cE to i8*), i8* @__dso_handle)
+  // FIXME: This is wrong. We should emit the call to __cxa_atexit prior to
+  // calling h(), in case h() throws.
   // CHECK: call {{.*}} @_ZN15BindToSubobject1hE
   // CHECK: getelementptr
+  // CHECK: call i32 @__cxa_atexit({{.*}} bitcast ({{.*}} @_ZN15BindToSubobject1AD1Ev to void (i8*)*), i8* bitcast ({{.*}} @_ZGRN15BindToSubobject1cE to i8*), i8* @__dso_handle)
   // CHECK: store i32* {{.*}}, i32** @_ZN15BindToSubobject1cE, align 8
   int &&c = (f(), (g(), A().*h()));
 
@@ -596,9 +598,9 @@
   };
 
   // CHECK: call void @_ZN15BindToSubobject1BC1Ev({{.*}} @_ZGRN15BindToSubobject1dE)
-  // CHECK: call i32 @__cxa_atexit({{.*}} bitcast ({{.*}} @_ZN15BindToSubobject1BD1Ev to void (i8*)*), i8* bitcast ({{.*}} @_ZGRN15BindToSubobject1dE to i8*), i8* @__dso_handle)
   // CHECK: call {{.*}} @_ZN15BindToSubobject1hE
   // CHECK: getelementptr {{.*}} getelementptr
+  // CHECK: call i32 @__cxa_atexit({{.*}} bitcast ({{.*}} @_ZN15BindToSubobject1BD1Ev to void (i8*)*), i8* bitcast ({{.*}} @_ZGRN15BindToSubobject1dE to i8*), i8* @__dso_handle)
   // CHECK: store i32* {{.*}}, i32** @_ZN15BindToSubobject1dE, align 8
   int &&d = (B().a).*h();
 }
@@ -609,7 +611,7 @@
   // Do not lifetime extend the S() temporary here.
   // CHECK: alloca
   // CHECK: call {{.*}}memset
-  // CHECK: store i32 {{.*}}, i32* @_ZGRN8Bitfield1rE
+  // CHECK: store i32 {{.*}}, i32* @_ZGRN8Bitfield1rE, align 4
   // CHECK: call void @_ZN8Bitfield1SD1
   // CHECK: store i32* @_ZGRN8Bitfield1rE, i32** @_ZN8Bitfield1rE, align 8
   int &&r = S().a;
@@ -624,91 +626,15 @@
   };
   // CHECK: alloca
   // CHECK: extractelement
-  // CHECK: store i32 {{.*}}, i32* @_ZGRN6Vector1rE
+  // CHECK: store i32 {{.*}}, i32* @_ZGRN6Vector1rE,
   // CHECK: store i32* @_ZGRN6Vector1rE, i32** @_ZN6Vector1rE,
   int &&r = S().v[1];
 
   // CHECK: alloca
   // CHECK: extractelement
-  // CHECK: store i32 {{.*}}, i32* @_ZGRN6Vector1sE
+  // CHECK: store i32 {{.*}}, i32* @_ZGRN6Vector1sE,
   // CHECK: store i32* @_ZGRN6Vector1sE, i32** @_ZN6Vector1sE,
   int &&s = S().w[1];
   // FIXME PR16204: The following code leads to an assertion in Sema.
   //int &&s = S().w.y;
 }
-
-namespace MultipleExtension {
-  struct A { A(); ~A(); };
-  struct B { B(); ~B(); };
-  struct C { C(); ~C(); };
-  struct D { D(); ~D(); int n; C c; };
-  struct E { const A &a; B b; const C &c; ~E(); };
-
-  E &&e1 = { A(), B(), D().c };
-
-  // CHECK: call void @_ZN17MultipleExtension1AC1Ev({{.*}} @[[TEMPA:_ZGRN17MultipleExtension2e1E.*]])
-  // CHECK: call i32 @__cxa_atexit({{.*}} @_ZN17MultipleExtension1AD1Ev {{.*}} @[[TEMPA]]
-  // CHECK: store {{.*}} @[[TEMPA]], {{.*}} getelementptr inbounds ({{.*}} @[[TEMPE:_ZGRN17MultipleExtension2e1E.*]], i32 0, i32 0)
-
-  // CHECK: call void @_ZN17MultipleExtension1BC1Ev({{.*}} getelementptr inbounds ({{.*}} @[[TEMPE]], i32 0, i32 1))
-
-  // CHECK: call void @_ZN17MultipleExtension1DC1Ev({{.*}} @[[TEMPD:_ZGRN17MultipleExtension2e1E.*]])
-  // CHECK: call i32 @__cxa_atexit({{.*}} @_ZN17MultipleExtension1DD1Ev {{.*}} @[[TEMPD]]
-  // CHECK: store {{.*}} @[[TEMPD]], {{.*}} getelementptr inbounds ({{.*}} @[[TEMPE]], i32 0, i32 2)
-  // CHECK: call i32 @__cxa_atexit({{.*}} @_ZN17MultipleExtension1ED1Ev {{.*}} @[[TEMPE]]
-  // CHECK: store {{.*}} @[[TEMPE]], %"struct.MultipleExtension::E"** @_ZN17MultipleExtension2e1E, align 8
-
-  E e2 = { A(), B(), D().c };
-
-  // CHECK: call void @_ZN17MultipleExtension1AC1Ev({{.*}} @[[TEMPA:_ZGRN17MultipleExtension2e2E.*]])
-  // CHECK: call i32 @__cxa_atexit({{.*}} @_ZN17MultipleExtension1AD1Ev {{.*}} @[[TEMPA]]
-  // CHECK: store {{.*}} @[[TEMPA]], {{.*}} getelementptr inbounds ({{.*}} @[[E:_ZN17MultipleExtension2e2E]], i32 0, i32 0)
-
-  // CHECK: call void @_ZN17MultipleExtension1BC1Ev({{.*}} getelementptr inbounds ({{.*}} @[[E]], i32 0, i32 1))
-
-  // CHECK: call void @_ZN17MultipleExtension1DC1Ev({{.*}} @[[TEMPD:_ZGRN17MultipleExtension2e2E.*]])
-  // CHECK: call i32 @__cxa_atexit({{.*}} @_ZN17MultipleExtension1DD1Ev {{.*}} @[[TEMPD]]
-  // CHECK: store {{.*}} @[[TEMPD]], {{.*}} getelementptr inbounds ({{.*}} @[[E]], i32 0, i32 2)
-  // CHECK: call i32 @__cxa_atexit({{.*}} @_ZN17MultipleExtension1ED1Ev {{.*}} @[[E]]
-
-
-  void g();
-  // CHECK: define void @[[NS:_ZN17MultipleExtension]]1fEv(
-  void f() {
-    E &&e1 = { A(), B(), D().c };
-    // CHECK: %[[TEMPE1_A:.*]] = getelementptr inbounds {{.*}} %[[TEMPE1:.*]], i32 0, i32 0
-    // CHECK: call void @[[NS]]1AC1Ev({{.*}} %[[TEMPA1:.*]])
-    // CHECK: store {{.*}} %[[TEMPA1]], {{.*}} %[[TEMPE1_A]]
-    // CHECK: %[[TEMPE1_B:.*]] = getelementptr inbounds {{.*}} %[[TEMPE1]], i32 0, i32 1
-    // CHECK: call void @[[NS]]1BC1Ev({{.*}} %[[TEMPE1_B]])
-    // CHECK: %[[TEMPE1_C:.*]] = getelementptr inbounds {{.*}} %[[TEMPE1]], i32 0, i32 2
-    // CHECK: call void @[[NS]]1DC1Ev({{.*}} %[[TEMPD1:.*]])
-    // CHECK: %[[TEMPD1_C:.*]] = getelementptr inbounds {{.*}} %[[TEMPD1]], i32 0, i32 1
-    // CHECK: store {{.*}} %[[TEMPD1_C]], {{.*}} %[[TEMPE1_C]]
-    // CHECK: store {{.*}} %[[TEMPE1]], {{.*}} %[[E1:.*]]
-
-    g();
-    // CHECK: call void @[[NS]]1gEv()
-
-    E e2 = { A(), B(), D().c };
-    // CHECK: %[[TEMPE2_A:.*]] = getelementptr inbounds {{.*}} %[[E2:.*]], i32 0, i32 0
-    // CHECK: call void @[[NS]]1AC1Ev({{.*}} %[[TEMPA2:.*]])
-    // CHECK: store {{.*}} %[[TEMPA2]], {{.*}} %[[TEMPE2_A]]
-    // CHECK: %[[TEMPE2_B:.*]] = getelementptr inbounds {{.*}} %[[E2]], i32 0, i32 1
-    // CHECK: call void @[[NS]]1BC1Ev({{.*}} %[[TEMPE2_B]])
-    // CHECK: %[[TEMPE2_C:.*]] = getelementptr inbounds {{.*}} %[[E2]], i32 0, i32 2
-    // CHECK: call void @[[NS]]1DC1Ev({{.*}} %[[TEMPD2:.*]])
-    // CHECK: %[[TEMPD2_C:.*]] = getelementptr inbounds {{.*}} %[[TEMPD2]], i32 0, i32 1
-    // CHECK: store {{.*}} %[[TEMPD2_C]], {{.*}}* %[[TEMPE2_C]]
-
-    g();
-    // CHECK: call void @[[NS]]1gEv()
-
-    // CHECK: call void @[[NS]]1ED1Ev({{.*}} %[[E2]])
-    // CHECK: call void @[[NS]]1DD1Ev({{.*}} %[[TEMPD2]])
-    // CHECK: call void @[[NS]]1AD1Ev({{.*}} %[[TEMPA2]])
-    // CHECK: call void @[[NS]]1ED1Ev({{.*}} %[[TEMPE1]])
-    // CHECK: call void @[[NS]]1DD1Ev({{.*}} %[[TEMPD1]])
-    // CHECK: call void @[[NS]]1AD1Ev({{.*}} %[[TEMPA1]])
-  }
-}