Merge pull request #19886 from atrick/silgen-critedge

SILGen: simplify cleanups and avoid critical edges.
diff --git a/include/swift/AST/Decl.h b/include/swift/AST/Decl.h
index 92aefaa..7299101 100644
--- a/include/swift/AST/Decl.h
+++ b/include/swift/AST/Decl.h
@@ -4699,7 +4699,11 @@
   bool isOwned() const { return getSpecifier() == Specifier::Owned; }
 
   ValueOwnership getValueOwnership() const {
-    switch (getSpecifier()) {
+    return getValueOwnershipForSpecifier(getSpecifier());
+  }
+
+  static ValueOwnership getValueOwnershipForSpecifier(Specifier specifier) {
+    switch (specifier) {
     case Specifier::Let:
       return ValueOwnership::Default;
     case Specifier::Var:
@@ -4714,6 +4718,21 @@
     llvm_unreachable("unhandled specifier");
   }
 
+  static Specifier
+  getParameterSpecifierForValueOwnership(ValueOwnership ownership) {
+    switch (ownership) {
+    case ValueOwnership::Default:
+      return Specifier::Let;
+    case ValueOwnership::Shared:
+      return Specifier::Shared;
+    case ValueOwnership::InOut:
+      return Specifier::InOut;
+    case ValueOwnership::Owned:
+      return Specifier::Owned;
+    }
+    llvm_unreachable("unhandled ownership");
+  }
+
   /// Is this an element in a capture list?
   bool isCaptureList() const { return Bits.VarDecl.IsCaptureList; }
 
diff --git a/include/swift/AST/Types.h b/include/swift/AST/Types.h
index 5205974..5f53c15 100644
--- a/include/swift/AST/Types.h
+++ b/include/swift/AST/Types.h
@@ -1658,9 +1658,8 @@
     Variadic    = 1 << 0,
     AutoClosure = 1 << 1,
     Escaping    = 1 << 2,
-    InOut       = 1 << 3,
-    Shared      = 1 << 4,
-    Owned       = 1 << 5,
+    OwnershipShift = 3,
+    Ownership   = 7 << OwnershipShift,
 
     NumBits = 6
   };
@@ -1679,9 +1678,7 @@
                      ValueOwnership ownership)
       : value((variadic ? Variadic : 0) | (autoclosure ? AutoClosure : 0) |
               (escaping ? Escaping : 0) |
-              (ownership == ValueOwnership::InOut ? InOut : 0) |
-              (ownership == ValueOwnership::Shared ? Shared : 0) |
-              (ownership == ValueOwnership::Owned ? Owned : 0)) {}
+              uint8_t(ownership) << OwnershipShift) {}
 
   /// Create one from what's present in the parameter type
   inline static ParameterTypeFlags
@@ -1691,19 +1688,12 @@
   bool isVariadic() const { return value.contains(Variadic); }
   bool isAutoClosure() const { return value.contains(AutoClosure); }
   bool isEscaping() const { return value.contains(Escaping); }
-  bool isInOut() const { return value.contains(InOut); }
-  bool isShared() const { return value.contains(Shared); }
-  bool isOwned() const { return value.contains(Owned); }
+  bool isInOut() const { return getValueOwnership() == ValueOwnership::InOut; }
+  bool isShared() const { return getValueOwnership() == ValueOwnership::Shared;}
+  bool isOwned() const { return getValueOwnership() == ValueOwnership::Owned; }
 
   ValueOwnership getValueOwnership() const {
-    if (isInOut())
-      return ValueOwnership::InOut;
-    else if (isShared())
-      return ValueOwnership::Shared;
-    else if (isOwned())
-      return ValueOwnership::Owned;
-
-    return ValueOwnership::Default;
+    return ValueOwnership((value.toRaw() & Ownership) >> OwnershipShift);
   }
 
   ParameterTypeFlags withVariadic(bool variadic) const {
@@ -1717,18 +1707,23 @@
   }
 
   ParameterTypeFlags withInOut(bool isInout) const {
-    return ParameterTypeFlags(isInout ? value | ParameterTypeFlags::InOut
-                                      : value - ParameterTypeFlags::InOut);
+    return withValueOwnership(isInout ? ValueOwnership::InOut
+                                      : ValueOwnership::Default);
   }
   
   ParameterTypeFlags withShared(bool isShared) const {
-    return ParameterTypeFlags(isShared ? value | ParameterTypeFlags::Shared
-                                       : value - ParameterTypeFlags::Shared);
+    return withValueOwnership(isShared ? ValueOwnership::Shared
+                                       : ValueOwnership::Default);
   }
 
   ParameterTypeFlags withOwned(bool isOwned) const {
-    return ParameterTypeFlags(isOwned ? value | ParameterTypeFlags::Owned
-                                      : value - ParameterTypeFlags::Owned);
+    return withValueOwnership(isOwned ? ValueOwnership::Owned
+                                      : ValueOwnership::Default);
+  }
+
+  ParameterTypeFlags withValueOwnership(ValueOwnership ownership) const {
+    return (value - ParameterTypeFlags::Ownership)
+            | ParameterFlags(uint8_t(ownership) << OwnershipShift);
   }
 
   bool operator ==(const ParameterTypeFlags &other) const {
@@ -1745,9 +1740,8 @@
 class YieldTypeFlags {
   enum YieldFlags : uint8_t {
     None        = 0,
-    InOut       = 1 << 1,
-    Shared      = 1 << 2,
-    Owned       = 1 << 3,
+    Ownership   = 7,
+    OwnershipShift = 0,
 
     NumBits = 3
   };
@@ -1764,35 +1758,34 @@
   }
 
   YieldTypeFlags(ValueOwnership ownership)
-      : value((ownership == ValueOwnership::InOut ? InOut : 0) |
-              (ownership == ValueOwnership::Shared ? Shared : 0) |
-              (ownership == ValueOwnership::Owned ? Owned : 0)) {}
+      : value(uint8_t(ownership) << OwnershipShift) {}
 
-  bool isInOut() const { return value.contains(InOut); }
-  bool isShared() const { return value.contains(Shared); }
-  bool isOwned() const { return value.contains(Owned); }
+  bool isInOut() const { return getValueOwnership() == ValueOwnership::InOut; }
+  bool isShared() const { return getValueOwnership() == ValueOwnership::Shared;}
+  bool isOwned() const { return getValueOwnership() == ValueOwnership::Owned; }
 
   ValueOwnership getValueOwnership() const {
-    if (isInOut())
-      return ValueOwnership::InOut;
-    else if (isShared())
-      return ValueOwnership::Shared;
-    else if (isOwned())
-      return ValueOwnership::Owned;
-
-    return ValueOwnership::Default;
+    return ValueOwnership((value.toRaw() & Ownership) >> OwnershipShift);
   }
 
   YieldTypeFlags withInOut(bool isInout) const {
-    return YieldTypeFlags(isInout ? value | InOut : value - InOut);
+    return withValueOwnership(isInout ? ValueOwnership::InOut
+                                      : ValueOwnership::Default);
   }
   
   YieldTypeFlags withShared(bool isShared) const {
-    return YieldTypeFlags(isShared ? value | Shared : value - Shared);
+    return withValueOwnership(isShared ? ValueOwnership::Shared
+                                       : ValueOwnership::Default);
   }
 
   YieldTypeFlags withOwned(bool isOwned) const {
-    return YieldTypeFlags(isOwned ? value | Owned : value - Owned);
+    return withValueOwnership(isOwned ? ValueOwnership::Owned
+                                      : ValueOwnership::Default);
+  }
+
+  YieldTypeFlags withValueOwnership(ValueOwnership ownership) const {
+    return (value - YieldTypeFlags::Ownership)
+            | YieldFlags(uint8_t(ownership) << OwnershipShift);
   }
 
   /// Return these flags interpreted as parameter flags.
diff --git a/lib/AST/ASTDumper.cpp b/lib/AST/ASTDumper.cpp
index bf7b568..ed44cfe 100644
--- a/lib/AST/ASTDumper.cpp
+++ b/lib/AST/ASTDumper.cpp
@@ -3166,6 +3166,12 @@
       printFlag(paramFlags.isVariadic(), "vararg");
       printFlag(paramFlags.isAutoClosure(), "autoclosure");
       printFlag(paramFlags.isEscaping(), "escaping");
+      switch (paramFlags.getValueOwnership()) {
+      case ValueOwnership::Default: break;
+      case ValueOwnership::Owned: printFlag("owned"); break;
+      case ValueOwnership::Shared: printFlag("shared"); break;
+      case ValueOwnership::InOut: printFlag("inout"); break;
+      }
     }
 
   public:
@@ -3417,7 +3423,7 @@
         if (param.hasLabel())
           printField("name", param.getLabel().str());
         dumpParameterFlags(param.getParameterFlags());
-        printRec(param.getOldType());
+        printRec(param.getPlainType());
         OS << ")";
       }
       Indent -= 2;
diff --git a/lib/AST/Builtins.cpp b/lib/AST/Builtins.cpp
index f9c9f07..e1cccf0 100644
--- a/lib/AST/Builtins.cpp
+++ b/lib/AST/Builtins.cpp
@@ -196,9 +196,9 @@
   SmallVector<ParamDecl*, 4> params;
   for (unsigned i = 0, e = ArgParamTypes.size(); i < e; i++) {
     auto paramIfaceType = ArgParamTypes[i].getPlainType();
-    auto specifier = (ArgParamTypes[i].getParameterFlags().isInOut())
-                         ? VarDecl::Specifier::InOut
-                         : VarDecl::Specifier::Default;
+    auto specifier =
+      VarDecl::getParameterSpecifierForValueOwnership(
+        ArgParamTypes[i].getParameterFlags().getValueOwnership());
     auto PD = new (Context) ParamDecl(specifier,
                                       SourceLoc(), SourceLoc(),
                                       Identifier(), SourceLoc(),
@@ -474,20 +474,14 @@
     }
 
     template <class G>
-    void addParameter(const G &generator) {
+    void addParameter(const G &generator,
+                      ValueOwnership ownership = ValueOwnership::Default) {
       Type gTyIface = generator.build(*this);
-      auto flags = ParameterTypeFlags();
+      auto flags = ParameterTypeFlags().withValueOwnership(ownership);
       InterfaceParams.emplace_back(gTyIface, Identifier(), flags);
     }
 
     template <class G>
-    void addInOutParameter(const G &generator) {
-      Type gTyIface = generator.build(*this);
-      auto flags = ParameterTypeFlags().withInOut(true);
-      InterfaceParams.emplace_back(gTyIface, Identifier(), flags);
-    }
-    
-    template <class G>
     void setResult(const G &generator) {
       InterfaceResult = generator.build(*this);
     }
@@ -591,7 +585,7 @@
 
 static ValueDecl *getStoreOperation(ASTContext &Context, Identifier Id) {
   BuiltinGenericSignatureBuilder builder(Context);
-  builder.addParameter(makeGenericParam());
+  builder.addParameter(makeGenericParam(), ValueOwnership::Owned);
   builder.addParameter(makeConcrete(Context.TheRawPointerType));
   builder.setResult(makeConcrete(TupleType::getEmpty(Context)));
   return builder.build(Id);
@@ -633,7 +627,7 @@
   Type Int1Ty = BuiltinIntegerType::get(1, Context);
 
   BuiltinGenericSignatureBuilder builder(Context);
-  builder.addInOutParameter(makeGenericParam());
+  builder.addParameter(makeGenericParam(), ValueOwnership::InOut);
   builder.setResult(makeConcrete(Int1Ty));
   return builder.build(Id);
 }
@@ -808,21 +802,35 @@
 
 static ValueDecl *getNativeObjectCast(ASTContext &Context, Identifier Id,
                                       BuiltinValueKind BV) {
-  Type BuiltinTy;
-  if (BV == BuiltinValueKind::BridgeToRawPointer ||
-      BV == BuiltinValueKind::BridgeFromRawPointer)
-    BuiltinTy = Context.TheRawPointerType;
-  else
-    BuiltinTy = Context.TheNativeObjectType;
+
+  ValueOwnership ownership;
+  Type builtinTy;
+  switch (BV) {
+  case BuiltinValueKind::CastToNativeObject:
+  case BuiltinValueKind::UnsafeCastToNativeObject:
+  case BuiltinValueKind::CastFromNativeObject:
+    builtinTy = Context.TheNativeObjectType;
+    ownership = ValueOwnership::Owned;
+    break;
+
+  case BuiltinValueKind::BridgeToRawPointer:
+  case BuiltinValueKind::BridgeFromRawPointer:
+    builtinTy = Context.TheRawPointerType;
+    ownership = ValueOwnership::Default;
+    break;
+
+  default:
+    llvm_unreachable("unexpected kind");
+  }
 
   BuiltinGenericSignatureBuilder builder(Context);
   if (BV == BuiltinValueKind::CastToNativeObject ||
       BV == BuiltinValueKind::UnsafeCastToNativeObject ||
       BV == BuiltinValueKind::BridgeToRawPointer) {
-    builder.addParameter(makeGenericParam());
-    builder.setResult(makeConcrete(BuiltinTy));
+    builder.addParameter(makeGenericParam(), ownership);
+    builder.setResult(makeConcrete(builtinTy));
   } else {
-    builder.addParameter(makeConcrete(BuiltinTy));
+    builder.addParameter(makeConcrete(builtinTy), ownership);
     builder.setResult(makeGenericParam());
   }
   return builder.build(Id);
@@ -833,7 +841,7 @@
   auto wordType = BuiltinIntegerType::get(BuiltinIntegerWidth::pointer(),
                                           C);
   BuiltinGenericSignatureBuilder builder(C);
-  builder.addParameter(makeGenericParam());
+  builder.addParameter(makeGenericParam(), ValueOwnership::Owned);
   builder.addParameter(makeConcrete(wordType));
   builder.setResult(makeConcrete(C.TheBridgeObjectType));
   return builder.build(Id);
@@ -846,7 +854,7 @@
   switch (BV) {
   case BuiltinValueKind::CastReferenceFromBridgeObject: {
     BuiltinGenericSignatureBuilder builder(C);
-    builder.addParameter(makeConcrete(BridgeTy));
+    builder.addParameter(makeConcrete(BridgeTy), ValueOwnership::Owned);
     builder.setResult(makeGenericParam());
     return builder.build(Id);
   }
@@ -906,7 +914,7 @@
   // <T, U> T -> U
   // SILGen and IRGen check additional constraints during lowering.
   BuiltinGenericSignatureBuilder builder(ctx, 2);
-  builder.addParameter(makeGenericParam(0));
+  builder.addParameter(makeGenericParam(0), ValueOwnership::Owned);
   builder.setResult(makeGenericParam(1));
   return builder.build(name);
 }
@@ -916,7 +924,7 @@
   // <T, U> T -> U
   // SILGen and IRGen check additional constraints during lowering.
   BuiltinGenericSignatureBuilder builder(ctx, 2);
-  builder.addParameter(makeGenericParam(0));
+  builder.addParameter(makeGenericParam(0), ValueOwnership::Owned);
   builder.setResult(makeGenericParam(1));
   return builder.build(name);
 }
@@ -949,7 +957,7 @@
 static ValueDecl *getAddressOfOperation(ASTContext &Context, Identifier Id) {
   // <T> (@inout T) -> RawPointer
   BuiltinGenericSignatureBuilder builder(Context);
-  builder.addInOutParameter(makeGenericParam());
+  builder.addParameter(makeGenericParam(), ValueOwnership::InOut);
   builder.setResult(makeConcrete(Context.TheRawPointerType));
   return builder.build(Id);
 }
@@ -976,7 +984,7 @@
                                             Identifier Id) {
   // <T,U,V> (inout T, U.Type) -> V.Type
   BuiltinGenericSignatureBuilder builder(Context, 3);
-  builder.addInOutParameter(makeGenericParam(0));
+  builder.addParameter(makeGenericParam(0), ValueOwnership::InOut);
   builder.addParameter(makeMetatype(makeGenericParam(1)));
   builder.setResult(makeMetatype(makeGenericParam(2)));
   return builder.build(Id);
diff --git a/lib/RemoteAST/RemoteAST.cpp b/lib/RemoteAST/RemoteAST.cpp
index 7a831fa..02c5035 100644
--- a/lib/RemoteAST/RemoteAST.cpp
+++ b/lib/RemoteAST/RemoteAST.cpp
@@ -364,9 +364,7 @@
       auto flags = param.getFlags();
       auto ownership = flags.getValueOwnership();
       auto parameterFlags = ParameterTypeFlags()
-                                .withInOut(ownership == ValueOwnership::InOut)
-                                .withShared(ownership == ValueOwnership::Shared)
-                                .withOwned(ownership == ValueOwnership::Owned)
+                                .withValueOwnership(ownership)
                                 .withVariadic(flags.isVariadic());
 
       funcParams.push_back(AnyFunctionType::Param(type, label, parameterFlags));
diff --git a/lib/SILGen/SILGenBuiltin.cpp b/lib/SILGen/SILGenBuiltin.cpp
index 2e85503..a716f93 100644
--- a/lib/SILGen/SILGenBuiltin.cpp
+++ b/lib/SILGen/SILGenBuiltin.cpp
@@ -30,6 +30,16 @@
 using namespace swift;
 using namespace Lowering;
 
+static bool isTrivialShuffle(TupleShuffleExpr *shuffle) {
+  // Each element must be mapped to the corresponding element of the input.
+  auto mapping = shuffle->getElementMapping();
+  for (auto index : indices(mapping)) {
+    if (mapping[index] < 0 || unsigned(mapping[index]) != index)
+      return false;
+  }
+  return true;
+}
+
 /// Break down an expression that's the formal argument expression to
 /// a builtin function, returning its individualized arguments.
 ///
@@ -43,7 +53,15 @@
   assert(arg->getType()->castTo<TupleType>()->getNumElements()
            == expectedCount);
 
-  auto tuple = dyn_cast<TupleExpr>(arg->getSemanticsProvidingExpr());
+  // The use of owned parameters can trip up CSApply enough to introduce
+  // a trivial tuple shuffle here.
+  arg = arg->getSemanticsProvidingExpr();
+  if (auto shuffle = dyn_cast<TupleShuffleExpr>(arg)) {
+    if (isTrivialShuffle(shuffle))
+      arg = shuffle->getSubExpr();
+  }
+
+  auto tuple = dyn_cast<TupleExpr>(arg);
   if (tuple && tuple->getElements().size() == expectedCount) {
     return tuple->getElements();
   }
diff --git a/test/SILGen/builtins.swift b/test/SILGen/builtins.swift
index c49b487..2e08a0b 100644
--- a/test/SILGen/builtins.swift
+++ b/test/SILGen/builtins.swift
@@ -199,61 +199,61 @@
 // CHECK-LABEL: sil hidden @$s8builtins22class_to_native_object{{[_0-9a-zA-Z]*}}F
 // CHECK: bb0([[ARG:%.*]] : @guaranteed $C):
 // CHECK-NEXT:   debug_value
-// CHECK-NEXT:   [[OBJ:%.*]] = unchecked_ref_cast [[ARG:%.*]] to $Builtin.NativeObject
-// CHECK-NEXT:   [[OBJ_COPY:%.*]] = copy_value [[OBJ]]
-// CHECK-NEXT:   return [[OBJ_COPY]]
+// CHECK-NEXT:   [[ARG_COPY:%.*]] = copy_value [[ARG]]
+// CHECK-NEXT:   [[OBJ:%.*]] = unchecked_ref_cast [[ARG_COPY]] : $C to $Builtin.NativeObject
+// CHECK-NEXT:   return [[OBJ]]
 func class_to_native_object(_ c:C) -> Builtin.NativeObject {
   return Builtin.castToNativeObject(c)
 }
 
 // CHECK-LABEL: sil hidden @$s8builtins32class_archetype_to_native_object{{[_0-9a-zA-Z]*}}F
 func class_archetype_to_native_object<T : C>(_ t: T) -> Builtin.NativeObject {
-  // CHECK: [[OBJ:%.*]] = unchecked_ref_cast [[C:%.*]] to $Builtin.NativeObject
-  // CHECK-NEXT: [[OBJ_COPY:%.*]] = copy_value [[OBJ]]
-  // CHECK-NOT: destroy_value [[C]]
+  // CHECK: [[COPY:%.*]] = copy_value %0
+  // CHECK: [[OBJ:%.*]] = unchecked_ref_cast [[COPY]] : $T to $Builtin.NativeObject
+  // CHECK-NOT: destroy_value [[COPY]]
   // CHECK-NOT: destroy_value [[OBJ]]
-  // CHECK: return [[OBJ_COPY]]
+  // CHECK: return [[OBJ]]
   return Builtin.castToNativeObject(t)
 }
 
 // CHECK-LABEL: sil hidden @$s8builtins34class_existential_to_native_object{{[_0-9a-zA-Z]*}}F
 // CHECK: bb0([[ARG:%.*]] : @guaranteed $ClassProto):
 // CHECK-NEXT:   debug_value
-// CHECK-NEXT:   [[REF:%[0-9]+]] = open_existential_ref [[ARG]] : $ClassProto
+// CHECK-NEXT:   [[ARG_COPY:%.*]] = copy_value [[ARG]]
+// CHECK-NEXT:   [[REF:%[0-9]+]] = open_existential_ref [[ARG_COPY]] : $ClassProto
 // CHECK-NEXT:   [[PTR:%[0-9]+]] = unchecked_ref_cast [[REF]] : $@opened({{.*}}) ClassProto to $Builtin.NativeObject
-// CHECK-NEXT:   [[PTR_COPY:%.*]] = copy_value [[PTR]]
-// CHECK-NEXT:   return [[PTR_COPY]]
+// CHECK-NEXT:   return [[PTR]]
 func class_existential_to_native_object(_ t:ClassProto) -> Builtin.NativeObject {
   return Builtin.unsafeCastToNativeObject(t)
 }
 
 // CHECK-LABEL: sil hidden @$s8builtins24class_from_native_object{{[_0-9a-zA-Z]*}}F
 func class_from_native_object(_ p: Builtin.NativeObject) -> C {
-  // CHECK: [[C:%.*]] = unchecked_ref_cast [[OBJ:%.*]] to $C
-  // CHECK: [[C_RETURN:%.*]] = copy_value [[C]]
-  // CHECK-NOT: destroy_value [[C]]
-  // CHECK-NOT: destroy_value [[OBJ]]
-  // CHECK: return [[C_RETURN]]
+  // CHECK: [[COPY:%.*]] = copy_value %0
+  // CHECK: [[CAST:%.*]] = unchecked_ref_cast [[COPY]] : $Builtin.NativeObject to $C
+  // CHECK-NOT: destroy_value [[COPY]]
+  // CHECK-NOT: destroy_value [[CAST]]
+  // CHECK: return [[CAST]]
   return Builtin.castFromNativeObject(p)
 }
 
 // CHECK-LABEL: sil hidden @$s8builtins34class_archetype_from_native_object{{[_0-9a-zA-Z]*}}F
 func class_archetype_from_native_object<T : C>(_ p: Builtin.NativeObject) -> T {
-  // CHECK: [[C:%.*]] = unchecked_ref_cast [[OBJ:%.*]] : $Builtin.NativeObject to $T
-  // CHECK: [[C_RETURN:%.*]] = copy_value [[C]]
-  // CHECK-NOT: destroy_value [[C]]
-  // CHECK-NOT: destroy_value [[OBJ]]
-  // CHECK: return [[C_RETURN]]
+  // CHECK: [[COPY:%.*]] = copy_value %0
+  // CHECK: [[CAST:%.*]] = unchecked_ref_cast [[COPY]] : $Builtin.NativeObject to $T
+  // CHECK-NOT: destroy_value [[COPY]]
+  // CHECK-NOT: destroy_value [[CAST]]
+  // CHECK: return [[CAST]]
   return Builtin.castFromNativeObject(p)
 }
 
 // CHECK-LABEL: sil hidden @$s8builtins41objc_class_existential_from_native_object{{[_0-9a-zA-Z]*}}F
 func objc_class_existential_from_native_object(_ p: Builtin.NativeObject) -> AnyObject {
-  // CHECK: [[C:%.*]] = unchecked_ref_cast [[OBJ:%.*]] : $Builtin.NativeObject to $AnyObject
-  // CHECK: [[C_RETURN:%.*]] = copy_value [[C]]
-  // CHECK-NOT: destroy_value [[C]]
-  // CHECK-NOT: destroy_value [[OBJ]]
-  // CHECK: return [[C_RETURN]]
+  // CHECK: [[COPY:%.*]] = copy_value %0
+  // CHECK: [[CAST:%.*]] = unchecked_ref_cast [[COPY]] : $Builtin.NativeObject to $AnyObject
+  // CHECK-NOT: destroy_value [[COPY]]
+  // CHECK-NOT: destroy_value [[CAST]]
+  // CHECK: return [[CAST]]
   return Builtin.castFromNativeObject(p)
 }
 
@@ -510,14 +510,16 @@
 // CHECK:       bb0([[ARG1:%.*]] : @guaranteed $C, [[ARG2:%.*]] : @trivial $Builtin.Word):
 // CHECK-NEXT:    debug_value
 // CHECK-NEXT:    debug_value
-// CHECK-NEXT:    [[ARG1_TRIVIAL:%.*]] = unchecked_trivial_bit_cast [[ARG1]] : $C to $Builtin.Word
-// CHECK-NEXT:    [[ARG1_D:%.*]] = unchecked_ref_cast [[ARG1]] : $C to $D
-// CHECK-NEXT:    [[ARG1_OPT:%.*]] = unchecked_ref_cast [[ARG1]] : $C to $Optional<C>
+// CHECK-NEXT:    [[ARG1_COPY1:%.*]] = copy_value [[ARG1]]
+// CHECK-NEXT:    [[ARG1_TRIVIAL:%.*]] = unchecked_trivial_bit_cast [[ARG1_COPY1]] : $C to $Builtin.Word
+// CHECK-NEXT:    [[ARG1_COPY2:%.*]] = copy_value [[ARG1]]
+// CHECK-NEXT:    [[ARG1_D:%.*]] = unchecked_ref_cast [[ARG1_COPY2]] : $C to $D
+// CHECK-NEXT:    [[ARG1_COPY3:%.*]] = copy_value [[ARG1]]
+// CHECK-NEXT:    [[ARG1_OPT:%.*]] = unchecked_ref_cast [[ARG1_COPY3]] : $C to $Optional<C>
 // CHECK-NEXT:    [[ARG2_FROM_WORD:%.*]] = unchecked_bitwise_cast [[ARG2]] : $Builtin.Word to $C
 // CHECK-NEXT:    [[ARG2_FROM_WORD_COPY:%.*]] = copy_value [[ARG2_FROM_WORD]]
-// CHECK-NEXT:    [[ARG1_D_COPY:%.*]] = copy_value [[ARG1_D]]
-// CHECK-NEXT:    [[ARG1_OPT_COPY:%.*]] = copy_value [[ARG1_OPT]]
-// CHECK-NEXT:    [[RESULT:%.*]] = tuple ([[ARG1_TRIVIAL]] : $Builtin.Word, [[ARG1_D_COPY]] : $D, [[ARG1_OPT_COPY]] : $Optional<C>, [[ARG2_FROM_WORD_COPY:%.*]] : $C)
+// CHECK-NEXT:    destroy_value [[ARG1_COPY1]]
+// CHECK-NEXT:    [[RESULT:%.*]] = tuple ([[ARG1_TRIVIAL]] : $Builtin.Word, [[ARG1_D]] : $D, [[ARG1_OPT]] : $Optional<C>, [[ARG2_FROM_WORD_COPY:%.*]] : $C)
 // CHECK:         return [[RESULT]]
 func reinterpretCast(_ c: C, x: Builtin.Word) -> (Builtin.Word, D, C?, C) {
   return (Builtin.reinterpretCast(c) as Builtin.Word,
@@ -534,9 +536,10 @@
 
 // CHECK-LABEL: sil hidden @$s8builtins28reinterpretAddrOnlyToTrivial{{[_0-9a-zA-Z]*}}F
 func reinterpretAddrOnlyToTrivial<T>(_ t: T) -> Int {
-  // CHECK: [[ADDR:%.*]] = unchecked_addr_cast [[INPUT:%.*]] : $*T to $*Int
+  // CHECK: copy_addr %0 to [initialization] [[INPUT:%.*]] : $*T
+  // CHECK: [[ADDR:%.*]] = unchecked_addr_cast [[INPUT]] : $*T to $*Int
   // CHECK: [[VALUE:%.*]] = load [trivial] [[ADDR]]
-  // CHECK-NOT: destroy_addr [[INPUT]]
+  // CHECK: destroy_addr [[INPUT]]
   return Builtin.reinterpretCast(t)
 }
 
@@ -553,9 +556,10 @@
 }
 
 // CHECK-LABEL: sil hidden @$s8builtins18castToBridgeObject{{[_0-9a-zA-Z]*}}F
-// CHECK:         [[BO:%.*]] = ref_to_bridge_object {{%.*}} : $C, {{%.*}} : $Builtin.Word
-// CHECK:         [[BO_COPY:%.*]] = copy_value [[BO]]
-// CHECK:         return [[BO_COPY]]
+// CHECK:         [[ARG_COPY:%.*]] = copy_value %0
+// CHECK:         [[BO:%.*]] = ref_to_bridge_object [[ARG_COPY]] : $C, {{%.*}} : $Builtin.Word
+// CHECK-NOT:     destroy_value [[ARG_COPY]]
+// CHECK:         return [[BO]]
 func castToBridgeObject(_ c: C, _ w: Builtin.Word) -> Builtin.BridgeObject {
   return Builtin.castToBridgeObject(c, w)
 }
@@ -653,10 +657,10 @@
 
 // CHECK-LABEL: sil hidden @$s8builtins17refcast_class_anyyyXlAA1ACF :
 // CHECK: bb0([[ARG:%.*]] : @guaranteed $A):
-// CHECK:   [[ARG_CASTED:%.*]] = unchecked_ref_cast [[ARG]] : $A to $AnyObject
-// CHECK:   [[ARG_CASTED_COPY:%.*]] = copy_value [[ARG_CASTED]]
+// CHECK:   [[ARG_COPY:%.*]] = copy_value [[ARG]]
+// CHECK:   [[ARG_CASTED:%.*]] = unchecked_ref_cast [[ARG_COPY]] : $A to $AnyObject
 // CHECK-NOT:   destroy_value [[ARG]]
-// CHECK:   return [[ARG_CASTED_COPY]]
+// CHECK:   return [[ARG_CASTED]]
 // CHECK: } // end sil function '$s8builtins17refcast_class_anyyyXlAA1ACF'
 func refcast_class_any(_ o: A) -> AnyObject {
   return Builtin.castReference(o)
@@ -670,9 +674,9 @@
 
 // CHECK-LABEL: sil hidden @$s8builtins18refcast_pclass_anyyyXlAA6PClass_pF :
 // CHECK: bb0([[ARG:%.*]] : @guaranteed $PClass):
-// CHECK:   [[ARG_CAST:%.*]] = unchecked_ref_cast [[ARG]] : $PClass to $AnyObject
-// CHECK:   [[ARG_CAST_COPY:%.*]] = copy_value [[ARG_CAST]]
-// CHECK:   return [[ARG_CAST_COPY]]
+// CHECK:   [[ARG_COPY:%.*]] = copy_value [[ARG]]
+// CHECK:   [[ARG_CAST:%.*]] = unchecked_ref_cast [[ARG_COPY]] : $PClass to $AnyObject
+// CHECK:   return [[ARG_CAST]]
 // CHECK: } // end sil function '$s8builtins18refcast_pclass_anyyyXlAA6PClass_pF'
 func refcast_pclass_any(_ o: PClass) -> AnyObject {
   return Builtin.castReference(o)
diff --git a/test/SILGen/guaranteed_normal_args.swift b/test/SILGen/guaranteed_normal_args.swift
index 8140c20..ada4cc6 100644
--- a/test/SILGen/guaranteed_normal_args.swift
+++ b/test/SILGen/guaranteed_normal_args.swift
@@ -133,15 +133,10 @@
   // CHECK:   [[BUF_BORROW:%.*]] = begin_borrow [[BUF]]
   // CHECK:   [[K:%.*]] = struct_extract [[BUF_BORROW]] : $Buffer, #Buffer.k
   // CHECK:   [[COPIED_K:%.*]] = copy_value [[K]]
-  //   FIXME: this borrow-and-copy is really dumb and unnecessary
-  // CHECK:   [[COPIED_BORROWED_K:%.*]] = begin_borrow [[COPIED_K]]
-  // CHECK:   [[CASTED_BORROWED_BUF_KLASS:%.*]] = unchecked_ref_cast [[COPIED_BORROWED_K]]
-  // CHECK:   [[COPY_CASTED_BORROWED_BUF_KLASS:%.*]] = copy_value [[CASTED_BORROWED_BUF_KLASS]]
-  // CHECK:   end_borrow  [[COPIED_BORROWED_K]]
-  // CHECK:   destroy_value [[COPIED_K]]
+  // CHECK:   [[CASTED_COPIED_K:%.*]] = unchecked_ref_cast [[COPIED_K]]
   // CHECK:   end_borrow [[BUF_BORROW]]
   // CHECK:   destroy_value [[BUF]]
-  // CHECK:   return [[COPY_CASTED_BORROWED_BUF_KLASS]]
+  // CHECK:   return [[CASTED_COPIED_K]]
   // CHECK: } // end sil function '$ss15KlassWithBufferC03getC14AsNativeObjectBoyF'
   func getBufferAsNativeObject() -> Builtin.NativeObject {
     return Builtin.unsafeCastToNativeObject(buffer.k)
@@ -154,9 +149,9 @@
   // CHECK-LABEL: sil hidden @$ss28StructContainingBridgeObjectV8swiftObjAByXl_tcfC : $@convention(method) (@owned AnyObject, @thin StructContainingBridgeObject.Type) -> @owned StructContainingBridgeObject {
   // CHECK: bb0([[ARG:%.*]] : @owned $AnyObject,
   // CHECK:   [[BORROWED_ARG:%.*]] = begin_borrow [[ARG]]
-  // CHECK:   [[CASTED_ARG:%.*]] = unchecked_ref_cast [[BORROWED_ARG]] : $AnyObject to $Builtin.BridgeObject
-  // CHECK:   [[COPY_CASTED_ARG:%.*]] = copy_value [[CASTED_ARG]]
-  // CHECK:   assign [[COPY_CASTED_ARG]] to
+  // CHECK:   [[COPIED_ARG:%.*]] = copy_value [[BORROWED_ARG]]
+  // CHECK:   [[CASTED_ARG:%.*]] = unchecked_ref_cast [[COPIED_ARG]] : $AnyObject to $Builtin.BridgeObject
+  // CHECK:   assign [[CASTED_ARG]] to
   // CHECK: } // end sil function '$ss28StructContainingBridgeObjectV8swiftObjAByXl_tcfC'
   init(swiftObj: AnyObject) {
     rawValue = Builtin.reinterpretCast(swiftObj)
diff --git a/test/SILGen/opaque_ownership.swift b/test/SILGen/opaque_ownership.swift
index 3292b69..f72028a 100644
--- a/test/SILGen/opaque_ownership.swift
+++ b/test/SILGen/opaque_ownership.swift
@@ -45,8 +45,10 @@
 // ---
 // CHECK-LABEL: sil @$ss13unsafeBitCast_2toq_x_q_mtr0_lF : $@convention(thin) <T, U> (@in_guaranteed T, @thick U.Type) -> @out U {
 // CHECK: bb0([[ARG0:%.*]] : @guaranteed $T, [[ARG1:%.*]] : @trivial $@thick U.Type):
-// CHECK:   [[RESULT:%.*]] = unchecked_bitwise_cast [[ARG0]] : $T to $U
+// CHECK:   [[ARG_COPY:%.*]] = copy_value [[ARG0]] : $T
+// CHECK:   [[RESULT:%.*]] = unchecked_bitwise_cast [[ARG_COPY]] : $T to $U
 // CHECK:   [[RESULT_COPY:%.*]] = copy_value [[RESULT]] : $U
+// CHECK:   destroy_value [[ARG_COPY]] : $T
 // CHECK:   return [[RESULT_COPY]] : $U
 // CHECK-LABEL: } // end sil function '$ss13unsafeBitCast_2toq_x_q_mtr0_lF'
 public func unsafeBitCast<T, U>(_ x: T, to type: U.Type) -> U {
diff --git a/test/SILGen/opaque_values_silgen_lib.swift b/test/SILGen/opaque_values_silgen_lib.swift
index 3518d6a..70d529a 100644
--- a/test/SILGen/opaque_values_silgen_lib.swift
+++ b/test/SILGen/opaque_values_silgen_lib.swift
@@ -29,9 +29,10 @@
 // ---
 // CHECK-LABEL: sil hidden @$ss21s020__________bitCast_2toq_x_q_mtr0_lF : $@convention(thin) <T, U> (@in_guaranteed T, @thick U.Type) -> @out U {
 // CHECK: bb0([[ARG:%.*]] : @guaranteed $T,
-// CHECK: [[CAST:%.*]] = unchecked_bitwise_cast [[ARG]] : $T to $U
+// CHECK: [[COPY:%.*]] = copy_value [[ARG]] : $T
+// CHECK: [[CAST:%.*]] = unchecked_bitwise_cast [[COPY]] : $T to $U
 // CHECK: [[RET:%.*]] = copy_value [[CAST]] : $U
-// CHECK-NOT: destroy_value [[COPY]] : $T
+// CHECK: destroy_value [[COPY]] : $T
 // CHECK: return [[RET]] : $U
 // CHECK-LABEL: } // end sil function '$ss21s020__________bitCast_2toq_x_q_mtr0_lF'
 func s020__________bitCast<T, U>(_ x: T, to type: U.Type) -> U {
diff --git a/test/Serialization/transparent-std.swift b/test/Serialization/transparent-std.swift
index 4565f70..5e5331e 100644
--- a/test/Serialization/transparent-std.swift
+++ b/test/Serialization/transparent-std.swift
@@ -31,8 +31,8 @@
 func test_conversion(c: C, t32: Builtin.Int32) {
 // SIL-LABEL: sil public_external [transparent] [serialized] [canonical] @$s19def_transparent_std22class_to_native_object1cBoAA1CC_tF : $@convention(thin) (@guaranteed C) -> @owned Builtin.NativeObject {
 // SIL: bb0(%0 : $C):
-// SIL: unchecked_ref_cast %0 : $C to $Builtin.NativeObject
 // SIL-NEXT: strong_retain
+// SIL-NEXT: unchecked_ref_cast %0 : $C to $Builtin.NativeObject
 // SIL-NEXT: return
   var b = class_to_native_object(c: c)