Auto merge of #66927 - RalfJung:engines-dont-panic, r=oli-obk

Miri core engine: use throw_ub instead of throw_panic

See https://github.com/rust-lang/rust/issues/66902 for context: panicking is not really an "interpreter error", but just part of a normal Rust execution. This is a first step towards removing the `InterpError::Panic` variant: the core Miri engine does not use it any more.

ConstProp and ConstEval still use it, though. This will be addressed in future PRs.

From what I can tell, all the error messages this removes are actually duplicates.

r? @oli-obk @wesleywiser
diff --git a/Cargo.lock b/Cargo.lock
index 67259ae..26727c5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -468,12 +468,14 @@
  "clippy_lints",
  "compiletest_rs",
  "derive-new",
+ "git2",
  "lazy_static 1.3.0",
  "regex",
  "rustc-workspace-hack",
  "rustc_tools_util 0.2.0",
  "semver",
  "serde",
+ "tempfile",
  "tester",
 ]
 
@@ -1292,7 +1294,7 @@
 dependencies = [
  "cfg-if",
  "libc",
- "wasi",
+ "wasi 0.7.0",
 ]
 
 [[package]]
@@ -3203,6 +3205,7 @@
  "rustc_fs_util",
  "rustc_index",
  "rustc_macros",
+ "rustc_session",
  "rustc_target",
  "scoped-tls",
  "serialize",
@@ -3516,6 +3519,7 @@
  "rustc_fs_util",
  "rustc_incremental",
  "rustc_index",
+ "rustc_session",
  "rustc_target",
  "serialize",
  "syntax",
@@ -3632,6 +3636,7 @@
  "rustc",
  "rustc_data_structures",
  "rustc_fs_util",
+ "rustc_session",
  "serialize",
  "syntax",
  "syntax_pos",
@@ -3695,6 +3700,7 @@
  "rustc_error_codes",
  "rustc_feature",
  "rustc_index",
+ "rustc_session",
  "rustc_target",
  "syntax",
  "syntax_pos",
@@ -3800,7 +3806,6 @@
  "rustc_errors",
  "rustc_feature",
  "rustc_lexer",
- "rustc_target",
  "smallvec 1.0.0",
  "syntax",
  "syntax_pos",
@@ -3883,6 +3888,22 @@
 ]
 
 [[package]]
+name = "rustc_session"
+version = "0.0.0"
+dependencies = [
+ "log",
+ "num_cpus",
+ "rustc_data_structures",
+ "rustc_errors",
+ "rustc_feature",
+ "rustc_fs_util",
+ "rustc_index",
+ "rustc_target",
+ "serialize",
+ "syntax_pos",
+]
+
+[[package]]
 name = "rustc_target"
 version = "0.0.0"
 dependencies = [
@@ -4299,7 +4320,7 @@
  "rustc_msan",
  "rustc_tsan",
  "unwind",
- "wasi",
+ "wasi 0.9.0+wasi-snapshot-preview1",
 ]
 
 [[package]]
@@ -4461,6 +4482,7 @@
  "rustc_index",
  "rustc_lexer",
  "rustc_macros",
+ "rustc_session",
  "scoped-tls",
  "serialize",
  "smallvec 1.0.0",
@@ -5170,6 +5192,12 @@
 version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
+
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index 730e8cf..bb16941 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -643,7 +643,9 @@
         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
             (os.pathsep + env["LIBRARY_PATH"]) \
             if "LIBRARY_PATH" in env else ""
-        env["RUSTFLAGS"] = "-Cdebuginfo=2 "
+        # preserve existing RUSTFLAGS
+        env.setdefault("RUSTFLAGS", "")
+        env["RUSTFLAGS"] += " -Cdebuginfo=2"
 
         build_section = "target.{}".format(self.build_triple())
         target_features = []
@@ -652,13 +654,13 @@
         elif self.get_toml("crt-static", build_section) == "false":
             target_features += ["-crt-static"]
         if target_features:
-            env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
+            env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
         target_linker = self.get_toml("linker", build_section)
         if target_linker is not None:
-            env["RUSTFLAGS"] += "-C linker=" + target_linker + " "
-        env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes "
+            env["RUSTFLAGS"] += " -C linker=" + target_linker
+        env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
         if self.get_toml("deny-warnings", "rust") != "false":
-            env["RUSTFLAGS"] += "-Dwarnings "
+            env["RUSTFLAGS"] += " -Dwarnings"
 
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
             os.pathsep + env["PATH"]
diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs
index 384219c..f8734eb 100644
--- a/src/bootstrap/install.rs
+++ b/src/bootstrap/install.rs
@@ -260,7 +260,7 @@
     };
     Rustc, "src/librustc", true, only_hosts: true, {
         builder.ensure(dist::Rustc {
-            compiler: self.compiler,
+            compiler: builder.compiler(builder.top_stage, self.target),
         });
         install_rustc(builder, self.compiler.stage, self.target);
     };
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index 39d7ea9..7ea2bb1 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -810,6 +810,7 @@
                   !target.contains("emscripten") &&
                   !target.contains("wasm32") &&
                   !target.contains("nvptx") &&
+                  !target.contains("fortanix") &&
                   !target.contains("fuchsia") {
             Some(self.cc(target))
         } else {
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index 2e89fd5..be13b9a 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -294,11 +294,11 @@
     let mut parts = version.split('.').take(2)
         .filter_map(|s| s.parse::<u32>().ok());
     if let (Some(major), Some(_minor)) = (parts.next(), parts.next()) {
-        if major >= 6 {
+        if major >= 7 {
             return
         }
     }
-    panic!("\n\nbad LLVM version: {}, need >=6.0\n\n", version)
+    panic!("\n\nbad LLVM version: {}, need >=7.0\n\n", version)
 }
 
 fn configure_cmake(builder: &Builder<'_>,
diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml
index bfe5174..70d6bad 100644
--- a/src/ci/azure-pipelines/auto.yml
+++ b/src/ci/azure-pipelines/auto.yml
@@ -18,7 +18,7 @@
   - template: steps/run.yml
   strategy:
     matrix:
-      x86_64-gnu-llvm-6.0:
+      x86_64-gnu-llvm-7:
         RUST_BACKTRACE: 1
       dist-x86_64-linux: {}
       dist-x86_64-linux-alt:
diff --git a/src/ci/azure-pipelines/pr.yml b/src/ci/azure-pipelines/pr.yml
index aee4d8d..1f0be53 100644
--- a/src/ci/azure-pipelines/pr.yml
+++ b/src/ci/azure-pipelines/pr.yml
@@ -18,7 +18,7 @@
     - template: steps/run.yml
   strategy:
     matrix:
-      x86_64-gnu-llvm-6.0: {}
+      x86_64-gnu-llvm-7: {}
       mingw-check: {}
       x86_64-gnu-tools:
         CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
diff --git a/src/ci/docker/README.md b/src/ci/docker/README.md
index a2d83ec..872f2c3 100644
--- a/src/ci/docker/README.md
+++ b/src/ci/docker/README.md
@@ -16,6 +16,13 @@
 
 Images will output artifacts in an `obj` dir at the root of a repository.
 
+**NOTE**: Re-using the same `obj` dir with different docker images with
+the same target triple (e.g. `dist-x86_64-linux` and `dist-various-1`)
+may result in strange linker errors, due shared library versions differing between platforms.
+
+If you encounter any issues when using multiple Docker images, try deleting your `obj` directory
+before running your command.
+
 ## Filesystem layout
 
 - Each directory, excluding `scripts` and `disabled`, corresponds to a docker image
diff --git a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
index 17aa789..925d5ca 100755
--- a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
+++ b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
@@ -12,7 +12,7 @@
 git clone https://github.com/CraneStation/wasi-libc
 
 cd wasi-libc
-git reset --hard a94d2d04e7722b323573da2bd04e909a5763d35b
+git reset --hard f645f498dfbbbc00a7a97874d33082d3605c3f21
 make -j$(nproc) INSTALL_DIR=/wasm32-wasi install
 
 cd ..
diff --git a/src/ci/docker/x86_64-gnu-llvm-6.0/Dockerfile b/src/ci/docker/x86_64-gnu-llvm-7/Dockerfile
similarity index 90%
rename from src/ci/docker/x86_64-gnu-llvm-6.0/Dockerfile
rename to src/ci/docker/x86_64-gnu-llvm-7/Dockerfile
index 6dbbb22..a1c9c13 100644
--- a/src/ci/docker/x86_64-gnu-llvm-6.0/Dockerfile
+++ b/src/ci/docker/x86_64-gnu-llvm-7/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
@@ -11,7 +11,7 @@
   cmake \
   sudo \
   gdb \
-  llvm-6.0-tools \
+  llvm-7-tools \
   libedit-dev \
   libssl-dev \
   pkg-config \
@@ -24,7 +24,7 @@
 # using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
-      --llvm-root=/usr/lib/llvm-6.0 \
+      --llvm-root=/usr/lib/llvm-7 \
       --enable-llvm-link-shared
 ENV SCRIPT python2.7 ../x.py test src/tools/tidy && python2.7 ../x.py test
 
diff --git a/src/ci/run.sh b/src/ci/run.sh
index ae5b224..38d1d2b 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -23,7 +23,7 @@
 ci_dir=`cd $(dirname $0) && pwd`
 source "$ci_dir/shared.sh"
 
-if [ ! isCI ] || isCiBranch auto || isCiBranch beta; then
+if ! isCI || isCiBranch auto || isCiBranch beta; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
 fi
 
diff --git a/src/doc/rustc/src/lints/listing/warn-by-default.md b/src/doc/rustc/src/lints/listing/warn-by-default.md
index 77642a8..386f600 100644
--- a/src/doc/rustc/src/lints/listing/warn-by-default.md
+++ b/src/doc/rustc/src/lints/listing/warn-by-default.md
@@ -307,18 +307,6 @@
   |
 ```
 
-## plugin-as-library
-
-This lint detects when compiler plugins are used as ordinary library in
-non-plugin crate. Some example code that triggers this lint:
-
-```rust,ignore
-#![feature(plugin)]
-#![plugin(macro_crate_test)]
-
-extern crate macro_crate_test;
-```
-
 ## private-in-public
 
 This lint detects private items in public interfaces not caught by the old implementation. Some
diff --git a/src/doc/unstable-book/src/language-features/cfg-sanitize.md b/src/doc/unstable-book/src/language-features/cfg-sanitize.md
new file mode 100644
index 0000000..949f24a
--- /dev/null
+++ b/src/doc/unstable-book/src/language-features/cfg-sanitize.md
@@ -0,0 +1,36 @@
+# `cfg_sanitize`
+
+The tracking issue for this feature is: [#39699]
+
+[#39699]: https://github.com/rust-lang/rust/issues/39699
+
+------------------------
+
+The `cfg_sanitize` feature makes it possible to execute different code
+depending on whether a particular sanitizer is enabled or not.
+
+## Examples
+
+``` rust
+#![feature(cfg_sanitize)]
+
+#[cfg(sanitize = "thread")]
+fn a() {
+  // ...
+}
+
+#[cfg(not(sanitize = "thread"))]
+fn a() {
+  // ...
+}
+
+fn b() {
+  if cfg!(sanitize = "leak") {
+    // ...
+  } else {
+    // ...
+  }
+}
+
+```
+
diff --git a/src/doc/unstable-book/src/language-features/plugin.md b/src/doc/unstable-book/src/language-features/plugin.md
index cd1137e..495cdee 100644
--- a/src/doc/unstable-book/src/language-features/plugin.md
+++ b/src/doc/unstable-book/src/language-features/plugin.md
@@ -21,15 +21,10 @@
 `rustc_driver::plugin` documentation for more about the
 mechanics of defining and loading a plugin.
 
-If present, arguments passed as `#![plugin(foo(... args ...))]` are not
-interpreted by rustc itself.  They are provided to the plugin through the
-`Registry`'s `args` method.
-
 In the vast majority of cases, a plugin should *only* be used through
 `#![plugin]` and not through an `extern crate` item.  Linking a plugin would
 pull in all of libsyntax and librustc as dependencies of your crate.  This is
-generally unwanted unless you are building another plugin.  The
-`plugin_as_library` lint checks these guidelines.
+generally unwanted unless you are building another plugin.
 
 The usual practice is to put compiler plugins in their own crate, separate from
 any `macro_rules!` macros or ordinary Rust code meant to be used by consumers
diff --git a/src/liballoc/rc.rs b/src/liballoc/rc.rs
index ec08965..1ff1c3c 100644
--- a/src/liballoc/rc.rs
+++ b/src/liballoc/rc.rs
@@ -926,7 +926,7 @@
         // reference (see #54908).
         let layout = Layout::new::<RcBox<()>>()
             .extend(value_layout).unwrap().0
-            .pad_to_align().unwrap();
+            .pad_to_align();
 
         // Allocate for the layout.
         let mem = Global.alloc(layout)
@@ -1648,10 +1648,8 @@
 
     /// Returns a raw pointer to the object `T` pointed to by this `Weak<T>`.
     ///
-    /// It is up to the caller to ensure that the object is still alive when accessing it through
-    /// the pointer.
-    ///
-    /// The pointer may be [`null`] or be dangling in case the object has already been destroyed.
+    /// The pointer is valid only if there are some strong references. The pointer may be dangling
+    /// or even [`null`] otherwise.
     ///
     /// # Examples
     ///
@@ -1731,14 +1729,18 @@
     /// This can be used to safely get a strong reference (by calling [`upgrade`]
     /// later) or to deallocate the weak count by dropping the `Weak<T>`.
     ///
-    /// It takes ownership of one weak count. In case a [`null`] is passed, a dangling [`Weak`] is
-    /// returned.
+    /// It takes ownership of one weak count (with the exception of pointers created by [`new`],
+    /// as these don't have any corresponding weak count).
     ///
     /// # Safety
     ///
-    /// The pointer must represent one valid weak count. In other words, it must point to `T` which
-    /// is or *was* managed by an [`Rc`] and the weak count of that [`Rc`] must not have reached
-    /// 0. It is allowed for the strong count to be 0.
+    /// The pointer must have originated from the [`into_raw`] (or [`as_raw`], provided there was
+    /// a corresponding [`forget`] on the `Weak<T>`) and must still own its potential weak reference
+    /// count.
+    ///
+    /// It is allowed for the strong count to be 0 at the time of calling this, but the weak count
+    /// must be non-zero or the pointer must have originated from a dangling `Weak<T>` (one created
+    /// by [`new`]).
     ///
     /// # Examples
     ///
@@ -1763,11 +1765,13 @@
     /// assert!(unsafe { Weak::from_raw(raw_2) }.upgrade().is_none());
     /// ```
     ///
-    /// [`null`]: ../../std/ptr/fn.null.html
     /// [`into_raw`]: struct.Weak.html#method.into_raw
     /// [`upgrade`]: struct.Weak.html#method.upgrade
     /// [`Rc`]: struct.Rc.html
     /// [`Weak`]: struct.Weak.html
+    /// [`as_raw`]: struct.Weak.html#method.as_raw
+    /// [`new`]: struct.Weak.html#method.new
+    /// [`forget`]: ../../std/mem/fn.forget.html
     #[unstable(feature = "weak_into_raw", issue = "60728")]
     pub unsafe fn from_raw(ptr: *const T) -> Self {
         if ptr.is_null() {
diff --git a/src/liballoc/sync.rs b/src/liballoc/sync.rs
index 0deb321..19b0086 100644
--- a/src/liballoc/sync.rs
+++ b/src/liballoc/sync.rs
@@ -780,7 +780,7 @@
         // reference (see #54908).
         let layout = Layout::new::<ArcInner<()>>()
             .extend(value_layout).unwrap().0
-            .pad_to_align().unwrap();
+            .pad_to_align();
 
         let mem = Global.alloc(layout)
             .unwrap_or_else(|_| handle_alloc_error(layout));
@@ -1324,10 +1324,8 @@
 
     /// Returns a raw pointer to the object `T` pointed to by this `Weak<T>`.
     ///
-    /// It is up to the caller to ensure that the object is still alive when accessing it through
-    /// the pointer.
-    ///
-    /// The pointer may be [`null`] or be dangling in case the object has already been destroyed.
+    /// The pointer is valid only if there are some strong references. The pointer may be dangling
+    /// or even [`null`] otherwise.
     ///
     /// # Examples
     ///
@@ -1408,14 +1406,18 @@
     /// This can be used to safely get a strong reference (by calling [`upgrade`]
     /// later) or to deallocate the weak count by dropping the `Weak<T>`.
     ///
-    /// It takes ownership of one weak count. In case a [`null`] is passed, a dangling [`Weak`] is
-    /// returned.
+    /// It takes ownership of one weak count (with the exception of pointers created by [`new`],
+    /// as these don't have any corresponding weak count).
     ///
     /// # Safety
     ///
-    /// The pointer must represent one valid weak count. In other words, it must point to `T` which
-    /// is or *was* managed by an [`Arc`] and the weak count of that [`Arc`] must not have reached
-    /// 0. It is allowed for the strong count to be 0.
+    /// The pointer must have originated from the [`into_raw`] (or [`as_raw'], provided there was
+    /// a corresponding [`forget`] on the `Weak<T>`) and must still own its potential weak reference
+    /// count.
+    ///
+    /// It is allowed for the strong count to be 0 at the time of calling this, but the weak count
+    /// must be non-zero or the pointer must have originated from a dangling `Weak<T>` (one created
+    /// by [`new`]).
     ///
     /// # Examples
     ///
@@ -1440,11 +1442,13 @@
     /// assert!(unsafe { Weak::from_raw(raw_2) }.upgrade().is_none());
     /// ```
     ///
-    /// [`null`]: ../../std/ptr/fn.null.html
+    /// [`as_raw`]: struct.Weak.html#method.as_raw
+    /// [`new`]: struct.Weak.html#method.new
     /// [`into_raw`]: struct.Weak.html#method.into_raw
     /// [`upgrade`]: struct.Weak.html#method.upgrade
     /// [`Weak`]: struct.Weak.html
     /// [`Arc`]: struct.Arc.html
+    /// [`forget`]: ../../std/mem/fn.forget.html
     #[unstable(feature = "weak_into_raw", issue = "60728")]
     pub unsafe fn from_raw(ptr: *const T) -> Self {
         if ptr.is_null() {
diff --git a/src/libcore/alloc.rs b/src/libcore/alloc.rs
index 1b06bae..20248f7 100644
--- a/src/libcore/alloc.rs
+++ b/src/libcore/alloc.rs
@@ -53,7 +53,7 @@
 
 impl Layout {
     /// Constructs a `Layout` from a given `size` and `align`,
-    /// or returns `LayoutErr` if either of the following conditions
+    /// or returns `LayoutErr` if any of the following conditions
     /// are not met:
     ///
     /// * `align` must not be zero,
@@ -137,7 +137,7 @@
     #[inline]
     pub fn for_value<T: ?Sized>(t: &T) -> Self {
         let (size, align) = (mem::size_of_val(t), mem::align_of_val(t));
-        // See rationale in `new` for why this us using an unsafe variant below
+        // See rationale in `new` for why this is using an unsafe variant below
         debug_assert!(Layout::from_size_align(size, align).is_ok());
         unsafe {
             Layout::from_size_align_unchecked(size, align)
@@ -196,7 +196,7 @@
         //    valid.
         //
         // 2. `len + align - 1` can overflow by at most `align - 1`,
-        //    so the &-mask wth `!(align - 1)` will ensure that in the
+        //    so the &-mask with `!(align - 1)` will ensure that in the
         //    case of overflow, `len_rounded_up` will itself be 0.
         //    Thus the returned padding, when added to `len`, yields 0,
         //    which trivially satisfies the alignment `align`.
@@ -213,18 +213,19 @@
     /// Creates a layout by rounding the size of this layout up to a multiple
     /// of the layout's alignment.
     ///
-    /// Returns `Err` if the padded size would overflow.
-    ///
     /// This is equivalent to adding the result of `padding_needed_for`
     /// to the layout's current size.
     #[unstable(feature = "alloc_layout_extra", issue = "55724")]
     #[inline]
-    pub fn pad_to_align(&self) -> Result<Layout, LayoutErr> {
+    pub fn pad_to_align(&self) -> Layout {
         let pad = self.padding_needed_for(self.align());
-        let new_size = self.size().checked_add(pad)
-            .ok_or(LayoutErr { private: () })?;
+        // This cannot overflow. Quoting from the invariant of Layout:
+        // > `size`, when rounded up to the nearest multiple of `align`,
+        // > must not overflow (i.e., the rounded value must be less than
+        // > `usize::MAX`)
+        let new_size = self.size() + pad;
 
-        Layout::from_size_align(new_size, self.align())
+        Layout::from_size_align(new_size, self.align()).unwrap()
     }
 
     /// Creates a layout describing the record for `n` instances of
diff --git a/src/libcore/bool.rs b/src/libcore/bool.rs
index 617bdd2..1b3c254 100644
--- a/src/libcore/bool.rs
+++ b/src/libcore/bool.rs
@@ -9,12 +9,12 @@
     /// ```
     /// #![feature(bool_to_option)]
     ///
-    /// assert_eq!(false.then(0), None);
-    /// assert_eq!(true.then(0), Some(0));
+    /// assert_eq!(false.then_some(0), None);
+    /// assert_eq!(true.then_some(0), Some(0));
     /// ```
     #[unstable(feature = "bool_to_option", issue = "64260")]
     #[inline]
-    pub fn then<T>(self, t: T) -> Option<T> {
+    pub fn then_some<T>(self, t: T) -> Option<T> {
         if self {
             Some(t)
         } else {
@@ -29,12 +29,12 @@
     /// ```
     /// #![feature(bool_to_option)]
     ///
-    /// assert_eq!(false.then_with(|| 0), None);
-    /// assert_eq!(true.then_with(|| 0), Some(0));
+    /// assert_eq!(false.then(|| 0), None);
+    /// assert_eq!(true.then(|| 0), Some(0));
     /// ```
     #[unstable(feature = "bool_to_option", issue = "64260")]
     #[inline]
-    pub fn then_with<T, F: FnOnce() -> T>(self, f: F) -> Option<T> {
+    pub fn then<T, F: FnOnce() -> T>(self, f: F) -> Option<T> {
         if self {
             Some(f())
         } else {
diff --git a/src/libcore/cmp.rs b/src/libcore/cmp.rs
index eea3dc3..a5f355c 100644
--- a/src/libcore/cmp.rs
+++ b/src/libcore/cmp.rs
@@ -534,7 +534,6 @@
 ///     }
 /// }
 /// ```
-#[lang = "ord"]
 #[doc(alias = "<")]
 #[doc(alias = ">")]
 #[doc(alias = "<=")]
diff --git a/src/libcore/convert.rs b/src/libcore/convert/mod.rs
similarity index 98%
rename from src/libcore/convert.rs
rename to src/libcore/convert/mod.rs
index 08802b3..16d5375 100644
--- a/src/libcore/convert.rs
+++ b/src/libcore/convert/mod.rs
@@ -40,6 +40,11 @@
 
 #![stable(feature = "rust1", since = "1.0.0")]
 
+mod num;
+
+#[unstable(feature = "convert_float_to_int", issue = "67057")]
+pub use num::FloatToInt;
+
 /// The identity function.
 ///
 /// Two things are important to note about this function:
diff --git a/src/libcore/convert/num.rs b/src/libcore/convert/num.rs
new file mode 100644
index 0000000..0877dac
--- /dev/null
+++ b/src/libcore/convert/num.rs
@@ -0,0 +1,369 @@
+use super::{From, TryFrom};
+use crate::num::TryFromIntError;
+
+mod private {
+    /// This trait being unreachable from outside the crate
+    /// prevents other implementations of the `FloatToInt` trait,
+    /// which allows potentially adding more trait methods after the trait is `#[stable]`.
+    #[unstable(feature = "convert_float_to_int", issue = "67057")]
+    pub trait Sealed {}
+}
+
+/// Supporting trait for inherent methods of `f32` and `f64` such as `round_unchecked_to`.
+/// Typically doesn’t need to be used directly.
+#[unstable(feature = "convert_float_to_int", issue = "67057")]
+pub trait FloatToInt<Int>: private::Sealed + Sized {
+    #[cfg(not(bootstrap))]
+    #[unstable(feature = "float_approx_unchecked_to", issue = "67058")]
+    #[doc(hidden)]
+    unsafe fn approx_unchecked(self) -> Int;
+}
+
+macro_rules! impl_float_to_int {
+    ( $Float: ident => $( $Int: ident )+ ) => {
+        #[unstable(feature = "convert_float_to_int", issue = "67057")]
+        impl private::Sealed for $Float {}
+        $(
+            #[unstable(feature = "convert_float_to_int", issue = "67057")]
+            impl FloatToInt<$Int> for $Float {
+                #[cfg(not(bootstrap))]
+                #[doc(hidden)]
+                #[inline]
+                unsafe fn approx_unchecked(self) -> $Int {
+                    crate::intrinsics::float_to_int_approx_unchecked(self)
+                }
+            }
+        )+
+    }
+}
+
+impl_float_to_int!(f32 => u8 u16 u32 u64 u128 usize i8 i16 i32 i64 i128 isize);
+impl_float_to_int!(f64 => u8 u16 u32 u64 u128 usize i8 i16 i32 i64 i128 isize);
+
+// Conversion traits for primitive integer and float types
+// Conversions T -> T are covered by a blanket impl and therefore excluded
+// Some conversions from and to usize/isize are not implemented due to portability concerns
+macro_rules! impl_from {
+    ($Small: ty, $Large: ty, #[$attr:meta], $doc: expr) => {
+        #[$attr]
+        #[doc = $doc]
+        impl From<$Small> for $Large {
+            #[inline]
+            fn from(small: $Small) -> $Large {
+                small as $Large
+            }
+        }
+    };
+    ($Small: ty, $Large: ty, #[$attr:meta]) => {
+        impl_from!($Small,
+                   $Large,
+                   #[$attr],
+                   concat!("Converts `",
+                           stringify!($Small),
+                           "` to `",
+                           stringify!($Large),
+                           "` losslessly."));
+    }
+}
+
+macro_rules! impl_from_bool {
+    ($target: ty, #[$attr:meta]) => {
+        impl_from!(bool, $target, #[$attr], concat!("Converts a `bool` to a `",
+            stringify!($target), "`. The resulting value is `0` for `false` and `1` for `true`
+values.
+
+# Examples
+
+```
+assert_eq!(", stringify!($target), "::from(true), 1);
+assert_eq!(", stringify!($target), "::from(false), 0);
+```"));
+    };
+}
+
+// Bool -> Any
+impl_from_bool! { u8, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { u16, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { u32, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { u64, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { u128, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { usize, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { i8, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { i16, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { i32, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { i64, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { i128, #[stable(feature = "from_bool", since = "1.28.0")] }
+impl_from_bool! { isize, #[stable(feature = "from_bool", since = "1.28.0")] }
+
+// Unsigned -> Unsigned
+impl_from! { u8, u16, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u8, u32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u8, u64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u8, u128, #[stable(feature = "i128", since = "1.26.0")] }
+impl_from! { u8, usize, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u16, u32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u16, u64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u16, u128, #[stable(feature = "i128", since = "1.26.0")] }
+impl_from! { u32, u64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u32, u128, #[stable(feature = "i128", since = "1.26.0")] }
+impl_from! { u64, u128, #[stable(feature = "i128", since = "1.26.0")] }
+
+// Signed -> Signed
+impl_from! { i8, i16, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { i8, i32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { i8, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { i8, i128, #[stable(feature = "i128", since = "1.26.0")] }
+impl_from! { i8, isize, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { i16, i32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { i16, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { i16, i128, #[stable(feature = "i128", since = "1.26.0")] }
+impl_from! { i32, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { i32, i128, #[stable(feature = "i128", since = "1.26.0")] }
+impl_from! { i64, i128, #[stable(feature = "i128", since = "1.26.0")] }
+
+// Unsigned -> Signed
+impl_from! { u8, i16, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u8, i32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u8, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u8, i128, #[stable(feature = "i128", since = "1.26.0")] }
+impl_from! { u16, i32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u16, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u16, i128, #[stable(feature = "i128", since = "1.26.0")] }
+impl_from! { u32, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
+impl_from! { u32, i128, #[stable(feature = "i128", since = "1.26.0")] }
+impl_from! { u64, i128, #[stable(feature = "i128", since = "1.26.0")] }
+
+// The C99 standard defines bounds on INTPTR_MIN, INTPTR_MAX, and UINTPTR_MAX
+// which imply that pointer-sized integers must be at least 16 bits:
+// https://port70.net/~nsz/c/c99/n1256.html#7.18.2.4
+impl_from! { u16, usize, #[stable(feature = "lossless_iusize_conv", since = "1.26.0")] }
+impl_from! { u8, isize, #[stable(feature = "lossless_iusize_conv", since = "1.26.0")] }
+impl_from! { i16, isize, #[stable(feature = "lossless_iusize_conv", since = "1.26.0")] }
+
+// RISC-V defines the possibility of a 128-bit address space (RV128).
+
+// CHERI proposes 256-bit “capabilities”. Unclear if this would be relevant to usize/isize.
+// https://www.cl.cam.ac.uk/research/security/ctsrd/pdfs/20171017a-cheri-poster.pdf
+// http://www.csl.sri.com/users/neumann/2012resolve-cheri.pdf
+
+
+// Note: integers can only be represented with full precision in a float if
+// they fit in the significand, which is 24 bits in f32 and 53 bits in f64.
+// Lossy float conversions are not implemented at this time.
+
+// Signed -> Float
+impl_from! { i8, f32, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+impl_from! { i8, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+impl_from! { i16, f32, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+impl_from! { i16, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+impl_from! { i32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+
+// Unsigned -> Float
+impl_from! { u8, f32, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+impl_from! { u8, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+impl_from! { u16, f32, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+impl_from! { u16, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+impl_from! { u32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+
+// Float -> Float
+impl_from! { f32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
+
+// no possible bounds violation
+macro_rules! try_from_unbounded {
+    ($source:ty, $($target:ty),*) => {$(
+        #[stable(feature = "try_from", since = "1.34.0")]
+        impl TryFrom<$source> for $target {
+            type Error = TryFromIntError;
+
+            /// Try to create the target number type from a source
+            /// number type. This returns an error if the source value
+            /// is outside of the range of the target type.
+            #[inline]
+            fn try_from(value: $source) -> Result<Self, Self::Error> {
+                Ok(value as $target)
+            }
+        }
+    )*}
+}
+
+// only negative bounds
+macro_rules! try_from_lower_bounded {
+    ($source:ty, $($target:ty),*) => {$(
+        #[stable(feature = "try_from", since = "1.34.0")]
+        impl TryFrom<$source> for $target {
+            type Error = TryFromIntError;
+
+            /// Try to create the target number type from a source
+            /// number type. This returns an error if the source value
+            /// is outside of the range of the target type.
+            #[inline]
+            fn try_from(u: $source) -> Result<$target, TryFromIntError> {
+                if u >= 0 {
+                    Ok(u as $target)
+                } else {
+                    Err(TryFromIntError(()))
+                }
+            }
+        }
+    )*}
+}
+
+// unsigned to signed (only positive bound)
+macro_rules! try_from_upper_bounded {
+    ($source:ty, $($target:ty),*) => {$(
+        #[stable(feature = "try_from", since = "1.34.0")]
+        impl TryFrom<$source> for $target {
+            type Error = TryFromIntError;
+
+            /// Try to create the target number type from a source
+            /// number type. This returns an error if the source value
+            /// is outside of the range of the target type.
+            #[inline]
+            fn try_from(u: $source) -> Result<$target, TryFromIntError> {
+                if u > (<$target>::max_value() as $source) {
+                    Err(TryFromIntError(()))
+                } else {
+                    Ok(u as $target)
+                }
+            }
+        }
+    )*}
+}
+
+// all other cases
+macro_rules! try_from_both_bounded {
+    ($source:ty, $($target:ty),*) => {$(
+        #[stable(feature = "try_from", since = "1.34.0")]
+        impl TryFrom<$source> for $target {
+            type Error = TryFromIntError;
+
+            /// Try to create the target number type from a source
+            /// number type. This returns an error if the source value
+            /// is outside of the range of the target type.
+            #[inline]
+            fn try_from(u: $source) -> Result<$target, TryFromIntError> {
+                let min = <$target>::min_value() as $source;
+                let max = <$target>::max_value() as $source;
+                if u < min || u > max {
+                    Err(TryFromIntError(()))
+                } else {
+                    Ok(u as $target)
+                }
+            }
+        }
+    )*}
+}
+
+macro_rules! rev {
+    ($mac:ident, $source:ty, $($target:ty),*) => {$(
+        $mac!($target, $source);
+    )*}
+}
+
+// intra-sign conversions
+try_from_upper_bounded!(u16, u8);
+try_from_upper_bounded!(u32, u16, u8);
+try_from_upper_bounded!(u64, u32, u16, u8);
+try_from_upper_bounded!(u128, u64, u32, u16, u8);
+
+try_from_both_bounded!(i16, i8);
+try_from_both_bounded!(i32, i16, i8);
+try_from_both_bounded!(i64, i32, i16, i8);
+try_from_both_bounded!(i128, i64, i32, i16, i8);
+
+// unsigned-to-signed
+try_from_upper_bounded!(u8, i8);
+try_from_upper_bounded!(u16, i8, i16);
+try_from_upper_bounded!(u32, i8, i16, i32);
+try_from_upper_bounded!(u64, i8, i16, i32, i64);
+try_from_upper_bounded!(u128, i8, i16, i32, i64, i128);
+
+// signed-to-unsigned
+try_from_lower_bounded!(i8, u8, u16, u32, u64, u128);
+try_from_lower_bounded!(i16, u16, u32, u64, u128);
+try_from_lower_bounded!(i32, u32, u64, u128);
+try_from_lower_bounded!(i64, u64, u128);
+try_from_lower_bounded!(i128, u128);
+try_from_both_bounded!(i16, u8);
+try_from_both_bounded!(i32, u16, u8);
+try_from_both_bounded!(i64, u32, u16, u8);
+try_from_both_bounded!(i128, u64, u32, u16, u8);
+
+// usize/isize
+try_from_upper_bounded!(usize, isize);
+try_from_lower_bounded!(isize, usize);
+
+#[cfg(target_pointer_width = "16")]
+mod ptr_try_from_impls {
+    use super::TryFromIntError;
+    use crate::convert::TryFrom;
+
+    try_from_upper_bounded!(usize, u8);
+    try_from_unbounded!(usize, u16, u32, u64, u128);
+    try_from_upper_bounded!(usize, i8, i16);
+    try_from_unbounded!(usize, i32, i64, i128);
+
+    try_from_both_bounded!(isize, u8);
+    try_from_lower_bounded!(isize, u16, u32, u64, u128);
+    try_from_both_bounded!(isize, i8);
+    try_from_unbounded!(isize, i16, i32, i64, i128);
+
+    rev!(try_from_upper_bounded, usize, u32, u64, u128);
+    rev!(try_from_lower_bounded, usize, i8, i16);
+    rev!(try_from_both_bounded, usize, i32, i64, i128);
+
+    rev!(try_from_upper_bounded, isize, u16, u32, u64, u128);
+    rev!(try_from_both_bounded, isize, i32, i64, i128);
+}
+
+#[cfg(target_pointer_width = "32")]
+mod ptr_try_from_impls {
+    use super::TryFromIntError;
+    use crate::convert::TryFrom;
+
+    try_from_upper_bounded!(usize, u8, u16);
+    try_from_unbounded!(usize, u32, u64, u128);
+    try_from_upper_bounded!(usize, i8, i16, i32);
+    try_from_unbounded!(usize, i64, i128);
+
+    try_from_both_bounded!(isize, u8, u16);
+    try_from_lower_bounded!(isize, u32, u64, u128);
+    try_from_both_bounded!(isize, i8, i16);
+    try_from_unbounded!(isize, i32, i64, i128);
+
+    rev!(try_from_unbounded, usize, u32);
+    rev!(try_from_upper_bounded, usize, u64, u128);
+    rev!(try_from_lower_bounded, usize, i8, i16, i32);
+    rev!(try_from_both_bounded, usize, i64, i128);
+
+    rev!(try_from_unbounded, isize, u16);
+    rev!(try_from_upper_bounded, isize, u32, u64, u128);
+    rev!(try_from_unbounded, isize, i32);
+    rev!(try_from_both_bounded, isize, i64, i128);
+}
+
+#[cfg(target_pointer_width = "64")]
+mod ptr_try_from_impls {
+    use super::TryFromIntError;
+    use crate::convert::TryFrom;
+
+    try_from_upper_bounded!(usize, u8, u16, u32);
+    try_from_unbounded!(usize, u64, u128);
+    try_from_upper_bounded!(usize, i8, i16, i32, i64);
+    try_from_unbounded!(usize, i128);
+
+    try_from_both_bounded!(isize, u8, u16, u32);
+    try_from_lower_bounded!(isize, u64, u128);
+    try_from_both_bounded!(isize, i8, i16, i32);
+    try_from_unbounded!(isize, i64, i128);
+
+    rev!(try_from_unbounded, usize, u32, u64);
+    rev!(try_from_upper_bounded, usize, u128);
+    rev!(try_from_lower_bounded, usize, i8, i16, i32, i64);
+    rev!(try_from_both_bounded, usize, i128);
+
+    rev!(try_from_unbounded, isize, u16, u32);
+    rev!(try_from_upper_bounded, isize, u64, u128);
+    rev!(try_from_unbounded, isize, i32, i64);
+    rev!(try_from_both_bounded, isize, i128);
+}
diff --git a/src/libcore/fmt/mod.rs b/src/libcore/fmt/mod.rs
index 4c941e2..e2f49ee 100644
--- a/src/libcore/fmt/mod.rs
+++ b/src/libcore/fmt/mod.rs
@@ -662,7 +662,7 @@
 ///     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 ///         let val = self.0;
 ///
-///         write!(f, "{:o}", val) // delegate to i32's implementation
+///         fmt::Octal::fmt(&val, f) // delegate to i32's implementation
 ///     }
 /// }
 ///
@@ -712,7 +712,7 @@
 ///     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 ///         let val = self.0;
 ///
-///         write!(f, "{:b}", val) // delegate to i32's implementation
+///         fmt::Binary::fmt(&val, f) // delegate to i32's implementation
 ///     }
 /// }
 ///
@@ -771,7 +771,7 @@
 ///     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 ///         let val = self.0;
 ///
-///         write!(f, "{:x}", val) // delegate to i32's implementation
+///         fmt::LowerHex::fmt(&val, f) // delegate to i32's implementation
 ///     }
 /// }
 ///
@@ -824,7 +824,7 @@
 ///     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 ///         let val = self.0;
 ///
-///         write!(f, "{:X}", val) // delegate to i32's implementation
+///         fmt::UpperHex::fmt(&val, f) // delegate to i32's implementation
 ///     }
 /// }
 ///
@@ -869,7 +869,8 @@
 ///     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 ///         // use `as` to convert to a `*const T`, which implements Pointer, which we can use
 ///
-///         write!(f, "{:p}", self as *const Length)
+///         let ptr = self as *const Self;
+///         fmt::Pointer::fmt(&ptr, f)
 ///     }
 /// }
 ///
diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs
index e3dc563..18aae59 100644
--- a/src/libcore/intrinsics.rs
+++ b/src/libcore/intrinsics.rs
@@ -939,6 +939,7 @@
     /// }
     /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
+    #[rustc_const_unstable(feature = "const_transmute")]
     pub fn transmute<T, U>(e: T) -> U;
 
     /// Returns `true` if the actual type given as `T` requires drop
@@ -1143,6 +1144,11 @@
     /// May assume inputs are finite.
     pub fn frem_fast<T>(a: T, b: T) -> T;
 
+    /// Convert with LLVM’s fptoui/fptosi, which may return undef for values out of range
+    /// https://github.com/rust-lang/rust/issues/10184
+    #[cfg(not(bootstrap))]
+    pub fn float_to_int_approx_unchecked<Float, Int>(value: Float) -> Int;
+
 
     /// Returns the number of bits set in an integer type `T`
     pub fn ctpop<T>(x: T) -> T;
@@ -1348,9 +1354,11 @@
     pub fn ptr_offset_from<T>(ptr: *const T, base: *const T) -> isize;
 
     /// Internal hook used by Miri to implement unwinding.
+    /// Compiles to a NOP during non-Miri codegen.
+    ///
     /// Perma-unstable: do not use
     #[cfg(not(bootstrap))]
-    pub fn miri_start_panic(data: *mut (dyn crate::any::Any + crate::marker::Send)) -> !;
+    pub fn miri_start_panic(data: *mut (dyn crate::any::Any + crate::marker::Send)) -> ();
 }
 
 // Some functions are defined here because they accidentally got made
diff --git a/src/libcore/iter/traits/collect.rs b/src/libcore/iter/traits/collect.rs
index bbdb169..d6ae5cf 100644
--- a/src/libcore/iter/traits/collect.rs
+++ b/src/libcore/iter/traits/collect.rs
@@ -91,9 +91,9 @@
 /// ```
 #[stable(feature = "rust1", since = "1.0.0")]
 #[rustc_on_unimplemented(
-    message="a collection of type `{Self}` cannot be built from an iterator \
+    message="a value of type `{Self}` cannot be built from an iterator \
              over elements of type `{A}`",
-    label="a collection of type `{Self}` cannot be built from `std::iter::Iterator<Item={A}>`",
+    label="value of type `{Self}` cannot be built from `std::iter::Iterator<Item={A}>`",
 )]
 pub trait FromIterator<A>: Sized {
     /// Creates a value from an iterator.
diff --git a/src/libcore/lib.rs b/src/libcore/lib.rs
index ec19392..188ea1a 100644
--- a/src/libcore/lib.rs
+++ b/src/libcore/lib.rs
@@ -74,6 +74,8 @@
 #![feature(const_fn)]
 #![feature(const_fn_union)]
 #![feature(const_generics)]
+#![cfg_attr(not(bootstrap), feature(const_ptr_offset_from))]
+#![cfg_attr(not(bootstrap), feature(const_type_name))]
 #![feature(custom_inner_attributes)]
 #![feature(decl_macro)]
 #![feature(doc_cfg)]
diff --git a/src/libcore/num/f32.rs b/src/libcore/num/f32.rs
index 913c0f9..ac06f95 100644
--- a/src/libcore/num/f32.rs
+++ b/src/libcore/num/f32.rs
@@ -7,9 +7,10 @@
 
 #![stable(feature = "rust1", since = "1.0.0")]
 
+#[cfg(not(bootstrap))]
+use crate::convert::FloatToInt;
 #[cfg(not(test))]
 use crate::intrinsics;
-
 use crate::mem;
 use crate::num::FpCategory;
 
@@ -400,6 +401,35 @@
         intrinsics::minnumf32(self, other)
     }
 
+    /// Rounds toward zero and converts to any primitive integer type,
+    /// assuming that the value is finite and fits in that type.
+    ///
+    /// ```
+    /// #![feature(float_approx_unchecked_to)]
+    ///
+    /// let value = 4.6_f32;
+    /// let rounded = unsafe { value.approx_unchecked_to::<u16>() };
+    /// assert_eq!(rounded, 4);
+    ///
+    /// let value = -128.9_f32;
+    /// let rounded = unsafe { value.approx_unchecked_to::<i8>() };
+    /// assert_eq!(rounded, std::i8::MIN);
+    /// ```
+    ///
+    /// # Safety
+    ///
+    /// The value must:
+    ///
+    /// * Not be `NaN`
+    /// * Not be infinite
+    /// * Be representable in the return type `Int`, after truncating off its fractional part
+    #[cfg(not(bootstrap))]
+    #[unstable(feature = "float_approx_unchecked_to", issue = "67058")]
+    #[inline]
+    pub unsafe fn approx_unchecked_to<Int>(self) -> Int where Self: FloatToInt<Int> {
+        FloatToInt::<Int>::approx_unchecked(self)
+    }
+
     /// Raw transmutation to `u32`.
     ///
     /// This is currently identical to `transmute::<f32, u32>(self)` on all platforms.
diff --git a/src/libcore/num/f64.rs b/src/libcore/num/f64.rs
index 6ca830b..794f77f 100644
--- a/src/libcore/num/f64.rs
+++ b/src/libcore/num/f64.rs
@@ -7,9 +7,10 @@
 
 #![stable(feature = "rust1", since = "1.0.0")]
 
+#[cfg(not(bootstrap))]
+use crate::convert::FloatToInt;
 #[cfg(not(test))]
 use crate::intrinsics;
-
 use crate::mem;
 use crate::num::FpCategory;
 
@@ -413,6 +414,35 @@
         intrinsics::minnumf64(self, other)
     }
 
+    /// Rounds toward zero and converts to any primitive integer type,
+    /// assuming that the value is finite and fits in that type.
+    ///
+    /// ```
+    /// #![feature(float_approx_unchecked_to)]
+    ///
+    /// let value = 4.6_f32;
+    /// let rounded = unsafe { value.approx_unchecked_to::<u16>() };
+    /// assert_eq!(rounded, 4);
+    ///
+    /// let value = -128.9_f32;
+    /// let rounded = unsafe { value.approx_unchecked_to::<i8>() };
+    /// assert_eq!(rounded, std::i8::MIN);
+    /// ```
+    ///
+    /// # Safety
+    ///
+    /// The value must:
+    ///
+    /// * Not be `NaN`
+    /// * Not be infinite
+    /// * Be representable in the return type `Int`, after truncating off its fractional part
+    #[cfg(not(bootstrap))]
+    #[unstable(feature = "float_approx_unchecked_to", issue = "67058")]
+    #[inline]
+    pub unsafe fn approx_unchecked_to<Int>(self) -> Int where Self: FloatToInt<Int> {
+        FloatToInt::<Int>::approx_unchecked(self)
+    }
+
     /// Raw transmutation to `u64`.
     ///
     /// This is currently identical to `transmute::<f64, u64>(self)` on all platforms.
diff --git a/src/libcore/num/mod.rs b/src/libcore/num/mod.rs
index 4313248..585f144 100644
--- a/src/libcore/num/mod.rs
+++ b/src/libcore/num/mod.rs
@@ -4,7 +4,6 @@
 
 #![stable(feature = "rust1", since = "1.0.0")]
 
-use crate::convert::TryFrom;
 use crate::fmt;
 use crate::intrinsics;
 use crate::mem;
@@ -4701,7 +4700,7 @@
 /// The error type returned when a checked integral type conversion fails.
 #[stable(feature = "try_from", since = "1.34.0")]
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
-pub struct TryFromIntError(());
+pub struct TryFromIntError(pub(crate) ());
 
 impl TryFromIntError {
     #[unstable(feature = "int_error_internals",
@@ -4728,206 +4727,6 @@
     }
 }
 
-// no possible bounds violation
-macro_rules! try_from_unbounded {
-    ($source:ty, $($target:ty),*) => {$(
-        #[stable(feature = "try_from", since = "1.34.0")]
-        impl TryFrom<$source> for $target {
-            type Error = TryFromIntError;
-
-            /// Try to create the target number type from a source
-            /// number type. This returns an error if the source value
-            /// is outside of the range of the target type.
-            #[inline]
-            fn try_from(value: $source) -> Result<Self, Self::Error> {
-                Ok(value as $target)
-            }
-        }
-    )*}
-}
-
-// only negative bounds
-macro_rules! try_from_lower_bounded {
-    ($source:ty, $($target:ty),*) => {$(
-        #[stable(feature = "try_from", since = "1.34.0")]
-        impl TryFrom<$source> for $target {
-            type Error = TryFromIntError;
-
-            /// Try to create the target number type from a source
-            /// number type. This returns an error if the source value
-            /// is outside of the range of the target type.
-            #[inline]
-            fn try_from(u: $source) -> Result<$target, TryFromIntError> {
-                if u >= 0 {
-                    Ok(u as $target)
-                } else {
-                    Err(TryFromIntError(()))
-                }
-            }
-        }
-    )*}
-}
-
-// unsigned to signed (only positive bound)
-macro_rules! try_from_upper_bounded {
-    ($source:ty, $($target:ty),*) => {$(
-        #[stable(feature = "try_from", since = "1.34.0")]
-        impl TryFrom<$source> for $target {
-            type Error = TryFromIntError;
-
-            /// Try to create the target number type from a source
-            /// number type. This returns an error if the source value
-            /// is outside of the range of the target type.
-            #[inline]
-            fn try_from(u: $source) -> Result<$target, TryFromIntError> {
-                if u > (<$target>::max_value() as $source) {
-                    Err(TryFromIntError(()))
-                } else {
-                    Ok(u as $target)
-                }
-            }
-        }
-    )*}
-}
-
-// all other cases
-macro_rules! try_from_both_bounded {
-    ($source:ty, $($target:ty),*) => {$(
-        #[stable(feature = "try_from", since = "1.34.0")]
-        impl TryFrom<$source> for $target {
-            type Error = TryFromIntError;
-
-            /// Try to create the target number type from a source
-            /// number type. This returns an error if the source value
-            /// is outside of the range of the target type.
-            #[inline]
-            fn try_from(u: $source) -> Result<$target, TryFromIntError> {
-                let min = <$target>::min_value() as $source;
-                let max = <$target>::max_value() as $source;
-                if u < min || u > max {
-                    Err(TryFromIntError(()))
-                } else {
-                    Ok(u as $target)
-                }
-            }
-        }
-    )*}
-}
-
-macro_rules! rev {
-    ($mac:ident, $source:ty, $($target:ty),*) => {$(
-        $mac!($target, $source);
-    )*}
-}
-
-// intra-sign conversions
-try_from_upper_bounded!(u16, u8);
-try_from_upper_bounded!(u32, u16, u8);
-try_from_upper_bounded!(u64, u32, u16, u8);
-try_from_upper_bounded!(u128, u64, u32, u16, u8);
-
-try_from_both_bounded!(i16, i8);
-try_from_both_bounded!(i32, i16, i8);
-try_from_both_bounded!(i64, i32, i16, i8);
-try_from_both_bounded!(i128, i64, i32, i16, i8);
-
-// unsigned-to-signed
-try_from_upper_bounded!(u8, i8);
-try_from_upper_bounded!(u16, i8, i16);
-try_from_upper_bounded!(u32, i8, i16, i32);
-try_from_upper_bounded!(u64, i8, i16, i32, i64);
-try_from_upper_bounded!(u128, i8, i16, i32, i64, i128);
-
-// signed-to-unsigned
-try_from_lower_bounded!(i8, u8, u16, u32, u64, u128);
-try_from_lower_bounded!(i16, u16, u32, u64, u128);
-try_from_lower_bounded!(i32, u32, u64, u128);
-try_from_lower_bounded!(i64, u64, u128);
-try_from_lower_bounded!(i128, u128);
-try_from_both_bounded!(i16, u8);
-try_from_both_bounded!(i32, u16, u8);
-try_from_both_bounded!(i64, u32, u16, u8);
-try_from_both_bounded!(i128, u64, u32, u16, u8);
-
-// usize/isize
-try_from_upper_bounded!(usize, isize);
-try_from_lower_bounded!(isize, usize);
-
-#[cfg(target_pointer_width = "16")]
-mod ptr_try_from_impls {
-    use super::TryFromIntError;
-    use crate::convert::TryFrom;
-
-    try_from_upper_bounded!(usize, u8);
-    try_from_unbounded!(usize, u16, u32, u64, u128);
-    try_from_upper_bounded!(usize, i8, i16);
-    try_from_unbounded!(usize, i32, i64, i128);
-
-    try_from_both_bounded!(isize, u8);
-    try_from_lower_bounded!(isize, u16, u32, u64, u128);
-    try_from_both_bounded!(isize, i8);
-    try_from_unbounded!(isize, i16, i32, i64, i128);
-
-    rev!(try_from_upper_bounded, usize, u32, u64, u128);
-    rev!(try_from_lower_bounded, usize, i8, i16);
-    rev!(try_from_both_bounded, usize, i32, i64, i128);
-
-    rev!(try_from_upper_bounded, isize, u16, u32, u64, u128);
-    rev!(try_from_both_bounded, isize, i32, i64, i128);
-}
-
-#[cfg(target_pointer_width = "32")]
-mod ptr_try_from_impls {
-    use super::TryFromIntError;
-    use crate::convert::TryFrom;
-
-    try_from_upper_bounded!(usize, u8, u16);
-    try_from_unbounded!(usize, u32, u64, u128);
-    try_from_upper_bounded!(usize, i8, i16, i32);
-    try_from_unbounded!(usize, i64, i128);
-
-    try_from_both_bounded!(isize, u8, u16);
-    try_from_lower_bounded!(isize, u32, u64, u128);
-    try_from_both_bounded!(isize, i8, i16);
-    try_from_unbounded!(isize, i32, i64, i128);
-
-    rev!(try_from_unbounded, usize, u32);
-    rev!(try_from_upper_bounded, usize, u64, u128);
-    rev!(try_from_lower_bounded, usize, i8, i16, i32);
-    rev!(try_from_both_bounded, usize, i64, i128);
-
-    rev!(try_from_unbounded, isize, u16);
-    rev!(try_from_upper_bounded, isize, u32, u64, u128);
-    rev!(try_from_unbounded, isize, i32);
-    rev!(try_from_both_bounded, isize, i64, i128);
-}
-
-#[cfg(target_pointer_width = "64")]
-mod ptr_try_from_impls {
-    use super::TryFromIntError;
-    use crate::convert::TryFrom;
-
-    try_from_upper_bounded!(usize, u8, u16, u32);
-    try_from_unbounded!(usize, u64, u128);
-    try_from_upper_bounded!(usize, i8, i16, i32, i64);
-    try_from_unbounded!(usize, i128);
-
-    try_from_both_bounded!(isize, u8, u16, u32);
-    try_from_lower_bounded!(isize, u64, u128);
-    try_from_both_bounded!(isize, i8, i16, i32);
-    try_from_unbounded!(isize, i64, i128);
-
-    rev!(try_from_unbounded, usize, u32, u64);
-    rev!(try_from_upper_bounded, usize, u128);
-    rev!(try_from_lower_bounded, usize, i8, i16, i32, i64);
-    rev!(try_from_both_bounded, usize, i128);
-
-    rev!(try_from_unbounded, isize, u16, u32);
-    rev!(try_from_upper_bounded, isize, u64, u128);
-    rev!(try_from_unbounded, isize, i32, i64);
-    rev!(try_from_both_bounded, isize, i128);
-}
-
 #[doc(hidden)]
 trait FromStrRadixHelper: PartialOrd + Copy {
     fn min_value() -> Self;
@@ -5110,131 +4909,3 @@
 
 #[stable(feature = "rust1", since = "1.0.0")]
 pub use crate::num::dec2flt::ParseFloatError;
-
-// Conversion traits for primitive integer and float types
-// Conversions T -> T are covered by a blanket impl and therefore excluded
-// Some conversions from and to usize/isize are not implemented due to portability concerns
-macro_rules! impl_from {
-    ($Small: ty, $Large: ty, #[$attr:meta], $doc: expr) => {
-        #[$attr]
-        #[doc = $doc]
-        impl From<$Small> for $Large {
-            #[inline]
-            fn from(small: $Small) -> $Large {
-                small as $Large
-            }
-        }
-    };
-    ($Small: ty, $Large: ty, #[$attr:meta]) => {
-        impl_from!($Small,
-                   $Large,
-                   #[$attr],
-                   concat!("Converts `",
-                           stringify!($Small),
-                           "` to `",
-                           stringify!($Large),
-                           "` losslessly."));
-    }
-}
-
-macro_rules! impl_from_bool {
-    ($target: ty, #[$attr:meta]) => {
-        impl_from!(bool, $target, #[$attr], concat!("Converts a `bool` to a `",
-            stringify!($target), "`. The resulting value is `0` for `false` and `1` for `true`
-values.
-
-# Examples
-
-```
-assert_eq!(", stringify!($target), "::from(true), 1);
-assert_eq!(", stringify!($target), "::from(false), 0);
-```"));
-    };
-}
-
-// Bool -> Any
-impl_from_bool! { u8, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { u16, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { u32, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { u64, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { u128, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { usize, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { i8, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { i16, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { i32, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { i64, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { i128, #[stable(feature = "from_bool", since = "1.28.0")] }
-impl_from_bool! { isize, #[stable(feature = "from_bool", since = "1.28.0")] }
-
-// Unsigned -> Unsigned
-impl_from! { u8, u16, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u8, u32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u8, u64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u8, u128, #[stable(feature = "i128", since = "1.26.0")] }
-impl_from! { u8, usize, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u16, u32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u16, u64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u16, u128, #[stable(feature = "i128", since = "1.26.0")] }
-impl_from! { u32, u64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u32, u128, #[stable(feature = "i128", since = "1.26.0")] }
-impl_from! { u64, u128, #[stable(feature = "i128", since = "1.26.0")] }
-
-// Signed -> Signed
-impl_from! { i8, i16, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { i8, i32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { i8, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { i8, i128, #[stable(feature = "i128", since = "1.26.0")] }
-impl_from! { i8, isize, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { i16, i32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { i16, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { i16, i128, #[stable(feature = "i128", since = "1.26.0")] }
-impl_from! { i32, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { i32, i128, #[stable(feature = "i128", since = "1.26.0")] }
-impl_from! { i64, i128, #[stable(feature = "i128", since = "1.26.0")] }
-
-// Unsigned -> Signed
-impl_from! { u8, i16, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u8, i32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u8, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u8, i128, #[stable(feature = "i128", since = "1.26.0")] }
-impl_from! { u16, i32, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u16, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u16, i128, #[stable(feature = "i128", since = "1.26.0")] }
-impl_from! { u32, i64, #[stable(feature = "lossless_int_conv", since = "1.5.0")] }
-impl_from! { u32, i128, #[stable(feature = "i128", since = "1.26.0")] }
-impl_from! { u64, i128, #[stable(feature = "i128", since = "1.26.0")] }
-
-// The C99 standard defines bounds on INTPTR_MIN, INTPTR_MAX, and UINTPTR_MAX
-// which imply that pointer-sized integers must be at least 16 bits:
-// https://port70.net/~nsz/c/c99/n1256.html#7.18.2.4
-impl_from! { u16, usize, #[stable(feature = "lossless_iusize_conv", since = "1.26.0")] }
-impl_from! { u8, isize, #[stable(feature = "lossless_iusize_conv", since = "1.26.0")] }
-impl_from! { i16, isize, #[stable(feature = "lossless_iusize_conv", since = "1.26.0")] }
-
-// RISC-V defines the possibility of a 128-bit address space (RV128).
-
-// CHERI proposes 256-bit “capabilities”. Unclear if this would be relevant to usize/isize.
-// https://www.cl.cam.ac.uk/research/security/ctsrd/pdfs/20171017a-cheri-poster.pdf
-// http://www.csl.sri.com/users/neumann/2012resolve-cheri.pdf
-
-
-// Note: integers can only be represented with full precision in a float if
-// they fit in the significand, which is 24 bits in f32 and 53 bits in f64.
-// Lossy float conversions are not implemented at this time.
-
-// Signed -> Float
-impl_from! { i8, f32, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-impl_from! { i8, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-impl_from! { i16, f32, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-impl_from! { i16, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-impl_from! { i32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-
-// Unsigned -> Float
-impl_from! { u8, f32, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-impl_from! { u8, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-impl_from! { u16, f32, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-impl_from! { u16, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-impl_from! { u32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
-
-// Float -> Float
-impl_from! { f32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }
diff --git a/src/libcore/ops/try.rs b/src/libcore/ops/try.rs
index 4f46520..a748ee8 100644
--- a/src/libcore/ops/try.rs
+++ b/src/libcore/ops/try.rs
@@ -5,19 +5,20 @@
 /// extracting those success or failure values from an existing instance and
 /// creating a new instance from a success or failure value.
 #[unstable(feature = "try_trait", issue = "42327")]
-#[rustc_on_unimplemented(
+#[cfg_attr(not(bootstrap), rustc_on_unimplemented(
 on(all(
 any(from_method="from_error", from_method="from_ok"),
 from_desugaring="QuestionMark"),
 message="the `?` operator can only be used in {ItemContext} \
                that returns `Result` or `Option` \
                (or another type that implements `{Try}`)",
-label="cannot use the `?` operator in {ItemContext} that returns `{Self}`"),
+label="cannot use the `?` operator in {ItemContext} that returns `{Self}`",
+enclosing_scope="this function should return `Result` or `Option` to accept `?`"),
 on(all(from_method="into_result", from_desugaring="QuestionMark"),
 message="the `?` operator can only be applied to values \
                that implement `{Try}`",
 label="the `?` operator cannot be applied to type `{Self}`")
-)]
+))]
 #[doc(alias = "?")]
 pub trait Try {
     /// The type of this value when viewed as successful.
diff --git a/src/libcore/str/mod.rs b/src/libcore/str/mod.rs
index 25b7eec..b2a420f 100644
--- a/src/libcore/str/mod.rs
+++ b/src/libcore/str/mod.rs
@@ -3371,8 +3371,8 @@
     /// An iterator over the disjoint matches of a pattern within the given string
     /// slice.
     ///
-    /// The pattern can be any type that implements the Pattern trait. Notable
-    /// examples are `&str`, [`char`], and closures that determines the split.
+    /// The pattern can be a `&str`, [`char`], or a closure that determines if
+    /// a character matches.
     ///
     /// # Iterator behavior
     ///
diff --git a/src/libcore/sync/atomic.rs b/src/libcore/sync/atomic.rs
index 251d49d..6e1aac0 100644
--- a/src/libcore/sync/atomic.rs
+++ b/src/libcore/sync/atomic.rs
@@ -27,7 +27,7 @@
 //!
 //! Atomic variables are safe to share between threads (they implement [`Sync`])
 //! but they do not themselves provide the mechanism for sharing and follow the
-//! [threading model](../../../std/thread/index.html#the-threading-model) of rust.
+//! [threading model](../../../std/thread/index.html#the-threading-model) of Rust.
 //! The most common way to share an atomic variable is to put it into an [`Arc`][arc] (an
 //! atomically-reference-counted shared pointer).
 //!
diff --git a/src/libcore/tests/bool.rs b/src/libcore/tests/bool.rs
index 0f1e6e8..e89eb2c 100644
--- a/src/libcore/tests/bool.rs
+++ b/src/libcore/tests/bool.rs
@@ -1,7 +1,7 @@
 #[test]
 fn test_bool_to_option() {
-    assert_eq!(false.then(0), None);
-    assert_eq!(true.then(0), Some(0));
-    assert_eq!(false.then_with(|| 0), None);
-    assert_eq!(true.then_with(|| 0), Some(0));
+    assert_eq!(false.then_some(0), None);
+    assert_eq!(true.then_some(0), Some(0));
+    assert_eq!(false.then(|| 0), None);
+    assert_eq!(true.then(|| 0), Some(0));
 }
diff --git a/src/libfmt_macros/lib.rs b/src/libfmt_macros/lib.rs
index 900c6ed..2ecbe77 100644
--- a/src/libfmt_macros/lib.rs
+++ b/src/libfmt_macros/lib.rs
@@ -11,6 +11,7 @@
 #![feature(nll)]
 #![feature(rustc_private)]
 #![feature(unicode_internals)]
+#![feature(bool_to_option)]
 
 pub use Piece::*;
 pub use Position::*;
@@ -644,11 +645,7 @@
                 break;
             }
         }
-        if found {
-            Some(cur)
-        } else {
-            None
-        }
+        found.then_some(cur)
     }
 }
 
diff --git a/src/libpanic_unwind/lib.rs b/src/libpanic_unwind/lib.rs
index 0c834e5..3a14197 100644
--- a/src/libpanic_unwind/lib.rs
+++ b/src/libpanic_unwind/lib.rs
@@ -36,10 +36,7 @@
 use core::panic::BoxMeUp;
 
 cfg_if::cfg_if! {
-    if #[cfg(miri)] {
-        #[path = "miri.rs"]
-        mod imp;
-    } else if #[cfg(target_os = "emscripten")] {
+    if #[cfg(target_os = "emscripten")] {
         #[path = "emcc.rs"]
         mod imp;
     } else if #[cfg(target_arch = "wasm32")] {
@@ -94,5 +91,14 @@
 #[unwind(allowed)]
 pub unsafe extern "C" fn __rust_start_panic(payload: usize) -> u32 {
     let payload = payload as *mut &mut dyn BoxMeUp;
-    imp::panic(Box::from_raw((*payload).take_box()))
+    let payload = (*payload).take_box();
+
+    // Miri panic support: cfg'd out of normal builds just to be sure.
+    // When going through normal codegen, `miri_start_panic` is a NOP, so the
+    // Miri-enabled sysroot still supports normal unwinding. But when executed in
+    // Miri, this line initiates unwinding.
+    #[cfg(miri)]
+    core::intrinsics::miri_start_panic(payload);
+
+    imp::panic(Box::from_raw(payload))
 }
diff --git a/src/libpanic_unwind/miri.rs b/src/libpanic_unwind/miri.rs
deleted file mode 100644
index f26c42f..0000000
--- a/src/libpanic_unwind/miri.rs
+++ /dev/null
@@ -1,42 +0,0 @@
-#![allow(nonstandard_style)]
-
-use core::any::Any;
-use alloc::boxed::Box;
-
-pub fn payload() -> *mut u8 {
-    core::ptr::null_mut()
-}
-
-pub unsafe fn panic(data: Box<dyn Any + Send>) -> ! {
-    core::intrinsics::miri_start_panic(Box::into_raw(data))
-}
-
-pub unsafe fn cleanup(ptr: *mut u8) -> Box<dyn Any + Send> {
-    Box::from_raw(ptr)
-}
-
-// This is required by the compiler to exist (e.g., it's a lang item),
-// but is never used by Miri. Therefore, we just use a stub here
-#[lang = "eh_personality"]
-#[cfg(not(test))]
-fn rust_eh_personality() {
-    unsafe { core::intrinsics::abort() }
-}
-
-// The rest is required on *some* targets to exist (specifically, MSVC targets that use SEH).
-// We just add it on all targets. Copied from `seh.rs`.
-#[repr(C)]
-pub struct _TypeDescriptor {
-    pub pVFTable: *const u8,
-    pub spare: *mut u8,
-    pub name: [u8; 11],
-}
-
-const TYPE_NAME: [u8; 11] = *b"rust_panic\0";
-
-#[cfg_attr(not(test), lang = "eh_catch_typeinfo")]
-static mut TYPE_DESCRIPTOR: _TypeDescriptor = _TypeDescriptor {
-    pVFTable: core::ptr::null(),
-    spare: core::ptr::null_mut(),
-    name: TYPE_NAME,
-};
diff --git a/src/librustc/Cargo.toml b/src/librustc/Cargo.toml
index fb30d6c..f8ad6f8 100644
--- a/src/librustc/Cargo.toml
+++ b/src/librustc/Cargo.toml
@@ -39,3 +39,4 @@
 smallvec = { version = "1.0", features = ["union", "may_dangle"] }
 measureme = "0.4"
 rustc_error_codes = { path = "../librustc_error_codes" }
+rustc_session = { path = "../librustc_session" }
diff --git a/src/librustc/arena.rs b/src/librustc/arena.rs
index 1d6a342..364a35f 100644
--- a/src/librustc/arena.rs
+++ b/src/librustc/arena.rs
@@ -23,17 +23,17 @@
             [] generics: rustc::ty::Generics,
             [] trait_def: rustc::ty::TraitDef,
             [] adt_def: rustc::ty::AdtDef,
-            [] steal_mir: rustc::ty::steal::Steal<rustc::mir::Body<$tcx>>,
-            [] mir: rustc::mir::Body<$tcx>,
+            [] steal_mir: rustc::ty::steal::Steal<rustc::mir::BodyCache<$tcx>>,
+            [] mir: rustc::mir::BodyCache<$tcx>,
             [] steal_promoted: rustc::ty::steal::Steal<
                 rustc_index::vec::IndexVec<
                     rustc::mir::Promoted,
-                    rustc::mir::Body<$tcx>
+                    rustc::mir::BodyCache<$tcx>
                 >
             >,
             [] promoted: rustc_index::vec::IndexVec<
                 rustc::mir::Promoted,
-                rustc::mir::Body<$tcx>
+                rustc::mir::BodyCache<$tcx>
             >,
             [] tables: rustc::ty::TypeckTables<$tcx>,
             [] const_allocs: rustc::mir::interpret::Allocation,
diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs
index 0104507..d952bf7 100644
--- a/src/librustc/dep_graph/graph.rs
+++ b/src/librustc/dep_graph/graph.rs
@@ -710,14 +710,25 @@
                                 return None
                             }
                             None => {
-                                if !tcx.sess.has_errors() {
+                                if !tcx.sess.has_errors_or_delayed_span_bugs() {
                                     bug!("try_mark_previous_green() - Forcing the DepNode \
                                           should have set its color")
                                 } else {
-                                    // If the query we just forced has resulted
-                                    // in some kind of compilation error, we
-                                    // don't expect that the corresponding
-                                    // dep-node color has been updated.
+                                    // If the query we just forced has resulted in
+                                    // some kind of compilation error, we cannot rely on
+                                    // the dep-node color having been properly updated.
+                                    // This means that the query system has reached an
+                                    // invalid state. We let the compiler continue (by
+                                    // returning `None`) so it can emit error messages
+                                    // and wind down, but rely on the fact that this
+                                    // invalid state will not be persisted to the
+                                    // incremental compilation cache because of
+                                    // compilation errors being present.
+                                    debug!("try_mark_previous_green({:?}) - END - \
+                                            dependency {:?} resulted in compilation error",
+                                           dep_node,
+                                           dep_dep_node);
+                                    return None
                                 }
                             }
                         }
diff --git a/src/librustc/dep_graph/mod.rs b/src/librustc/dep_graph/mod.rs
index 43f3d7e..a1321d5 100644
--- a/src/librustc/dep_graph/mod.rs
+++ b/src/librustc/dep_graph/mod.rs
@@ -5,7 +5,6 @@
 mod query;
 mod safe;
 mod serialized;
-pub mod cgu_reuse_tracker;
 
 pub use self::dep_node::{DepNode, DepKind, DepConstructor, WorkProductId, RecoverKey, label_strs};
 pub use self::graph::{DepGraph, WorkProduct, DepNodeIndex, DepNodeColor, TaskDeps, hash_result};
diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs
index ac8173f..e13f6ca 100644
--- a/src/librustc/hir/lowering.rs
+++ b/src/librustc/hir/lowering.rs
@@ -1003,7 +1003,7 @@
             AttrKind::Normal(ref item) => {
                 AttrKind::Normal(AttrItem {
                     path: item.path.clone(),
-                    tokens: self.lower_token_stream(item.tokens.clone()),
+                    args: self.lower_mac_args(&item.args),
                 })
             }
             AttrKind::DocComment(comment) => AttrKind::DocComment(comment)
@@ -1017,6 +1017,16 @@
         }
     }
 
+    fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs {
+        match *args {
+            MacArgs::Empty => MacArgs::Empty,
+            MacArgs::Delimited(dspan, delim, ref tokens) =>
+                MacArgs::Delimited(dspan, delim, self.lower_token_stream(tokens.clone())),
+            MacArgs::Eq(eq_span, ref tokens) =>
+                MacArgs::Eq(eq_span, self.lower_token_stream(tokens.clone())),
+        }
+    }
+
     fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
         tokens
             .into_trees()
diff --git a/src/librustc/hir/lowering/item.rs b/src/librustc/hir/lowering/item.rs
index f689e7f..ff9d8c8 100644
--- a/src/librustc/hir/lowering/item.rs
+++ b/src/librustc/hir/lowering/item.rs
@@ -233,7 +233,7 @@
 
         if let ItemKind::MacroDef(ref def) = i.kind {
             if !def.legacy || attr::contains_name(&i.attrs, sym::macro_export) {
-                let body = self.lower_token_stream(def.stream());
+                let body = self.lower_token_stream(def.body.inner_tokens());
                 let hir_id = self.lower_node_id(i.id);
                 self.exported_macros.push(hir::MacroDef {
                     name: ident.name,
diff --git a/src/librustc/hir/map/blocks.rs b/src/librustc/hir/map/blocks.rs
index f25f3b5..8f9f398 100644
--- a/src/librustc/hir/map/blocks.rs
+++ b/src/librustc/hir/map/blocks.rs
@@ -147,13 +147,7 @@
             map::Node::Expr(e) => e.is_fn_like(),
             _ => false
         };
-        if fn_like {
-            Some(FnLikeNode {
-                node,
-            })
-        } else {
-            None
-        }
+        fn_like.then_some(FnLikeNode { node })
     }
 
     pub fn body(self) -> ast::BodyId {
diff --git a/src/librustc/infer/error_reporting/mod.rs b/src/librustc/infer/error_reporting/mod.rs
index 5a940f2..58c1498 100644
--- a/src/librustc/infer/error_reporting/mod.rs
+++ b/src/librustc/infer/error_reporting/mod.rs
@@ -1809,12 +1809,17 @@
                             sub_region,
                             "...",
                         );
-                        err.note(&format!(
-                            "...so that the {}:\nexpected {}\n   found {}",
-                            sup_trace.cause.as_requirement_str(),
-                            sup_expected.content(),
-                            sup_found.content()
+                        err.span_note(sup_trace.cause.span, &format!(
+                            "...so that the {}",
+                            sup_trace.cause.as_requirement_str()
                         ));
+
+                        err.note_expected_found(
+                            &"",
+                            sup_expected,
+                            &"",
+                            sup_found
+                        );
                         err.emit();
                         return;
                     }
diff --git a/src/librustc/infer/error_reporting/note.rs b/src/librustc/infer/error_reporting/note.rs
index c1f840a..4b93373 100644
--- a/src/librustc/infer/error_reporting/note.rs
+++ b/src/librustc/infer/error_reporting/note.rs
@@ -13,12 +13,20 @@
         match *origin {
             infer::Subtype(ref trace) => {
                 if let Some((expected, found)) = self.values_str(&trace.values) {
-                    let expected = expected.content();
-                    let found = found.content();
-                    err.note(&format!("...so that the {}:\nexpected {}\n   found {}",
-                                      trace.cause.as_requirement_str(),
-                                      expected,
-                                      found));
+                    err.span_note(
+                        trace.cause.span,
+                        &format!(
+                            "...so that the {}",
+                            trace.cause.as_requirement_str()
+                        )
+                    );
+
+                    err.note_expected_found(
+                        &"",
+                        expected,
+                        &"",
+                        found
+                    );
                 } else {
                     // FIXME: this really should be handled at some earlier stage. Our
                     // handling of region checking when type errors are present is
diff --git a/src/librustc/infer/outlives/verify.rs b/src/librustc/infer/outlives/verify.rs
index 3110b02..3e28145 100644
--- a/src/librustc/infer/outlives/verify.rs
+++ b/src/librustc/infer/outlives/verify.rs
@@ -211,11 +211,7 @@
                 (r, p)
             );
             let p_ty = p.to_ty(tcx);
-            if compare_ty(p_ty) {
-                Some(ty::OutlivesPredicate(p_ty, r))
-            } else {
-                None
-            }
+            compare_ty(p_ty).then_some(ty::OutlivesPredicate(p_ty, r))
         });
 
         param_bounds
diff --git a/src/librustc/lib.rs b/src/librustc/lib.rs
index 481ae3b..24b87ff 100644
--- a/src/librustc/lib.rs
+++ b/src/librustc/lib.rs
@@ -29,6 +29,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/")]
 
 #![feature(arbitrary_self_types)]
+#![feature(bool_to_option)]
 #![feature(box_patterns)]
 #![feature(box_syntax)]
 #![feature(const_fn)]
@@ -64,7 +65,6 @@
 #![recursion_limit="512"]
 
 #[macro_use] extern crate bitflags;
-extern crate getopts;
 #[macro_use] extern crate scoped_tls;
 #[cfg(windows)]
 extern crate libc;
@@ -74,10 +74,6 @@
 #[macro_use] extern crate syntax;
 #[macro_use] extern crate smallvec;
 
-// Use the test crate here so we depend on getopts through it. This allow tools to link to both
-// librustc_driver and libtest.
-extern crate test as _;
-
 #[cfg(test)]
 mod tests;
 
@@ -113,7 +109,7 @@
 }
 
 pub mod mir;
-pub mod session;
+pub use rustc_session as session;
 pub mod traits;
 pub mod ty;
 
diff --git a/src/librustc/lint/builtin.rs b/src/librustc/lint/builtin.rs
index f8a592d..1aba73e 100644
--- a/src/librustc/lint/builtin.rs
+++ b/src/librustc/lint/builtin.rs
@@ -12,6 +12,8 @@
 use syntax::edition::Edition;
 use syntax::source_map::Span;
 use syntax::symbol::Symbol;
+use syntax::early_buffered_lints::{ILL_FORMED_ATTRIBUTE_INPUT, META_VARIABLE_MISUSE};
+use rustc_session::declare_lint;
 
 declare_lint! {
     pub EXCEEDING_BITSHIFTS,
@@ -404,31 +406,6 @@
     };
 }
 
-/// Some lints that are buffered from `libsyntax`. See `syntax::early_buffered_lints`.
-pub mod parser {
-    declare_lint! {
-        pub ILL_FORMED_ATTRIBUTE_INPUT,
-        Deny,
-        "ill-formed attribute inputs that were previously accepted and used in practice",
-        @future_incompatible = super::FutureIncompatibleInfo {
-            reference: "issue #57571 <https://github.com/rust-lang/rust/issues/57571>",
-            edition: None,
-        };
-    }
-
-    declare_lint! {
-        pub META_VARIABLE_MISUSE,
-        Allow,
-        "possible meta-variable misuse at macro definition"
-    }
-
-    declare_lint! {
-        pub INCOMPLETE_INCLUDE,
-        Deny,
-        "trailing content in included file"
-    }
-}
-
 declare_lint! {
     pub DEPRECATED_IN_FUTURE,
     Allow,
@@ -520,8 +497,8 @@
         PROC_MACRO_DERIVE_RESOLUTION_FALLBACK,
         MACRO_USE_EXTERN_CRATE,
         MACRO_EXPANDED_MACRO_EXPORTS_ACCESSED_BY_ABSOLUTE_PATHS,
-        parser::ILL_FORMED_ATTRIBUTE_INPUT,
-        parser::META_VARIABLE_MISUSE,
+        ILL_FORMED_ATTRIBUTE_INPUT,
+        META_VARIABLE_MISUSE,
         DEPRECATED_IN_FUTURE,
         AMBIGUOUS_ASSOCIATED_ITEMS,
         MUTABLE_BORROW_RESERVATION_CONFLICT,
diff --git a/src/librustc/lint/context.rs b/src/librustc/lint/context.rs
index 05543f1..7f72154 100644
--- a/src/librustc/lint/context.rs
+++ b/src/librustc/lint/context.rs
@@ -47,8 +47,7 @@
 /// This is basically the subset of `Context` that we can
 /// build early in the compile pipeline.
 pub struct LintStore {
-    /// Registered lints. The bool is true if the lint was
-    /// added by a plugin.
+    /// Registered lints.
     lints: Vec<&'static Lint>,
 
     /// Constructor functions for each variety of lint pass.
diff --git a/src/librustc/lint/internal.rs b/src/librustc/lint/internal.rs
index 1c5f86f..10c0c63 100644
--- a/src/librustc/lint/internal.rs
+++ b/src/librustc/lint/internal.rs
@@ -9,6 +9,7 @@
 use rustc_data_structures::fx::FxHashMap;
 use syntax::ast::{Ident, Item, ItemKind};
 use syntax::symbol::{sym, Symbol};
+use rustc_session::declare_tool_lint;
 
 declare_tool_lint! {
     pub rustc::DEFAULT_HASH_TYPES,
diff --git a/src/librustc/lint/levels.rs b/src/librustc/lint/levels.rs
index 619ca72..f29d1a3 100644
--- a/src/librustc/lint/levels.rs
+++ b/src/librustc/lint/levels.rs
@@ -8,7 +8,7 @@
 use crate::session::Session;
 use crate::util::nodemap::FxHashMap;
 use errors::{Applicability, DiagnosticBuilder};
-use rustc_data_structures::stable_hasher::{HashStable, ToStableHashKey, StableHasher};
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use syntax::ast;
 use syntax::attr;
 use syntax::feature_gate;
@@ -93,7 +93,7 @@
 
         // If `level` is none then we actually assume the default level for this
         // lint.
-        let mut level = level.unwrap_or_else(|| lint.default_level(sess));
+        let mut level = level.unwrap_or_else(|| lint.default_level(sess.edition()));
 
         // If we're about to issue a warning, check at the last minute for any
         // directives against the warnings "lint". If, for example, there's an
@@ -566,19 +566,3 @@
         })
     }
 }
-
-impl<HCX> HashStable<HCX> for LintId {
-    #[inline]
-    fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
-        self.lint_name_raw().hash_stable(hcx, hasher);
-    }
-}
-
-impl<HCX> ToStableHashKey<HCX> for LintId {
-    type KeyType = &'static str;
-
-    #[inline]
-    fn to_stable_hash_key(&self, _: &HCX) -> &'static str {
-        self.lint_name_raw()
-    }
-}
diff --git a/src/librustc/lint/mod.rs b/src/librustc/lint/mod.rs
index d84102f..a8d8868 100644
--- a/src/librustc/lint/mod.rs
+++ b/src/librustc/lint/mod.rs
@@ -27,19 +27,14 @@
 use crate::hir::intravisit;
 use crate::hir;
 use crate::lint::builtin::BuiltinLintDiagnostics;
-use crate::lint::builtin::parser::{ILL_FORMED_ATTRIBUTE_INPUT, META_VARIABLE_MISUSE};
-use crate::lint::builtin::parser::INCOMPLETE_INCLUDE;
 use crate::session::{Session, DiagnosticMessageId};
 use crate::ty::TyCtxt;
 use crate::ty::query::Providers;
 use crate::util::nodemap::NodeMap;
 use errors::{DiagnosticBuilder, DiagnosticId};
-use std::{hash, ptr};
 use syntax::ast;
 use syntax::source_map::{MultiSpan, ExpnKind, DesugaringKind};
-use syntax::early_buffered_lints::BufferedEarlyLintId;
-use syntax::edition::Edition;
-use syntax::symbol::{Symbol, sym};
+use syntax::symbol::Symbol;
 use syntax_pos::hygiene::MacroKind;
 use syntax_pos::Span;
 
@@ -47,150 +42,7 @@
                         check_crate, check_ast_crate, late_lint_mod, CheckLintNameResult,
                         BufferedEarlyLint,};
 
-/// Specification of a single lint.
-#[derive(Copy, Clone, Debug)]
-pub struct Lint {
-    /// A string identifier for the lint.
-    ///
-    /// This identifies the lint in attributes and in command-line arguments.
-    /// In those contexts it is always lowercase, but this field is compared
-    /// in a way which is case-insensitive for ASCII characters. This allows
-    /// `declare_lint!()` invocations to follow the convention of upper-case
-    /// statics without repeating the name.
-    ///
-    /// The name is written with underscores, e.g., "unused_imports".
-    /// On the command line, underscores become dashes.
-    pub name: &'static str,
-
-    /// Default level for the lint.
-    pub default_level: Level,
-
-    /// Description of the lint or the issue it detects.
-    ///
-    /// e.g., "imports that are never used"
-    pub desc: &'static str,
-
-    /// Starting at the given edition, default to the given lint level. If this is `None`, then use
-    /// `default_level`.
-    pub edition_lint_opts: Option<(Edition, Level)>,
-
-    /// `true` if this lint is reported even inside expansions of external macros.
-    pub report_in_external_macro: bool,
-
-    pub future_incompatible: Option<FutureIncompatibleInfo>,
-
-    pub is_plugin: bool,
-}
-
-/// Extra information for a future incompatibility lint.
-#[derive(Copy, Clone, Debug)]
-pub struct FutureIncompatibleInfo {
-    /// e.g., a URL for an issue/PR/RFC or error code
-    pub reference: &'static str,
-    /// If this is an edition fixing lint, the edition in which
-    /// this lint becomes obsolete
-    pub edition: Option<Edition>,
-}
-
-impl Lint {
-    pub const fn default_fields_for_macro() -> Self {
-        Lint {
-            name: "",
-            default_level: Level::Forbid,
-            desc: "",
-            edition_lint_opts: None,
-            is_plugin: false,
-            report_in_external_macro: false,
-            future_incompatible: None,
-        }
-    }
-
-    /// Returns the `rust::lint::Lint` for a `syntax::early_buffered_lints::BufferedEarlyLintId`.
-    pub fn from_parser_lint_id(lint_id: BufferedEarlyLintId) -> &'static Self {
-        match lint_id {
-            BufferedEarlyLintId::IllFormedAttributeInput => ILL_FORMED_ATTRIBUTE_INPUT,
-            BufferedEarlyLintId::MetaVariableMisuse => META_VARIABLE_MISUSE,
-            BufferedEarlyLintId::IncompleteInclude => INCOMPLETE_INCLUDE,
-        }
-    }
-
-    /// Gets the lint's name, with ASCII letters converted to lowercase.
-    pub fn name_lower(&self) -> String {
-        self.name.to_ascii_lowercase()
-    }
-
-    pub fn default_level(&self, session: &Session) -> Level {
-        self.edition_lint_opts
-            .filter(|(e, _)| *e <= session.edition())
-            .map(|(_, l)| l)
-            .unwrap_or(self.default_level)
-    }
-}
-
-/// Declares a static item of type `&'static Lint`.
-#[macro_export]
-macro_rules! declare_lint {
-    ($vis: vis $NAME: ident, $Level: ident, $desc: expr) => (
-        declare_lint!(
-            $vis $NAME, $Level, $desc,
-        );
-    );
-    ($vis: vis $NAME: ident, $Level: ident, $desc: expr,
-     $(@future_incompatible = $fi:expr;)? $($v:ident),*) => (
-        $vis static $NAME: &$crate::lint::Lint = &$crate::lint::Lint {
-            name: stringify!($NAME),
-            default_level: $crate::lint::$Level,
-            desc: $desc,
-            edition_lint_opts: None,
-            is_plugin: false,
-            $($v: true,)*
-            $(future_incompatible: Some($fi),)*
-            ..$crate::lint::Lint::default_fields_for_macro()
-        };
-    );
-    ($vis: vis $NAME: ident, $Level: ident, $desc: expr,
-     $lint_edition: expr => $edition_level: ident
-    ) => (
-        $vis static $NAME: &$crate::lint::Lint = &$crate::lint::Lint {
-            name: stringify!($NAME),
-            default_level: $crate::lint::$Level,
-            desc: $desc,
-            edition_lint_opts: Some(($lint_edition, $crate::lint::Level::$edition_level)),
-            report_in_external_macro: false,
-            is_plugin: false,
-        };
-    );
-}
-
-#[macro_export]
-macro_rules! declare_tool_lint {
-    (
-        $(#[$attr:meta])* $vis:vis $tool:ident ::$NAME:ident, $Level: ident, $desc: expr
-    ) => (
-        declare_tool_lint!{$(#[$attr])* $vis $tool::$NAME, $Level, $desc, false}
-    );
-    (
-        $(#[$attr:meta])* $vis:vis $tool:ident ::$NAME:ident, $Level:ident, $desc:expr,
-        report_in_external_macro: $rep:expr
-    ) => (
-         declare_tool_lint!{$(#[$attr])* $vis $tool::$NAME, $Level, $desc, $rep}
-    );
-    (
-        $(#[$attr:meta])* $vis:vis $tool:ident ::$NAME:ident, $Level:ident, $desc:expr,
-        $external:expr
-    ) => (
-        $(#[$attr])*
-        $vis static $NAME: &$crate::lint::Lint = &$crate::lint::Lint {
-            name: &concat!(stringify!($tool), "::", stringify!($NAME)),
-            default_level: $crate::lint::$Level,
-            desc: $desc,
-            edition_lint_opts: None,
-            report_in_external_macro: $external,
-            future_incompatible: None,
-            is_plugin: true,
-        };
-    );
-}
+pub use rustc_session::lint::{Lint, LintId, Level, FutureIncompatibleInfo};
 
 /// Declares a static `LintArray` and return it as an expression.
 #[macro_export]
@@ -502,86 +354,6 @@
 pub type LateLintPassObject = Box<dyn for<'a, 'tcx> LateLintPass<'a, 'tcx> + sync::Send
                                                                            + sync::Sync + 'static>;
 
-/// Identifies a lint known to the compiler.
-#[derive(Clone, Copy, Debug)]
-pub struct LintId {
-    // Identity is based on pointer equality of this field.
-    lint: &'static Lint,
-}
-
-impl PartialEq for LintId {
-    fn eq(&self, other: &LintId) -> bool {
-        ptr::eq(self.lint, other.lint)
-    }
-}
-
-impl Eq for LintId { }
-
-impl hash::Hash for LintId {
-    fn hash<H: hash::Hasher>(&self, state: &mut H) {
-        let ptr = self.lint as *const Lint;
-        ptr.hash(state);
-    }
-}
-
-impl LintId {
-    /// Gets the `LintId` for a `Lint`.
-    pub fn of(lint: &'static Lint) -> LintId {
-        LintId {
-            lint,
-        }
-    }
-
-    pub fn lint_name_raw(&self) -> &'static str {
-        self.lint.name
-    }
-
-    /// Gets the name of the lint.
-    pub fn to_string(&self) -> String {
-        self.lint.name_lower()
-    }
-}
-
-/// Setting for how to handle a lint.
-#[derive(Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Debug, Hash, HashStable)]
-pub enum Level {
-    Allow, Warn, Deny, Forbid,
-}
-
-impl Level {
-    /// Converts a level to a lower-case string.
-    pub fn as_str(self) -> &'static str {
-        match self {
-            Allow => "allow",
-            Warn => "warn",
-            Deny => "deny",
-            Forbid => "forbid",
-        }
-    }
-
-    /// Converts a lower-case string to a level.
-    pub fn from_str(x: &str) -> Option<Level> {
-        match x {
-            "allow" => Some(Allow),
-            "warn" => Some(Warn),
-            "deny" => Some(Deny),
-            "forbid" => Some(Forbid),
-            _ => None,
-        }
-    }
-
-    /// Converts a symbol to a level.
-    pub fn from_symbol(x: Symbol) -> Option<Level> {
-        match x {
-            sym::allow => Some(Allow),
-            sym::warn => Some(Warn),
-            sym::deny => Some(Deny),
-            sym::forbid => Some(Forbid),
-            _ => None,
-        }
-    }
-}
-
 /// How a lint level was set.
 #[derive(Clone, Copy, PartialEq, Eq, HashStable)]
 pub enum LintSource {
diff --git a/src/librustc/middle/cstore.rs b/src/librustc/middle/cstore.rs
index 44c6f6b..324b013 100644
--- a/src/librustc/middle/cstore.rs
+++ b/src/librustc/middle/cstore.rs
@@ -20,6 +20,7 @@
 use rustc_data_structures::sync::{self, MetadataRef};
 use rustc_macros::HashStable;
 
+pub use rustc_session::utils::NativeLibraryKind;
 pub use self::NativeLibraryKind::*;
 
 // lonely orphan structs and enums looking for a better home
@@ -94,21 +95,6 @@
     RequireStatic,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash,
-         RustcEncodable, RustcDecodable, HashStable)]
-pub enum NativeLibraryKind {
-    /// native static library (.a archive)
-    NativeStatic,
-    /// native static library, which doesn't get bundled into .rlibs
-    NativeStaticNobundle,
-    /// macOS-specific
-    NativeFramework,
-    /// Windows dynamic library without import library.
-    NativeRawDylib,
-    /// default way to specify a dynamic library
-    NativeUnknown,
-}
-
 #[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
 pub struct NativeLibrary {
     pub kind: NativeLibraryKind,
diff --git a/src/librustc/middle/lang_items.rs b/src/librustc/middle/lang_items.rs
index f6cd9b1..6f7a022 100644
--- a/src/librustc/middle/lang_items.rs
+++ b/src/librustc/middle/lang_items.rs
@@ -358,7 +358,6 @@
     // Don't be fooled by the naming here: this lang item denotes `PartialEq`, not `Eq`.
     EqTraitLangItem,             "eq",                 eq_trait,                Target::Trait;
     PartialOrdTraitLangItem,     "partial_ord",        partial_ord_trait,       Target::Trait;
-    OrdTraitLangItem,            "ord",                ord_trait,               Target::Trait;
 
     // A number of panic-related lang items. The `panic` item corresponds to
     // divide-by-zero and various panic cases with `match`. The
diff --git a/src/librustc/mir/cache.rs b/src/librustc/mir/cache.rs
index 9b41366..95e9f09 100644
--- a/src/librustc/mir/cache.rs
+++ b/src/librustc/mir/cache.rs
@@ -1,16 +1,19 @@
 use rustc_index::vec::IndexVec;
-use rustc_data_structures::sync::{RwLock, MappedReadGuard, ReadGuard};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_serialize::{Encodable, Encoder, Decodable, Decoder};
 use crate::ich::StableHashingContext;
-use crate::mir::{Body, BasicBlock};
+use crate::mir::{BasicBlock, BasicBlockData, Body, LocalDecls, Location, Successors};
+use rustc_data_structures::graph::{self, GraphPredecessors, GraphSuccessors};
+use rustc_data_structures::graph::dominators::{dominators, Dominators};
+use std::iter;
+use std::ops::{Deref, DerefMut, Index, IndexMut};
+use std::vec::IntoIter;
 
 #[derive(Clone, Debug)]
 pub struct Cache {
-    predecessors: RwLock<Option<IndexVec<BasicBlock, Vec<BasicBlock>>>>
+    predecessors: Option<IndexVec<BasicBlock, Vec<BasicBlock>>>,
 }
 
-
 impl rustc_serialize::Encodable for Cache {
     fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
         Encodable::encode(&(), s)
@@ -31,39 +34,256 @@
 
 impl Cache {
     pub fn new() -> Self {
-        Cache {
-            predecessors: RwLock::new(None)
+        Self {
+            predecessors: None,
         }
     }
 
-    pub fn invalidate(&self) {
+    pub fn invalidate_predecessors(&mut self) {
         // FIXME: consider being more fine-grained
-        *self.predecessors.borrow_mut() = None;
+        self.predecessors = None;
     }
 
-    pub fn predecessors(
-        &self,
-        body: &Body<'_>
-    ) -> MappedReadGuard<'_, IndexVec<BasicBlock, Vec<BasicBlock>>> {
-        if self.predecessors.borrow().is_none() {
-            *self.predecessors.borrow_mut() = Some(calculate_predecessors(body));
-        }
+    pub fn ensure_predecessors(&mut self, body: &Body<'_>) {
+        if self.predecessors.is_none() {
+            let mut result = IndexVec::from_elem(vec![], body.basic_blocks());
+            for (bb, data) in body.basic_blocks().iter_enumerated() {
+                if let Some(ref term) = data.terminator {
+                    for &tgt in term.successors() {
+                        result[tgt].push(bb);
+                    }
+                }
+            }
 
-        ReadGuard::map(self.predecessors.borrow(), |p| p.as_ref().unwrap())
+            self.predecessors = Some(result)
+        }
+    }
+
+    /// This will recompute the predecessors cache if it is not available
+    fn predecessors(&mut self, body: &Body<'_>) -> &IndexVec<BasicBlock, Vec<BasicBlock>> {
+        self.ensure_predecessors(body);
+        self.predecessors.as_ref().unwrap()
+    }
+
+    fn unwrap_predecessors_for(&self, bb: BasicBlock) -> &[BasicBlock] {
+        &self.predecessors.as_ref().unwrap()[bb]
+    }
+
+    fn unwrap_predecessor_locations<'a>(
+        &'a self,
+        loc: Location,
+        body: &'a Body<'a>
+    ) -> impl Iterator<Item = Location> + 'a {
+        let if_zero_locations = if loc.statement_index == 0 {
+            let predecessor_blocks = self.unwrap_predecessors_for(loc.block);
+            let num_predecessor_blocks = predecessor_blocks.len();
+            Some(
+                (0..num_predecessor_blocks)
+                    .map(move |i| predecessor_blocks[i])
+                    .map(move |bb| body.terminator_loc(bb)),
+            )
+        } else {
+            None
+        };
+
+        let if_not_zero_locations = if loc.statement_index == 0 {
+            None
+        } else {
+            Some(Location { block: loc.block, statement_index: loc.statement_index - 1 })
+        };
+
+        if_zero_locations.into_iter().flatten().chain(if_not_zero_locations)
+    }
+
+    pub fn basic_blocks_mut<'a, 'tcx>(
+        &mut self,
+        body: &'a mut Body<'tcx>
+    ) -> &'a mut IndexVec<BasicBlock, BasicBlockData<'tcx>> {
+        debug!("bbm: Clearing predecessors cache for body at: {:?}", body.span.data());
+        self.invalidate_predecessors();
+        &mut body.basic_blocks
+    }
+
+    pub fn basic_blocks_and_local_decls_mut<'a, 'tcx>(
+        &mut self,
+        body: &'a mut Body<'tcx>
+    ) -> (&'a mut IndexVec<BasicBlock, BasicBlockData<'tcx>>, &'a mut LocalDecls<'tcx>) {
+        debug!("bbaldm: Clearing predecessors cache for body at: {:?}", body.span.data());
+        self.invalidate_predecessors();
+        (&mut body.basic_blocks, &mut body.local_decls)
     }
 }
 
-fn calculate_predecessors(body: &Body<'_>) -> IndexVec<BasicBlock, Vec<BasicBlock>> {
-    let mut result = IndexVec::from_elem(vec![], body.basic_blocks());
-    for (bb, data) in body.basic_blocks().iter_enumerated() {
-        if let Some(ref term) = data.terminator {
-            for &tgt in term.successors() {
-                result[tgt].push(bb);
-            }
+#[derive(Clone, Debug, HashStable, RustcEncodable, RustcDecodable, TypeFoldable)]
+pub struct BodyCache<'tcx> {
+    cache: Cache,
+    body: Body<'tcx>,
+}
+
+impl BodyCache<'tcx> {
+    pub fn new(body: Body<'tcx>) -> Self {
+        Self {
+            cache: Cache::new(),
+            body,
+        }
+    }
+}
+
+#[macro_export]
+macro_rules! read_only {
+    ($body:expr) => {
+        {
+            $body.ensure_predecessors();
+            $body.unwrap_read_only()
+        }
+    };
+}
+
+impl BodyCache<'tcx> {
+    pub fn ensure_predecessors(&mut self) {
+        self.cache.ensure_predecessors(&self.body);
+    }
+
+    pub fn predecessors(&mut self) -> &IndexVec<BasicBlock, Vec<BasicBlock>> {
+        self.cache.predecessors(&self.body)
+    }
+
+    pub fn unwrap_read_only(&self) -> ReadOnlyBodyCache<'_, 'tcx> {
+        ReadOnlyBodyCache::new(&self.cache, &self.body)
+    }
+
+    pub fn basic_blocks_mut(&mut self) -> &mut IndexVec<BasicBlock, BasicBlockData<'tcx>> {
+        self.cache.basic_blocks_mut(&mut self.body)
+    }
+
+    pub fn basic_blocks_and_local_decls_mut(
+        &mut self
+    ) -> (&mut IndexVec<BasicBlock, BasicBlockData<'tcx>>, &mut LocalDecls<'tcx>) {
+        self.cache.basic_blocks_and_local_decls_mut(&mut self.body)
+    }
+}
+
+impl<'tcx> Index<BasicBlock> for BodyCache<'tcx> {
+    type Output = BasicBlockData<'tcx>;
+
+    fn index(&self, index: BasicBlock) -> &BasicBlockData<'tcx> {
+        &self.body[index]
+    }
+}
+
+impl<'tcx> IndexMut<BasicBlock> for BodyCache<'tcx> {
+    fn index_mut(&mut self, index: BasicBlock) -> &mut Self::Output {
+        &mut self.basic_blocks_mut()[index]
+    }
+}
+
+impl<'tcx> Deref for BodyCache<'tcx> {
+    type Target = Body<'tcx>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.body
+    }
+}
+
+impl<'tcx> DerefMut for BodyCache<'tcx> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.body
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+pub struct ReadOnlyBodyCache<'a, 'tcx> {
+    cache: &'a Cache,
+    body: &'a Body<'tcx>,
+}
+
+impl ReadOnlyBodyCache<'a, 'tcx> {
+    fn new(cache: &'a Cache, body: &'a Body<'tcx>) -> Self {
+        assert!(
+            cache.predecessors.is_some(),
+            "Cannot construct ReadOnlyBodyCache without computed predecessors");
+        Self {
+            cache,
+            body,
         }
     }
 
-    result
+    pub fn predecessors(&self) -> &IndexVec<BasicBlock, Vec<BasicBlock>> {
+        self.cache.predecessors.as_ref().unwrap()
+    }
+
+    pub fn predecessors_for(&self, bb: BasicBlock) -> &[BasicBlock] {
+        self.cache.unwrap_predecessors_for(bb)
+    }
+
+    pub fn predecessor_locations(&self, loc: Location) -> impl Iterator<Item = Location> + '_ {
+        self.cache.unwrap_predecessor_locations(loc, self.body)
+    }
+
+    pub fn body(&self) -> &'a Body<'tcx> {
+        self.body
+    }
+
+    pub fn basic_blocks(&self) -> &IndexVec<BasicBlock, BasicBlockData<'tcx>> {
+        &self.body.basic_blocks
+    }
+
+    pub fn dominators(&self) -> Dominators<BasicBlock> {
+        dominators(self)
+    }
+}
+
+impl graph::DirectedGraph for ReadOnlyBodyCache<'a, 'tcx> {
+    type Node = BasicBlock;
+}
+
+impl graph::GraphPredecessors<'graph> for ReadOnlyBodyCache<'a, 'tcx> {
+    type Item = BasicBlock;
+    type Iter = IntoIter<BasicBlock>;
+}
+
+impl graph::WithPredecessors for ReadOnlyBodyCache<'a, 'tcx> {
+    fn predecessors(
+        &self,
+        node: Self::Node,
+    ) -> <Self as GraphPredecessors<'_>>::Iter {
+        self.cache.unwrap_predecessors_for(node).to_vec().into_iter()
+    }
+}
+
+impl graph::WithNumNodes for ReadOnlyBodyCache<'a, 'tcx> {
+    fn num_nodes(&self) -> usize {
+        self.body.num_nodes()
+    }
+}
+
+impl graph::WithStartNode for ReadOnlyBodyCache<'a, 'tcx> {
+    fn start_node(&self) -> Self::Node {
+        self.body.start_node()
+    }
+}
+
+impl graph::WithSuccessors for ReadOnlyBodyCache<'a, 'tcx> {
+    fn successors(
+        &self,
+        node: Self::Node,
+    ) -> <Self as GraphSuccessors<'_>>::Iter {
+        self.body.successors(node)
+    }
+}
+
+impl<'a, 'b, 'tcx> graph::GraphSuccessors<'b> for ReadOnlyBodyCache<'a, 'tcx> {
+    type Item = BasicBlock;
+    type Iter = iter::Cloned<Successors<'b>>;
+}
+
+
+impl Deref for ReadOnlyBodyCache<'a, 'tcx> {
+    type Target = &'a Body<'tcx>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.body
+    }
 }
 
 CloneTypeFoldableAndLiftImpls! {
diff --git a/src/librustc/mir/interpret/error.rs b/src/librustc/mir/interpret/error.rs
index 41c22de..0dec7c0 100644
--- a/src/librustc/mir/interpret/error.rs
+++ b/src/librustc/mir/interpret/error.rs
@@ -14,7 +14,7 @@
 use syntax_pos::{Pos, Span};
 use syntax::symbol::Symbol;
 use hir::GeneratorKind;
-use std::{fmt, env};
+use std::{fmt, env, any::Any};
 
 use rustc_error_codes::*;
 
@@ -44,14 +44,14 @@
 pub type ConstEvalRawResult<'tcx> = Result<RawConst<'tcx>, ErrorHandled>;
 pub type ConstEvalResult<'tcx> = Result<&'tcx ty::Const<'tcx>, ErrorHandled>;
 
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub struct ConstEvalErr<'tcx> {
     pub span: Span,
     pub error: crate::mir::interpret::InterpError<'tcx>,
     pub stacktrace: Vec<FrameInfo<'tcx>>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub struct FrameInfo<'tcx> {
     /// This span is in the caller.
     pub call_site: Span,
@@ -138,6 +138,7 @@
         lint_root: Option<hir::HirId>,
     ) -> Result<DiagnosticBuilder<'tcx>, ErrorHandled> {
         let must_error = match self.error {
+            InterpError::MachineStop(_) => bug!("CTFE does not stop"),
             err_inval!(Layout(LayoutError::Unknown(_))) |
             err_inval!(TooGeneric) =>
                 return Err(ErrorHandled::TooGeneric),
@@ -189,7 +190,7 @@
 /// Thsese should always be constructed by calling `.into()` on
 /// a `InterpError`. In `librustc_mir::interpret`, we have `throw_err_*`
 /// macros for this.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct InterpErrorInfo<'tcx> {
     pub kind: InterpError<'tcx>,
     backtrace: Option<Box<Backtrace>>,
@@ -331,7 +332,6 @@
 /// Error information for when the program we executed turned out not to actually be a valid
 /// program. This cannot happen in stand-alone Miri, but it can happen during CTFE/ConstProp
 /// where we work on generic code or execution does not have all information available.
-#[derive(Clone, HashStable)]
 pub enum InvalidProgramInfo<'tcx> {
     /// Resolution can fail if we are in a too generic context.
     TooGeneric,
@@ -361,7 +361,6 @@
 }
 
 /// Error information for when the program caused Undefined Behavior.
-#[derive(Clone, HashStable)]
 pub enum UndefinedBehaviorInfo {
     /// Free-form case. Only for errors that are never caught!
     Ub(String),
@@ -411,7 +410,6 @@
 ///
 /// Currently, we also use this as fall-back error kind for errors that have not been
 /// categorized yet.
-#[derive(Clone, HashStable)]
 pub enum UnsupportedOpInfo<'tcx> {
     /// Free-form case. Only for errors that are never caught!
     Unsupported(String),
@@ -588,7 +586,6 @@
 
 /// Error information for when the program exhausted the resources granted to it
 /// by the interpreter.
-#[derive(Clone, HashStable)]
 pub enum ResourceExhaustionInfo {
     /// The stack grew too big.
     StackFrameLimitReached,
@@ -609,7 +606,6 @@
     }
 }
 
-#[derive(Clone, HashStable)]
 pub enum InterpError<'tcx> {
     /// The program panicked.
     Panic(PanicInfo<u64>),
@@ -618,14 +614,14 @@
     /// The program did something the interpreter does not support (some of these *might* be UB
     /// but the interpreter is not sure).
     Unsupported(UnsupportedOpInfo<'tcx>),
-    /// The program was invalid (ill-typed, not sufficiently monomorphized, ...).
+    /// The program was invalid (ill-typed, bad MIR, not sufficiently monomorphized, ...).
     InvalidProgram(InvalidProgramInfo<'tcx>),
     /// The program exhausted the interpreter's resources (stack/heap too big,
-    /// execution takes too long, ..).
+    /// execution takes too long, ...).
     ResourceExhaustion(ResourceExhaustionInfo),
-    /// Not actually an interpreter error -- used to signal that execution has exited
-    /// with the given status code.  Used by Miri, but not by CTFE.
-    Exit(i32),
+    /// Stop execution for a machine-controlled reason. This is never raised by
+    /// the core engine itself.
+    MachineStop(Box<dyn Any + Send>),
 }
 
 pub type InterpResult<'tcx, T = ()> = Result<T, InterpErrorInfo<'tcx>>;
@@ -651,8 +647,8 @@
                 write!(f, "{:?}", msg),
             Panic(ref msg) =>
                 write!(f, "{:?}", msg),
-            Exit(code) =>
-                write!(f, "exited with status code {}", code),
+            MachineStop(_) =>
+                write!(f, "machine caused execution to stop"),
         }
     }
 }
diff --git a/src/librustc/mir/interpret/mod.rs b/src/librustc/mir/interpret/mod.rs
index 65f4ee8..fff8767 100644
--- a/src/librustc/mir/interpret/mod.rs
+++ b/src/librustc/mir/interpret/mod.rs
@@ -90,6 +90,13 @@
     ($($tt:tt)*) => { return Err(err_exhaust!($($tt)*).into()) };
 }
 
+#[macro_export]
+macro_rules! throw_machine_stop {
+    ($($tt:tt)*) => {
+        return Err($crate::mir::interpret::InterpError::MachineStop(Box::new($($tt)*)).into())
+    };
+}
+
 mod error;
 mod value;
 mod allocation;
diff --git a/src/librustc/mir/interpret/value.rs b/src/librustc/mir/interpret/value.rs
index f82af62..a038ca2 100644
--- a/src/librustc/mir/interpret/value.rs
+++ b/src/librustc/mir/interpret/value.rs
@@ -471,6 +471,13 @@
     }
 }
 
+impl<Tag> From<Pointer<Tag>> for ScalarMaybeUndef<Tag> {
+    #[inline(always)]
+    fn from(s: Pointer<Tag>) -> Self {
+        ScalarMaybeUndef::Scalar(s.into())
+    }
+}
+
 impl<Tag: fmt::Debug, Id: fmt::Debug> fmt::Debug for ScalarMaybeUndef<Tag, Id> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
diff --git a/src/librustc/mir/mod.rs b/src/librustc/mir/mod.rs
index 8c1690a..814c244 100644
--- a/src/librustc/mir/mod.rs
+++ b/src/librustc/mir/mod.rs
@@ -21,25 +21,25 @@
 use polonius_engine::Atom;
 use rustc_index::bit_set::BitMatrix;
 use rustc_data_structures::fx::FxHashSet;
-use rustc_data_structures::graph::dominators::{dominators, Dominators};
-use rustc_data_structures::graph::{self, GraphPredecessors, GraphSuccessors};
+use rustc_data_structures::graph::dominators::Dominators;
+use rustc_data_structures::graph::{self, GraphSuccessors};
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_data_structures::sync::Lrc;
-use rustc_data_structures::sync::MappedReadGuard;
 use rustc_macros::HashStable;
 use rustc_serialize::{Encodable, Decodable};
 use smallvec::SmallVec;
 use std::borrow::Cow;
 use std::fmt::{self, Debug, Display, Formatter, Write};
-use std::ops::{Index, IndexMut};
+use std::ops::Index;
 use std::slice;
-use std::vec::IntoIter;
 use std::{iter, mem, option, u32};
 use syntax::ast::Name;
 use syntax::symbol::Symbol;
 use syntax_pos::{Span, DUMMY_SP};
 
 pub use crate::mir::interpret::AssertMessage;
+pub use crate::mir::cache::{BodyCache, ReadOnlyBodyCache};
+pub use crate::read_only;
 
 mod cache;
 pub mod interpret;
@@ -104,15 +104,11 @@
     /// and used for debuginfo. Indexed by a `SourceScope`.
     pub source_scopes: IndexVec<SourceScope, SourceScopeData>,
 
-    /// Crate-local information for each source scope, that can't (and
-    /// needn't) be tracked across crates.
-    pub source_scope_local_data: ClearCrossCrate<IndexVec<SourceScope, SourceScopeLocalData>>,
-
     /// The yield type of the function, if it is a generator.
     pub yield_ty: Option<Ty<'tcx>>,
 
     /// Generator drop glue.
-    pub generator_drop: Option<Box<Body<'tcx>>>,
+    pub generator_drop: Option<Box<BodyCache<'tcx>>>,
 
     /// The layout of a generator. Produced by the state transformation.
     pub generator_layout: Option<GeneratorLayout<'tcx>>,
@@ -158,16 +154,12 @@
 
     /// A span representing this MIR, for error reporting.
     pub span: Span,
-
-    /// A cache for various calculations.
-    cache: cache::Cache,
 }
 
 impl<'tcx> Body<'tcx> {
     pub fn new(
         basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
         source_scopes: IndexVec<SourceScope, SourceScopeData>,
-        source_scope_local_data: ClearCrossCrate<IndexVec<SourceScope, SourceScopeLocalData>>,
         local_decls: LocalDecls<'tcx>,
         user_type_annotations: CanonicalUserTypeAnnotations<'tcx>,
         arg_count: usize,
@@ -188,7 +180,6 @@
             phase: MirPhase::Build,
             basic_blocks,
             source_scopes,
-            source_scope_local_data,
             yield_ty: None,
             generator_drop: None,
             generator_layout: None,
@@ -199,7 +190,6 @@
             spread_arg: None,
             var_debug_info,
             span,
-            cache: cache::Cache::new(),
             control_flow_destroyed,
         }
     }
@@ -209,58 +199,6 @@
         &self.basic_blocks
     }
 
-    #[inline]
-    pub fn basic_blocks_mut(&mut self) -> &mut IndexVec<BasicBlock, BasicBlockData<'tcx>> {
-        self.cache.invalidate();
-        &mut self.basic_blocks
-    }
-
-    #[inline]
-    pub fn basic_blocks_and_local_decls_mut(
-        &mut self,
-    ) -> (&mut IndexVec<BasicBlock, BasicBlockData<'tcx>>, &mut LocalDecls<'tcx>) {
-        self.cache.invalidate();
-        (&mut self.basic_blocks, &mut self.local_decls)
-    }
-
-    #[inline]
-    pub fn predecessors(&self) -> MappedReadGuard<'_, IndexVec<BasicBlock, Vec<BasicBlock>>> {
-        self.cache.predecessors(self)
-    }
-
-    #[inline]
-    pub fn predecessors_for(&self, bb: BasicBlock) -> MappedReadGuard<'_, Vec<BasicBlock>> {
-        MappedReadGuard::map(self.predecessors(), |p| &p[bb])
-    }
-
-    #[inline]
-    pub fn predecessor_locations(&self, loc: Location) -> impl Iterator<Item = Location> + '_ {
-        let if_zero_locations = if loc.statement_index == 0 {
-            let predecessor_blocks = self.predecessors_for(loc.block);
-            let num_predecessor_blocks = predecessor_blocks.len();
-            Some(
-                (0..num_predecessor_blocks)
-                    .map(move |i| predecessor_blocks[i])
-                    .map(move |bb| self.terminator_loc(bb)),
-            )
-        } else {
-            None
-        };
-
-        let if_not_zero_locations = if loc.statement_index == 0 {
-            None
-        } else {
-            Some(Location { block: loc.block, statement_index: loc.statement_index - 1 })
-        };
-
-        if_zero_locations.into_iter().flatten().chain(if_not_zero_locations)
-    }
-
-    #[inline]
-    pub fn dominators(&self) -> Dominators<BasicBlock> {
-        dominators(self)
-    }
-
     /// Returns `true` if a cycle exists in the control-flow graph that is reachable from the
     /// `START_BLOCK`.
     pub fn is_cfg_cyclic(&self) -> bool {
@@ -304,11 +242,7 @@
     pub fn vars_iter<'a>(&'a self) -> impl Iterator<Item = Local> + 'a {
         (self.arg_count + 1..self.local_decls.len()).filter_map(move |index| {
             let local = Local::new(index);
-            if self.local_decls[local].is_user_variable() {
-                Some(local)
-            } else {
-                None
-            }
+            self.local_decls[local].is_user_variable().then_some(local)
         })
     }
 
@@ -361,7 +295,7 @@
     /// Changes a statement to a nop. This is both faster than deleting instructions and avoids
     /// invalidating statement indices in `Location`s.
     pub fn make_statement_nop(&mut self, location: Location) {
-        let block = &mut self[location.block];
+        let block = &mut self.basic_blocks[location.block];
         debug_assert!(location.statement_index < block.statements.len());
         block.statements[location.statement_index].make_nop()
     }
@@ -421,13 +355,6 @@
     }
 }
 
-impl<'tcx> IndexMut<BasicBlock> for Body<'tcx> {
-    #[inline]
-    fn index_mut(&mut self, index: BasicBlock) -> &mut BasicBlockData<'tcx> {
-        &mut self.basic_blocks_mut()[index]
-    }
-}
-
 #[derive(Copy, Clone, Debug, HashStable, TypeFoldable)]
 pub enum ClearCrossCrate<T> {
     Clear,
@@ -435,6 +362,13 @@
 }
 
 impl<T> ClearCrossCrate<T> {
+    pub fn as_ref(&'a self) -> ClearCrossCrate<&'a T> {
+        match self {
+            ClearCrossCrate::Clear => ClearCrossCrate::Clear,
+            ClearCrossCrate::Set(v) => ClearCrossCrate::Set(v),
+        }
+    }
+
     pub fn assert_crate_local(self) -> T {
         match self {
             ClearCrossCrate::Clear => bug!("unwrapping cross-crate data"),
@@ -2027,6 +1961,10 @@
 pub struct SourceScopeData {
     pub span: Span,
     pub parent_scope: Option<SourceScope>,
+
+    /// Crate-local information for this source scope, that can't (and
+    /// needn't) be tracked across crates.
+    pub local_data: ClearCrossCrate<SourceScopeLocalData>,
 }
 
 #[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
@@ -2308,10 +2246,14 @@
                         }
                     }
 
-                    AggregateKind::Closure(def_id, _) => ty::tls::with(|tcx| {
+                    AggregateKind::Closure(def_id, substs) => ty::tls::with(|tcx| {
                         if let Some(hir_id) = tcx.hir().as_local_hir_id(def_id) {
                             let name = if tcx.sess.opts.debugging_opts.span_free_formats {
-                                format!("[closure@{:?}]", hir_id)
+                                let substs = tcx.lift(&substs).unwrap();
+                                format!(
+                                    "[closure@{}]",
+                                    tcx.def_path_str_with_substs(def_id, substs),
+                                )
                             } else {
                                 format!("[closure@{:?}]", tcx.hir().span(hir_id))
                             };
@@ -2609,15 +2551,6 @@
     }
 }
 
-impl<'tcx> graph::WithPredecessors for Body<'tcx> {
-    fn predecessors(
-        &self,
-        node: Self::Node,
-    ) -> <Self as GraphPredecessors<'_>>::Iter {
-        self.predecessors_for(node).clone().into_iter()
-    }
-}
-
 impl<'tcx> graph::WithSuccessors for Body<'tcx> {
     fn successors(
         &self,
@@ -2627,11 +2560,6 @@
     }
 }
 
-impl<'a, 'b> graph::GraphPredecessors<'b> for Body<'a> {
-    type Item = BasicBlock;
-    type Iter = IntoIter<BasicBlock>;
-}
-
 impl<'a, 'b> graph::GraphSuccessors<'b> for Body<'a> {
     type Item = BasicBlock;
     type Iter = iter::Cloned<Successors<'b>>;
@@ -2666,7 +2594,11 @@
     }
 
     /// Returns `true` if `other` is earlier in the control flow graph than `self`.
-    pub fn is_predecessor_of<'tcx>(&self, other: Location, body: &Body<'tcx>) -> bool {
+    pub fn is_predecessor_of<'tcx>(
+        &self,
+        other: Location,
+        body: ReadOnlyBodyCache<'_, 'tcx>
+    ) -> bool {
         // If we are in the same block as the other location and are an earlier statement
         // then we are a predecessor of `other`.
         if self.block == other.block && self.statement_index < other.statement_index {
@@ -2674,13 +2606,13 @@
         }
 
         // If we're in another block, then we want to check that block is a predecessor of `other`.
-        let mut queue: Vec<BasicBlock> = body.predecessors_for(other.block).clone();
+        let mut queue: Vec<BasicBlock> = body.predecessors_for(other.block).to_vec();
         let mut visited = FxHashSet::default();
 
         while let Some(block) = queue.pop() {
             // If we haven't visited this block before, then make sure we visit it's predecessors.
             if visited.insert(block) {
-                queue.append(&mut body.predecessors_for(block).clone());
+                queue.extend(body.predecessors_for(block).iter().cloned());
             } else {
                 continue;
             }
diff --git a/src/librustc/mir/visit.rs b/src/librustc/mir/visit.rs
index 58c12ef..47a1d67 100644
--- a/src/librustc/mir/visit.rs
+++ b/src/librustc/mir/visit.rs
@@ -65,13 +65,25 @@
 // variant argument) that does not require visiting, as in
 // `is_cleanup` above.
 
+macro_rules! body_cache_type {
+    (mut $a:lifetime, $tcx:lifetime) => {
+        &mut BodyCache<$tcx>
+    };
+    ($a:lifetime, $tcx:lifetime) => {
+        ReadOnlyBodyCache<$a, $tcx>
+    };
+}
+
 macro_rules! make_mir_visitor {
     ($visitor_trait_name:ident, $($mutability:ident)?) => {
         pub trait $visitor_trait_name<'tcx> {
             // Override these, and call `self.super_xxx` to revert back to the
             // default behavior.
 
-            fn visit_body(&mut self, body: & $($mutability)? Body<'tcx>) {
+            fn visit_body(
+                &mut self,
+                body: body_cache_type!($($mutability)? '_, 'tcx)
+            ) {
                 self.super_body(body);
             }
 
@@ -240,11 +252,14 @@
             // The `super_xxx` methods comprise the default behavior and are
             // not meant to be overridden.
 
-            fn super_body(&mut self,
-                         body: & $($mutability)? Body<'tcx>) {
+            fn super_body(
+                &mut self,
+                $($mutability)? body: body_cache_type!($($mutability)? '_, 'tcx)
+            ) {
+                let span = body.span;
                 if let Some(yield_ty) = &$($mutability)? body.yield_ty {
                     self.visit_ty(yield_ty, TyContext::YieldTy(SourceInfo {
-                        span: body.span,
+                        span,
                         scope: OUTERMOST_SOURCE_SCOPE,
                     }));
                 }
@@ -260,6 +275,7 @@
                     self.visit_basic_block_data(bb, data);
                 }
 
+                let body: & $($mutability)? Body<'_> = & $($mutability)? body;
                 for scope in &$($mutability)? body.source_scopes {
                     self.visit_source_scope_data(scope);
                 }
@@ -317,6 +333,7 @@
                 let SourceScopeData {
                     span,
                     parent_scope,
+                    local_data: _,
                 } = scope_data;
 
                 self.visit_span(span);
@@ -789,7 +806,11 @@
 
             // Convenience methods
 
-            fn visit_location(&mut self, body: & $($mutability)? Body<'tcx>, location: Location) {
+            fn visit_location(
+                &mut self,
+                body: body_cache_type!($($mutability)? '_, 'tcx),
+                location: Location
+            ) {
                 let basic_block = & $($mutability)? body[location.block];
                 if basic_block.statements.len() == location.statement_index {
                     if let Some(ref $($mutability)? terminator) = basic_block.terminator {
diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs
index d715ddb..a6d7e5c 100644
--- a/src/librustc/query/mod.rs
+++ b/src/librustc/query/mod.rs
@@ -106,44 +106,54 @@
 
         /// Fetch the MIR for a given `DefId` right after it's built - this includes
         /// unreachable code.
-        query mir_built(_: DefId) -> &'tcx Steal<mir::Body<'tcx>> {}
+        query mir_built(_: DefId) -> &'tcx Steal<mir::BodyCache<'tcx>> {}
 
         /// Fetch the MIR for a given `DefId` up till the point where it is
         /// ready for const evaluation.
         ///
         /// See the README for the `mir` module for details.
-        query mir_const(_: DefId) -> &'tcx Steal<mir::Body<'tcx>> {
+        query mir_const(_: DefId) -> &'tcx Steal<mir::BodyCache<'tcx>> {
             no_hash
         }
 
         query mir_validated(_: DefId) ->
             (
-                &'tcx Steal<mir::Body<'tcx>>,
-                &'tcx Steal<IndexVec<mir::Promoted, mir::Body<'tcx>>>
+                &'tcx Steal<mir::BodyCache<'tcx>>,
+                &'tcx Steal<IndexVec<mir::Promoted, mir::BodyCache<'tcx>>>
             ) {
             no_hash
         }
 
         /// MIR after our optimization passes have run. This is MIR that is ready
         /// for codegen. This is also the only query that can fetch non-local MIR, at present.
-        query optimized_mir(key: DefId) -> &'tcx mir::Body<'tcx> {
+        query optimized_mir(key: DefId) -> &'tcx mir::BodyCache<'tcx> {
             cache_on_disk_if { key.is_local() }
             load_cached(tcx, id) {
-                let mir: Option<crate::mir::Body<'tcx>> = tcx.queries.on_disk_cache
-                                                            .try_load_query_result(tcx, id);
-                mir.map(|x| &*tcx.arena.alloc(x))
+                let mir: Option<crate::mir::BodyCache<'tcx>>
+                    = tcx.queries.on_disk_cache.try_load_query_result(tcx, id);
+                mir.map(|x| {
+                    let cache = tcx.arena.alloc(x);
+                    cache.ensure_predecessors();
+                    &*cache
+                })
             }
         }
 
-        query promoted_mir(key: DefId) -> &'tcx IndexVec<mir::Promoted, mir::Body<'tcx>> {
+        query promoted_mir(key: DefId) -> &'tcx IndexVec<mir::Promoted, mir::BodyCache<'tcx>> {
             cache_on_disk_if { key.is_local() }
             load_cached(tcx, id) {
                 let promoted: Option<
                     rustc_index::vec::IndexVec<
                         crate::mir::Promoted,
-                        crate::mir::Body<'tcx>
+                        crate::mir::BodyCache<'tcx>
                     >> = tcx.queries.on_disk_cache.try_load_query_result(tcx, id);
-                promoted.map(|p| &*tcx.arena.alloc(p))
+                promoted.map(|p| {
+                    let cache = tcx.arena.alloc(p);
+                    for body in cache.iter_mut() {
+                        body.ensure_predecessors();
+                    }
+                    &*cache
+                })
             }
         }
     }
@@ -502,7 +512,7 @@
         /// in the case of closures, this will be redirected to the enclosing function.
         query region_scope_tree(_: DefId) -> &'tcx region::ScopeTree {}
 
-        query mir_shims(key: ty::InstanceDef<'tcx>) -> &'tcx mir::Body<'tcx> {
+        query mir_shims(key: ty::InstanceDef<'tcx>) -> &'tcx mir::BodyCache<'tcx> {
             no_force
             desc { |tcx| "generating MIR shim for `{}`", tcx.def_path_str(key.def_id()) }
         }
diff --git a/src/librustc/traits/error_reporting.rs b/src/librustc/traits/error_reporting.rs
index 90db1fe..35017d6 100644
--- a/src/librustc/traits/error_reporting.rs
+++ b/src/librustc/traits/error_reporting.rs
@@ -363,11 +363,7 @@
             return None
         };
 
-        if tcx.has_attr(impl_def_id, sym::rustc_on_unimplemented) {
-            Some(impl_def_id)
-        } else {
-            None
-        }
+        tcx.has_attr(impl_def_id, sym::rustc_on_unimplemented).then_some(impl_def_id)
     }
 
     fn describe_generator(&self, body_id: hir::BodyId) -> Option<&'static str> {
@@ -521,7 +517,7 @@
         ) {
             command.evaluate(self.tcx, trait_ref, &flags[..])
         } else {
-            OnUnimplementedNote::empty()
+            OnUnimplementedNote::default()
         }
     }
 
@@ -697,6 +693,7 @@
         fallback_has_occurred: bool,
         points_at_arg: bool,
     ) {
+        let tcx = self.tcx;
         let span = obligation.cause.span;
 
         let mut err = match *error {
@@ -732,6 +729,7 @@
                             message,
                             label,
                             note,
+                            enclosing_scope,
                         } = self.on_unimplemented_note(trait_ref, obligation);
                         let have_alt_message = message.is_some() || label.is_some();
                         let is_try = self.tcx.sess.source_map().span_to_snippet(span)
@@ -798,6 +796,19 @@
                             // If it has a custom `#[rustc_on_unimplemented]` note, let's display it
                             err.note(s.as_str());
                         }
+                        if let Some(ref s) = enclosing_scope {
+                            let enclosing_scope_span = tcx.def_span(
+                                tcx.hir()
+                                    .opt_local_def_id(obligation.cause.body_id)
+                                    .unwrap_or_else(|| {
+                                        tcx.hir().body_owner_def_id(hir::BodyId {
+                                            hir_id: obligation.cause.body_id,
+                                        })
+                                    }),
+                            );
+
+                            err.span_label(enclosing_scope_span, s.as_str());
+                        }
 
                         self.suggest_borrow_on_unsized_slice(&obligation.cause.code, &mut err);
                         self.suggest_fn_call(&obligation, &mut err, &trait_ref, points_at_arg);
@@ -2199,6 +2210,10 @@
         }
 
         let span = self.tcx.def_span(generator_did);
+        // Do not ICE on closure typeck (#66868).
+        if let None = self.tcx.hir().as_local_hir_id(generator_did) {
+            return false;
+        }
         let tables = self.tcx.typeck_tables_of(generator_did);
         debug!("note_obligation_cause_for_async_await: generator_did={:?} span={:?} ",
                generator_did, span);
diff --git a/src/librustc/traits/on_unimplemented.rs b/src/librustc/traits/on_unimplemented.rs
index 59aa081..604f39d 100644
--- a/src/librustc/traits/on_unimplemented.rs
+++ b/src/librustc/traits/on_unimplemented.rs
@@ -22,18 +22,15 @@
     pub message: Option<OnUnimplementedFormatString>,
     pub label: Option<OnUnimplementedFormatString>,
     pub note: Option<OnUnimplementedFormatString>,
+    pub enclosing_scope: Option<OnUnimplementedFormatString>,
 }
 
+#[derive(Default)]
 pub struct OnUnimplementedNote {
     pub message: Option<String>,
     pub label: Option<String>,
     pub note: Option<String>,
-}
-
-impl OnUnimplementedNote {
-    pub fn empty() -> Self {
-        OnUnimplementedNote { message: None, label: None, note: None }
-    }
+    pub enclosing_scope: Option<String>,
 }
 
 fn parse_error(
@@ -85,24 +82,33 @@
         let mut message = None;
         let mut label = None;
         let mut note = None;
+        let mut enclosing_scope = None;
         let mut subcommands = vec![];
+
+        let parse_value = |value_str| {
+                OnUnimplementedFormatString::try_parse(tcx, trait_def_id, value_str, span)
+                    .map(Some)
+            };
+
         for item in item_iter {
             if item.check_name(sym::message) && message.is_none() {
                 if let Some(message_) = item.value_str() {
-                    message = Some(OnUnimplementedFormatString::try_parse(
-                        tcx, trait_def_id, message_, span)?);
+                    message = parse_value(message_)?;
                     continue;
                 }
             } else if item.check_name(sym::label) && label.is_none() {
                 if let Some(label_) = item.value_str() {
-                    label = Some(OnUnimplementedFormatString::try_parse(
-                        tcx, trait_def_id, label_, span)?);
+                    label = parse_value(label_)?;
                     continue;
                 }
             } else if item.check_name(sym::note) && note.is_none() {
                 if let Some(note_) = item.value_str() {
-                    note = Some(OnUnimplementedFormatString::try_parse(
-                        tcx, trait_def_id, note_, span)?);
+                    note = parse_value(note_)?;
+                    continue;
+                }
+            } else if item.check_name(sym::enclosing_scope) && enclosing_scope.is_none() {
+                if let Some(enclosing_scope_) = item.value_str() {
+                    enclosing_scope = parse_value(enclosing_scope_)?;
                     continue;
                 }
             } else if item.check_name(sym::on) && is_root &&
@@ -130,7 +136,14 @@
         if errored {
             Err(ErrorReported)
         } else {
-            Ok(OnUnimplementedDirective { condition, message, label, subcommands, note })
+            Ok(OnUnimplementedDirective {
+                condition,
+                subcommands,
+                message,
+                label,
+                note,
+                enclosing_scope
+            })
         }
     }
 
@@ -157,6 +170,7 @@
                 label: Some(OnUnimplementedFormatString::try_parse(
                     tcx, trait_def_id, value, attr.span)?),
                 note: None,
+                enclosing_scope: None,
             }))
         } else {
             return Err(ErrorReported);
@@ -174,6 +188,7 @@
         let mut message = None;
         let mut label = None;
         let mut note = None;
+        let mut enclosing_scope = None;
         info!("evaluate({:?}, trait_ref={:?}, options={:?})", self, trait_ref, options);
 
         for command in self.subcommands.iter().chain(Some(self)).rev() {
@@ -202,6 +217,10 @@
             if let Some(ref note_) = command.note {
                 note = Some(note_.clone());
             }
+
+            if let Some(ref enclosing_scope_) = command.enclosing_scope {
+                enclosing_scope = Some(enclosing_scope_.clone());
+            }
         }
 
         let options: FxHashMap<Symbol, String> = options.into_iter()
@@ -211,6 +230,7 @@
             label: label.map(|l| l.format(tcx, trait_ref, &options)),
             message: message.map(|m| m.format(tcx, trait_ref, &options)),
             note: note.map(|n| n.format(tcx, trait_ref, &options)),
+            enclosing_scope: enclosing_scope.map(|e_s| e_s.format(tcx, trait_ref, &options)),
         }
     }
 }
diff --git a/src/librustc/ty/constness.rs b/src/librustc/ty/constness.rs
index 676916f..268015a 100644
--- a/src/librustc/ty/constness.rs
+++ b/src/librustc/ty/constness.rs
@@ -2,7 +2,8 @@
 use crate::hir::def_id::DefId;
 use crate::hir;
 use crate::ty::TyCtxt;
-use syntax_pos::symbol::Symbol;
+use syntax_pos::symbol::{sym, Symbol};
+use rustc_target::spec::abi::Abi;
 use crate::hir::map::blocks::FnLikeNode;
 use syntax::attr;
 
@@ -35,12 +36,51 @@
         }
     }
 
+    /// Returns `true` if the `def_id` refers to an intrisic which we've whitelisted
+    /// for being called from stable `const fn`s (`min_const_fn`).
+    ///
+    /// Adding more intrinsics requires sign-off from @rust-lang/lang.
+    ///
+    /// This list differs from the list in `is_const_intrinsic` in the sense that any item on this
+    /// list must be on the `is_const_intrinsic` list, too, because if an intrinsic is callable from
+    /// stable, it must be callable at all.
+    fn is_intrinsic_min_const_fn(self, def_id: DefId) -> bool {
+        match self.item_name(def_id) {
+            | sym::size_of
+            | sym::min_align_of
+            | sym::needs_drop
+            // Arithmetic:
+            | sym::add_with_overflow // ~> .overflowing_add
+            | sym::sub_with_overflow // ~> .overflowing_sub
+            | sym::mul_with_overflow // ~> .overflowing_mul
+            | sym::wrapping_add // ~> .wrapping_add
+            | sym::wrapping_sub // ~> .wrapping_sub
+            | sym::wrapping_mul // ~> .wrapping_mul
+            | sym::saturating_add // ~> .saturating_add
+            | sym::saturating_sub // ~> .saturating_sub
+            | sym::unchecked_shl // ~> .wrapping_shl
+            | sym::unchecked_shr // ~> .wrapping_shr
+            | sym::rotate_left // ~> .rotate_left
+            | sym::rotate_right // ~> .rotate_right
+            | sym::ctpop // ~> .count_ones
+            | sym::ctlz // ~> .leading_zeros
+            | sym::cttz // ~> .trailing_zeros
+            | sym::bswap // ~> .swap_bytes
+            | sym::bitreverse // ~> .reverse_bits
+            => true,
+            _ => false,
+        }
+    }
+
     /// Returns `true` if this function must conform to `min_const_fn`
     pub fn is_min_const_fn(self, def_id: DefId) -> bool {
         // Bail out if the signature doesn't contain `const`
         if !self.is_const_fn_raw(def_id) {
             return false;
         }
+        if let Abi::RustIntrinsic = self.fn_sig(def_id).abi() {
+            return self.is_intrinsic_min_const_fn(def_id);
+        }
 
         if self.features().staged_api {
             // in order for a libstd function to be considered min_const_fn
@@ -63,13 +103,82 @@
 
 
 pub fn provide(providers: &mut Providers<'_>) {
-    /// only checks whether the function has a `const` modifier
+    /// Const evaluability whitelist is here to check evaluability at the
+    /// top level beforehand.
+    fn is_const_intrinsic(tcx: TyCtxt<'_>, def_id: DefId) -> Option<bool> {
+        match tcx.fn_sig(def_id).abi() {
+            Abi::RustIntrinsic |
+            Abi::PlatformIntrinsic => {
+                // FIXME: deduplicate these two lists as much as possible
+                match tcx.item_name(def_id) {
+                    // Keep this list in the same order as the match patterns in
+                    // `librustc_mir/interpret/intrinsics.rs`
+
+                    // This whitelist is a list of intrinsics that have a miri-engine implementation
+                    // and can thus be called when enabling enough feature gates. The similar
+                    // whitelist in `is_intrinsic_min_const_fn` (in this file), exists for allowing
+                    // the intrinsics to be called by stable const fns.
+                    | sym::caller_location
+
+                    | sym::min_align_of
+                    | sym::pref_align_of
+                    | sym::needs_drop
+                    | sym::size_of
+                    | sym::type_id
+                    | sym::type_name
+
+                    | sym::ctpop
+                    | sym::cttz
+                    | sym::cttz_nonzero
+                    | sym::ctlz
+                    | sym::ctlz_nonzero
+                    | sym::bswap
+                    | sym::bitreverse
+
+                    | sym::wrapping_add
+                    | sym::wrapping_sub
+                    | sym::wrapping_mul
+                    | sym::add_with_overflow
+                    | sym::sub_with_overflow
+                    | sym::mul_with_overflow
+
+                    | sym::saturating_add
+                    | sym::saturating_sub
+
+                    | sym::unchecked_shl
+                    | sym::unchecked_shr
+
+                    | sym::rotate_left
+                    | sym::rotate_right
+
+                    | sym::ptr_offset_from
+
+                    | sym::transmute
+
+                    | sym::simd_insert
+
+                    | sym::simd_extract
+
+                    => Some(true),
+
+                    _ => Some(false)
+                }
+            }
+            _ => None
+        }
+    }
+
+    /// Checks whether the function has a `const` modifier or, in case it is an intrinsic, whether
+    /// said intrinsic is on the whitelist for being const callable.
     fn is_const_fn_raw(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
         let hir_id = tcx.hir().as_local_hir_id(def_id)
                               .expect("Non-local call to local provider is_const_fn");
 
         let node = tcx.hir().get(hir_id);
-        if let Some(fn_like) = FnLikeNode::from_node(node) {
+
+        if let Some(whitelisted) = is_const_intrinsic(tcx, def_id) {
+            whitelisted
+        } else if let Some(fn_like) = FnLikeNode::from_node(node) {
             fn_like.constness() == hir::Constness::Const
         } else if let hir::Node::Ctor(_) = node {
             true
diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs
index 776ae7d..6a0002c 100644
--- a/src/librustc/ty/context.rs
+++ b/src/librustc/ty/context.rs
@@ -22,7 +22,7 @@
 use crate::middle::lang_items;
 use crate::middle::resolve_lifetime::{self, ObjectLifetimeDefault};
 use crate::middle::stability;
-use crate::mir::{Body, Field, interpret, Local, Place, PlaceElem, ProjectionKind, Promoted};
+use crate::mir::{BodyCache, Field, interpret, Local, Place, PlaceElem, ProjectionKind, Promoted};
 use crate::mir::interpret::{ConstValue, Allocation, Scalar};
 use crate::ty::subst::{GenericArg, InternalSubsts, SubstsRef, Subst};
 use crate::ty::ReprOptions;
@@ -1083,17 +1083,17 @@
         &self.hir_map
     }
 
-    pub fn alloc_steal_mir(self, mir: Body<'tcx>) -> &'tcx Steal<Body<'tcx>> {
+    pub fn alloc_steal_mir(self, mir: BodyCache<'tcx>) -> &'tcx Steal<BodyCache<'tcx>> {
         self.arena.alloc(Steal::new(mir))
     }
 
-    pub fn alloc_steal_promoted(self, promoted: IndexVec<Promoted, Body<'tcx>>) ->
-        &'tcx Steal<IndexVec<Promoted, Body<'tcx>>> {
+    pub fn alloc_steal_promoted(self, promoted: IndexVec<Promoted, BodyCache<'tcx>>) ->
+        &'tcx Steal<IndexVec<Promoted, BodyCache<'tcx>>> {
         self.arena.alloc(Steal::new(promoted))
     }
 
-    pub fn intern_promoted(self, promoted: IndexVec<Promoted, Body<'tcx>>) ->
-        &'tcx IndexVec<Promoted, Body<'tcx>> {
+    pub fn intern_promoted(self, promoted: IndexVec<Promoted, BodyCache<'tcx>>) ->
+        &'tcx IndexVec<Promoted, BodyCache<'tcx>> {
         self.arena.alloc(promoted)
     }
 
diff --git a/src/librustc/ty/instance.rs b/src/librustc/ty/instance.rs
index 777db38..801dfa8 100644
--- a/src/librustc/ty/instance.rs
+++ b/src/librustc/ty/instance.rs
@@ -1,8 +1,7 @@
 use crate::hir::CodegenFnAttrFlags;
-use crate::hir::Unsafety;
 use crate::hir::def::Namespace;
 use crate::hir::def_id::DefId;
-use crate::ty::{self, Ty, PolyFnSig, TypeFoldable, SubstsRef, TyCtxt};
+use crate::ty::{self, Ty, TypeFoldable, SubstsRef, TyCtxt};
 use crate::ty::print::{FmtPrinter, Printer};
 use crate::traits;
 use crate::middle::lang_items::DropInPlaceFnLangItem;
@@ -10,7 +9,6 @@
 use rustc_macros::HashStable;
 
 use std::fmt;
-use std::iter;
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)]
 #[derive(HashStable, Lift)]
@@ -29,17 +27,26 @@
 
     /// `fn()` pointer where the function itself cannot be turned into a pointer.
     ///
-    /// One example in the compiler today is functions annotated with `#[track_caller]`, which
-    /// must have their implicit caller location argument populated for a call. Because this is a
-    /// required part of the function's ABI but can't be tracked as a property of the function
-    /// pointer, we create a single "caller location" at the site where the function is reified.
+    /// One example is `<dyn Trait as Trait>::fn`, where the shim contains
+    /// a virtual call, which codegen supports only via a direct call to the
+    /// `<dyn Trait as Trait>::fn` instance (an `InstanceDef::Virtual`).
+    ///
+    /// Another example is functions annotated with `#[track_caller]`, which
+    /// must have their implicit caller location argument populated for a call.
+    /// Because this is a required part of the function's ABI but can't be tracked
+    /// as a property of the function pointer, we use a single "caller location"
+    /// (the definition of the function itself).
     ReifyShim(DefId),
 
     /// `<fn() as FnTrait>::call_*`
     /// `DefId` is `FnTrait::call_*`.
     FnPtrShim(DefId, Ty<'tcx>),
 
-    /// `<dyn Trait as Trait>::fn`
+    /// `<dyn Trait as Trait>::fn`, "direct calls" of which are implicitly
+    /// codegen'd as virtual calls.
+    ///
+    /// NB: if this is reified to a `fn` pointer, a `ReifyShim` is used
+    /// (see `ReifyShim` above for more details on that).
     Virtual(DefId, usize),
 
     /// `<[mut closure] as FnOnce>::call_once`
@@ -61,70 +68,6 @@
             &ty,
         )
     }
-
-    fn fn_sig_noadjust(&self, tcx: TyCtxt<'tcx>) -> PolyFnSig<'tcx> {
-        let ty = self.ty(tcx);
-        match ty.kind {
-            ty::FnDef(..) |
-            // Shims currently have type FnPtr. Not sure this should remain.
-            ty::FnPtr(_) => ty.fn_sig(tcx),
-            ty::Closure(def_id, substs) => {
-                let sig = substs.as_closure().sig(def_id, tcx);
-
-                let env_ty = tcx.closure_env_ty(def_id, substs).unwrap();
-                sig.map_bound(|sig| tcx.mk_fn_sig(
-                    iter::once(*env_ty.skip_binder()).chain(sig.inputs().iter().cloned()),
-                    sig.output(),
-                    sig.c_variadic,
-                    sig.unsafety,
-                    sig.abi
-                ))
-            }
-            ty::Generator(def_id, substs, _) => {
-                let sig = substs.as_generator().poly_sig(def_id, tcx);
-
-                let env_region = ty::ReLateBound(ty::INNERMOST, ty::BrEnv);
-                let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty);
-
-                let pin_did = tcx.lang_items().pin_type().unwrap();
-                let pin_adt_ref = tcx.adt_def(pin_did);
-                let pin_substs = tcx.intern_substs(&[env_ty.into()]);
-                let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs);
-
-                sig.map_bound(|sig| {
-                    let state_did = tcx.lang_items().gen_state().unwrap();
-                    let state_adt_ref = tcx.adt_def(state_did);
-                    let state_substs = tcx.intern_substs(&[
-                        sig.yield_ty.into(),
-                        sig.return_ty.into(),
-                    ]);
-                    let ret_ty = tcx.mk_adt(state_adt_ref, state_substs);
-
-                    tcx.mk_fn_sig(iter::once(env_ty),
-                        ret_ty,
-                        false,
-                        Unsafety::Normal,
-                        Abi::Rust
-                    )
-                })
-            }
-            _ => bug!("unexpected type {:?} in Instance::fn_sig_noadjust", ty)
-        }
-    }
-
-    pub fn fn_sig(&self, tcx: TyCtxt<'tcx>) -> ty::PolyFnSig<'tcx> {
-        let mut fn_sig = self.fn_sig_noadjust(tcx);
-        if let InstanceDef::VtableShim(..) = self.def {
-            // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`.
-            fn_sig = fn_sig.map_bound(|mut fn_sig| {
-                let mut inputs_and_output = fn_sig.inputs_and_output.to_vec();
-                inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]);
-                fn_sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
-                fn_sig
-            });
-        }
-        fn_sig
-    }
 }
 
 impl<'tcx> InstanceDef<'tcx> {
@@ -196,7 +139,7 @@
                 write!(f, " - intrinsic")
             }
             InstanceDef::Virtual(_, num) => {
-                write!(f, " - shim(#{})", num)
+                write!(f, " - virtual#{}", num)
             }
             InstanceDef::FnPtrShim(_, ty) => {
                 write!(f, " - shim({:?})", ty)
@@ -311,20 +254,23 @@
         substs: SubstsRef<'tcx>,
     ) -> Option<Instance<'tcx>> {
         debug!("resolve(def_id={:?}, substs={:?})", def_id, substs);
-        Instance::resolve(tcx, param_env, def_id, substs).map(|resolved| {
+        Instance::resolve(tcx, param_env, def_id, substs).map(|mut resolved| {
             let has_track_caller = |def| tcx.codegen_fn_attrs(def).flags
                 .contains(CodegenFnAttrFlags::TRACK_CALLER);
 
             match resolved.def {
                 InstanceDef::Item(def_id) if has_track_caller(def_id) => {
                     debug!(" => fn pointer created for function with #[track_caller]");
-                    Instance {
-                        def: InstanceDef::ReifyShim(def_id),
-                        substs,
-                    }
-                },
-                _ => resolved,
+                    resolved.def = InstanceDef::ReifyShim(def_id);
+                }
+                InstanceDef::Virtual(def_id, _) => {
+                    debug!(" => fn pointer created for virtual call");
+                    resolved.def = InstanceDef::ReifyShim(def_id);
+                }
+                _ => {}
             }
+
+            resolved
         })
     }
 
diff --git a/src/librustc/ty/layout.rs b/src/librustc/ty/layout.rs
index c67e6a0..7f93e8c 100644
--- a/src/librustc/ty/layout.rs
+++ b/src/librustc/ty/layout.rs
@@ -2339,6 +2339,76 @@
     }
 }
 
+
+impl<'tcx> ty::Instance<'tcx> {
+    // NOTE(eddyb) this is private to avoid using it from outside of
+    // `FnAbi::of_instance` - any other uses are either too high-level
+    // for `Instance` (e.g. typeck would use `Ty::fn_sig` instead),
+    // or should go through `FnAbi` instead, to avoid losing any
+    // adjustments `FnAbi::of_instance` might be performing.
+    fn fn_sig_for_fn_abi(&self, tcx: TyCtxt<'tcx>) -> ty::PolyFnSig<'tcx> {
+        let ty = self.ty(tcx);
+        match ty.kind {
+            ty::FnDef(..) |
+            // Shims currently have type FnPtr. Not sure this should remain.
+            ty::FnPtr(_) => {
+                let mut sig = ty.fn_sig(tcx);
+                if let ty::InstanceDef::VtableShim(..) = self.def {
+                    // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`.
+                    sig = sig.map_bound(|mut sig| {
+                        let mut inputs_and_output = sig.inputs_and_output.to_vec();
+                        inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]);
+                        sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
+                        sig
+                    });
+                }
+                sig
+            }
+            ty::Closure(def_id, substs) => {
+                let sig = substs.as_closure().sig(def_id, tcx);
+
+                let env_ty = tcx.closure_env_ty(def_id, substs).unwrap();
+                sig.map_bound(|sig| tcx.mk_fn_sig(
+                    iter::once(*env_ty.skip_binder()).chain(sig.inputs().iter().cloned()),
+                    sig.output(),
+                    sig.c_variadic,
+                    sig.unsafety,
+                    sig.abi
+                ))
+            }
+            ty::Generator(def_id, substs, _) => {
+                let sig = substs.as_generator().poly_sig(def_id, tcx);
+
+                let env_region = ty::ReLateBound(ty::INNERMOST, ty::BrEnv);
+                let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty);
+
+                let pin_did = tcx.lang_items().pin_type().unwrap();
+                let pin_adt_ref = tcx.adt_def(pin_did);
+                let pin_substs = tcx.intern_substs(&[env_ty.into()]);
+                let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs);
+
+                sig.map_bound(|sig| {
+                    let state_did = tcx.lang_items().gen_state().unwrap();
+                    let state_adt_ref = tcx.adt_def(state_did);
+                    let state_substs = tcx.intern_substs(&[
+                        sig.yield_ty.into(),
+                        sig.return_ty.into(),
+                    ]);
+                    let ret_ty = tcx.mk_adt(state_adt_ref, state_substs);
+
+                    tcx.mk_fn_sig(iter::once(env_ty),
+                        ret_ty,
+                        false,
+                        hir::Unsafety::Normal,
+                        rustc_target::spec::abi::Abi::Rust
+                    )
+                })
+            }
+            _ => bug!("unexpected type {:?} in Instance::fn_sig", ty)
+        }
+    }
+}
+
 pub trait FnAbiExt<'tcx, C>
 where
     C: LayoutOf<Ty = Ty<'tcx>, TyLayout = TyLayout<'tcx>>
@@ -2347,12 +2417,22 @@
         + HasTyCtxt<'tcx>
         + HasParamEnv<'tcx>,
 {
-    fn of_instance(cx: &C, instance: ty::Instance<'tcx>) -> Self;
-    fn new(cx: &C, sig: ty::FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self;
-    fn new_vtable(cx: &C, sig: ty::FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self;
+    /// Compute a `FnAbi` suitable for indirect calls, i.e. to `fn` pointers.
+    ///
+    /// NB: this doesn't handle virtual calls - those should use `FnAbi::of_instance`
+    /// instead, where the instance is a `InstanceDef::Virtual`.
+    fn of_fn_ptr(cx: &C, sig: ty::PolyFnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self;
+
+    /// Compute a `FnAbi` suitable for declaring/defining an `fn` instance, and for
+    /// direct calls to an `fn`.
+    ///
+    /// NB: that includes virtual calls, which are represented by "direct calls"
+    /// to a `InstanceDef::Virtual` instance (of `<dyn Trait as Trait>::fn`).
+    fn of_instance(cx: &C, instance: ty::Instance<'tcx>, extra_args: &[Ty<'tcx>]) -> Self;
+
     fn new_internal(
         cx: &C,
-        sig: ty::FnSig<'tcx>,
+        sig: ty::PolyFnSig<'tcx>,
         extra_args: &[Ty<'tcx>],
         mk_arg_type: impl Fn(Ty<'tcx>, Option<usize>) -> ArgAbi<'tcx, Ty<'tcx>>,
     ) -> Self;
@@ -2367,25 +2447,19 @@
         + HasTyCtxt<'tcx>
         + HasParamEnv<'tcx>,
 {
-    fn of_instance(cx: &C, instance: ty::Instance<'tcx>) -> Self {
-        let sig = instance.fn_sig(cx.tcx());
-        let sig = cx
-            .tcx()
-            .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig);
-        call::FnAbi::new(cx, sig, &[])
-    }
-
-    fn new(cx: &C, sig: ty::FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self {
+    fn of_fn_ptr(cx: &C, sig: ty::PolyFnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self {
         call::FnAbi::new_internal(cx, sig, extra_args, |ty, _| ArgAbi::new(cx.layout_of(ty)))
     }
 
-    fn new_vtable(cx: &C, sig: ty::FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self {
-        FnAbiExt::new_internal(cx, sig, extra_args, |ty, arg_idx| {
+    fn of_instance(cx: &C, instance: ty::Instance<'tcx>, extra_args: &[Ty<'tcx>]) -> Self {
+        let sig = instance.fn_sig_for_fn_abi(cx.tcx());
+
+        call::FnAbi::new_internal(cx, sig, extra_args, |ty, arg_idx| {
             let mut layout = cx.layout_of(ty);
             // Don't pass the vtable, it's not an argument of the virtual fn.
             // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait`
             // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen
-            if arg_idx == Some(0) {
+            if let (ty::InstanceDef::Virtual(..), Some(0)) = (&instance.def, arg_idx) {
                 let fat_pointer_ty = if layout.is_unsized() {
                     // unsized `self` is passed as a pointer to `self`
                     // FIXME (mikeyhew) change this to use &own if it is ever added to the language
@@ -2436,15 +2510,19 @@
 
     fn new_internal(
         cx: &C,
-        sig: ty::FnSig<'tcx>,
+        sig: ty::PolyFnSig<'tcx>,
         extra_args: &[Ty<'tcx>],
         mk_arg_type: impl Fn(Ty<'tcx>, Option<usize>) -> ArgAbi<'tcx, Ty<'tcx>>,
     ) -> Self {
         debug!("FnAbi::new_internal({:?}, {:?})", sig, extra_args);
 
+        let sig = cx
+            .tcx()
+            .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig);
+
         use rustc_target::spec::abi::Abi::*;
         let conv = match cx.tcx().sess.target.target.adjust_abi(sig.abi) {
-            RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::C,
+            RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust,
 
             // It's the ABI's job to select this, not ours.
             System => bug!("system abi should be selected elsewhere"),
diff --git a/src/librustc/ty/mod.rs b/src/librustc/ty/mod.rs
index 8ccfc46..05a2704 100644
--- a/src/librustc/ty/mod.rs
+++ b/src/librustc/ty/mod.rs
@@ -18,7 +18,7 @@
 use crate::middle::cstore::CrateStoreDyn;
 use crate::middle::lang_items::{FnTraitLangItem, FnMutTraitLangItem, FnOnceTraitLangItem};
 use crate::middle::resolve_lifetime::ObjectLifetimeDefault;
-use crate::mir::Body;
+use crate::mir::ReadOnlyBodyCache;
 use crate::mir::interpret::{GlobalId, ErrorHandled};
 use crate::mir::GeneratorLayout;
 use crate::session::CrateDisambiguator;
@@ -2784,11 +2784,7 @@
             }
         };
 
-        if is_associated_item {
-            Some(self.associated_item(def_id))
-        } else {
-            None
-        }
+        is_associated_item.then(|| self.associated_item(def_id))
     }
 
     fn associated_item_from_trait_item_ref(self,
@@ -2985,10 +2981,10 @@
     }
 
     /// Returns the possibly-auto-generated MIR of a `(DefId, Subst)` pair.
-    pub fn instance_mir(self, instance: ty::InstanceDef<'tcx>) -> &'tcx Body<'tcx> {
+    pub fn instance_mir(self, instance: ty::InstanceDef<'tcx>) -> ReadOnlyBodyCache<'tcx, 'tcx> {
         match instance {
             ty::InstanceDef::Item(did) => {
-                self.optimized_mir(did)
+                self.optimized_mir(did).unwrap_read_only()
             }
             ty::InstanceDef::VtableShim(..) |
             ty::InstanceDef::ReifyShim(..) |
@@ -2998,7 +2994,7 @@
             ty::InstanceDef::ClosureOnceShim { .. } |
             ty::InstanceDef::DropGlue(..) |
             ty::InstanceDef::CloneShim(..) => {
-                self.mir_shims(instance)
+                self.mir_shims(instance).unwrap_read_only()
             }
         }
     }
@@ -3253,7 +3249,7 @@
     let unnormalized_env = ty::ParamEnv::new(
         tcx.intern_predicates(&predicates),
         traits::Reveal::UserFacing,
-        if tcx.sess.opts.debugging_opts.chalk { Some(def_id) } else { None }
+        tcx.sess.opts.debugging_opts.chalk.then_some(def_id),
     );
 
     let body_id = tcx.hir().as_local_hir_id(def_id).map_or(hir::DUMMY_HIR_ID, |id| {
diff --git a/src/librustc/ty/print/pretty.rs b/src/librustc/ty/print/pretty.rs
index c941b3e..745f7d0 100644
--- a/src/librustc/ty/print/pretty.rs
+++ b/src/librustc/ty/print/pretty.rs
@@ -682,7 +682,7 @@
                 // FIXME(eddyb) should use `def_span`.
                 if let Some(hir_id) = self.tcx().hir().as_local_hir_id(did) {
                     if self.tcx().sess.opts.debugging_opts.span_free_formats {
-                        p!(write("@{:?}", hir_id));
+                        p!(write("@"), print_def_path(did, substs));
                     } else {
                         p!(write("@{:?}", self.tcx().hir().span(hir_id)));
                     }
@@ -1282,6 +1282,9 @@
             if !self.empty_path {
                 write!(self, "::")?;
             }
+            if ast::Ident::from_str(&name).is_raw_guess() {
+                write!(self, "r#")?;
+            }
             write!(self, "{}", name)?;
 
             // FIXME(eddyb) this will print e.g. `{{closure}}#3`, but it
diff --git a/src/librustc/ty/query/job.rs b/src/librustc/ty/query/job.rs
index e5f4e79..2f30b79 100644
--- a/src/librustc/ty/query/job.rs
+++ b/src/librustc/ty/query/job.rs
@@ -303,13 +303,8 @@
         return true;
     }
 
-    visit_waiters(query, |_, successor| {
-        if connected_to_root(successor, visited) {
-            Some(None)
-        } else {
-            None
-        }
-    }).is_some()
+    visit_waiters(query, |_, successor| connected_to_root(successor, visited).then_some(None))
+        .is_some()
 }
 
 // Deterministically pick an query from a list
diff --git a/src/librustc/ty/structural_impls.rs b/src/librustc/ty/structural_impls.rs
index ffbf881..ce76a4c 100644
--- a/src/librustc/ty/structural_impls.rs
+++ b/src/librustc/ty/structural_impls.rs
@@ -3,6 +3,7 @@
 //! hand, though we've recently added some macros and proc-macros to help with the tedium.
 
 use crate::hir::def::Namespace;
+use crate::hir::def_id::CRATE_DEF_INDEX;
 use crate::mir::ProjectionKind;
 use crate::mir::interpret;
 use crate::ty::{self, Lift, Ty, TyCtxt, InferConst};
@@ -95,8 +96,11 @@
         match *self {
             ty::BrAnon(n) => write!(f, "BrAnon({:?})", n),
             ty::BrNamed(did, name) => {
-                write!(f, "BrNamed({:?}:{:?}, {})",
-                        did.krate, did.index, name)
+                if did.index == CRATE_DEF_INDEX {
+                    write!(f, "BrNamed({})", name)
+                } else {
+                    write!(f, "BrNamed({:?}, {})", did, name)
+                }
             }
             ty::BrEnv => write!(f, "BrEnv"),
         }
diff --git a/src/librustc/util/common.rs b/src/librustc/util/common.rs
index 8581a5b..7515d30 100644
--- a/src/librustc/util/common.rs
+++ b/src/librustc/util/common.rs
@@ -88,15 +88,7 @@
              what);
 }
 
-// Hack up our own formatting for the duration to make it easier for scripts
-// to parse (always use the same number of decimal places and the same unit).
-pub fn duration_to_secs_str(dur: Duration) -> String {
-    const NANOS_PER_SEC: f64 = 1_000_000_000.0;
-    let secs = dur.as_secs() as f64 +
-               dur.subsec_nanos() as f64 / NANOS_PER_SEC;
-
-    format!("{:.3}", secs)
-}
+pub use rustc_session::utils::duration_to_secs_str;
 
 pub fn to_readable_str(mut val: usize) -> String {
     let mut groups = vec![];
diff --git a/src/librustc_codegen_llvm/abi.rs b/src/librustc_codegen_llvm/abi.rs
index 804a7af..1f3c8e1 100644
--- a/src/librustc_codegen_llvm/abi.rs
+++ b/src/librustc_codegen_llvm/abi.rs
@@ -372,7 +372,7 @@
 
     fn llvm_cconv(&self) -> llvm::CallConv {
         match self.conv {
-            Conv::C => llvm::CCallConv,
+            Conv::C | Conv::Rust => llvm::CCallConv,
             Conv::AmdGpuKernel => llvm::AmdGpuKernel,
             Conv::ArmAapcs => llvm::ArmAapcsCallConv,
             Conv::Msp430Intr => llvm::Msp430Intr,
@@ -388,6 +388,11 @@
     }
 
     fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) {
+        // FIXME(eddyb) can this also be applied to callsites?
+        if self.ret.layout.abi.is_uninhabited() {
+            llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn);
+        }
+
         let mut i = 0;
         let mut apply = |attrs: &ArgAttributes, ty: Option<&Type>| {
             attrs.apply_llfn(llvm::AttributePlace::Argument(i), llfn, ty);
diff --git a/src/librustc_codegen_llvm/attributes.rs b/src/librustc_codegen_llvm/attributes.rs
index 6f4e7d0..1ea9362 100644
--- a/src/librustc_codegen_llvm/attributes.rs
+++ b/src/librustc_codegen_llvm/attributes.rs
@@ -2,19 +2,20 @@
 
 use std::ffi::CString;
 
-use rustc::hir::{CodegenFnAttrFlags, CodegenFnAttrs};
+use rustc::hir::CodegenFnAttrFlags;
 use rustc::hir::def_id::{DefId, LOCAL_CRATE};
 use rustc::session::Session;
 use rustc::session::config::{Sanitizer, OptLevel};
-use rustc::ty::{self, TyCtxt, PolyFnSig};
+use rustc::ty::{self, TyCtxt, Ty};
 use rustc::ty::layout::HasTyCtxt;
 use rustc::ty::query::Providers;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_data_structures::fx::FxHashMap;
+use rustc_target::abi::call::Conv;
 use rustc_target::spec::PanicStrategy;
 use rustc_codegen_ssa::traits::*;
 
-use crate::abi::Abi;
+use crate::abi::FnAbi;
 use crate::attributes;
 use crate::llvm::{self, Attribute};
 use crate::llvm::AttributePlace::Function;
@@ -26,7 +27,7 @@
 
 /// Mark LLVM function to use provided inline heuristic.
 #[inline]
-pub fn inline(cx: &CodegenCx<'ll, '_>, val: &'ll Value, inline: InlineAttr) {
+fn inline(cx: &CodegenCx<'ll, '_>, val: &'ll Value, inline: InlineAttr) {
     use self::InlineAttr::*;
     match inline {
         Hint   => Attribute::InlineHint.apply_llfn(Function, val),
@@ -58,7 +59,7 @@
 
 /// Tell LLVM if this function should be 'naked', i.e., skip the epilogue and prologue.
 #[inline]
-pub fn naked(val: &'ll Value, is_naked: bool) {
+fn naked(val: &'ll Value, is_naked: bool) {
     Attribute::Naked.toggle_llfn(Function, val, is_naked);
 }
 
@@ -72,7 +73,7 @@
 
 /// Tell LLVM what instrument function to insert.
 #[inline]
-pub fn set_instrument_function(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
+fn set_instrument_function(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
     if cx.sess().instrument_mcount() {
         // Similar to `clang -pg` behavior. Handled by the
         // `post-inline-ee-instrument` LLVM pass.
@@ -88,7 +89,7 @@
     }
 }
 
-pub fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
+fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
     // Only use stack probes if the target specification indicates that we
     // should be using stack probes
     if !cx.sess().target.target.options.stack_probes {
@@ -202,11 +203,10 @@
 pub fn from_fn_attrs(
     cx: &CodegenCx<'ll, 'tcx>,
     llfn: &'ll Value,
-    id: Option<DefId>,
-    sig: PolyFnSig<'tcx>,
+    instance: ty::Instance<'tcx>,
+    fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
 ) {
-    let codegen_fn_attrs = id.map(|id| cx.tcx.codegen_fn_attrs(id))
-        .unwrap_or_else(|| CodegenFnAttrs::new());
+    let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id());
 
     match codegen_fn_attrs.optimize {
         OptimizeAttr::None => {
@@ -224,6 +224,11 @@
         }
     }
 
+    // FIXME(eddyb) consolidate these two `inline` calls (and avoid overwrites).
+    if instance.def.is_inline(cx.tcx) {
+        inline(cx, llfn, attributes::InlineAttr::Hint);
+    }
+
     inline(cx, llfn, codegen_fn_attrs.inline);
 
     // The `uwtable` attribute according to LLVM is:
@@ -276,8 +281,7 @@
         // Special attribute for allocator functions, which can't unwind.
         false
     } else {
-        let sig = cx.tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig);
-        if sig.abi == Abi::Rust || sig.abi == Abi::RustCall {
+        if fn_abi.conv == Conv::Rust {
             // Any Rust method (or `extern "Rust" fn` or `extern
             // "rust-call" fn`) is explicitly allowed to unwind
             // (unless it has no-unwind attribute, handled above).
@@ -331,16 +335,14 @@
     // Note that currently the `wasm-import-module` doesn't do anything, but
     // eventually LLVM 7 should read this and ferry the appropriate import
     // module to the output file.
-    if let Some(id) = id {
-        if cx.tcx.sess.target.target.arch == "wasm32" {
-            if let Some(module) = wasm_import_module(cx.tcx, id) {
-                llvm::AddFunctionAttrStringValue(
-                    llfn,
-                    llvm::AttributePlace::Function,
-                    const_cstr!("wasm-import-module"),
-                    &module,
-                );
-            }
+    if cx.tcx.sess.target.target.arch == "wasm32" {
+        if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) {
+            llvm::AddFunctionAttrStringValue(
+                llfn,
+                llvm::AttributePlace::Function,
+                const_cstr!("wasm-import-module"),
+                &module,
+            );
         }
     }
 }
@@ -373,11 +375,7 @@
         let native_libs = tcx.native_libraries(cnum);
 
         let def_id_to_native_lib = native_libs.iter().filter_map(|lib|
-            if let Some(id) = lib.foreign_module {
-                Some((id, lib))
-            } else {
-                None
-            }
+            lib.foreign_module.map(|id| (id, lib))
         ).collect::<FxHashMap<_, _>>();
 
         let mut ret = FxHashMap::default();
diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs
index d0b065c..858dd59 100644
--- a/src/librustc_codegen_llvm/back/lto.rs
+++ b/src/librustc_codegen_llvm/back/lto.rs
@@ -10,7 +10,7 @@
 use rustc_codegen_ssa::traits::*;
 use errors::{FatalError, Handler};
 use rustc::dep_graph::WorkProduct;
-use rustc::dep_graph::cgu_reuse_tracker::CguReuse;
+use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc::hir::def_id::LOCAL_CRATE;
 use rustc::middle::exported_symbols::SymbolExportLevel;
 use rustc::session::config::{self, Lto};
diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs
index 5dfb04a..ada29c3 100644
--- a/src/librustc_codegen_llvm/back/write.rs
+++ b/src/librustc_codegen_llvm/back/write.rs
@@ -22,7 +22,7 @@
 use rustc_data_structures::small_c_str::SmallCStr;
 use errors::{Handler, FatalError};
 
-use std::ffi::{CString, CStr};
+use std::ffi::CString;
 use std::fs;
 use std::io::{self, Write};
 use std::path::{Path, PathBuf};
@@ -167,7 +167,7 @@
     let emit_stack_size_section = sess.opts.debugging_opts.emit_stack_sizes;
 
     let asm_comments = sess.asm_comments();
-
+    let relax_elf_relocations = sess.target.target.options.relax_elf_relocations;
     Arc::new(move || {
         let tm = unsafe {
             llvm::LLVMRustCreateTargetMachine(
@@ -183,6 +183,7 @@
                 singlethread,
                 asm_comments,
                 emit_stack_size_section,
+                relax_elf_relocations,
             )
         };
 
@@ -365,20 +366,6 @@
 
             add_sanitizer_passes(config, &mut extra_passes);
 
-            for pass_name in &cgcx.plugin_passes {
-                if let Some(pass) = find_pass(pass_name) {
-                    extra_passes.push(pass);
-                } else {
-                    diag_handler.err(&format!("a plugin asked for LLVM pass \
-                                               `{}` but LLVM does not \
-                                               recognize it", pass_name));
-                }
-
-                if pass_name == "name-anon-globals" {
-                    have_name_anon_globals_pass = true;
-                }
-            }
-
             // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need
             // to make sure we run LLVM's NameAnonGlobals pass when emitting bitcode; otherwise
             // we'll get errors in LLVM.
@@ -601,14 +588,11 @@
                     cursor.position() as size_t
                 }
 
-                with_codegen(tm, llmod, config.no_builtins, |cpm| {
-                    let result =
-                        llvm::LLVMRustPrintModule(cpm, llmod, out_c.as_ptr(), demangle_callback);
-                    llvm::LLVMDisposePassManager(cpm);
-                    result.into_result().map_err(|()| {
-                        let msg = format!("failed to write LLVM IR to {}", out.display());
-                        llvm_err(diag_handler, &msg)
-                    })
+                let result =
+                    llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback);
+                result.into_result().map_err(|()| {
+                    let msg = format!("failed to write LLVM IR to {}", out.display());
+                    llvm_err(diag_handler, &msg)
                 })?;
             }
 
@@ -849,8 +833,8 @@
             })
             .filter_map(|val| {
                 // Exclude some symbols that we know are not Rust symbols.
-                let name = CStr::from_ptr(llvm::LLVMGetValueName(val));
-                if ignored(name.to_bytes()) {
+                let name = llvm::get_value_name(val);
+                if ignored(name) {
                     None
                 } else {
                     Some((val, name))
@@ -858,7 +842,7 @@
             })
             .map(move |(val, name)| {
                 let mut imp_name = prefix.as_bytes().to_vec();
-                imp_name.extend(name.to_bytes());
+                imp_name.extend(name);
                 let imp_name = CString::new(imp_name).unwrap();
                 (imp_name, val)
             })
diff --git a/src/librustc_codegen_llvm/callee.rs b/src/librustc_codegen_llvm/callee.rs
index 08fa23f..e0db7ca 100644
--- a/src/librustc_codegen_llvm/callee.rs
+++ b/src/librustc_codegen_llvm/callee.rs
@@ -4,6 +4,7 @@
 //! and methods are represented as just a fn ptr and not a full
 //! closure.
 
+use crate::abi::{FnAbi, FnAbiLlvmExt};
 use crate::attributes;
 use crate::llvm;
 use crate::context::CodegenCx;
@@ -11,7 +12,7 @@
 use rustc_codegen_ssa::traits::*;
 
 use rustc::ty::{TypeFoldable, Instance};
-use rustc::ty::layout::{LayoutOf, HasTyCtxt};
+use rustc::ty::layout::{FnAbiExt, HasTyCtxt};
 
 /// Codegens a reference to a fn/method item, monomorphizing and
 /// inlining as it goes.
@@ -32,19 +33,19 @@
     assert!(!instance.substs.has_escaping_bound_vars());
     assert!(!instance.substs.has_param_types());
 
-    let sig = instance.fn_sig(cx.tcx());
     if let Some(&llfn) = cx.instances.borrow().get(&instance) {
         return llfn;
     }
 
     let sym = tcx.symbol_name(instance).name.as_str();
-    debug!("get_fn({:?}: {:?}) => {}", instance, sig, sym);
+    debug!("get_fn({:?}: {:?}) => {}", instance, instance.ty(cx.tcx()), sym);
 
-    // Create a fn pointer with the substituted signature.
-    let fn_ptr_ty = tcx.mk_fn_ptr(sig);
-    let llptrty = cx.backend_type(cx.layout_of(fn_ptr_ty));
+    let fn_abi = FnAbi::of_instance(cx, instance, &[]);
 
     let llfn = if let Some(llfn) = cx.get_declared_value(&sym) {
+        // Create a fn pointer with the new signature.
+        let llptrty = fn_abi.ptr_to_llvm_type(cx);
+
         // This is subtle and surprising, but sometimes we have to bitcast
         // the resulting fn pointer.  The reason has to do with external
         // functions.  If you have two crates that both bind the same C
@@ -76,14 +77,10 @@
             llfn
         }
     } else {
-        let llfn = cx.declare_fn(&sym, sig);
-        assert_eq!(cx.val_ty(llfn), llptrty);
+        let llfn = cx.declare_fn(&sym, &fn_abi);
         debug!("get_fn: not casting pointer!");
 
-        if instance.def.is_inline(tcx) {
-            attributes::inline(cx, llfn, attributes::InlineAttr::Hint);
-        }
-        attributes::from_fn_attrs(cx, llfn, Some(instance.def.def_id()), sig);
+        attributes::from_fn_attrs(cx, llfn, instance, &fn_abi);
 
         let instance_def_id = instance.def_id();
 
diff --git a/src/librustc_codegen_llvm/common.rs b/src/librustc_codegen_llvm/common.rs
index f38f9df..419e99d 100644
--- a/src/librustc_codegen_llvm/common.rs
+++ b/src/librustc_codegen_llvm/common.rs
@@ -245,11 +245,7 @@
             let (mut lo, mut hi) = (0u64, 0u64);
             let success = llvm::LLVMRustConstInt128Get(v, sign_ext,
                                                         &mut hi, &mut lo);
-            if success {
-                Some(hi_lo_to_u128(lo, hi))
-            } else {
-                None
-            }
+            success.then_some(hi_lo_to_u128(lo, hi))
         })
     }
 
diff --git a/src/librustc_codegen_llvm/consts.rs b/src/librustc_codegen_llvm/consts.rs
index 541b3d9..297aff9 100644
--- a/src/librustc_codegen_llvm/consts.rs
+++ b/src/librustc_codegen_llvm/consts.rs
@@ -21,7 +21,7 @@
 
 use rustc::hir::{self, CodegenFnAttrs, CodegenFnAttrFlags};
 
-use std::ffi::{CStr, CString};
+use std::ffi::CStr;
 
 pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll Value {
     let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
@@ -392,16 +392,14 @@
             } else {
                 // If we created the global with the wrong type,
                 // correct the type.
-                let empty_string = const_cstr!("");
-                let name_str_ref = CStr::from_ptr(llvm::LLVMGetValueName(g));
-                let name_string = CString::new(name_str_ref.to_bytes()).unwrap();
-                llvm::LLVMSetValueName(g, empty_string.as_ptr());
+                let name = llvm::get_value_name(g).to_vec();
+                llvm::set_value_name(g, b"");
 
                 let linkage = llvm::LLVMRustGetLinkage(g);
                 let visibility = llvm::LLVMRustGetVisibility(g);
 
                 let new_g = llvm::LLVMRustGetOrInsertGlobal(
-                    self.llmod, name_string.as_ptr(), val_llty);
+                    self.llmod, name.as_ptr().cast(), name.len(), val_llty);
 
                 llvm::LLVMRustSetLinkage(new_g, linkage);
                 llvm::LLVMRustSetVisibility(new_g, visibility);
diff --git a/src/librustc_codegen_llvm/context.rs b/src/librustc_codegen_llvm/context.rs
index 4a40349..39ea1f6 100644
--- a/src/librustc_codegen_llvm/context.rs
+++ b/src/librustc_codegen_llvm/context.rs
@@ -1,3 +1,4 @@
+use crate::abi::FnAbi;
 use crate::attributes;
 use crate::llvm;
 use crate::llvm_util;
@@ -15,7 +16,7 @@
 use rustc::session::config::{self, DebugInfo};
 use rustc::session::Session;
 use rustc::ty::layout::{
-    LayoutError, LayoutOf, PointeeInfo, Size, TyLayout, VariantIdx, HasParamEnv
+    FnAbiExt, LayoutError, LayoutOf, PointeeInfo, Size, TyLayout, VariantIdx, HasParamEnv
 };
 use rustc::ty::{self, Ty, TyCtxt, Instance};
 use rustc::util::nodemap::FxHashMap;
@@ -420,7 +421,8 @@
             Abi::C
         ));
 
-        let llfn = self.declare_fn("rust_eh_unwind_resume", sig);
+        let fn_abi = FnAbi::of_fn_ptr(self, sig, &[]);
+        let llfn = self.declare_fn("rust_eh_unwind_resume", &fn_abi);
         attributes::apply_target_cpu_attr(self, llfn);
         unwresume.set(Some(llfn));
         llfn
diff --git a/src/librustc_codegen_llvm/debuginfo/mod.rs b/src/librustc_codegen_llvm/debuginfo/mod.rs
index 7713fe4..a3782ec 100644
--- a/src/librustc_codegen_llvm/debuginfo/mod.rs
+++ b/src/librustc_codegen_llvm/debuginfo/mod.rs
@@ -16,7 +16,7 @@
 use rustc::hir::def_id::{DefId, CrateNum, LOCAL_CRATE};
 use rustc::ty::subst::{SubstsRef, GenericArgKind};
 
-use crate::abi::Abi;
+use crate::abi::FnAbi;
 use crate::common::CodegenCx;
 use crate::builder::Builder;
 use crate::value::Value;
@@ -32,7 +32,7 @@
 
 use libc::c_uint;
 use std::cell::RefCell;
-use std::ffi::{CStr, CString};
+use std::ffi::CString;
 
 use smallvec::SmallVec;
 use syntax_pos::{self, BytePos, Span, Pos};
@@ -255,23 +255,11 @@
             return;
         }
 
-        let old_name = unsafe {
-            CStr::from_ptr(llvm::LLVMGetValueName(value))
-        };
-        match old_name.to_str() {
-            Ok("") => {}
-            Ok(_) => {
-                // Avoid replacing the name if it already exists.
-                // While we could combine the names somehow, it'd
-                // get noisy quick, and the usefulness is dubious.
-                return;
-            }
-            Err(_) => return,
-        }
-
-        let cname = SmallCStr::new(name);
-        unsafe {
-            llvm::LLVMSetValueName(value, cname.as_ptr());
+        // Avoid replacing the name if it already exists.
+        // While we could combine the names somehow, it'd
+        // get noisy quick, and the usefulness is dubious.
+        if llvm::get_value_name(value).is_empty() {
+            llvm::set_value_name(value, name.as_bytes());
         }
     }
 }
@@ -280,7 +268,7 @@
     fn create_function_debug_context(
         &self,
         instance: Instance<'tcx>,
-        sig: ty::FnSig<'tcx>,
+        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         llfn: &'ll Value,
         mir: &mir::Body<'_>,
     ) -> Option<FunctionDebugContext<&'ll DIScope>> {
@@ -308,7 +296,7 @@
         let file_metadata = file_metadata(self, &loc.file.name, def_id.krate);
 
         let function_type_metadata = unsafe {
-            let fn_signature = get_function_signature(self, sig);
+            let fn_signature = get_function_signature(self, fn_abi);
             llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(self), file_metadata, fn_signature)
         };
 
@@ -338,7 +326,7 @@
 
         let mut flags = DIFlags::FlagPrototyped;
 
-        if self.layout_of(sig.output()).abi.is_uninhabited() {
+        if fn_abi.ret.layout.abi.is_uninhabited() {
             flags |= DIFlags::FlagNoReturn;
         }
 
@@ -392,25 +380,20 @@
 
         fn get_function_signature<'ll, 'tcx>(
             cx: &CodegenCx<'ll, 'tcx>,
-            sig: ty::FnSig<'tcx>,
+            fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         ) -> &'ll DIArray {
             if cx.sess().opts.debuginfo == DebugInfo::Limited {
                 return create_DIArray(DIB(cx), &[]);
             }
 
-            let mut signature = Vec::with_capacity(sig.inputs().len() + 1);
+            let mut signature = Vec::with_capacity(fn_abi.args.len() + 1);
 
             // Return type -- llvm::DIBuilder wants this at index 0
-            signature.push(match sig.output().kind {
-                ty::Tuple(ref tys) if tys.is_empty() => None,
-                _ => Some(type_metadata(cx, sig.output(), syntax_pos::DUMMY_SP))
-            });
-
-            let inputs = if sig.abi == Abi::RustCall {
-                &sig.inputs()[..sig.inputs().len() - 1]
+            signature.push(if fn_abi.ret.is_ignore() {
+                None
             } else {
-                sig.inputs()
-            };
+                Some(type_metadata(cx, fn_abi.ret.layout.ty, syntax_pos::DUMMY_SP))
+            });
 
             // Arguments types
             if cx.sess().target.target.options.is_like_msvc {
@@ -424,7 +407,8 @@
                 // and a function `fn bar(x: [(); 7])` as `fn bar(x: *const ())`.
                 // This transformed type is wrong, but these function types are
                 // already inaccurate due to ABI adjustments (see #42800).
-                signature.extend(inputs.iter().map(|&t| {
+                signature.extend(fn_abi.args.iter().map(|arg| {
+                    let t = arg.layout.ty;
                     let t = match t.kind {
                         ty::Array(ct, _)
                             if (ct == cx.tcx.types.u8) || cx.layout_of(ct).is_zst() => {
@@ -435,21 +419,11 @@
                     Some(type_metadata(cx, t, syntax_pos::DUMMY_SP))
                 }));
             } else {
-                signature.extend(inputs.iter().map(|t| {
-                    Some(type_metadata(cx, t, syntax_pos::DUMMY_SP))
+                signature.extend(fn_abi.args.iter().map(|arg| {
+                    Some(type_metadata(cx, arg.layout.ty, syntax_pos::DUMMY_SP))
                 }));
             }
 
-            if sig.abi == Abi::RustCall && !sig.inputs().is_empty() {
-                if let ty::Tuple(args) = sig.inputs()[sig.inputs().len() - 1].kind {
-                    signature.extend(
-                        args.iter().map(|argument_type| {
-                            Some(type_metadata(cx, argument_type.expect_ty(), syntax_pos::DUMMY_SP))
-                        })
-                    );
-                }
-            }
-
             create_DIArray(DIB(cx), &signature[..])
         }
 
diff --git a/src/librustc_codegen_llvm/declare.rs b/src/librustc_codegen_llvm/declare.rs
index 46cdd2a..fa9fc46 100644
--- a/src/librustc_codegen_llvm/declare.rs
+++ b/src/librustc_codegen_llvm/declare.rs
@@ -18,8 +18,7 @@
 use crate::context::CodegenCx;
 use crate::type_::Type;
 use crate::value::Value;
-use rustc::ty::{self, PolyFnSig};
-use rustc::ty::layout::{FnAbiExt, LayoutOf};
+use rustc::ty::Ty;
 use rustc::session::config::Sanitizer;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_codegen_ssa::traits::*;
@@ -77,9 +76,8 @@
         name: &str, ty: &'ll Type
     ) -> &'ll Value {
         debug!("declare_global(name={:?})", name);
-        let namebuf = SmallCStr::new(name);
         unsafe {
-            llvm::LLVMRustGetOrInsertGlobal(self.llmod, namebuf.as_ptr(), ty)
+            llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_ptr().cast(), name.len(), ty)
         }
     }
 
@@ -94,21 +92,12 @@
     fn declare_fn(
         &self,
         name: &str,
-        sig: PolyFnSig<'tcx>,
+        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
     ) -> &'ll Value {
-        debug!("declare_rust_fn(name={:?}, sig={:?})", name, sig);
-        let sig = self.tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig);
-        debug!("declare_rust_fn (after region erasure) sig={:?}", sig);
+        debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi);
 
-        let fn_abi = FnAbi::new(self, sig, &[]);
         let llfn = declare_raw_fn(self, name, fn_abi.llvm_cconv(), fn_abi.llvm_type(self));
-
-        if self.layout_of(sig.output()).abi.is_uninhabited() {
-            llvm::Attribute::NoReturn.apply_llfn(Function, llfn);
-        }
-
         fn_abi.apply_attrs_llfn(self, llfn);
-
         llfn
     }
 
@@ -130,28 +119,6 @@
         }
     }
 
-    fn define_fn(
-        &self,
-        name: &str,
-        fn_sig: PolyFnSig<'tcx>,
-    ) -> &'ll Value {
-        if self.get_defined_value(name).is_some() {
-            self.sess().fatal(&format!("symbol `{}` already defined", name))
-        } else {
-            self.declare_fn(name, fn_sig)
-        }
-    }
-
-    fn define_internal_fn(
-        &self,
-        name: &str,
-        fn_sig: PolyFnSig<'tcx>,
-    ) -> &'ll Value {
-        let llfn = self.define_fn(name, fn_sig);
-        unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) };
-        llfn
-    }
-
     fn get_declared_value(&self, name: &str) -> Option<&'ll Value> {
         debug!("get_declared_value(name={:?})", name);
         let namebuf = SmallCStr::new(name);
diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs
index aa55f3a..1767ad1 100644
--- a/src/librustc_codegen_llvm/intrinsic.rs
+++ b/src/librustc_codegen_llvm/intrinsic.rs
@@ -1,4 +1,3 @@
-use crate::attributes;
 use crate::llvm;
 use crate::llvm_util;
 use crate::abi::{Abi, FnAbi, LlvmType, PassMode};
@@ -14,7 +13,7 @@
 use rustc_codegen_ssa::glue;
 use rustc_codegen_ssa::base::{to_immediate, wants_msvc_seh, compare_simd_types};
 use rustc::ty::{self, Ty};
-use rustc::ty::layout::{self, LayoutOf, HasTyCtxt, Primitive};
+use rustc::ty::layout::{self, FnAbiExt, LayoutOf, HasTyCtxt, Primitive};
 use rustc::mir::interpret::GlobalId;
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
 use rustc::hir;
@@ -442,32 +441,11 @@
                                 let is_left = name == "rotate_left";
                                 let val = args[0].immediate();
                                 let raw_shift = args[1].immediate();
-                                if llvm_util::get_major_version() >= 7 {
-                                    // rotate = funnel shift with first two args the same
-                                    let llvm_name = &format!("llvm.fsh{}.i{}",
-                                                            if is_left { 'l' } else { 'r' }, width);
-                                    let llfn = self.get_intrinsic(llvm_name);
-                                    self.call(llfn, &[val, val, raw_shift], None)
-                                } else {
-                                    // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
-                                    // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
-                                    let width = self.const_uint(
-                                        self.type_ix(width),
-                                        width,
-                                    );
-                                    let shift = self.urem(raw_shift, width);
-                                    let width_minus_raw_shift = self.sub(width, raw_shift);
-                                    let inv_shift = self.urem(width_minus_raw_shift, width);
-                                    let shift1 = self.shl(
-                                        val,
-                                        if is_left { shift } else { inv_shift },
-                                    );
-                                    let shift2 = self.lshr(
-                                        val,
-                                        if !is_left { shift } else { inv_shift },
-                                    );
-                                    self.or(shift1, shift2)
-                                }
+                                // rotate = funnel shift with first two args the same
+                                let llvm_name = &format!("llvm.fsh{}.i{}",
+                                                        if is_left { 'l' } else { 'r' }, width);
+                                let llfn = self.get_intrinsic(llvm_name);
+                                self.call(llfn, &[val, val, raw_shift], None)
                             },
                             "saturating_add" | "saturating_sub" => {
                                 let is_add = name == "saturating_add";
@@ -538,9 +516,36 @@
                         return;
                     }
                 }
-
             },
 
+            "float_to_int_approx_unchecked" => {
+                if float_type_width(arg_tys[0]).is_none() {
+                    span_invalid_monomorphization_error(
+                        tcx.sess, span,
+                        &format!("invalid monomorphization of `float_to_int_approx_unchecked` \
+                                  intrinsic: expected basic float type, \
+                                  found `{}`", arg_tys[0]));
+                    return;
+                }
+                match int_type_width_signed(ret_ty, self.cx) {
+                    Some((width, signed)) => {
+                        if signed {
+                            self.fptosi(args[0].immediate(), self.cx.type_ix(width))
+                        } else {
+                            self.fptoui(args[0].immediate(), self.cx.type_ix(width))
+                        }
+                    }
+                    None => {
+                        span_invalid_monomorphization_error(
+                            tcx.sess, span,
+                            &format!("invalid monomorphization of `float_to_int_approx_unchecked` \
+                                      intrinsic:  expected basic integer type, \
+                                      found `{}`", ret_ty));
+                        return;
+                    }
+                }
+            }
+
             "discriminant_value" => {
                 args[0].deref(self.cx()).codegen_get_discr(self, ret_ty)
             }
@@ -1013,8 +1018,10 @@
         hir::Unsafety::Unsafe,
         Abi::Rust
     ));
-    let llfn = cx.define_internal_fn(name, rust_fn_sig);
-    attributes::from_fn_attrs(cx, llfn, None, rust_fn_sig);
+    let fn_abi = FnAbi::of_fn_ptr(cx, rust_fn_sig, &[]);
+    let llfn = cx.declare_fn(name, &fn_abi);
+    // FIXME(eddyb) find a nicer way to do this.
+    unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) };
     let bx = Builder::new_block(cx, llfn, "entry-block");
     codegen(bx);
     llfn
diff --git a/src/librustc_codegen_llvm/lib.rs b/src/librustc_codegen_llvm/lib.rs
index acc221f..00a84f8 100644
--- a/src/librustc_codegen_llvm/lib.rs
+++ b/src/librustc_codegen_llvm/lib.rs
@@ -6,6 +6,7 @@
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/")]
 
+#![feature(bool_to_option)]
 #![feature(box_patterns)]
 #![feature(box_syntax)]
 #![feature(const_cstr_unchecked)]
@@ -43,6 +44,7 @@
 extern crate syntax;
 extern crate syntax_pos;
 extern crate rustc_errors as errors;
+extern crate rustc_session;
 
 use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::back::write::{CodegenContext, ModuleConfig, FatLTOInput};
diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs
index a49e863..5da3275 100644
--- a/src/librustc_codegen_llvm/llvm/ffi.rs
+++ b/src/librustc_codegen_llvm/llvm/ffi.rs
@@ -701,8 +701,8 @@
 
     // Operations on all values
     pub fn LLVMTypeOf(Val: &Value) -> &Type;
-    pub fn LLVMGetValueName(Val: &Value) -> *const c_char;
-    pub fn LLVMSetValueName(Val: &Value, Name: *const c_char);
+    pub fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char;
+    pub fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t);
     pub fn LLVMReplaceAllUsesWith(OldVal: &'a Value, NewVal: &'a Value);
     pub fn LLVMSetMetadata(Val: &'a Value, KindID: c_uint, Node: &'a Value);
 
@@ -774,7 +774,8 @@
     pub fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>;
     pub fn LLVMAddGlobal(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value;
     pub fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>;
-    pub fn LLVMRustGetOrInsertGlobal(M: &'a Module, Name: *const c_char, T: &'a Type) -> &'a Value;
+    pub fn LLVMRustGetOrInsertGlobal(M: &'a Module, Name: *const c_char, NameLen: size_t,
+                                     T: &'a Type) -> &'a Value;
     pub fn LLVMRustInsertPrivateGlobal(M: &'a Module, T: &'a Type) -> &'a Value;
     pub fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>;
     pub fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>;
@@ -1702,7 +1703,8 @@
                                        TrapUnreachable: bool,
                                        Singlethread: bool,
                                        AsmComments: bool,
-                                       EmitStackSizeSection: bool)
+                                       EmitStackSizeSection: bool,
+                                       RelaxELFRelocations: bool)
                                        -> Option<&'static mut TargetMachine>;
     pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine);
     pub fn LLVMRustAddBuilderLibraryInfo(PMB: &'a PassManagerBuilder,
@@ -1726,8 +1728,7 @@
                                    Output: *const c_char,
                                    FileType: FileType)
                                    -> LLVMRustResult;
-    pub fn LLVMRustPrintModule(PM: &PassManager<'a>,
-                               M: &'a Module,
+    pub fn LLVMRustPrintModule(M: &'a Module,
                                Output: *const c_char,
                                Demangle: extern fn(*const c_char,
                                                    size_t,
@@ -1811,7 +1812,7 @@
 
     pub fn LLVMRustPositionBuilderAtStart(B: &Builder<'a>, BB: &'a BasicBlock);
 
-    pub fn LLVMRustSetComdat(M: &'a Module, V: &'a Value, Name: *const c_char);
+    pub fn LLVMRustSetComdat(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t);
     pub fn LLVMRustUnsetComdat(V: &Value);
     pub fn LLVMRustSetModulePICLevel(M: &Module);
     pub fn LLVMRustSetModulePIELevel(M: &Module);
diff --git a/src/librustc_codegen_llvm/llvm/mod.rs b/src/librustc_codegen_llvm/llvm/mod.rs
index 5781593..d2d4187 100644
--- a/src/librustc_codegen_llvm/llvm/mod.rs
+++ b/src/librustc_codegen_llvm/llvm/mod.rs
@@ -115,7 +115,8 @@
 // For more details on COMDAT sections see e.g., http://www.airs.com/blog/archives/52
 pub fn SetUniqueComdat(llmod: &Module, val: &'a Value) {
     unsafe {
-        LLVMRustSetComdat(llmod, val, LLVMGetValueName(val));
+        let name = get_value_name(val);
+        LLVMRustSetComdat(llmod, val, name.as_ptr().cast(), name.len());
     }
 }
 
@@ -217,6 +218,23 @@
     }
 }
 
+/// Safe wrapper for `LLVMGetValueName2` into a byte slice
+pub fn get_value_name(value: &'a Value) -> &'a [u8] {
+    unsafe {
+        let mut len = 0;
+        let data = LLVMGetValueName2(value, &mut len);
+        std::slice::from_raw_parts(data.cast(), len)
+    }
+}
+
+/// Safe wrapper for `LLVMSetValueName2` from a byte slice
+pub fn set_value_name(value: &Value, name: &[u8]) {
+    unsafe {
+        let data = name.as_ptr().cast();
+        LLVMSetValueName2(value, data, name.len());
+    }
+}
+
 pub fn build_string(f: impl FnOnce(&RustString)) -> Result<String, FromUtf8Error> {
     let sr = RustString {
         bytes: RefCell::new(Vec::new()),
diff --git a/src/librustc_codegen_llvm/llvm_util.rs b/src/librustc_codegen_llvm/llvm_util.rs
index 7bff9e6..72612c4 100644
--- a/src/librustc_codegen_llvm/llvm_util.rs
+++ b/src/librustc_codegen_llvm/llvm_util.rs
@@ -108,6 +108,8 @@
     ("rclass", Some(sym::arm_target_feature)),
     ("dsp", Some(sym::arm_target_feature)),
     ("neon", Some(sym::arm_target_feature)),
+    ("crc", Some(sym::arm_target_feature)),
+    ("crypto", Some(sym::arm_target_feature)),
     ("v5te", Some(sym::arm_target_feature)),
     ("v6", Some(sym::arm_target_feature)),
     ("v6k", Some(sym::arm_target_feature)),
diff --git a/src/librustc_codegen_llvm/mono_item.rs b/src/librustc_codegen_llvm/mono_item.rs
index c1703ff..cbc8af4 100644
--- a/src/librustc_codegen_llvm/mono_item.rs
+++ b/src/librustc_codegen_llvm/mono_item.rs
@@ -1,3 +1,4 @@
+use crate::abi::FnAbi;
 use crate::attributes;
 use crate::base;
 use crate::context::CodegenCx;
@@ -6,7 +7,7 @@
 use rustc::hir::def_id::{DefId, LOCAL_CRATE};
 use rustc::mir::mono::{Linkage, Visibility};
 use rustc::ty::{TypeFoldable, Instance};
-use rustc::ty::layout::{LayoutOf, HasTyCtxt};
+use rustc::ty::layout::{FnAbiExt, LayoutOf};
 use rustc_codegen_ssa::traits::*;
 
 pub use rustc::mir::mono::MonoItem;
@@ -42,10 +43,10 @@
         assert!(!instance.substs.needs_infer() &&
                 !instance.substs.has_param_types());
 
-        let mono_sig = instance.fn_sig(self.tcx());
-        let attrs = self.tcx.codegen_fn_attrs(instance.def_id());
-        let lldecl = self.declare_fn(symbol_name, mono_sig);
+        let fn_abi = FnAbi::of_instance(self, instance, &[]);
+        let lldecl = self.declare_fn(symbol_name, &fn_abi);
         unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) };
+        let attrs = self.tcx.codegen_fn_attrs(instance.def_id());
         base::set_link_section(lldecl, &attrs);
         if linkage == Linkage::LinkOnceODR ||
             linkage == Linkage::WeakODR {
@@ -67,16 +68,9 @@
             }
         }
 
-        debug!("predefine_fn: mono_sig = {:?} instance = {:?}", mono_sig, instance);
-        if instance.def.is_inline(self.tcx) {
-            attributes::inline(self, lldecl, attributes::InlineAttr::Hint);
-        }
-        attributes::from_fn_attrs(
-            self,
-            lldecl,
-            Some(instance.def.def_id()),
-            mono_sig,
-        );
+        debug!("predefine_fn: instance = {:?}", instance);
+
+        attributes::from_fn_attrs(self, lldecl, instance, &fn_abi);
 
         self.instances.borrow_mut().insert(instance, lldecl);
     }
diff --git a/src/librustc_codegen_llvm/type_of.rs b/src/librustc_codegen_llvm/type_of.rs
index c21e62e..d77bbb2 100644
--- a/src/librustc_codegen_llvm/type_of.rs
+++ b/src/librustc_codegen_llvm/type_of.rs
@@ -235,11 +235,7 @@
                     cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx))
                 }
                 ty::FnPtr(sig) => {
-                    let sig = cx.tcx.normalize_erasing_late_bound_regions(
-                        ty::ParamEnv::reveal_all(),
-                        &sig,
-                    );
-                    cx.fn_ptr_backend_type(&FnAbi::new(cx, sig, &[]))
+                    cx.fn_ptr_backend_type(&FnAbi::of_fn_ptr(cx, sig, &[]))
                 }
                 _ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO)
             };
diff --git a/src/librustc_codegen_ssa/Cargo.toml b/src/librustc_codegen_ssa/Cargo.toml
index 478c3a9..53d3c51 100644
--- a/src/librustc_codegen_ssa/Cargo.toml
+++ b/src/librustc_codegen_ssa/Cargo.toml
@@ -32,3 +32,4 @@
 rustc_index = { path = "../librustc_index" }
 rustc_target = { path = "../librustc_target" }
 rustc_error_codes = { path = "../librustc_error_codes" }
+rustc_session = { path = "../librustc_session" }
diff --git a/src/librustc_codegen_ssa/back/linker.rs b/src/librustc_codegen_ssa/back/linker.rs
index 999cc40..4278852 100644
--- a/src/librustc_codegen_ssa/back/linker.rs
+++ b/src/librustc_codegen_ssa/back/linker.rs
@@ -398,7 +398,8 @@
 
     fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType) {
         // Symbol visibility in object files typically takes care of this.
-        if crate_type == CrateType::Executable {
+        if crate_type == CrateType::Executable &&
+            self.sess.target.target.options.override_export_symbols.is_none() {
             return;
         }
 
diff --git a/src/librustc_codegen_ssa/back/rpath.rs b/src/librustc_codegen_ssa/back/rpath.rs
index e27cb6d..cd3d999 100644
--- a/src/librustc_codegen_ssa/back/rpath.rs
+++ b/src/librustc_codegen_ssa/back/rpath.rs
@@ -119,11 +119,7 @@
     use std::path::Component;
 
     if path.is_absolute() != base.is_absolute() {
-        if path.is_absolute() {
-            Some(PathBuf::from(path))
-        } else {
-            None
-        }
+        path.is_absolute().then(|| PathBuf::from(path))
     } else {
         let mut ita = path.components();
         let mut itb = base.components();
diff --git a/src/librustc_codegen_ssa/back/symbol_export.rs b/src/librustc_codegen_ssa/back/symbol_export.rs
index f8b3e0f..cea5dc1 100644
--- a/src/librustc_codegen_ssa/back/symbol_export.rs
+++ b/src/librustc_codegen_ssa/back/symbol_export.rs
@@ -85,11 +85,7 @@
             match tcx.hir().get(hir_id) {
                 Node::ForeignItem(..) => {
                     let def_id = tcx.hir().local_def_id(hir_id);
-                    if tcx.is_statically_included_foreign_item(def_id) {
-                        Some(def_id)
-                    } else {
-                        None
-                    }
+                    tcx.is_statically_included_foreign_item(def_id).then_some(def_id)
                 }
 
                 // Only consider nodes that actually have exported symbols.
diff --git a/src/librustc_codegen_ssa/back/write.rs b/src/librustc_codegen_ssa/back/write.rs
index 9d3e574..283295c 100644
--- a/src/librustc_codegen_ssa/back/write.rs
+++ b/src/librustc_codegen_ssa/back/write.rs
@@ -10,7 +10,7 @@
 use rustc_incremental::{copy_cgu_workproducts_to_incr_comp_cache_dir,
                         in_incr_comp_dir, in_incr_comp_dir_sess};
 use rustc::dep_graph::{WorkProduct, WorkProductId, WorkProductFileKind};
-use rustc::dep_graph::cgu_reuse_tracker::CguReuseTracker;
+use rustc_session::cgu_reuse_tracker::CguReuseTracker;
 use rustc::middle::cstore::EncodedMetadata;
 use rustc::session::config::{self, OutputFilenames, OutputType, Passes, Lto,
                              Sanitizer, SwitchWithOptPath};
@@ -231,8 +231,6 @@
     pub total_cgus: usize,
     // Handler to use for diagnostics produced during codegen.
     pub diag_emitter: SharedEmitter,
-    // LLVM passes added by plugins.
-    pub plugin_passes: Vec<String>,
     // LLVM optimizations for which we want to print remarks.
     pub remark: Passes,
     // Worker thread number
@@ -1028,7 +1026,6 @@
         time_passes: sess.time_extended(),
         prof: sess.prof.clone(),
         exported_symbols,
-        plugin_passes: sess.plugin_llvm_passes.borrow().clone(),
         remark: sess.opts.cg.remark.clone(),
         worker: 0,
         incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
@@ -1755,7 +1752,7 @@
             }
         };
 
-        sess.cgu_reuse_tracker.check_expected_reuse(sess);
+        sess.cgu_reuse_tracker.check_expected_reuse(sess.diagnostic());
 
         sess.abort_if_errors();
 
diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs
index 22693be..f6725e6 100644
--- a/src/librustc_codegen_ssa/base.rs
+++ b/src/librustc_codegen_ssa/base.rs
@@ -25,8 +25,8 @@
 use crate::mir::place::PlaceRef;
 use crate::traits::*;
 
-use rustc::dep_graph::cgu_reuse_tracker::CguReuse;
 use rustc::hir;
+use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc::hir::def_id::{DefId, LOCAL_CRATE};
 use rustc::middle::cstore::EncodedMetadata;
 use rustc::middle::lang_items::StartFnLangItem;
@@ -368,13 +368,7 @@
     // release builds.
     info!("codegen_instance({})", instance);
 
-    let sig = instance.fn_sig(cx.tcx());
-    let sig = cx.tcx().normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig);
-
-    let lldecl = cx.get_fn(instance);
-
-    let mir = cx.tcx().instance_mir(instance.def);
-    mir::codegen_mir::<Bx>(cx, lldecl, &mir, instance, sig);
+    mir::codegen_mir::<Bx>(cx, instance);
 }
 
 /// Creates the `main` function which will initialize the rust runtime and call
diff --git a/src/librustc_codegen_ssa/lib.rs b/src/librustc_codegen_ssa/lib.rs
index 9784d87..9919666 100644
--- a/src/librustc_codegen_ssa/lib.rs
+++ b/src/librustc_codegen_ssa/lib.rs
@@ -1,5 +1,6 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/")]
 
+#![feature(bool_to_option)]
 #![feature(box_patterns)]
 #![feature(box_syntax)]
 #![feature(core_intrinsics)]
@@ -68,22 +69,14 @@
                             emit_bc: bool,
                             emit_bc_compressed: bool,
                             outputs: &OutputFilenames) -> CompiledModule {
-        let object = if emit_obj {
-            Some(outputs.temp_path(OutputType::Object, Some(&self.name)))
-        } else {
-            None
-        };
-        let bytecode = if emit_bc {
-            Some(outputs.temp_path(OutputType::Bitcode, Some(&self.name)))
-        } else {
-            None
-        };
-        let bytecode_compressed = if emit_bc_compressed {
-            Some(outputs.temp_path(OutputType::Bitcode, Some(&self.name))
-                    .with_extension(RLIB_BYTECODE_EXTENSION))
-        } else {
-            None
-        };
+        let object = emit_obj
+            .then(|| outputs.temp_path(OutputType::Object, Some(&self.name)));
+        let bytecode = emit_bc
+            .then(|| outputs.temp_path(OutputType::Bitcode, Some(&self.name)));
+        let bytecode_compressed = emit_bc_compressed.then(|| {
+            outputs.temp_path(OutputType::Bitcode, Some(&self.name))
+                .with_extension(RLIB_BYTECODE_EXTENSION)
+        });
 
         CompiledModule {
             name: self.name.clone(),
diff --git a/src/librustc_codegen_ssa/mir/analyze.rs b/src/librustc_codegen_ssa/mir/analyze.rs
index e44551f..6c62708 100644
--- a/src/librustc_codegen_ssa/mir/analyze.rs
+++ b/src/librustc_codegen_ssa/mir/analyze.rs
@@ -29,7 +29,7 @@
         // FIXME(eddyb): We should figure out how to use llvm.dbg.value instead
         // of putting everything in allocas just so we can use llvm.dbg.declare.
         if fx.cx.sess().opts.debuginfo == DebugInfo::Full {
-            if mir.local_kind(local) == mir::LocalKind::Arg {
+            if fx.mir.local_kind(local) == mir::LocalKind::Arg {
                 analyzer.not_ssa(local);
                 continue;
             }
@@ -70,9 +70,10 @@
     fn new(fx: &'mir FunctionCx<'a, 'tcx, Bx>) -> Self {
         let invalid_location =
             mir::BasicBlock::new(fx.mir.basic_blocks().len()).start_location();
+        let dominators = fx.mir.dominators();
         let mut analyzer = LocalAnalyzer {
             fx,
-            dominators: fx.mir.dominators(),
+            dominators,
             non_ssa_locals: BitSet::new_empty(fx.mir.local_decls.len()),
             first_assignment: IndexVec::from_elem(invalid_location, &fx.mir.local_decls)
         };
@@ -130,7 +131,7 @@
             };
             if is_consume {
                 let base_ty =
-                    mir::Place::ty_from(place_ref.base, proj_base, self.fx.mir, cx.tcx());
+                    mir::Place::ty_from(place_ref.base, proj_base, *self.fx.mir, cx.tcx());
                 let base_ty = self.fx.monomorphize(&base_ty);
 
                 // ZSTs don't require any actual memory access.
diff --git a/src/librustc_codegen_ssa/mir/block.rs b/src/librustc_codegen_ssa/mir/block.rs
index 14be0e8..ce703f2 100644
--- a/src/librustc_codegen_ssa/mir/block.rs
+++ b/src/librustc_codegen_ssa/mir/block.rs
@@ -24,28 +24,28 @@
 
 /// Used by `FunctionCx::codegen_terminator` for emitting common patterns
 /// e.g., creating a basic block, calling a function, etc.
-struct TerminatorCodegenHelper<'a, 'tcx> {
-    bb: &'a mir::BasicBlock,
-    terminator: &'a mir::Terminator<'tcx>,
+struct TerminatorCodegenHelper<'tcx> {
+    bb: mir::BasicBlock,
+    terminator: &'tcx mir::Terminator<'tcx>,
     funclet_bb: Option<mir::BasicBlock>,
 }
 
-impl<'a, 'tcx> TerminatorCodegenHelper<'a, 'tcx> {
+impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// Returns the associated funclet from `FunctionCx::funclets` for the
     /// `funclet_bb` member if it is not `None`.
-    fn funclet<'c, 'b, Bx: BuilderMethods<'b, 'tcx>>(
+    fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &'c mut FunctionCx<'b, 'tcx, Bx>,
-    ) -> Option<&'c Bx::Funclet> {
+        fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
+    ) -> Option<&'b Bx::Funclet> {
         match self.funclet_bb {
             Some(funcl) => fx.funclets[funcl].as_ref(),
             None => None,
         }
     }
 
-    fn lltarget<'b, 'c, Bx: BuilderMethods<'b, 'tcx>>(
+    fn lltarget<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &'c mut FunctionCx<'b, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> (Bx::BasicBlock, bool) {
         let span = self.terminator.source_info.span;
@@ -63,9 +63,9 @@
     }
 
     /// Create a basic block.
-    fn llblock<'c, 'b, Bx: BuilderMethods<'b, 'tcx>>(
+    fn llblock<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &'c mut FunctionCx<'b, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> Bx::BasicBlock {
         let (lltarget, is_cleanupret) = self.lltarget(fx, target);
@@ -83,9 +83,9 @@
         }
     }
 
-    fn funclet_br<'c, 'b, Bx: BuilderMethods<'b, 'tcx>>(
+    fn funclet_br<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &'c mut FunctionCx<'b, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, 'tcx, Bx>,
         bx: &mut Bx,
         target: mir::BasicBlock,
     ) {
@@ -101,9 +101,9 @@
 
     /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional
     /// return destination `destination` and the cleanup function `cleanup`.
-    fn do_call<'c, 'b, Bx: BuilderMethods<'b, 'tcx>>(
+    fn do_call<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &'c mut FunctionCx<'b, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, 'tcx, Bx>,
         bx: &mut Bx,
         fn_abi: FnAbi<'tcx, Ty<'tcx>>,
         fn_ptr: Bx::Value,
@@ -132,7 +132,7 @@
         } else {
             let llret = bx.call(fn_ptr, &llargs, self.funclet(fx));
             bx.apply_attrs_callsite(&fn_abi, llret);
-            if fx.mir[*self.bb].is_cleanup {
+            if fx.mir[self.bb].is_cleanup {
                 // Cleanup is always the cold path. Don't inline
                 // drop glue. Also, when there is a deeply-nested
                 // struct, there are "symmetry" issues that cause
@@ -151,15 +151,15 @@
 
     // Generate sideeffect intrinsic if jumping to any of the targets can form
     // a loop.
-    fn maybe_sideeffect<'b, 'tcx2: 'b, Bx: BuilderMethods<'b, 'tcx2>>(
+    fn maybe_sideeffect<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        mir: &'b mir::Body<'tcx>,
+        mir: mir::ReadOnlyBodyCache<'tcx, 'tcx>,
         bx: &mut Bx,
         targets: &[mir::BasicBlock],
     ) {
         if bx.tcx().sess.opts.debugging_opts.insert_sideeffect {
-            if targets.iter().any(|target| {
-                *target <= *self.bb
+            if targets.iter().any(|&target| {
+                target <= self.bb
                     && target
                         .start_location()
                         .is_predecessor_of(self.bb.start_location(), mir)
@@ -173,9 +173,9 @@
 /// Codegen implementations for some terminator variants.
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     /// Generates code for a `Resume` terminator.
-    fn codegen_resume_terminator<'b>(
+    fn codegen_resume_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'b, 'tcx>,
+        helper: TerminatorCodegenHelper<'tcx>,
         mut bx: Bx,
     ) {
         if let Some(funclet) = helper.funclet(self) {
@@ -201,9 +201,9 @@
         }
     }
 
-    fn codegen_switchint_terminator<'b>(
+    fn codegen_switchint_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'b, 'tcx>,
+        helper: TerminatorCodegenHelper<'tcx>,
         mut bx: Bx,
         discr: &mir::Operand<'tcx>,
         switch_ty: Ty<'tcx>,
@@ -261,7 +261,11 @@
         if self.fn_abi.ret.layout.abi.is_uninhabited() {
             // Functions with uninhabited return values are marked `noreturn`,
             // so we should make sure that we never actually do.
+            // We play it safe by using a well-defined `abort`, but we could go for immediate UB
+            // if that turns out to be helpful.
             bx.abort();
+            // `abort` does not terminate the block, so we still need to generate
+            // an `unreachable` terminator after it.
             bx.unreachable();
             return;
         }
@@ -316,15 +320,15 @@
     }
 
 
-    fn codegen_drop_terminator<'b>(
+    fn codegen_drop_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'b, 'tcx>,
+        helper: TerminatorCodegenHelper<'tcx>,
         mut bx: Bx,
         location: &mir::Place<'tcx>,
         target: mir::BasicBlock,
         unwind: Option<mir::BasicBlock>,
     ) {
-        let ty = location.ty(self.mir, bx.tcx()).ty;
+        let ty = location.ty(*self.mir, bx.tcx()).ty;
         let ty = self.monomorphize(&ty);
         let drop_fn = Instance::resolve_drop_in_place(bx.tcx(), ty);
 
@@ -345,20 +349,21 @@
             &args1[..]
         };
         let (drop_fn, fn_abi) = match ty.kind {
+            // FIXME(eddyb) perhaps move some of this logic into
+            // `Instance::resolve_drop_in_place`?
             ty::Dynamic(..) => {
-                let sig = drop_fn.fn_sig(self.cx.tcx());
-                let sig = self.cx.tcx().normalize_erasing_late_bound_regions(
-                    ty::ParamEnv::reveal_all(),
-                    &sig,
-                );
-                let fn_abi = FnAbi::new_vtable(&bx, sig, &[]);
+                let virtual_drop = Instance {
+                    def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+                    substs: drop_fn.substs,
+                };
+                let fn_abi = FnAbi::of_instance(&bx, virtual_drop, &[]);
                 let vtable = args[1];
                 args = &args[..1];
                 (meth::DESTRUCTOR.get_fn(&mut bx, vtable, &fn_abi), fn_abi)
             }
             _ => {
                 (bx.get_fn_addr(drop_fn),
-                 FnAbi::of_instance(&bx, drop_fn))
+                 FnAbi::of_instance(&bx, drop_fn, &[]))
             }
         };
         helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
@@ -367,9 +372,9 @@
                        unwind);
     }
 
-    fn codegen_assert_terminator<'b>(
+    fn codegen_assert_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'b, 'tcx>,
+        helper: TerminatorCodegenHelper<'tcx>,
         mut bx: Bx,
         terminator: &mir::Terminator<'tcx>,
         cond: &mir::Operand<'tcx>,
@@ -439,16 +444,16 @@
         // Obtain the panic entry point.
         let def_id = common::langcall(bx.tcx(), Some(span), "", lang_item);
         let instance = ty::Instance::mono(bx.tcx(), def_id);
-        let fn_abi = FnAbi::of_instance(&bx, instance);
+        let fn_abi = FnAbi::of_instance(&bx, instance, &[]);
         let llfn = bx.get_fn_addr(instance);
 
         // Codegen the actual panic invoke/call.
         helper.do_call(self, &mut bx, fn_abi, llfn, &args, None, cleanup);
     }
 
-    fn codegen_call_terminator<'b>(
+    fn codegen_call_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'b, 'tcx>,
+        helper: TerminatorCodegenHelper<'tcx>,
         mut bx: Bx,
         terminator: &mir::Terminator<'tcx>,
         func: &mir::Operand<'tcx>,
@@ -474,12 +479,20 @@
             _ => bug!("{} is not callable", callee.layout.ty),
         };
         let def = instance.map(|i| i.def);
+
+        if let Some(ty::InstanceDef::DropGlue(_, None)) = def {
+            // Empty drop glue; a no-op.
+            let &(_, target) = destination.as_ref().unwrap();
+            helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+            helper.funclet_br(self, &mut bx, target);
+            return;
+        }
+
+        // FIXME(eddyb) avoid computing this if possible, when `instance` is
+        // available - right now `sig` is only needed for getting the `abi`
+        // and figuring out how many extra args were passed to a C-variadic `fn`.
         let sig = callee.layout.ty.fn_sig(bx.tcx());
-        let sig = bx.tcx().normalize_erasing_late_bound_regions(
-            ty::ParamEnv::reveal_all(),
-            &sig,
-        );
-        let abi = sig.abi;
+        let abi = sig.abi();
 
         // Handle intrinsics old codegen wants Expr's for, ourselves.
         let intrinsic = match def {
@@ -489,6 +502,17 @@
         };
         let intrinsic = intrinsic.as_ref().map(|s| &s[..]);
 
+        let extra_args = &args[sig.inputs().skip_binder().len()..];
+        let extra_args = extra_args.iter().map(|op_arg| {
+            let op_ty = op_arg.ty(*self.mir, bx.tcx());
+            self.monomorphize(&op_ty)
+        }).collect::<Vec<_>>();
+
+        let fn_abi = match instance {
+            Some(instance) => FnAbi::of_instance(&bx, instance, &extra_args),
+            None => FnAbi::of_fn_ptr(&bx, sig, &extra_args)
+        };
+
         if intrinsic == Some("transmute") {
             if let Some(destination_ref) = destination.as_ref() {
                 let &(ref dest, target) = destination_ref;
@@ -502,44 +526,17 @@
                 // we can do what we like. Here, we declare that transmuting
                 // into an uninhabited type is impossible, so anything following
                 // it must be unreachable.
-                assert_eq!(bx.layout_of(sig.output()).abi, layout::Abi::Uninhabited);
+                assert_eq!(fn_abi.ret.layout.abi, layout::Abi::Uninhabited);
                 bx.unreachable();
             }
             return;
         }
 
-        let extra_args = &args[sig.inputs().len()..];
-        let extra_args = extra_args.iter().map(|op_arg| {
-            let op_ty = op_arg.ty(self.mir, bx.tcx());
-            self.monomorphize(&op_ty)
-        }).collect::<Vec<_>>();
-
-        let fn_abi = match def {
-            Some(ty::InstanceDef::Virtual(..)) => {
-                FnAbi::new_vtable(&bx, sig, &extra_args)
-            }
-            Some(ty::InstanceDef::DropGlue(_, None)) => {
-                // Empty drop glue; a no-op.
-                let &(_, target) = destination.as_ref().unwrap();
-                helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
-                helper.funclet_br(self, &mut bx, target);
-                return;
-            }
-            _ => FnAbi::new(&bx, sig, &extra_args)
-        };
-
-        // This should never be reachable at runtime:
-        // We should only emit a call to this intrinsic in #[cfg(miri)] mode,
-        // which means that we will never actually use the generate object files
-        // (we will just be interpreting the MIR)
-        //
-        // Note that we still need to be able to codegen *something* for this intrisnic:
-        // Miri currently uses Xargo to build a special libstd. As a side effect,
-        // we generate normal object files for libstd - while these are never used,
-        // we still need to be able to build them.
+        // For normal codegen, this Miri-specific intrinsic is just a NOP.
         if intrinsic == Some("miri_start_panic") {
-            bx.abort();
-            bx.unreachable();
+            let target = destination.as_ref().unwrap().1;
+            helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+            helper.funclet_br(self, &mut bx, target);
             return;
         }
 
@@ -556,7 +553,7 @@
                 let def_id =
                     common::langcall(bx.tcx(), Some(span), "", lang_items::PanicFnLangItem);
                 let instance = ty::Instance::mono(bx.tcx(), def_id);
-                let fn_abi = FnAbi::of_instance(&bx, instance);
+                let fn_abi = FnAbi::of_instance(&bx, instance, &[]);
                 let llfn = bx.get_fn_addr(instance);
 
                 if let Some((_, target)) = destination.as_ref() {
@@ -576,7 +573,7 @@
                 // a NOP
                 let target = destination.as_ref().unwrap().1;
                 helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
-                helper.funclet_br(self, &mut bx, target);
+                helper.funclet_br(self, &mut bx, target)
             }
             return;
         }
@@ -798,7 +795,8 @@
         bb: mir::BasicBlock,
     ) {
         let mut bx = self.build_block(bb);
-        let data = &self.mir[bb];
+        let mir = self.mir;
+        let data = &mir[bb];
 
         debug!("codegen_block({:?}={:?})", bb, data);
 
@@ -813,14 +811,14 @@
         &mut self,
         mut bx: Bx,
         bb: mir::BasicBlock,
-        terminator: &mir::Terminator<'tcx>
+        terminator: &'tcx mir::Terminator<'tcx>
     ) {
         debug!("codegen_terminator: {:?}", terminator);
 
         // Create the cleanup bundle, if needed.
         let funclet_bb = self.cleanup_kinds[bb].funclet_bb(bb);
         let helper = TerminatorCodegenHelper {
-            bb: &bb, terminator, funclet_bb
+            bb, terminator, funclet_bb
         };
 
         self.set_debug_loc(&mut bx, terminator.source_info);
@@ -831,6 +829,8 @@
 
             mir::TerminatorKind::Abort => {
                 bx.abort();
+                // `abort` does not terminate the block, so we still need to generate
+                // an `unreachable` terminator after it.
                 bx.unreachable();
             }
 
@@ -1057,7 +1057,7 @@
 
     fn landing_pad_uncached(
         &mut self,
-        target_bb: Bx::BasicBlock
+        target_bb: Bx::BasicBlock,
     ) -> Bx::BasicBlock {
         if base::wants_msvc_seh(self.cx.sess()) {
             span_bug!(self.mir.span, "landing pad was not inserted?")
diff --git a/src/librustc_codegen_ssa/mir/mod.rs b/src/librustc_codegen_ssa/mir/mod.rs
index 6041232..3a157ca 100644
--- a/src/librustc_codegen_ssa/mir/mod.rs
+++ b/src/librustc_codegen_ssa/mir/mod.rs
@@ -1,6 +1,6 @@
 use rustc::ty::{self, Ty, TypeFoldable, Instance};
 use rustc::ty::layout::{TyLayout, HasTyCtxt, FnAbiExt};
-use rustc::mir::{self, Body};
+use rustc::mir;
 use rustc_target::abi::call::{FnAbi, PassMode};
 use crate::base;
 use crate::traits::*;
@@ -21,7 +21,7 @@
 pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
     instance: Instance<'tcx>,
 
-    mir: &'a mir::Body<'tcx>,
+    mir: mir::ReadOnlyBodyCache<'tcx, 'tcx>,
 
     debug_context: Option<FunctionDebugContext<Bx::DIScope>>,
 
@@ -76,7 +76,7 @@
 
     /// All `VarDebuginfo` from the MIR body, partitioned by `Local`.
     /// This is `None` if no variable debuginfo/names are needed.
-    per_local_var_debug_info: Option<IndexVec<mir::Local, Vec<&'a mir::VarDebugInfo<'tcx>>>>,
+    per_local_var_debug_info: Option<IndexVec<mir::Local, Vec<&'tcx mir::VarDebugInfo<'tcx>>>>,
 }
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
@@ -121,18 +121,18 @@
 
 pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     cx: &'a Bx::CodegenCx,
-    llfn: Bx::Function,
-    mir: &'a Body<'tcx>,
     instance: Instance<'tcx>,
-    sig: ty::FnSig<'tcx>,
 ) {
     assert!(!instance.substs.needs_infer());
 
-    let fn_abi = FnAbi::new(cx, sig, &[]);
+    let llfn = cx.get_fn(instance);
+
+    let mir = cx.tcx().instance_mir(instance.def);
+
+    let fn_abi = FnAbi::of_instance(cx, instance, &[]);
     debug!("fn_abi: {:?}", fn_abi);
 
-    let debug_context =
-        cx.create_function_debug_context(instance, sig, llfn, mir);
+    let debug_context = cx.create_function_debug_context(instance, &fn_abi, llfn, &mir);
 
     let mut bx = Bx::new_block(cx, llfn, "start");
 
@@ -155,8 +155,8 @@
             }
         }).collect();
 
-    let (landing_pads, funclets) = create_funclets(mir, &mut bx, &cleanup_kinds, &block_bxs);
-
+    let (landing_pads, funclets) = create_funclets(&mir, &mut bx, &cleanup_kinds, &block_bxs);
+    let mir_body: &mir::Body<'_> = mir.body();
     let mut fx = FunctionCx {
         instance,
         mir,
@@ -171,7 +171,7 @@
         funclets,
         locals: IndexVec::new(),
         debug_context,
-        per_local_var_debug_info: debuginfo::per_local_var_debug_info(cx.tcx(), mir),
+        per_local_var_debug_info: debuginfo::per_local_var_debug_info(cx.tcx(), mir_body),
     };
 
     let memory_locals = analyze::non_ssa_locals(&fx);
@@ -181,7 +181,7 @@
         let args = arg_local_refs(&mut bx, &fx, &memory_locals);
 
         let mut allocate_local = |local| {
-            let decl = &mir.local_decls[local];
+            let decl = &mir_body.local_decls[local];
             let layout = bx.layout_of(fx.monomorphize(&decl.ty));
             assert!(!layout.ty.has_erasable_regions());
 
@@ -207,7 +207,7 @@
         let retptr = allocate_local(mir::RETURN_PLACE);
         iter::once(retptr)
             .chain(args.into_iter())
-            .chain(mir.vars_and_temps_iter().map(allocate_local))
+            .chain(mir_body.vars_and_temps_iter().map(allocate_local))
             .collect()
     };
 
@@ -226,8 +226,8 @@
         debug_context.source_locations_enabled = true;
     }
 
-    let rpo = traversal::reverse_postorder(&mir);
-    let mut visited = BitSet::new_empty(mir.basic_blocks().len());
+    let rpo = traversal::reverse_postorder(&mir_body);
+    let mut visited = BitSet::new_empty(mir_body.basic_blocks().len());
 
     // Codegen the body of each block using reverse postorder
     for (bb, _) in rpo {
@@ -237,7 +237,7 @@
 
     // Remove blocks that haven't been visited, or have no
     // predecessors.
-    for bb in mir.basic_blocks().indices() {
+    for bb in mir_body.basic_blocks().indices() {
         // Unreachable block
         if !visited.contains(bb.index()) {
             debug!("codegen_mir: block {:?} was not visited", bb);
@@ -249,7 +249,7 @@
 }
 
 fn create_funclets<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
-    mir: &'a Body<'tcx>,
+    mir: &'tcx mir::Body<'tcx>,
     bx: &mut Bx,
     cleanup_kinds: &IndexVec<mir::BasicBlock, CleanupKind>,
     block_bxs: &IndexVec<mir::BasicBlock, Bx::BasicBlock>,
diff --git a/src/librustc_codegen_ssa/mir/operand.rs b/src/librustc_codegen_ssa/mir/operand.rs
index 310b8ae..a6dec81 100644
--- a/src/librustc_codegen_ssa/mir/operand.rs
+++ b/src/librustc_codegen_ssa/mir/operand.rs
@@ -475,9 +475,10 @@
                             },
                         }
                         // Allow RalfJ to sleep soundly knowing that even refactorings that remove
-                        // the above error (or silence it under some conditions) will not cause UB
+                        // the above error (or silence it under some conditions) will not cause UB.
                         bx.abort();
-                        // We've errored, so we don't have to produce working code.
+                        // We still have to return an operand but it doesn't matter,
+                        // this code is unreachable.
                         let ty = self.monomorphize(&constant.literal.ty);
                         let layout = bx.cx().layout_of(ty);
                         bx.load_operand(PlaceRef::new_sized(
diff --git a/src/librustc_codegen_ssa/mir/place.rs b/src/librustc_codegen_ssa/mir/place.rs
index 4df5bce..e60b886 100644
--- a/src/librustc_codegen_ssa/mir/place.rs
+++ b/src/librustc_codegen_ssa/mir/place.rs
@@ -333,6 +333,9 @@
         variant_index: VariantIdx
     ) {
         if self.layout.for_variant(bx.cx(), variant_index).abi.is_uninhabited() {
+            // We play it safe by using a well-defined `abort`, but we could go for immediate UB
+            // if that turns out to be helpful.
+            bx.abort();
             return;
         }
         match self.layout.variants {
@@ -488,10 +491,12 @@
                     },
                     Err(_) => {
                         // This is unreachable as long as runtime
-                        // and compile-time agree on values
+                        // and compile-time agree perfectly.
                         // With floats that won't always be true,
-                        // so we generate an abort.
+                        // so we generate a (safe) abort.
                         bx.abort();
+                        // We still have to return a place but it doesn't matter,
+                        // this code is unreachable.
                         let llval = bx.cx().const_undef(
                             bx.cx().type_ptr_to(bx.cx().backend_type(layout))
                         );
@@ -591,7 +596,12 @@
 
     pub fn monomorphized_place_ty(&self, place_ref: &mir::PlaceRef<'_, 'tcx>) -> Ty<'tcx> {
         let tcx = self.cx.tcx();
-        let place_ty = mir::Place::ty_from(place_ref.base, place_ref.projection, self.mir, tcx);
+        let place_ty = mir::Place::ty_from(
+            place_ref.base,
+            place_ref.projection,
+            *self.mir,
+            tcx,
+        );
         self.monomorphize(&place_ty.ty)
     }
 }
diff --git a/src/librustc_codegen_ssa/mir/rvalue.rs b/src/librustc_codegen_ssa/mir/rvalue.rs
index f7fb4a5..488ae8d 100644
--- a/src/librustc_codegen_ssa/mir/rvalue.rs
+++ b/src/librustc_codegen_ssa/mir/rvalue.rs
@@ -460,7 +460,7 @@
             }
 
             mir::Rvalue::Discriminant(ref place) => {
-                let discr_ty = rvalue.ty(&*self.mir, bx.tcx());
+                let discr_ty = rvalue.ty(*self.mir, bx.tcx());
                 let discr =  self.codegen_place(&mut bx, &place.as_ref())
                     .codegen_get_discr(&mut bx, discr_ty);
                 (bx, OperandRef {
@@ -513,7 +513,7 @@
             mir::Rvalue::Aggregate(..) => {
                 // According to `rvalue_creates_operand`, only ZST
                 // aggregate rvalues are allowed to be operands.
-                let ty = rvalue.ty(self.mir, self.cx.tcx());
+                let ty = rvalue.ty(*self.mir, self.cx.tcx());
                 let operand = OperandRef::new_zst(
                     &mut bx,
                     self.cx.layout_of(self.monomorphize(&ty)),
@@ -710,7 +710,7 @@
                 true,
             mir::Rvalue::Repeat(..) |
             mir::Rvalue::Aggregate(..) => {
-                let ty = rvalue.ty(self.mir, self.cx.tcx());
+                let ty = rvalue.ty(*self.mir, self.cx.tcx());
                 let ty = self.monomorphize(&ty);
                 self.cx.spanned_layout_of(ty, span).is_zst()
             }
diff --git a/src/librustc_codegen_ssa/traits/debuginfo.rs b/src/librustc_codegen_ssa/traits/debuginfo.rs
index 802eaaa..e67201b 100644
--- a/src/librustc_codegen_ssa/traits/debuginfo.rs
+++ b/src/librustc_codegen_ssa/traits/debuginfo.rs
@@ -2,8 +2,9 @@
 use crate::mir::debuginfo::{FunctionDebugContext, VariableKind};
 use rustc::hir::def_id::CrateNum;
 use rustc::mir;
-use rustc::ty::{self, Ty, Instance};
+use rustc::ty::{Ty, Instance};
 use rustc::ty::layout::Size;
+use rustc_target::abi::call::FnAbi;
 use syntax::ast::Name;
 use syntax_pos::{SourceFile, Span};
 
@@ -17,7 +18,7 @@
     fn create_function_debug_context(
         &self,
         instance: Instance<'tcx>,
-        sig: ty::FnSig<'tcx>,
+        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         llfn: Self::Function,
         mir: &mir::Body<'_>,
     ) -> Option<FunctionDebugContext<Self::DIScope>>;
diff --git a/src/librustc_codegen_ssa/traits/declare.rs b/src/librustc_codegen_ssa/traits/declare.rs
index cd42044..1dd2c74 100644
--- a/src/librustc_codegen_ssa/traits/declare.rs
+++ b/src/librustc_codegen_ssa/traits/declare.rs
@@ -1,7 +1,8 @@
 use super::BackendTypes;
 use rustc::hir::def_id::DefId;
 use rustc::mir::mono::{Linkage, Visibility};
-use rustc::ty::{self, Instance};
+use rustc::ty::{Instance, Ty};
+use rustc_target::abi::call::FnAbi;
 
 pub trait DeclareMethods<'tcx>: BackendTypes {
     /// Declare a global value.
@@ -23,7 +24,7 @@
     ///
     /// If there’s a value with the same name already declared, the function will
     /// update the declaration and return existing Value instead.
-    fn declare_fn(&self, name: &str, sig: ty::PolyFnSig<'tcx>) -> Self::Function;
+    fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Function;
 
     /// Declare a global with an intention to define it.
     ///
@@ -38,20 +39,6 @@
     /// Use this function when you intend to define a global without a name.
     fn define_private_global(&self, ty: Self::Type) -> Self::Value;
 
-    /// Declare a Rust function with an intention to define it.
-    ///
-    /// Use this function when you intend to define a function. This function will
-    /// return panic if the name already has a definition associated with it. This
-    /// can happen with #[no_mangle] or #[export_name], for example.
-    fn define_fn(&self, name: &str, fn_sig: ty::PolyFnSig<'tcx>) -> Self::Value;
-
-    /// Declare a Rust function with an intention to define it.
-    ///
-    /// Use this function when you intend to define a function. This function will
-    /// return panic if the name already has a definition associated with it. This
-    /// can happen with #[no_mangle] or #[export_name], for example.
-    fn define_internal_fn(&self, name: &str, fn_sig: ty::PolyFnSig<'tcx>) -> Self::Value;
-
     /// Gets declared value by name.
     fn get_declared_value(&self, name: &str) -> Option<Self::Value>;
 
diff --git a/src/librustc_data_structures/graph/dominators/mod.rs b/src/librustc_data_structures/graph/dominators/mod.rs
index 444463c..5fb58ee 100644
--- a/src/librustc_data_structures/graph/dominators/mod.rs
+++ b/src/librustc_data_structures/graph/dominators/mod.rs
@@ -7,32 +7,33 @@
 use rustc_index::vec::{Idx, IndexVec};
 use super::iterate::reverse_post_order;
 use super::ControlFlowGraph;
+use std::borrow::BorrowMut;
 
 #[cfg(test)]
 mod tests;
 
-pub fn dominators<G: ControlFlowGraph>(graph: &G) -> Dominators<G::Node> {
+pub fn dominators<G: ControlFlowGraph>(graph: G) -> Dominators<G::Node> {
     let start_node = graph.start_node();
-    let rpo = reverse_post_order(graph, start_node);
+    let rpo = reverse_post_order(&graph, start_node);
     dominators_given_rpo(graph, &rpo)
 }
 
-fn dominators_given_rpo<G: ControlFlowGraph>(
-    graph: &G,
+fn dominators_given_rpo<G: ControlFlowGraph + BorrowMut<G>>(
+    mut graph: G,
     rpo: &[G::Node],
 ) -> Dominators<G::Node> {
-    let start_node = graph.start_node();
+    let start_node = graph.borrow().start_node();
     assert_eq!(rpo[0], start_node);
 
     // compute the post order index (rank) for each node
     let mut post_order_rank: IndexVec<G::Node, usize> =
-        (0..graph.num_nodes()).map(|_| 0).collect();
+        (0..graph.borrow().num_nodes()).map(|_| 0).collect();
     for (index, node) in rpo.iter().rev().cloned().enumerate() {
         post_order_rank[node] = index;
     }
 
     let mut immediate_dominators: IndexVec<G::Node, Option<G::Node>> =
-        (0..graph.num_nodes()).map(|_| None).collect();
+        (0..graph.borrow().num_nodes()).map(|_| None).collect();
     immediate_dominators[start_node] = Some(start_node);
 
     let mut changed = true;
@@ -41,7 +42,7 @@
 
         for &node in &rpo[1..] {
             let mut new_idom = None;
-            for pred in graph.predecessors(node) {
+            for pred in graph.borrow_mut().predecessors(node) {
                 if immediate_dominators[pred].is_some() {
                     // (*) dominators for `pred` have been calculated
                     new_idom = Some(if let Some(new_idom) = new_idom {
diff --git a/src/librustc_data_structures/jobserver.rs b/src/librustc_data_structures/jobserver.rs
index b42ccb9..a811c88 100644
--- a/src/librustc_data_structures/jobserver.rs
+++ b/src/librustc_data_structures/jobserver.rs
@@ -1,4 +1,4 @@
-use jobserver_crate::Client;
+pub use jobserver_crate::Client;
 use lazy_static::lazy_static;
 
 lazy_static! {
diff --git a/src/librustc_data_structures/obligation_forest/mod.rs b/src/librustc_data_structures/obligation_forest/mod.rs
index 958ab61..8a5badd 100644
--- a/src/librustc_data_structures/obligation_forest/mod.rs
+++ b/src/librustc_data_structures/obligation_forest/mod.rs
@@ -395,7 +395,16 @@
         let mut errors = vec![];
         let mut stalled = true;
 
-        for index in 0..self.nodes.len() {
+        // Note that the loop body can append new nodes, and those new nodes
+        // will then be processed by subsequent iterations of the loop.
+        //
+        // We can't use an iterator for the loop because `self.nodes` is
+        // appended to and the borrow checker would complain. We also can't use
+        // `for index in 0..self.nodes.len() { ... }` because the range would
+        // be computed with the initial length, and we would miss the appended
+        // nodes. Therefore we use a `while` loop.
+        let mut index = 0;
+        while index < self.nodes.len() {
             let node = &mut self.nodes[index];
 
             debug!("process_obligations: node {} == {:?}", index, node);
@@ -406,6 +415,7 @@
             // out of sync with `nodes`. It's not very common, but it does
             // happen, and code in `compress` has to allow for it.
             if node.state.get() != NodeState::Pending {
+                index += 1;
                 continue;
             }
             let result = processor.process_obligation(&mut node.obligation);
@@ -441,6 +451,7 @@
                     });
                 }
             }
+            index += 1;
         }
 
         if stalled {
diff --git a/src/librustc_data_structures/obligation_forest/tests.rs b/src/librustc_data_structures/obligation_forest/tests.rs
index 54b6f6d..995c99b 100644
--- a/src/librustc_data_structures/obligation_forest/tests.rs
+++ b/src/librustc_data_structures/obligation_forest/tests.rs
@@ -70,6 +70,7 @@
                 "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]),
                 "B" => ProcessResult::Error("B is for broken"),
                 "C" => ProcessResult::Changed(vec![]),
+                "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_| {}), DoCompleted::Yes);
@@ -94,6 +95,7 @@
                 "A.2" => ProcessResult::Unchanged,
                 "A.3" => ProcessResult::Changed(vec!["A.3.i"]),
                 "D" => ProcessResult::Changed(vec!["D.1", "D.2"]),
+                "A.3.i" | "D.1" | "D.2" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_| {}), DoCompleted::Yes);
@@ -113,6 +115,7 @@
                 "A.3.i" => ProcessResult::Changed(vec![]),
                 "D.1" => ProcessResult::Changed(vec!["D.1.i"]),
                 "D.2" => ProcessResult::Changed(vec!["D.2.i"]),
+                "D.1.i" | "D.2.i" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_| {}), DoCompleted::Yes);
@@ -161,18 +164,10 @@
         forest.process_obligations(&mut C(|obligation| {
             match *obligation {
                 "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]),
-                _ => unreachable!(),
-            }
-        }, |_| {}), DoCompleted::Yes);
-    assert!(ok.unwrap().is_empty());
-    assert!(err.is_empty());
-
-    let Outcome { completed: ok, errors: err, .. } =
-        forest.process_obligations(&mut C(|obligation| {
-            match *obligation {
                 "A.1" => ProcessResult::Changed(vec![]),
                 "A.2" => ProcessResult::Changed(vec!["A.2.i", "A.2.ii"]),
                 "A.3" => ProcessResult::Changed(vec![]),
+                "A.2.i" | "A.2.ii" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_| {}), DoCompleted::Yes);
@@ -184,7 +179,7 @@
     let Outcome { completed: ok, errors: err, .. } =
         forest.process_obligations(&mut C(|obligation| {
             match *obligation {
-                "A.2.i" => ProcessResult::Changed(vec!["A.2.i.a"]),
+                "A.2.i" => ProcessResult::Unchanged,
                 "A.2.ii" => ProcessResult::Changed(vec![]),
                 _ => unreachable!(),
             }
@@ -195,6 +190,17 @@
     let Outcome { completed: ok, errors: err, .. } =
         forest.process_obligations(&mut C(|obligation| {
             match *obligation {
+                "A.2.i" => ProcessResult::Changed(vec!["A.2.i.a"]),
+                "A.2.i.a" => ProcessResult::Unchanged,
+                _ => unreachable!(),
+            }
+        }, |_| {}), DoCompleted::Yes);
+    assert!(ok.unwrap().is_empty());
+    assert!(err.is_empty());
+
+    let Outcome { completed: ok, errors: err, .. } =
+        forest.process_obligations(&mut C(|obligation| {
+            match *obligation {
                 "A.2.i.a" => ProcessResult::Changed(vec![]),
                 _ => unreachable!(),
             }
@@ -222,6 +228,7 @@
         forest.process_obligations(&mut C(|obligation| {
             match *obligation {
                 "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]),
+                "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_|{}), DoCompleted::Yes);
@@ -243,6 +250,7 @@
         forest.process_obligations(&mut C(|obligation| {
             match *obligation {
                 "A" => ProcessResult::Changed(vec!["A.1", "A.2"]),
+                "A.1" | "A.2" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_|{}), DoCompleted::Yes);
@@ -254,6 +262,7 @@
             match *obligation {
                 "A.1" => ProcessResult::Changed(vec!["D"]),
                 "A.2" => ProcessResult::Changed(vec!["D"]),
+                "D" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_|{}), DoCompleted::Yes);
@@ -282,6 +291,7 @@
         forest.process_obligations(&mut C(|obligation| {
             match *obligation {
                 "A'" => ProcessResult::Changed(vec!["A'.1", "A'.2"]),
+                "A'.1" | "A'.2" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_|{}), DoCompleted::Yes);
@@ -293,6 +303,7 @@
             match *obligation {
                 "A'.1" => ProcessResult::Changed(vec!["D'", "A'"]),
                 "A'.2" => ProcessResult::Changed(vec!["D'"]),
+                "D'" | "A'" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_|{}), DoCompleted::Yes);
@@ -370,6 +381,7 @@
                 "B" => ProcessResult::Unchanged,
                 "C1" => ProcessResult::Changed(vec![]),
                 "C2" => ProcessResult::Changed(vec![]),
+                "D" | "E" => ProcessResult::Unchanged,
                 _ => unreachable!(),
             }
         }, |_|{}), DoCompleted::Yes);
diff --git a/src/librustc_driver/lib.rs b/src/librustc_driver/lib.rs
index 93f4e73..0594550 100644
--- a/src/librustc_driver/lib.rs
+++ b/src/librustc_driver/lib.rs
@@ -828,7 +828,7 @@
 
     fn sort_lints(sess: &Session, mut lints: Vec<&'static Lint>) -> Vec<&'static Lint> {
         // The sort doesn't case-fold but it's doubtful we care.
-        lints.sort_by_cached_key(|x: &&Lint| (x.default_level(sess), x.name));
+        lints.sort_by_cached_key(|x: &&Lint| (x.default_level(sess.edition()), x.name));
         lints
     }
 
diff --git a/src/librustc_error_codes/error_codes.rs b/src/librustc_error_codes/error_codes.rs
index 7f111b4..9c1bec3 100644
--- a/src/librustc_error_codes/error_codes.rs
+++ b/src/librustc_error_codes/error_codes.rs
@@ -18,7 +18,6 @@
 E0013: include_str!("./error_codes/E0013.md"),
 E0014: include_str!("./error_codes/E0014.md"),
 E0015: include_str!("./error_codes/E0015.md"),
-E0017: include_str!("./error_codes/E0017.md"),
 E0019: include_str!("./error_codes/E0019.md"),
 E0023: include_str!("./error_codes/E0023.md"),
 E0025: include_str!("./error_codes/E0025.md"),
@@ -347,6 +346,7 @@
 E0623: include_str!("./error_codes/E0623.md"),
 E0624: include_str!("./error_codes/E0624.md"),
 E0626: include_str!("./error_codes/E0626.md"),
+E0631: include_str!("./error_codes/E0631.md"),
 E0633: include_str!("./error_codes/E0633.md"),
 E0635: include_str!("./error_codes/E0635.md"),
 E0636: include_str!("./error_codes/E0636.md"),
@@ -580,7 +580,6 @@
     // rustc_const_unstable attribute must be paired with stable/unstable
     // attribute
     E0630,
-    E0631, // type mismatch in closure arguments
     E0632, // cannot provide explicit generic arguments when `impl Trait` is
            // used in argument position
     E0634, // type has conflicting packed representaton hints
diff --git a/src/librustc_error_codes/error_codes/E0017.md b/src/librustc_error_codes/error_codes/E0017.md
deleted file mode 100644
index d5e6857..0000000
--- a/src/librustc_error_codes/error_codes/E0017.md
+++ /dev/null
@@ -1,20 +0,0 @@
-References in statics and constants may only refer to immutable values.
-
-Erroneous code example:
-
-```compile_fail,E0017
-static X: i32 = 1;
-const C: i32 = 2;
-
-// these three are not allowed:
-const CR: &mut i32 = &mut C;
-static STATIC_REF: &'static mut i32 = &mut X;
-static CONST_REF: &'static mut i32 = &mut C;
-```
-
-Statics are shared everywhere, and if they refer to mutable data one might
-violate memory safety since holding multiple mutable references to shared data
-is not allowed.
-
-If you really want global mutable state, try using `static mut` or a global
-`UnsafeCell`.
diff --git a/src/librustc_error_codes/error_codes/E0092.md b/src/librustc_error_codes/error_codes/E0092.md
index 2750a7d..e289534 100644
--- a/src/librustc_error_codes/error_codes/E0092.md
+++ b/src/librustc_error_codes/error_codes/E0092.md
@@ -1,4 +1,5 @@
-You tried to declare an undefined atomic operation function.
+An undefined atomic operation function was declared.
+
 Erroneous code example:
 
 ```compile_fail,E0092
@@ -11,8 +12,8 @@
 ```
 
 Please check you didn't make a mistake in the function's name. All intrinsic
-functions are defined in librustc_codegen_llvm/intrinsic.rs and in
-libcore/intrinsics.rs in the Rust source code. Example:
+functions are defined in `librustc_codegen_llvm/intrinsic.rs` and in
+`libcore/intrinsics.rs` in the Rust source code. Example:
 
 ```
 #![feature(intrinsics)]
diff --git a/src/librustc_error_codes/error_codes/E0093.md b/src/librustc_error_codes/error_codes/E0093.md
index 9633f79..8e7de1a 100644
--- a/src/librustc_error_codes/error_codes/E0093.md
+++ b/src/librustc_error_codes/error_codes/E0093.md
@@ -1,4 +1,6 @@
-You declared an unknown intrinsic function. Erroneous code example:
+An unknown intrinsic function was declared.
+
+Erroneous code example:
 
 ```compile_fail,E0093
 #![feature(intrinsics)]
@@ -15,8 +17,8 @@
 ```
 
 Please check you didn't make a mistake in the function's name. All intrinsic
-functions are defined in librustc_codegen_llvm/intrinsic.rs and in
-libcore/intrinsics.rs in the Rust source code. Example:
+functions are defined in `librustc_codegen_llvm/intrinsic.rs` and in
+`libcore/intrinsics.rs` in the Rust source code. Example:
 
 ```
 #![feature(intrinsics)]
diff --git a/src/librustc_error_codes/error_codes/E0094.md b/src/librustc_error_codes/error_codes/E0094.md
index 4d27f61..42baa65 100644
--- a/src/librustc_error_codes/error_codes/E0094.md
+++ b/src/librustc_error_codes/error_codes/E0094.md
@@ -1,4 +1,5 @@
-You gave an invalid number of type parameters to an intrinsic function.
+An invalid number of type parameters was given to an intrinsic function.
+
 Erroneous code example:
 
 ```compile_fail,E0094
diff --git a/src/librustc_error_codes/error_codes/E0106.md b/src/librustc_error_codes/error_codes/E0106.md
index 8a49c1f..60ca1dd 100644
--- a/src/librustc_error_codes/error_codes/E0106.md
+++ b/src/librustc_error_codes/error_codes/E0106.md
@@ -2,7 +2,7 @@
 inside a function signature, the problem may be with failing to adhere to the
 lifetime elision rules (see below).
 
-Here are some simple examples of where you'll run into this error:
+Erroneous code examples:
 
 ```compile_fail,E0106
 struct Foo1 { x: &bool }
@@ -27,7 +27,7 @@
 For more background on lifetime elision see [the book][book-le].
 
 The lifetime elision rules require that any function signature with an elided
-output lifetime must either have
+output lifetime must either have:
 
  - exactly one input lifetime
  - or, multiple input lifetimes, but the function must also be a method with a
diff --git a/src/librustc_error_codes/error_codes/E0107.md b/src/librustc_error_codes/error_codes/E0107.md
index bfe0d21..4d22b17 100644
--- a/src/librustc_error_codes/error_codes/E0107.md
+++ b/src/librustc_error_codes/error_codes/E0107.md
@@ -1,4 +1,6 @@
-This error means that an incorrect number of generic arguments were provided:
+An incorrect number of generic arguments were provided.
+
+Erroneous code example:
 
 ```compile_fail,E0107
 struct Foo<T> { x: T }
@@ -9,6 +11,7 @@
                                   //        expected 1, found 2
 
 fn foo<T, U>(x: T, y: U) {}
+fn f() {}
 
 fn main() {
     let x: bool = true;
@@ -16,12 +19,26 @@
                                     //        expected 2, found 1
     foo::<bool, i32, i32>(x, 2, 4); // error: wrong number of type arguments:
                                     //        expected 2, found 3
+    f::<'static>();                 // error: wrong number of lifetime arguments
+                                    //        expected 0, found 1
 }
+```
 
+When using/declaring an item with generic arguments, you must provide the exact
+same number:
+
+```
+struct Foo<T> { x: T }
+
+struct Bar<T> { x: Foo<T> }               // ok!
+struct Baz<S, T> { x: Foo<S>, y: Foo<T> } // ok!
+
+fn foo<T, U>(x: T, y: U) {}
 fn f() {}
 
 fn main() {
-    f::<'static>(); // error: wrong number of lifetime arguments:
-                    //        expected 0, found 1
+    let x: bool = true;
+    foo::<bool, u32>(x, 12);              // ok!
+    f();                                  // ok!
 }
 ```
diff --git a/src/librustc_error_codes/error_codes/E0109.md b/src/librustc_error_codes/error_codes/E0109.md
index 5bc229a..2eab972 100644
--- a/src/librustc_error_codes/error_codes/E0109.md
+++ b/src/librustc_error_codes/error_codes/E0109.md
@@ -1,4 +1,5 @@
 You tried to provide a generic argument to a type which doesn't need it.
+
 Erroneous code example:
 
 ```compile_fail,E0109
diff --git a/src/librustc_error_codes/error_codes/E0116.md b/src/librustc_error_codes/error_codes/E0116.md
index 27759a4..ca849c2 100644
--- a/src/librustc_error_codes/error_codes/E0116.md
+++ b/src/librustc_error_codes/error_codes/E0116.md
@@ -1,11 +1,15 @@
-You can only define an inherent implementation for a type in the same crate
-where the type was defined. For example, an `impl` block as below is not allowed
-since `Vec` is defined in the standard library:
+An inherent implementation was defined for a type outside the current crate.
+
+Erroneous code example:
 
 ```compile_fail,E0116
 impl Vec<u8> { } // error
 ```
 
+You can only define an inherent implementation for a type in the same crate
+where the type was defined. For example, an `impl` block as above is not allowed
+since `Vec` is defined in the standard library.
+
 To fix this problem, you can do either of these things:
 
  - define a trait that has the desired associated functions/types/constants and
diff --git a/src/librustc_error_codes/error_codes/E0117.md b/src/librustc_error_codes/error_codes/E0117.md
index bd36230..7fa211d 100644
--- a/src/librustc_error_codes/error_codes/E0117.md
+++ b/src/librustc_error_codes/error_codes/E0117.md
@@ -1,3 +1,11 @@
+The `Drop` trait was implemented on a non-struct type.
+
+Erroneous code example:
+
+```compile_fail,E0117
+impl Drop for u32 {}
+```
+
 This error indicates a violation of one of Rust's orphan rules for trait
 implementations. The rule prohibits any implementation of a foreign trait (a
 trait defined in another crate) where
@@ -6,12 +14,6 @@
  - all of the parameters being passed to the trait (if there are any) are also
    foreign.
 
-Here's one example of this error:
-
-```compile_fail,E0117
-impl Drop for u32 {}
-```
-
 To avoid this kind of error, ensure that at least one local type is referenced
 by the `impl`:
 
diff --git a/src/librustc_error_codes/error_codes/E0118.md b/src/librustc_error_codes/error_codes/E0118.md
index baf35ff..5cb5f50 100644
--- a/src/librustc_error_codes/error_codes/E0118.md
+++ b/src/librustc_error_codes/error_codes/E0118.md
@@ -1,5 +1,7 @@
-You're trying to write an inherent implementation for something which isn't a
-struct nor an enum. Erroneous code example:
+An inherent implementation was defined for something which isn't a struct nor
+an enum.
+
+Erroneous code example:
 
 ```compile_fail,E0118
 impl (u8, u8) { // error: no base type found for inherent implementation
diff --git a/src/librustc_error_codes/error_codes/E0119.md b/src/librustc_error_codes/error_codes/E0119.md
index 0af3bd4..e596349 100644
--- a/src/librustc_error_codes/error_codes/E0119.md
+++ b/src/librustc_error_codes/error_codes/E0119.md
@@ -1,5 +1,6 @@
 There are conflicting trait implementations for the same type.
-Example of erroneous code:
+
+Erroneous code example:
 
 ```compile_fail,E0119
 trait MyTrait {
diff --git a/src/librustc_error_codes/error_codes/E0631.md b/src/librustc_error_codes/error_codes/E0631.md
new file mode 100644
index 0000000..6188d5f
--- /dev/null
+++ b/src/librustc_error_codes/error_codes/E0631.md
@@ -0,0 +1,27 @@
+This error indicates a type mismatch in closure arguments.
+
+Erroneous code example:
+
+```compile_fail,E0631
+fn foo<F: Fn(i32)>(f: F) {
+}
+
+fn main() {
+    foo(|x: &str| {});
+}
+```
+
+The error occurs because `foo` accepts a closure that takes an `i32` argument,
+but in `main`, it is passed a closure with a `&str` argument.
+
+This can be resolved by changing the type annotation or removing it entirely
+if it can be inferred.
+
+```
+fn foo<F: Fn(i32)>(f: F) {
+}
+
+fn main() {
+    foo(|x: i32| {});
+}
+```
diff --git a/src/librustc_feature/active.rs b/src/librustc_feature/active.rs
index 7c0d399..b1ae7c6 100644
--- a/src/librustc_feature/active.rs
+++ b/src/librustc_feature/active.rs
@@ -408,10 +408,6 @@
     /// Allows using `#[doc(keyword = "...")]`.
     (active, doc_keyword, "1.28.0", Some(51315), None),
 
-    /// Allows reinterpretation of the bits of a value of one type as another
-    /// type during const eval.
-    (active, const_transmute, "1.29.0", Some(53605), None),
-
     /// Allows using `try {...}` expressions.
     (active, try_blocks, "1.29.0", Some(31436), None),
 
@@ -524,6 +520,12 @@
     /// Allows the use of `if` and `match` in constants.
     (active, const_if_match, "1.41.0", Some(49146), None),
 
+    /// Allows the use of `#[cfg(sanitize = "option")]`; set when -Zsanitizer is used.
+    (active, cfg_sanitize, "1.41.0", Some(39699), None),
+
+    /// Allows using `&mut` in constant functions.
+    (active, const_mut_refs, "1.41.0", Some(57349), None),
+
     // -------------------------------------------------------------------------
     // feature-group-end: actual feature gates
     // -------------------------------------------------------------------------
diff --git a/src/librustc_feature/builtin_attrs.rs b/src/librustc_feature/builtin_attrs.rs
index f72df00..4fa0198 100644
--- a/src/librustc_feature/builtin_attrs.rs
+++ b/src/librustc_feature/builtin_attrs.rs
@@ -25,6 +25,7 @@
     (sym::target_thread_local, sym::cfg_target_thread_local, cfg_fn!(cfg_target_thread_local)),
     (sym::target_has_atomic, sym::cfg_target_has_atomic, cfg_fn!(cfg_target_has_atomic)),
     (sym::target_has_atomic_load_store, sym::cfg_target_has_atomic, cfg_fn!(cfg_target_has_atomic)),
+    (sym::sanitize, sym::cfg_sanitize, cfg_fn!(cfg_sanitize)),
 ];
 
 /// Find a gated cfg determined by the `pred`icate which is given the cfg's name.
@@ -283,7 +284,7 @@
         )
     ),
     (
-        sym::plugin, CrateLevel, template!(List: "name|name(args)"),
+        sym::plugin, CrateLevel, template!(List: "name"),
         Gated(
             Stability::Deprecated(
                 "https://github.com/rust-lang/rust/pull/64675",
diff --git a/src/librustc_incremental/Cargo.toml b/src/librustc_incremental/Cargo.toml
index 659c4c8..8dac726 100644
--- a/src/librustc_incremental/Cargo.toml
+++ b/src/librustc_incremental/Cargo.toml
@@ -19,3 +19,4 @@
 syntax = { path = "../libsyntax" }
 syntax_pos = { path = "../libsyntax_pos" }
 rustc_fs_util = { path = "../librustc_fs_util" }
+rustc_session = { path = "../librustc_session" }
diff --git a/src/librustc_incremental/assert_module_sources.rs b/src/librustc_incremental/assert_module_sources.rs
index 483b515..c2e3fa8 100644
--- a/src/librustc_incremental/assert_module_sources.rs
+++ b/src/librustc_incremental/assert_module_sources.rs
@@ -22,7 +22,7 @@
 //! was re-used.
 
 use rustc::hir::def_id::LOCAL_CRATE;
-use rustc::dep_graph::cgu_reuse_tracker::*;
+use rustc_session::cgu_reuse_tracker::*;
 use rustc::mir::mono::CodegenUnitNameBuilder;
 use rustc::ty::TyCtxt;
 use std::collections::BTreeSet;
diff --git a/src/librustc_interface/lib.rs b/src/librustc_interface/lib.rs
index 76af434..9bb1878 100644
--- a/src/librustc_interface/lib.rs
+++ b/src/librustc_interface/lib.rs
@@ -1,3 +1,4 @@
+#![feature(bool_to_option)]
 #![feature(box_syntax)]
 #![feature(set_stdio)]
 #![feature(nll)]
diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs
index e953a64..2a4bc41 100644
--- a/src/librustc_interface/passes.rs
+++ b/src/librustc_interface/passes.rs
@@ -30,7 +30,6 @@
 use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str};
 use rustc_passes::{self, ast_validation, hir_stats, layout_test};
 use rustc_plugin_impl as plugin;
-use rustc_plugin_impl::registry::Registry;
 use rustc_privacy;
 use rustc_resolve::{Resolver, ResolverArenas};
 use rustc_traits;
@@ -106,8 +105,7 @@
     (&mut Resolver<'_>) -> (Result<ast::Crate>, ResolverOutputs)
 );
 
-/// Runs the "early phases" of the compiler: initial `cfg` processing,
-/// loading compiler plugins (including those from `addl_plugins`),
+/// Runs the "early phases" of the compiler: initial `cfg` processing, loading compiler plugins,
 /// syntax expansion, secondary `cfg` expansion, synthesis of a test
 /// harness if one is to be provided, injection of a dependency on the
 /// standard library and prelude, and name resolution.
@@ -209,33 +207,22 @@
         middle::recursion_limit::update_limits(sess, &krate);
     });
 
-    let registrars = time(sess, "plugin loading", || {
-        plugin::load::load_plugins(
-            sess,
-            metadata_loader,
-            &krate,
-            Some(sess.opts.debugging_opts.extra_plugins.clone()),
-        )
-    });
-
     let mut lint_store = rustc_lint::new_lint_store(
         sess.opts.debugging_opts.no_interleave_lints,
         sess.unstable_options(),
     );
+    register_lints(&sess, &mut lint_store);
 
-    (register_lints)(&sess, &mut lint_store);
-
-    let mut registry = Registry::new(sess, &mut lint_store, krate.span);
-
+    let registrars = time(sess, "plugin loading", || {
+        plugin::load::load_plugins(sess, metadata_loader, &krate)
+    });
     time(sess, "plugin registration", || {
+        let mut registry = plugin::Registry { lint_store: &mut lint_store };
         for registrar in registrars {
-            registry.args_hidden = Some(registrar.args);
-            (registrar.fun)(&mut registry);
+            registrar(&mut registry);
         }
     });
 
-    *sess.plugin_llvm_passes.borrow_mut() = registry.llvm_passes;
-
     Ok((krate, Lrc::new(lint_store)))
 }
 
@@ -452,8 +439,7 @@
     sess.parse_sess.buffered_lints.with_lock(|buffered_lints| {
         info!("{} parse sess buffered_lints", buffered_lints.len());
         for BufferedEarlyLint{id, span, msg, lint_id} in buffered_lints.drain(..) {
-            let lint = lint::Lint::from_parser_lint_id(lint_id);
-            resolver.lint_buffer().buffer_lint(lint, id, span, &msg);
+            resolver.lint_buffer().buffer_lint(lint_id, id, span, &msg);
         }
     });
 
@@ -561,13 +547,7 @@
 }
 
 fn output_conflicts_with_dir(output_paths: &[PathBuf]) -> Option<PathBuf> {
-    let check = |output_path: &PathBuf| {
-        if output_path.is_dir() {
-            Some(output_path.clone())
-        } else {
-            None
-        }
-    };
+    let check = |output_path: &PathBuf| output_path.is_dir().then(|| output_path.clone());
     check_output(output_paths, check)
 }
 
diff --git a/src/librustc_interface/queries.rs b/src/librustc_interface/queries.rs
index 6103d42..e429b4d 100644
--- a/src/librustc_interface/queries.rs
+++ b/src/librustc_interface/queries.rs
@@ -22,7 +22,7 @@
 use syntax::{self, ast};
 
 /// Represent the result of a query.
-/// This result can be stolen with the `take` method and returned with the `give` method.
+/// This result can be stolen with the `take` method and generated with the `compute` method.
 pub struct Query<T> {
     result: RefCell<Option<Result<T>>>,
 }
@@ -37,7 +37,7 @@
     }
 
     /// Takes ownership of the query result. Further attempts to take or peek the query
-    /// result will panic unless it is returned by calling the `give` method.
+    /// result will panic unless it is generated by calling the `compute` method.
     pub fn take(&self) -> T {
         self.result
             .borrow_mut()
@@ -117,11 +117,9 @@
 
     pub fn dep_graph_future(&self) -> Result<&Query<Option<DepGraphFuture>>> {
         self.dep_graph_future.compute(|| {
-            Ok(if self.session().opts.build_dep_graph() {
-                Some(rustc_incremental::load_dep_graph(self.session()))
-            } else {
-                None
-            })
+            Ok(self.session().opts.build_dep_graph().then(|| {
+                rustc_incremental::load_dep_graph(self.session())
+            }))
         })
     }
 
diff --git a/src/librustc_interface/tests.rs b/src/librustc_interface/tests.rs
index d39a5d3..4c630b5 100644
--- a/src/librustc_interface/tests.rs
+++ b/src/librustc_interface/tests.rs
@@ -651,10 +651,6 @@
     assert!(reference.dep_tracking_hash() != opts.dep_tracking_hash());
 
     opts = reference.clone();
-    opts.debugging_opts.extra_plugins = vec![String::from("plugin1"), String::from("plugin2")];
-    assert!(reference.dep_tracking_hash() != opts.dep_tracking_hash());
-
-    opts = reference.clone();
     opts.debugging_opts.force_overflow_checks = Some(true);
     assert!(reference.dep_tracking_hash() != opts.dep_tracking_hash());
 
diff --git a/src/librustc_interface/util.rs b/src/librustc_interface/util.rs
index d8e20e1..8c225b8 100644
--- a/src/librustc_interface/util.rs
+++ b/src/librustc_interface/util.rs
@@ -107,11 +107,7 @@
 fn get_stack_size() -> Option<usize> {
     // FIXME: Hacks on hacks. If the env is trying to override the stack size
     // then *don't* set it explicitly.
-    if env::var_os("RUST_MIN_STACK").is_none() {
-        Some(STACK_SIZE)
-    } else {
-        None
-    }
+    env::var_os("RUST_MIN_STACK").is_none().then_some(STACK_SIZE)
 }
 
 struct Sink(Arc<Mutex<Vec<u8>>>);
@@ -285,11 +281,7 @@
             } else {
                 "rustc"
             });
-            if candidate.exists() {
-                Some(candidate)
-            } else {
-                None
-            }
+            candidate.exists().then_some(candidate)
         })
         .next()
 }
diff --git a/src/librustc_lint/Cargo.toml b/src/librustc_lint/Cargo.toml
index ed38243..e834b87 100644
--- a/src/librustc_lint/Cargo.toml
+++ b/src/librustc_lint/Cargo.toml
@@ -18,3 +18,4 @@
 rustc_feature = { path = "../librustc_feature" }
 rustc_index = { path = "../librustc_index" }
 rustc_error_codes = { path = "../librustc_error_codes" }
+rustc_session = { path = "../librustc_session" }
diff --git a/src/librustc_lint/builtin.rs b/src/librustc_lint/builtin.rs
index 5d3a6cc..10b00d3 100644
--- a/src/librustc_lint/builtin.rs
+++ b/src/librustc_lint/builtin.rs
@@ -24,7 +24,7 @@
 use std::fmt::Write;
 
 use rustc::hir::def::{Res, DefKind};
-use rustc::hir::def_id::{DefId, LOCAL_CRATE};
+use rustc::hir::def_id::DefId;
 use rustc::ty::{self, Ty, TyCtxt, layout::VariantIdx};
 use rustc::{lint, util};
 use rustc::lint::FutureIncompatibleInfo;
@@ -801,45 +801,6 @@
 }
 
 declare_lint! {
-    PLUGIN_AS_LIBRARY,
-    Warn,
-    "compiler plugin used as ordinary library in non-plugin crate"
-}
-
-declare_lint_pass!(PluginAsLibrary => [PLUGIN_AS_LIBRARY]);
-
-impl<'a, 'tcx> LateLintPass<'a, 'tcx> for PluginAsLibrary {
-    fn check_item(&mut self, cx: &LateContext<'_, '_>, it: &hir::Item) {
-        if cx.tcx.plugin_registrar_fn(LOCAL_CRATE).is_some() {
-            // We're compiling a plugin; it's fine to link other plugins.
-            return;
-        }
-
-        match it.kind {
-            hir::ItemKind::ExternCrate(..) => (),
-            _ => return,
-        };
-
-        let def_id = cx.tcx.hir().local_def_id(it.hir_id);
-        let prfn = match cx.tcx.extern_mod_stmt_cnum(def_id) {
-            Some(cnum) => cx.tcx.plugin_registrar_fn(cnum),
-            None => {
-                // Probably means we aren't linking the crate for some reason.
-                //
-                // Not sure if / when this could happen.
-                return;
-            }
-        };
-
-        if prfn.is_some() {
-            cx.span_lint(PLUGIN_AS_LIBRARY,
-                         it.span,
-                         "compiler plugin used as an ordinary library");
-        }
-    }
-}
-
-declare_lint! {
     NO_MANGLE_CONST_ITEMS,
     Deny,
     "const items will not have their symbols exported"
@@ -1268,7 +1229,6 @@
         MISSING_DEBUG_IMPLEMENTATIONS,
         ANONYMOUS_PARAMETERS,
         UNUSED_DOC_COMMENTS,
-        PLUGIN_AS_LIBRARY,
         NO_MANGLE_CONST_ITEMS,
         NO_MANGLE_GENERIC_ITEMS,
         MUTABLE_TRANSMUTES,
@@ -1490,10 +1450,10 @@
 
 impl EarlyLintPass for KeywordIdents {
     fn check_mac_def(&mut self, cx: &EarlyContext<'_>, mac_def: &ast::MacroDef, _id: ast::NodeId) {
-        self.check_tokens(cx, mac_def.stream());
+        self.check_tokens(cx, mac_def.body.inner_tokens());
     }
     fn check_mac(&mut self, cx: &EarlyContext<'_>, mac: &ast::Mac) {
-        self.check_tokens(cx, mac.tts.clone().into());
+        self.check_tokens(cx, mac.args.inner_tokens());
     }
     fn check_ident(&mut self, cx: &EarlyContext<'_>, ident: ast::Ident) {
         self.check_ident_token(cx, UnderMacro(false), ident);
@@ -1531,11 +1491,7 @@
             match pred {
                 ty::Predicate::TypeOutlives(outlives) => {
                     let outlives = outlives.skip_binder();
-                    if outlives.0.is_param(index) {
-                        Some(outlives.1)
-                    } else {
-                        None
-                    }
+                    outlives.0.is_param(index).then_some(outlives.1)
                 }
                 _ => None
             }
@@ -1594,11 +1550,7 @@
                             }),
                         _ => false,
                     };
-                    if is_inferred {
-                        Some((i, bound.span()))
-                    } else {
-                        None
-                    }
+                    is_inferred.then_some((i, bound.span()))
                 } else {
                     None
                 }
diff --git a/src/librustc_lint/lib.rs b/src/librustc_lint/lib.rs
index 7e8dc1d..12aab4b 100644
--- a/src/librustc_lint/lib.rs
+++ b/src/librustc_lint/lib.rs
@@ -12,6 +12,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/")]
 
 #![cfg_attr(test, feature(test))]
+#![feature(bool_to_option)]
 #![feature(box_patterns)]
 #![feature(box_syntax)]
 #![feature(nll)]
@@ -21,6 +22,8 @@
 
 #[macro_use]
 extern crate rustc;
+#[macro_use]
+extern crate rustc_session;
 
 mod array_into_iter;
 mod nonstandard_style;
@@ -157,8 +160,6 @@
             // Depends on types used in type definitions
             MissingCopyImplementations: MissingCopyImplementations,
 
-            PluginAsLibrary: PluginAsLibrary,
-
             // Depends on referenced function signatures in expressions
             MutableTransmutes: MutableTransmutes,
 
@@ -350,6 +351,7 @@
         "converted into hard error, see https://github.com/rust-lang/rust/issues/35896");
     store.register_removed("nested_impl_trait",
         "converted into hard error, see https://github.com/rust-lang/rust/issues/59014");
+    store.register_removed("plugin_as_library", "plugins have been deprecated and retired");
 }
 
 fn register_internals(store: &mut lint::LintStore) {
diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs
index dbf2dcf..25bd2c4 100644
--- a/src/librustc_metadata/creader.rs
+++ b/src/librustc_metadata/creader.rs
@@ -802,11 +802,8 @@
         // First up we check for global allocators. Look at the crate graph here
         // and see what's a global allocator, including if we ourselves are a
         // global allocator.
-        let mut global_allocator = if self.cstore.has_global_allocator {
-            Some(Symbol::intern("this crate"))
-        } else {
-            None
-        };
+        let mut global_allocator = self.cstore.has_global_allocator
+            .then(|| Symbol::intern("this crate"));
         self.cstore.iter_crate_data(|_, data| {
             if !data.has_global_allocator() {
                 return
diff --git a/src/librustc_metadata/lib.rs b/src/librustc_metadata/lib.rs
index 8c0b734..aaaff7e 100644
--- a/src/librustc_metadata/lib.rs
+++ b/src/librustc_metadata/lib.rs
@@ -1,5 +1,6 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/")]
 
+#![feature(bool_to_option)]
 #![feature(box_patterns)]
 #![feature(core_intrinsics)]
 #![feature(crate_visibility_modifier)]
diff --git a/src/librustc_metadata/rmeta/decoder.rs b/src/librustc_metadata/rmeta/decoder.rs
index 820783b..6edd17f 100644
--- a/src/librustc_metadata/rmeta/decoder.rs
+++ b/src/librustc_metadata/rmeta/decoder.rs
@@ -18,12 +18,11 @@
 use rustc_data_structures::svh::Svh;
 use rustc::dep_graph::{self, DepNodeIndex};
 use rustc::middle::lang_items;
-use rustc::mir::{self, interpret};
+use rustc::mir::{self, BodyCache, interpret, Promoted};
 use rustc::mir::interpret::{AllocDecodingSession, AllocDecodingState};
 use rustc::session::Session;
 use rustc::ty::{self, Ty, TyCtxt};
 use rustc::ty::codec::TyDecoder;
-use rustc::mir::{Body, Promoted};
 use rustc::util::common::record_time;
 use rustc::util::captures::Captures;
 
@@ -1080,26 +1079,32 @@
             self.root.per_def.mir.get(self, id).is_some()
     }
 
-    fn get_optimized_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> Body<'tcx> {
-        self.root.per_def.mir.get(self, id)
+    fn get_optimized_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> BodyCache<'tcx> {
+        let mut cache = self.root.per_def.mir.get(self, id)
             .filter(|_| !self.is_proc_macro(id))
             .unwrap_or_else(|| {
                 bug!("get_optimized_mir: missing MIR for `{:?}`", self.local_def_id(id))
             })
-            .decode((self, tcx))
+            .decode((self, tcx));
+        cache.ensure_predecessors();
+        cache
     }
 
     fn get_promoted_mir(
         &self,
         tcx: TyCtxt<'tcx>,
         id: DefIndex,
-    ) -> IndexVec<Promoted, Body<'tcx>> {
-        self.root.per_def.promoted_mir.get(self, id)
+    ) -> IndexVec<Promoted, BodyCache<'tcx>> {
+        let mut cache = self.root.per_def.promoted_mir.get(self, id)
             .filter(|_| !self.is_proc_macro(id))
             .unwrap_or_else(|| {
                 bug!("get_promoted_mir: missing MIR for `{:?}`", self.local_def_id(id))
             })
-            .decode((self, tcx))
+            .decode((self, tcx));
+        for body in cache.iter_mut() {
+            body.ensure_predecessors();
+        }
+        cache
     }
 
     fn mir_const_qualif(&self, id: DefIndex) -> mir::ConstQualifs {
@@ -1355,10 +1360,16 @@
         }
     }
 
+    // This replicates some of the logic of the crate-local `is_const_fn_raw` query, because we
+    // don't serialize constness for tuple variant and tuple struct constructors.
     fn is_const_fn_raw(&self, id: DefIndex) -> bool {
         let constness = match self.kind(id) {
             EntryKind::Method(data) => data.decode(self).fn_data.constness,
             EntryKind::Fn(data) => data.decode(self).constness,
+            // Some intrinsics can be const fn. While we could recompute this (at least until we
+            // stop having hardcoded whitelists and move to stability attributes), it seems cleaner
+            // to treat all const fns equally.
+            EntryKind::ForeignFn(data) => data.decode(self).constness,
             EntryKind::Variant(..) | EntryKind::Struct(..) => hir::Constness::Const,
             _ => hir::Constness::NotConst,
         };
diff --git a/src/librustc_metadata/rmeta/decoder/cstore_impl.rs b/src/librustc_metadata/rmeta/decoder/cstore_impl.rs
index 8214153..13db9a6 100644
--- a/src/librustc_metadata/rmeta/decoder/cstore_impl.rs
+++ b/src/librustc_metadata/rmeta/decoder/cstore_impl.rs
@@ -32,6 +32,8 @@
 use syntax::source_map::Spanned;
 use syntax::symbol::Symbol;
 use syntax::expand::allocator::AllocatorKind;
+use syntax::ptr::P;
+use syntax::tokenstream::DelimSpan;
 use syntax_pos::{Span, FileName};
 
 macro_rules! provide {
@@ -427,6 +429,7 @@
 
         let source_file = sess.parse_sess.source_map().new_source_file(source_name, def.body);
         let local_span = Span::with_root_ctxt(source_file.start_pos, source_file.end_pos);
+        let dspan = DelimSpan::from_single(local_span);
         let (body, mut errors) = source_file_to_stream(&sess.parse_sess, source_file, None);
         emit_unclosed_delims(&mut errors, &sess.parse_sess);
 
@@ -448,7 +451,7 @@
             span: local_span,
             attrs: attrs.iter().cloned().collect(),
             kind: ast::ItemKind::MacroDef(ast::MacroDef {
-                tokens: body.into(),
+                body: P(ast::MacArgs::Delimited(dspan, ast::MacDelimiter::Brace, body)),
                 legacy: def.legacy,
             }),
             vis: source_map::respan(local_span.shrink_to_lo(), ast::VisibilityKind::Inherited),
diff --git a/src/librustc_metadata/rmeta/encoder.rs b/src/librustc_metadata/rmeta/encoder.rs
index 3a318dd..fb70e10 100644
--- a/src/librustc_metadata/rmeta/encoder.rs
+++ b/src/librustc_metadata/rmeta/encoder.rs
@@ -1525,7 +1525,11 @@
             hir::ForeignItemKind::Fn(_, ref names, _) => {
                 let data = FnData {
                     asyncness: hir::IsAsync::NotAsync,
-                    constness: hir::Constness::NotConst,
+                    constness: if self.tcx.is_const_fn_raw(def_id) {
+                        hir::Constness::Const
+                    } else {
+                        hir::Constness::NotConst
+                    },
                     param_names: self.encode_fn_param_names(names),
                 };
                 EntryKind::ForeignFn(self.lazy(data))
diff --git a/src/librustc_metadata/rmeta/mod.rs b/src/librustc_metadata/rmeta/mod.rs
index 4ea562f..fdf43f0 100644
--- a/src/librustc_metadata/rmeta/mod.rs
+++ b/src/librustc_metadata/rmeta/mod.rs
@@ -276,8 +276,8 @@
     // Also, as an optimization, a missing entry indicates an empty `&[]`.
     inferred_outlives: Table<DefIndex, Lazy!(&'tcx [(ty::Predicate<'tcx>, Span)])>,
     super_predicates: Table<DefIndex, Lazy!(ty::GenericPredicates<'tcx>)>,
-    mir: Table<DefIndex, Lazy!(mir::Body<'tcx>)>,
-    promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::Body<'tcx>>)>,
+    mir: Table<DefIndex, Lazy!(mir::BodyCache<'tcx>)>,
+    promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::BodyCache<'tcx>>)>,
 }
 
 #[derive(Copy, Clone, RustcEncodable, RustcDecodable)]
diff --git a/src/librustc_mir/borrow_check/borrow_set.rs b/src/librustc_mir/borrow_check/borrow_set.rs
index 9432343..802464c 100644
--- a/src/librustc_mir/borrow_check/borrow_set.rs
+++ b/src/librustc_mir/borrow_check/borrow_set.rs
@@ -5,7 +5,7 @@
 use crate::dataflow::move_paths::MoveData;
 use rustc::mir::traversal;
 use rustc::mir::visit::{PlaceContext, Visitor, NonUseContext, MutatingUseContext};
-use rustc::mir::{self, Location, Body, Local};
+use rustc::mir::{self, Location, Body, Local, ReadOnlyBodyCache};
 use rustc::ty::{RegionVid, TyCtxt};
 use rustc::util::nodemap::{FxHashMap, FxHashSet};
 use rustc_index::vec::IndexVec;
@@ -90,7 +90,7 @@
 impl LocalsStateAtExit {
     fn build(
         locals_are_invalidated_at_exit: bool,
-        body: &Body<'tcx>,
+        body: ReadOnlyBodyCache<'_, 'tcx>,
         move_data: &MoveData<'tcx>
     ) -> Self {
         struct HasStorageDead(BitSet<Local>);
@@ -106,7 +106,8 @@
         if locals_are_invalidated_at_exit {
             LocalsStateAtExit::AllAreInvalidated
         } else {
-            let mut has_storage_dead = HasStorageDead(BitSet::new_empty(body.local_decls.len()));
+            let mut has_storage_dead
+                = HasStorageDead(BitSet::new_empty(body.local_decls.len()));
             has_storage_dead.visit_body(body);
             let mut has_storage_dead_or_moved = has_storage_dead.0;
             for move_out in &move_data.moves {
@@ -123,13 +124,13 @@
 impl<'tcx> BorrowSet<'tcx> {
     pub fn build(
         tcx: TyCtxt<'tcx>,
-        body: &Body<'tcx>,
+        body: ReadOnlyBodyCache<'_, 'tcx>,
         locals_are_invalidated_at_exit: bool,
         move_data: &MoveData<'tcx>,
     ) -> Self {
         let mut visitor = GatherBorrows {
             tcx,
-            body,
+            body: &body,
             idx_vec: IndexVec::new(),
             location_map: Default::default(),
             activation_map: Default::default(),
@@ -139,7 +140,7 @@
                 LocalsStateAtExit::build(locals_are_invalidated_at_exit, body, move_data),
         };
 
-        for (block, block_data) in traversal::preorder(body) {
+        for (block, block_data) in traversal::preorder(&body) {
             visitor.visit_basic_block_data(block, block_data);
         }
 
diff --git a/src/librustc_mir/borrow_check/conflict_errors.rs b/src/librustc_mir/borrow_check/diagnostics/conflict_errors.rs
similarity index 96%
rename from src/librustc_mir/borrow_check/conflict_errors.rs
rename to src/librustc_mir/borrow_check/diagnostics/conflict_errors.rs
index 48f8ad9..d14957b 100644
--- a/src/librustc_mir/borrow_check/conflict_errors.rs
+++ b/src/librustc_mir/borrow_check/diagnostics/conflict_errors.rs
@@ -14,18 +14,22 @@
 use syntax_pos::Span;
 use syntax::source_map::DesugaringKind;
 
-use super::nll::explain_borrow::BorrowExplanation;
-use super::nll::region_infer::{RegionName, RegionNameSource};
-use super::prefixes::IsPrefixOf;
-use super::WriteKind;
-use super::borrow_set::BorrowData;
-use super::MirBorrowckCtxt;
-use super::{InitializationRequiringAction, PrefixSet};
-use super::error_reporting::{IncludingDowncast, UseSpans};
 use crate::dataflow::drop_flag_effects;
 use crate::dataflow::indexes::{MovePathIndex, MoveOutIndex};
 use crate::util::borrowck_errors;
 
+use crate::borrow_check::{
+    prefixes::IsPrefixOf,
+    WriteKind,
+    borrow_set::BorrowData,
+    MirBorrowckCtxt, InitializationRequiringAction, PrefixSet
+};
+
+use super::{
+    IncludingDowncast, UseSpans, RegionName, RegionNameSource,
+    explain_borrow::BorrowExplanation,
+};
+
 #[derive(Debug)]
 struct MoveSite {
     /// Index of the "move out" that we found. The `MoveData` can
@@ -46,7 +50,7 @@
 }
 
 impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
-    pub(super) fn report_use_of_moved_or_uninitialized(
+    pub(in crate::borrow_check) fn report_use_of_moved_or_uninitialized(
         &mut self,
         location: Location,
         desired_action: InitializationRequiringAction,
@@ -205,9 +209,12 @@
                 );
             }
 
-            let ty =
-                Place::ty_from(used_place.base, used_place.projection, self.body, self.infcx.tcx)
-                    .ty;
+            let ty = Place::ty_from(
+                used_place.base,
+                used_place.projection,
+                *self.body,
+                self.infcx.tcx
+            ).ty;
             let needs_note = match ty.kind {
                 ty::Closure(id, _) => {
                     let tables = self.infcx.tcx.typeck_tables_of(id);
@@ -222,7 +229,7 @@
                 let mpi = self.move_data.moves[move_out_indices[0]].path;
                 let place = &self.move_data.move_paths[mpi].place;
 
-                let ty = place.ty(self.body, self.infcx.tcx).ty;
+                let ty = place.ty(*self.body, self.infcx.tcx).ty;
                 let opt_name =
                     self.describe_place_with_options(place.as_ref(), IncludingDowncast(true));
                 let note_msg = match opt_name {
@@ -266,7 +273,7 @@
         }
     }
 
-    pub(super) fn report_move_out_while_borrowed(
+    pub(in crate::borrow_check) fn report_move_out_while_borrowed(
         &mut self,
         location: Location,
         (place, span): (&Place<'tcx>, Span),
@@ -314,7 +321,7 @@
             None,
         ).add_explanation_to_diagnostic(
             self.infcx.tcx,
-            self.body,
+            &self.body,
             &self.local_names,
             &mut err,
             "",
@@ -323,7 +330,7 @@
         err.buffer(&mut self.errors_buffer);
     }
 
-    pub(super) fn report_use_while_mutably_borrowed(
+    pub(in crate::borrow_check) fn report_use_while_mutably_borrowed(
         &mut self,
         location: Location,
         (place, _span): (&Place<'tcx>, Span),
@@ -356,7 +363,7 @@
         self.explain_why_borrow_contains_point(location, borrow, None)
             .add_explanation_to_diagnostic(
                 self.infcx.tcx,
-                self.body,
+                &self.body,
                 &self.local_names,
                 &mut err,
                 "",
@@ -365,7 +372,7 @@
         err
     }
 
-    pub(super) fn report_conflicting_borrow(
+    pub(in crate::borrow_check) fn report_conflicting_borrow(
         &mut self,
         location: Location,
         (place, span): (&Place<'tcx>, Span),
@@ -578,7 +585,7 @@
 
         explanation.add_explanation_to_diagnostic(
             self.infcx.tcx,
-            self.body,
+            &self.body,
             &self.local_names,
             &mut err,
             first_borrow_desc,
@@ -611,7 +618,7 @@
     ///
     /// >  cannot borrow `a.u` (via `a.u.z.c`) as immutable because it is also borrowed as
     /// >  mutable (via `a.u.s.b`) [E0502]
-    pub(super) fn describe_place_for_conflicting_borrow(
+    pub(in crate::borrow_check) fn describe_place_for_conflicting_borrow(
         &self,
         first_borrowed_place: &Place<'tcx>,
         second_borrowed_place: &Place<'tcx>,
@@ -619,7 +626,12 @@
         // Define a small closure that we can use to check if the type of a place
         // is a union.
         let union_ty = |place_base, place_projection| {
-            let ty = Place::ty_from(place_base, place_projection, self.body, self.infcx.tcx).ty;
+            let ty = Place::ty_from(
+                place_base,
+                place_projection,
+                *self.body,
+                self.infcx.tcx
+            ).ty;
             ty.ty_adt_def().filter(|adt| adt.is_union()).map(|_| ty)
         };
         let describe_place = |place| self.describe_place(place).unwrap_or_else(|| "_".to_owned());
@@ -714,7 +726,7 @@
     /// short a lifetime. (But sometimes it is more useful to report
     /// it as a more direct conflict between the execution of a
     /// `Drop::drop` with an aliasing borrow.)
-    pub(super) fn report_borrowed_value_does_not_live_long_enough(
+    pub(in crate::borrow_check) fn report_borrowed_value_does_not_live_long_enough(
         &mut self,
         location: Location,
         borrow: &BorrowData<'tcx>,
@@ -965,7 +977,7 @@
             } else {
                 explanation.add_explanation_to_diagnostic(
                     self.infcx.tcx,
-                    self.body,
+                    &self.body,
                     &self.local_names,
                     &mut err,
                     "",
@@ -991,7 +1003,7 @@
             );
 
             explanation.add_explanation_to_diagnostic(
-                self.infcx.tcx, self.body, &self.local_names, &mut err, "", None);
+                self.infcx.tcx, &self.body, &self.local_names, &mut err, "", None);
         }
 
         err
@@ -1051,7 +1063,7 @@
 
         explanation.add_explanation_to_diagnostic(
             self.infcx.tcx,
-            self.body,
+            &self.body,
             &self.local_names,
             &mut err,
             "",
@@ -1138,7 +1150,7 @@
         }
         explanation.add_explanation_to_diagnostic(
             self.infcx.tcx,
-            self.body,
+            &self.body,
             &self.local_names,
             &mut err,
             "",
@@ -1174,11 +1186,12 @@
         };
 
         // FIXME use a better heuristic than Spans
-        let reference_desc = if return_span == self.body.source_info(borrow.reserve_location).span {
-            "reference to"
-        } else {
-            "value referencing"
-        };
+        let reference_desc
+            = if return_span == self.body.source_info(borrow.reserve_location).span {
+                "reference to"
+            } else {
+                "value referencing"
+            };
 
         let (place_desc, note) = if let Some(place_desc) = opt_place_desc {
             let local_kind = if let Some(local) = borrow.borrowed_place.as_local() {
@@ -1372,10 +1385,8 @@
     }
 
     fn get_moved_indexes(&mut self, location: Location, mpi: MovePathIndex) -> Vec<MoveSite> {
-        let body = self.body;
-
         let mut stack = Vec::new();
-        stack.extend(body.predecessor_locations(location).map(|predecessor| {
+        stack.extend(self.body.predecessor_locations(location).map(|predecessor| {
             let is_back_edge = location.dominates(predecessor, &self.dominators);
             (predecessor, is_back_edge)
         }));
@@ -1394,7 +1405,7 @@
             }
 
             // check for moves
-            let stmt_kind = body[location.block]
+            let stmt_kind = self.body[location.block]
                 .statements
                 .get(location.statement_index)
                 .map(|s| &s.kind);
@@ -1449,7 +1460,7 @@
             let mut any_match = false;
             drop_flag_effects::for_location_inits(
                 self.infcx.tcx,
-                self.body,
+                &self.body,
                 self.move_data,
                 location,
                 |m| {
@@ -1462,7 +1473,7 @@
                 continue 'dfs;
             }
 
-            stack.extend(body.predecessor_locations(location).map(|predecessor| {
+            stack.extend(self.body.predecessor_locations(location).map(|predecessor| {
                 let back_edge = location.dominates(predecessor, &self.dominators);
                 (predecessor, is_back_edge || back_edge)
             }));
@@ -1471,7 +1482,7 @@
         result
     }
 
-    pub(super) fn report_illegal_mutation_of_borrowed(
+    pub(in crate::borrow_check) fn report_illegal_mutation_of_borrowed(
         &mut self,
         location: Location,
         (place, span): (&Place<'tcx>, Span),
@@ -1514,7 +1525,7 @@
         self.explain_why_borrow_contains_point(location, loan, None)
             .add_explanation_to_diagnostic(
                 self.infcx.tcx,
-                self.body,
+                &self.body,
                 &self.local_names,
                 &mut err,
                 "",
@@ -1530,7 +1541,7 @@
     /// assigned; `err_place` is a place providing a reason why
     /// `place` is not mutable (e.g., the non-`mut` local `x` in an
     /// assignment to `x.f`).
-    pub(super) fn report_illegal_reassignment(
+    pub(in crate::borrow_check) fn report_illegal_reassignment(
         &mut self,
         _location: Location,
         (place, span): (&Place<'tcx>, Span),
@@ -1625,7 +1636,12 @@
                         StorageDeadOrDrop::LocalStorageDead
                         | StorageDeadOrDrop::BoxedStorageDead => {
                             assert!(
-                                Place::ty_from(&place.base, proj_base, self.body, tcx).ty.is_box(),
+                                Place::ty_from(
+                                    &place.base,
+                                    proj_base,
+                                    *self.body,
+                                    tcx
+                                ).ty.is_box(),
                                 "Drop of value behind a reference or raw pointer"
                             );
                             StorageDeadOrDrop::BoxedStorageDead
@@ -1633,7 +1649,12 @@
                         StorageDeadOrDrop::Destructor(_) => base_access,
                     },
                     ProjectionElem::Field(..) | ProjectionElem::Downcast(..) => {
-                        let base_ty = Place::ty_from(&place.base, proj_base, self.body, tcx).ty;
+                        let base_ty = Place::ty_from(
+                            &place.base,
+                            proj_base,
+                            *self.body,
+                            tcx
+                        ).ty;
                         match base_ty.kind {
                             ty::Adt(def, _) if def.has_dtor(tcx) => {
                                 // Report the outermost adt with a destructor
@@ -1678,7 +1699,7 @@
             }
         }
         let mut visitor = FakeReadCauseFinder { place, cause: None };
-        visitor.visit_body(&self.body);
+        visitor.visit_body(self.body);
         match visitor.cause {
             Some(FakeReadCause::ForMatchGuard) => Some("match guard"),
             Some(FakeReadCause::ForIndex) => Some("indexing expression"),
@@ -1736,7 +1757,8 @@
             // Next, look through the rest of the block, checking if we are assigning the
             // `target` (that is, the place that contains our borrow) to anything.
             let mut annotated_closure = None;
-            for stmt in &self.body[location.block].statements[location.statement_index + 1..] {
+            for stmt in &self.body[location.block].statements[location.statement_index + 1..]
+            {
                 debug!(
                     "annotate_argument_and_return_for_borrow: target={:?} stmt={:?}",
                     target, stmt
@@ -2062,7 +2084,7 @@
 impl<'tcx> AnnotatedBorrowFnSignature<'tcx> {
     /// Annotate the provided diagnostic with information about borrow from the fn signature that
     /// helps explain.
-    pub(super) fn emit(
+    pub(in crate::borrow_check) fn emit(
         &self,
         cx: &mut MirBorrowckCtxt<'_, 'tcx>,
         diag: &mut DiagnosticBuilder<'_>,
diff --git a/src/librustc_mir/borrow_check/nll/explain_borrow/mod.rs b/src/librustc_mir/borrow_check/diagnostics/explain_borrow.rs
similarity index 98%
rename from src/librustc_mir/borrow_check/nll/explain_borrow/mod.rs
rename to src/librustc_mir/borrow_check/diagnostics/explain_borrow.rs
index c705853..67c3c36 100644
--- a/src/librustc_mir/borrow_check/nll/explain_borrow/mod.rs
+++ b/src/librustc_mir/borrow_check/diagnostics/explain_borrow.rs
@@ -1,8 +1,7 @@
 use std::collections::VecDeque;
 
 use crate::borrow_check::borrow_set::BorrowData;
-use crate::borrow_check::error_reporting::UseSpans;
-use crate::borrow_check::nll::region_infer::{Cause, RegionName};
+use crate::borrow_check::nll::region_infer::Cause;
 use crate::borrow_check::nll::ConstraintDescription;
 use crate::borrow_check::{MirBorrowckCtxt, WriteKind};
 use rustc::mir::{
@@ -17,7 +16,7 @@
 use syntax_pos::Span;
 use syntax_pos::symbol::Symbol;
 
-mod find_use;
+use super::{UseSpans, find_use, RegionName};
 
 #[derive(Debug)]
 pub(in crate::borrow_check) enum BorrowExplanation {
@@ -237,7 +236,7 @@
         );
 
         let regioncx = &self.nonlexical_regioncx;
-        let body = self.body;
+        let body: &Body<'_> = &self.body;
         let tcx = self.infcx.tcx;
 
         let borrow_region_vid = borrow.region;
@@ -297,9 +296,9 @@
                 if let Some(region) = regioncx.to_error_region_vid(borrow_region_vid) {
                     let (category, from_closure, span, region_name) =
                         self.nonlexical_regioncx.free_region_constraint_info(
-                            self.body,
-                        &self.local_names,
-                        &self.upvars,
+                            &self.body,
+                            &self.local_names,
+                            &self.upvars,
                             self.mir_def_id,
                             self.infcx,
                             borrow_region_vid,
diff --git a/src/librustc_mir/borrow_check/nll/explain_borrow/find_use.rs b/src/librustc_mir/borrow_check/diagnostics/find_use.rs
similarity index 100%
rename from src/librustc_mir/borrow_check/nll/explain_borrow/find_use.rs
rename to src/librustc_mir/borrow_check/diagnostics/find_use.rs
diff --git a/src/librustc_mir/borrow_check/error_reporting.rs b/src/librustc_mir/borrow_check/diagnostics/mod.rs
similarity index 96%
rename from src/librustc_mir/borrow_check/error_reporting.rs
rename to src/librustc_mir/borrow_check/diagnostics/mod.rs
index a555e0b..1a76265 100644
--- a/src/librustc_mir/borrow_check/error_reporting.rs
+++ b/src/librustc_mir/borrow_check/diagnostics/mod.rs
@@ -1,3 +1,5 @@
+//! Borrow checker diagnostics.
+
 use rustc::hir;
 use rustc::hir::def::Namespace;
 use rustc::hir::def_id::DefId;
@@ -17,6 +19,22 @@
 use super::MirBorrowckCtxt;
 use crate::dataflow::move_paths::{InitLocation, LookupResult};
 
+mod find_use;
+mod var_name;
+mod region_name;
+mod outlives_suggestion;
+
+mod conflict_errors;
+mod move_errors;
+mod mutability_errors;
+mod region_errors;
+mod explain_borrow;
+
+crate use mutability_errors::AccessKind;
+crate use region_name::{RegionName, RegionNameSource, RegionErrorNamingCtx};
+crate use region_errors::{ErrorReportingCtx, ErrorConstraintInfo};
+crate use outlives_suggestion::OutlivesSuggestionBuilder;
+
 pub(super) struct IncludingDowncast(pub(super) bool);
 
 impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
@@ -369,8 +387,11 @@
                     }, field)
                 }
                 ProjectionElem::Downcast(_, variant_index) => {
-                    let base_ty =
-                        Place::ty_from(place.base, place.projection, self.body, self.infcx.tcx).ty;
+                    let base_ty = Place::ty_from(
+                        place.base,
+                        place.projection,
+                        *self.body,
+                        self.infcx.tcx).ty;
                     self.describe_field_from_ty(&base_ty, field, Some(*variant_index))
                 }
                 ProjectionElem::Field(_, field_type) => {
@@ -498,9 +519,10 @@
                         },
                         ..
                     }) = bbd.terminator {
-                        if let Some(source)
-                            = BorrowedContentSource::from_call(func.ty(self.body, tcx), tcx)
-                        {
+                        if let Some(source) = BorrowedContentSource::from_call(
+                            func.ty(*self.body, tcx),
+                            tcx
+                        ) {
                             return source;
                         }
                     }
@@ -512,7 +534,12 @@
 
         // If we didn't find an overloaded deref or index, then assume it's a
         // built in deref and check the type of the base.
-        let base_ty = Place::ty_from(deref_base.base, deref_base.projection, self.body, tcx).ty;
+        let base_ty = Place::ty_from(
+            deref_base.base,
+            deref_base.projection,
+            *self.body,
+            tcx
+        ).ty;
         if base_ty.is_unsafe_ptr() {
             BorrowedContentSource::DerefRawPointer
         } else if base_ty.is_mutable_ptr() {
diff --git a/src/librustc_mir/borrow_check/move_errors.rs b/src/librustc_mir/borrow_check/diagnostics/move_errors.rs
similarity index 98%
rename from src/librustc_mir/borrow_check/move_errors.rs
rename to src/librustc_mir/borrow_check/diagnostics/move_errors.rs
index bf61eb9..938836d 100644
--- a/src/librustc_mir/borrow_check/move_errors.rs
+++ b/src/librustc_mir/borrow_check/diagnostics/move_errors.rs
@@ -5,7 +5,7 @@
 
 use crate::borrow_check::MirBorrowckCtxt;
 use crate::borrow_check::prefixes::PrefixSet;
-use crate::borrow_check::error_reporting::UseSpans;
+use crate::borrow_check::diagnostics::UseSpans;
 use crate::dataflow::move_paths::{
     IllegalMoveOrigin, IllegalMoveOriginKind,
     LookupResult, MoveError, MovePathIndex,
@@ -300,7 +300,7 @@
         // Inspect the type of the content behind the
         // borrow to provide feedback about why this
         // was a move rather than a copy.
-        let ty = deref_target_place.ty(self.body, self.infcx.tcx).ty;
+        let ty = deref_target_place.ty(*self.body, self.infcx.tcx).ty;
         let upvar_field = self.prefixes(move_place.as_ref(), PrefixSet::All)
             .find_map(|p| self.is_upvar_field_projection(p));
 
@@ -411,7 +411,7 @@
         };
         let move_ty = format!(
             "{:?}",
-            move_place.ty(self.body, self.infcx.tcx).ty,
+            move_place.ty(*self.body, self.infcx.tcx).ty,
         );
         if let Ok(snippet) = self.infcx.tcx.sess.source_map().span_to_snippet(span) {
             let is_option = move_ty.starts_with("std::option::Option");
@@ -454,7 +454,7 @@
                 }
 
                 if binds_to.is_empty() {
-                    let place_ty = move_from.ty(self.body, self.infcx.tcx).ty;
+                    let place_ty = move_from.ty(*self.body, self.infcx.tcx).ty;
                     let place_desc = match self.describe_place(move_from.as_ref()) {
                         Some(desc) => format!("`{}`", desc),
                         None => format!("value"),
@@ -482,7 +482,7 @@
             // No binding. Nothing to suggest.
             GroupedMoveError::OtherIllegalMove { ref original_path, use_spans, .. } => {
                 let span = use_spans.var_or_use();
-                let place_ty = original_path.ty(self.body, self.infcx.tcx).ty;
+                let place_ty = original_path.ty(*self.body, self.infcx.tcx).ty;
                 let place_desc = match self.describe_place(original_path.as_ref()) {
                     Some(desc) => format!("`{}`", desc),
                     None => format!("value"),
diff --git a/src/librustc_mir/borrow_check/mutability_errors.rs b/src/librustc_mir/borrow_check/diagnostics/mutability_errors.rs
similarity index 97%
rename from src/librustc_mir/borrow_check/mutability_errors.rs
rename to src/librustc_mir/borrow_check/diagnostics/mutability_errors.rs
index bf070c3..6449ae3 100644
--- a/src/librustc_mir/borrow_check/mutability_errors.rs
+++ b/src/librustc_mir/borrow_check/diagnostics/mutability_errors.rs
@@ -1,6 +1,6 @@
 use rustc::hir;
 use rustc::hir::Node;
-use rustc::mir::{self, Body, ClearCrossCrate, Local, LocalInfo, Location};
+use rustc::mir::{self, ClearCrossCrate, Local, LocalInfo, Location, ReadOnlyBodyCache};
 use rustc::mir::{Mutability, Place, PlaceRef, PlaceBase, ProjectionElem};
 use rustc::ty::{self, Ty, TyCtxt};
 use rustc_index::vec::Idx;
@@ -8,18 +8,18 @@
 use syntax_pos::symbol::kw;
 
 use crate::borrow_check::MirBorrowckCtxt;
-use crate::borrow_check::error_reporting::BorrowedContentSource;
+use crate::borrow_check::diagnostics::BorrowedContentSource;
 use crate::util::collect_writes::FindAssignments;
 use rustc_errors::Applicability;
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
-pub(super) enum AccessKind {
+pub(crate) enum AccessKind {
     MutableBorrow,
     Mutate,
 }
 
 impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
-    pub(super) fn report_mutability_error(
+    pub(crate) fn report_mutability_error(
         &mut self,
         access_place: &Place<'tcx>,
         span: Span,
@@ -61,8 +61,12 @@
                 projection: [proj_base @ .., ProjectionElem::Field(upvar_index, _)],
             } => {
                 debug_assert!(is_closure_or_generator(
-                    Place::ty_from(&the_place_err.base, proj_base, self.body, self.infcx.tcx).ty
-                ));
+                    Place::ty_from(
+                        &the_place_err.base,
+                        proj_base,
+                        *self.body,
+                        self.infcx.tcx
+                    ).ty));
 
                 item_msg = format!("`{}`", access_place_desc.unwrap());
                 if self.is_upvar_field_projection(access_place.as_ref()).is_some() {
@@ -111,7 +115,7 @@
                         Place::ty_from(
                             the_place_err.base,
                             the_place_err.projection,
-                            self.body,
+                            *self.body,
                             self.infcx.tcx
                         )
                         .ty
@@ -225,7 +229,7 @@
 
                 if let Some((span, message)) = annotate_struct_field(
                     self.infcx.tcx,
-                    Place::ty_from(base, proj_base, self.body, self.infcx.tcx).ty,
+                    Place::ty_from(base, proj_base, *self.body, self.infcx.tcx).ty,
                     field,
                 ) {
                     err.span_suggestion(
@@ -300,7 +304,7 @@
                 projection: [proj_base @ .., ProjectionElem::Field(upvar_index, _)],
             } => {
                 debug_assert!(is_closure_or_generator(
-                    Place::ty_from(base, proj_base, self.body, self.infcx.tcx).ty
+                    Place::ty_from(base, proj_base, *self.body, self.infcx.tcx).ty
                 ));
 
                 err.span_label(span, format!("cannot {ACT}", ACT = act));
@@ -529,7 +533,7 @@
 // by trying (3.), then (2.) and finally falling back on (1.).
 fn suggest_ampmut<'tcx>(
     tcx: TyCtxt<'tcx>,
-    body: &Body<'tcx>,
+    body: ReadOnlyBodyCache<'_, 'tcx>,
     local: Local,
     local_decl: &mir::LocalDecl<'tcx>,
     opt_ty_info: Option<Span>,
diff --git a/src/librustc_mir/borrow_check/nll/region_infer/error_reporting/outlives_suggestion.rs b/src/librustc_mir/borrow_check/diagnostics/outlives_suggestion.rs
similarity index 97%
rename from src/librustc_mir/borrow_check/nll/region_infer/error_reporting/outlives_suggestion.rs
rename to src/librustc_mir/borrow_check/diagnostics/outlives_suggestion.rs
index 938059c..7aecada 100644
--- a/src/librustc_mir/borrow_check/nll/region_infer/error_reporting/outlives_suggestion.rs
+++ b/src/librustc_mir/borrow_check/diagnostics/outlives_suggestion.rs
@@ -13,12 +13,10 @@
 
 use smallvec::SmallVec;
 
-use crate::borrow_check::nll::region_infer::{
-    error_reporting::{
-        region_name::{RegionName, RegionNameSource},
-        ErrorConstraintInfo, ErrorReportingCtx, RegionErrorNamingCtx,
-    },
-    RegionInferenceContext,
+use crate::borrow_check::nll::region_infer::RegionInferenceContext;
+
+use super::{
+    RegionName, RegionNameSource, ErrorConstraintInfo, ErrorReportingCtx, RegionErrorNamingCtx,
 };
 
 /// The different things we could suggest.
diff --git a/src/librustc_mir/borrow_check/nll/region_infer/error_reporting/mod.rs b/src/librustc_mir/borrow_check/diagnostics/region_errors.rs
similarity index 97%
rename from src/librustc_mir/borrow_check/nll/region_infer/error_reporting/mod.rs
rename to src/librustc_mir/borrow_check/diagnostics/region_errors.rs
index 5e79a2f..66f0330 100644
--- a/src/librustc_mir/borrow_check/nll/region_infer/error_reporting/mod.rs
+++ b/src/librustc_mir/borrow_check/diagnostics/region_errors.rs
@@ -19,14 +19,7 @@
 use syntax_pos::Span;
 use syntax_pos::symbol::Symbol;
 
-use self::outlives_suggestion::OutlivesSuggestionBuilder;
-
-pub mod outlives_suggestion;
-
-mod region_name;
-mod var_name;
-
-crate use self::region_name::{RegionName, RegionNameSource, RegionErrorNamingCtx};
+use super::{OutlivesSuggestionBuilder, RegionName, RegionNameSource, RegionErrorNamingCtx};
 
 impl ConstraintDescription for ConstraintCategory {
     fn description(&self) -> &'static str {
@@ -61,36 +54,36 @@
 /// Various pieces of state used when reporting borrow checker errors.
 pub struct ErrorReportingCtx<'a, 'b, 'tcx> {
     /// The region inference context used for borrow chekcing this MIR body.
-    region_infcx: &'b RegionInferenceContext<'tcx>,
+    pub(super) region_infcx: &'b RegionInferenceContext<'tcx>,
 
     /// The inference context used for type checking.
-    infcx: &'b InferCtxt<'a, 'tcx>,
+    pub(super) infcx: &'b InferCtxt<'a, 'tcx>,
 
     /// The MIR def we are reporting errors on.
-    mir_def_id: DefId,
+    pub(super) mir_def_id: DefId,
 
     /// The MIR body we are reporting errors on (for convenience).
-    body: &'b Body<'tcx>,
+    pub(super) body: &'b Body<'tcx>,
 
     /// User variable names for MIR locals (where applicable).
-    local_names: &'b IndexVec<Local, Option<Symbol>>,
+    pub(super) local_names: &'b IndexVec<Local, Option<Symbol>>,
 
     /// Any upvars for the MIR body we have kept track of during borrow checking.
-    upvars: &'b [Upvar],
+    pub(super) upvars: &'b [Upvar],
 }
 
 /// Information about the various region constraints involved in a borrow checker error.
 #[derive(Clone, Debug)]
 pub struct ErrorConstraintInfo {
     // fr: outlived_fr
-    fr: RegionVid,
-    fr_is_local: bool,
-    outlived_fr: RegionVid,
-    outlived_fr_is_local: bool,
+    pub(super) fr: RegionVid,
+    pub(super) fr_is_local: bool,
+    pub(super) outlived_fr: RegionVid,
+    pub(super) outlived_fr_is_local: bool,
 
     // Category and span for best blame constraint
-    category: ConstraintCategory,
-    span: Span,
+    pub(super) category: ConstraintCategory,
+    pub(super) span: Span,
 }
 
 impl<'tcx> RegionInferenceContext<'tcx> {
@@ -368,7 +361,7 @@
     /// ```
     ///
     /// Here we would be invoked with `fr = 'a` and `outlived_fr = `'b`.
-    pub(super) fn report_error<'a>(
+    pub(in crate::borrow_check) fn report_error<'a>(
         &'a self,
         body: &Body<'tcx>,
         local_names: &IndexVec<Local, Option<Symbol>>,
diff --git a/src/librustc_mir/borrow_check/nll/region_infer/error_reporting/region_name.rs b/src/librustc_mir/borrow_check/diagnostics/region_name.rs
similarity index 98%
rename from src/librustc_mir/borrow_check/nll/region_infer/error_reporting/region_name.rs
rename to src/librustc_mir/borrow_check/diagnostics/region_name.rs
index 0f5d1c5..e2e7596 100644
--- a/src/librustc_mir/borrow_check/nll/region_infer/error_reporting/region_name.rs
+++ b/src/librustc_mir/borrow_check/diagnostics/region_name.rs
@@ -1,12 +1,5 @@
 use std::fmt::{self, Display};
 
-use crate::borrow_check::nll::region_infer::{
-    RegionInferenceContext,
-    error_reporting::ErrorReportingCtx,
-};
-use crate::borrow_check::nll::universal_regions::DefiningTy;
-use crate::borrow_check::nll::ToRegionVid;
-use crate::borrow_check::Upvar;
 use rustc::hir;
 use rustc::hir::def::{Res, DefKind};
 use rustc::hir::def_id::DefId;
@@ -21,6 +14,15 @@
 use rustc_data_structures::fx::FxHashMap;
 use syntax_pos::{Span, symbol::Symbol, DUMMY_SP};
 
+use crate::borrow_check::{
+    nll::region_infer::RegionInferenceContext,
+    nll::universal_regions::DefiningTy,
+    nll::ToRegionVid,
+    Upvar,
+};
+
+use super::region_errors::ErrorReportingCtx;
+
 /// A name for a particular region used in emitting diagnostics. This name could be a generated
 /// name like `'1`, a name used by the user like `'a`, or a name like `'static`.
 #[derive(Debug, Clone)]
diff --git a/src/librustc_mir/borrow_check/nll/region_infer/error_reporting/var_name.rs b/src/librustc_mir/borrow_check/diagnostics/var_name.rs
similarity index 100%
rename from src/librustc_mir/borrow_check/nll/region_infer/error_reporting/var_name.rs
rename to src/librustc_mir/borrow_check/diagnostics/var_name.rs
diff --git a/src/librustc_mir/borrow_check/mod.rs b/src/librustc_mir/borrow_check/mod.rs
index 649aeac..a1932b5 100644
--- a/src/librustc_mir/borrow_check/mod.rs
+++ b/src/librustc_mir/borrow_check/mod.rs
@@ -9,8 +9,8 @@
 use rustc::lint::builtin::{MUTABLE_BORROW_RESERVATION_CONFLICT};
 use rustc::mir::{AggregateKind, BasicBlock, BorrowCheckResult, BorrowKind};
 use rustc::mir::{
-    ClearCrossCrate, Local, Location, Body, Mutability, Operand, Place, PlaceBase, PlaceElem,
-    PlaceRef, Static, StaticKind
+    ClearCrossCrate, Local, Location, Body, BodyCache, Mutability, Operand, Place, PlaceBase,
+    PlaceElem, PlaceRef, ReadOnlyBodyCache, Static, StaticKind, read_only
 };
 use rustc::mir::{Field, ProjectionElem, Promoted, Rvalue, Statement, StatementKind};
 use rustc::mir::{Terminator, TerminatorKind};
@@ -46,17 +46,14 @@
 use self::location::LocationTable;
 use self::prefixes::PrefixSet;
 use self::MutateMode::{JustWrite, WriteAndRead};
-use self::mutability_errors::AccessKind;
+use self::diagnostics::AccessKind;
 
 use self::path_utils::*;
 
 crate mod borrow_set;
-mod error_reporting;
+mod diagnostics;
 mod flows;
 mod location;
-mod conflict_errors;
-mod move_errors;
-mod mutability_errors;
 mod path_utils;
 crate mod place_ext;
 crate mod places_conflict;
@@ -102,7 +99,7 @@
 fn do_mir_borrowck<'a, 'tcx>(
     infcx: &InferCtxt<'a, 'tcx>,
     input_body: &Body<'tcx>,
-    input_promoted: &IndexVec<Promoted, Body<'tcx>>,
+    input_promoted: &IndexVec<Promoted, BodyCache<'tcx>>,
     def_id: DefId,
 ) -> BorrowCheckResult<'tcx> {
     debug!("do_mir_borrowck(def_id = {:?})", def_id);
@@ -162,16 +159,22 @@
     // requires first making our own copy of the MIR. This copy will
     // be modified (in place) to contain non-lexical lifetimes. It
     // will have a lifetime tied to the inference context.
-    let mut body: Body<'tcx> = input_body.clone();
-    let mut promoted: IndexVec<Promoted, Body<'tcx>> = input_promoted.clone();
+    let body_clone: Body<'tcx> = input_body.clone();
+    let mut promoted = input_promoted.clone();
+    let mut body = BodyCache::new(body_clone);
     let free_regions =
         nll::replace_regions_in_mir(infcx, def_id, param_env, &mut body, &mut promoted);
-    let body = &body; // no further changes
-    let location_table = &LocationTable::new(body);
+    let body = read_only!(body); // no further changes
+    let promoted: IndexVec<_, _> = promoted
+        .iter_mut()
+        .map(|body| read_only!(body))
+        .collect();
+
+    let location_table = &LocationTable::new(&body);
 
     let mut errors_buffer = Vec::new();
     let (move_data, move_errors): (MoveData<'tcx>, Option<Vec<(Place<'tcx>, MoveError<'tcx>)>>) =
-        match MoveData::gather_moves(body, tcx) {
+        match MoveData::gather_moves(&body, tcx) {
             Ok(move_data) => (move_data, None),
             Err((move_data, move_errors)) => (move_data, Some(move_errors)),
         };
@@ -184,17 +187,17 @@
     let dead_unwinds = BitSet::new_empty(body.basic_blocks().len());
     let mut flow_inits = FlowAtLocation::new(do_dataflow(
         tcx,
-        body,
+        &body,
         def_id,
         &attributes,
         &dead_unwinds,
-        MaybeInitializedPlaces::new(tcx, body, &mdpe),
+        MaybeInitializedPlaces::new(tcx, &body, &mdpe),
         |bd, i| DebugFormatted::new(&bd.move_data().move_paths[i]),
     ));
 
     let locals_are_invalidated_at_exit = tcx.hir().body_owner_kind(id).is_fn_or_closure();
     let borrow_set = Rc::new(BorrowSet::build(
-            tcx, body, locals_are_invalidated_at_exit, &mdpe.move_data));
+        tcx, body, locals_are_invalidated_at_exit, &mdpe.move_data));
 
     // If we are in non-lexical mode, compute the non-lexical lifetimes.
     let (regioncx, polonius_output, opt_closure_req) = nll::compute_regions(
@@ -222,29 +225,29 @@
 
     let flow_borrows = FlowAtLocation::new(do_dataflow(
         tcx,
-        body,
+        &body,
         def_id,
         &attributes,
         &dead_unwinds,
-        Borrows::new(tcx, body, param_env, regioncx.clone(), &borrow_set),
+        Borrows::new(tcx, &body, param_env, regioncx.clone(), &borrow_set),
         |rs, i| DebugFormatted::new(&rs.location(i)),
     ));
     let flow_uninits = FlowAtLocation::new(do_dataflow(
         tcx,
-        body,
+        &body,
         def_id,
         &attributes,
         &dead_unwinds,
-        MaybeUninitializedPlaces::new(tcx, body, &mdpe),
+        MaybeUninitializedPlaces::new(tcx, &body, &mdpe),
         |bd, i| DebugFormatted::new(&bd.move_data().move_paths[i]),
     ));
     let flow_ever_inits = FlowAtLocation::new(do_dataflow(
         tcx,
-        body,
+        &body,
         def_id,
         &attributes,
         &dead_unwinds,
-        EverInitializedPlaces::new(tcx, body, &mdpe),
+        EverInitializedPlaces::new(tcx, &body, &mdpe),
         |bd, i| DebugFormatted::new(&bd.move_data().inits[i]),
     ));
 
@@ -300,11 +303,10 @@
         let mut initial_diag =
             mbcx.report_conflicting_borrow(location, (&place, span), bk, &borrow);
 
-        let lint_root = if let ClearCrossCrate::Set(ref vsi) = mbcx.body.source_scope_local_data {
-            let scope = mbcx.body.source_info(location).scope;
-            vsi[scope].lint_root
-        } else {
-            id
+        let scope = mbcx.body.source_info(location).scope;
+        let lint_root = match &mbcx.body.source_scopes[scope].local_data {
+            ClearCrossCrate::Set(data) => data.lint_root,
+            _ => id,
         };
 
         // Span and message don't matter; we overwrite them below anyway
@@ -337,39 +339,42 @@
 
     debug!("mbcx.used_mut: {:?}", mbcx.used_mut);
     let used_mut = mbcx.used_mut;
-    for local in mbcx.body.mut_vars_and_args_iter().filter(|local| !used_mut.contains(local)) {
-        if let ClearCrossCrate::Set(ref vsi) = mbcx.body.source_scope_local_data {
-            let local_decl = &mbcx.body.local_decls[local];
+    for local in mbcx.body.mut_vars_and_args_iter().filter(|local| !used_mut.contains(local))
+    {
+        let local_decl = &mbcx.body.local_decls[local];
+        let lint_root = match &mbcx.body.source_scopes[local_decl.source_info.scope].local_data {
+            ClearCrossCrate::Set(data) => data.lint_root,
+            _ => continue,
+        };
 
-            // Skip over locals that begin with an underscore or have no name
-            match mbcx.local_names[local] {
-                Some(name) => if name.as_str().starts_with("_") {
-                    continue;
-                },
-                None => continue,
-            }
-
-            let span = local_decl.source_info.span;
-            if span.desugaring_kind().is_some() {
-                // If the `mut` arises as part of a desugaring, we should ignore it.
+        // Skip over locals that begin with an underscore or have no name
+        match mbcx.local_names[local] {
+            Some(name) => if name.as_str().starts_with("_") {
                 continue;
-            }
-
-            let mut_span = tcx.sess.source_map().span_until_non_whitespace(span);
-            tcx.struct_span_lint_hir(
-                UNUSED_MUT,
-                vsi[local_decl.source_info.scope].lint_root,
-                span,
-                "variable does not need to be mutable",
-            )
-            .span_suggestion_short(
-                mut_span,
-                "remove this `mut`",
-                String::new(),
-                Applicability::MachineApplicable,
-            )
-            .emit();
+            },
+            None => continue,
         }
+
+        let span = local_decl.source_info.span;
+        if span.desugaring_kind().is_some() {
+            // If the `mut` arises as part of a desugaring, we should ignore it.
+            continue;
+        }
+
+        let mut_span = tcx.sess.source_map().span_until_non_whitespace(span);
+        tcx.struct_span_lint_hir(
+            UNUSED_MUT,
+            lint_root,
+            span,
+            "variable does not need to be mutable",
+        )
+        .span_suggestion_short(
+            mut_span,
+            "remove this `mut`",
+            String::new(),
+            Applicability::MachineApplicable,
+        )
+        .emit();
     }
 
     // Buffer any move errors that we collected and de-duplicated.
@@ -397,7 +402,7 @@
 
 crate struct MirBorrowckCtxt<'cx, 'tcx> {
     crate infcx: &'cx InferCtxt<'cx, 'tcx>,
-    body: &'cx Body<'tcx>,
+    body: ReadOnlyBodyCache<'cx, 'tcx>,
     mir_def_id: DefId,
     param_env: ty::ParamEnv<'tcx>,
     move_data: &'cx MoveData<'tcx>,
@@ -488,7 +493,7 @@
     type FlowState = Flows<'cx, 'tcx>;
 
     fn body(&self) -> &'cx Body<'tcx> {
-        self.body
+        self.body.body()
     }
 
     fn visit_block_entry(&mut self, bb: BasicBlock, flow_state: &Self::FlowState) {
@@ -638,7 +643,7 @@
                 let tcx = self.infcx.tcx;
 
                 // Compute the type with accurate region information.
-                let drop_place_ty = drop_place.ty(self.body, self.infcx.tcx);
+                let drop_place_ty = drop_place.ty(*self.body, self.infcx.tcx);
 
                 // Erase the regions.
                 let drop_place_ty = self.infcx.tcx.erase_regions(&drop_place_ty).ty;
@@ -983,6 +988,7 @@
         let mut error_reported = false;
         let tcx = self.infcx.tcx;
         let body = self.body;
+        let body: &Body<'_> = &body;
         let param_env = self.param_env;
         let location_table = self.location_table.start_index(location);
         let borrow_set = self.borrow_set.clone();
@@ -1333,7 +1339,8 @@
                             _ => bug!("temporary initialized in arguments"),
                         };
 
-                        let bbd = &self.body[loc.block];
+                        let body = self.body;
+                        let bbd = &body[loc.block];
                         let stmt = &bbd.statements[loc.statement_index];
                         debug!("temporary assigned in: stmt={:?}", stmt);
 
@@ -1452,7 +1459,7 @@
         if places_conflict::borrow_conflicts_with_place(
             self.infcx.tcx,
             self.param_env,
-            self.body,
+            &self.body,
             place,
             borrow.kind,
             root_place,
@@ -1743,7 +1750,7 @@
                     // assigning to `P.f` requires `P` itself
                     // be already initialized
                     let tcx = self.infcx.tcx;
-                    let base_ty = Place::ty_from(&place.base, proj_base, self.body, tcx).ty;
+                    let base_ty = Place::ty_from(&place.base, proj_base, self.body(), tcx).ty;
                     match base_ty.kind {
                         ty::Adt(def, _) if def.has_dtor(tcx) => {
                             self.check_if_path_or_subpath_is_moved(
@@ -1850,7 +1857,7 @@
                 // of the union - we should error in that case.
                 let tcx = this.infcx.tcx;
                 if let ty::Adt(def, _) =
-                    Place::ty_from(base.base, base.projection, this.body, tcx).ty.kind
+                    Place::ty_from(base.base, base.projection, this.body(), tcx).ty.kind
                 {
                     if def.is_union() {
                         if this.move_data.path_map[mpi].iter().any(|moi| {
@@ -2120,7 +2127,7 @@
                 match elem {
                     ProjectionElem::Deref => {
                         let base_ty =
-                            Place::ty_from(place.base, proj_base, self.body, self.infcx.tcx).ty;
+                            Place::ty_from(place.base, proj_base, self.body(), self.infcx.tcx).ty;
 
                         // Check the kind of deref to decide
                         match base_ty.kind {
@@ -2260,7 +2267,7 @@
         match place_projection {
             [base @ .., ProjectionElem::Field(field, _ty)] => {
                 let tcx = self.infcx.tcx;
-                let base_ty = Place::ty_from(place_ref.base, base, self.body, tcx).ty;
+                let base_ty = Place::ty_from(place_ref.base, base, self.body(), tcx).ty;
 
                 if (base_ty.is_closure() || base_ty.is_generator()) &&
                     (!by_ref || self.upvars[field.index()].by_ref) {
diff --git a/src/librustc_mir/borrow_check/nll/invalidation.rs b/src/librustc_mir/borrow_check/nll/invalidation.rs
index 1d429e3..98679f2 100644
--- a/src/librustc_mir/borrow_check/nll/invalidation.rs
+++ b/src/librustc_mir/borrow_check/nll/invalidation.rs
@@ -11,7 +11,7 @@
 use crate::dataflow::indexes::BorrowIndex;
 use rustc::ty::{self, TyCtxt};
 use rustc::mir::visit::Visitor;
-use rustc::mir::{BasicBlock, Location, Body, Place, Rvalue};
+use rustc::mir::{BasicBlock, Location, Body, Place, ReadOnlyBodyCache, Rvalue};
 use rustc::mir::{Statement, StatementKind};
 use rustc::mir::TerminatorKind;
 use rustc::mir::{Operand, BorrowKind};
@@ -22,7 +22,7 @@
     param_env: ty::ParamEnv<'tcx>,
     all_facts: &mut Option<AllFacts>,
     location_table: &LocationTable,
-    body: &Body<'tcx>,
+    body: ReadOnlyBodyCache<'_, 'tcx>,
     borrow_set: &BorrowSet<'tcx>,
 ) {
     if all_facts.is_none() {
@@ -38,7 +38,7 @@
             param_env,
             tcx,
             location_table,
-            body,
+            body: &body,
             dominators,
         };
         ig.visit_body(body);
diff --git a/src/librustc_mir/borrow_check/nll/mod.rs b/src/librustc_mir/borrow_check/nll/mod.rs
index 4d67b72..7a5bbb4 100644
--- a/src/librustc_mir/borrow_check/nll/mod.rs
+++ b/src/librustc_mir/borrow_check/nll/mod.rs
@@ -12,7 +12,8 @@
 use rustc::hir::def_id::DefId;
 use rustc::infer::InferCtxt;
 use rustc::mir::{ClosureOutlivesSubject, ClosureRegionRequirements,
-                 Local, Location, Body, LocalKind, BasicBlock, Promoted};
+                 Local, Location, Body, BodyCache, LocalKind, BasicBlock,
+                 Promoted, ReadOnlyBodyCache};
 use rustc::ty::{self, RegionKind, RegionVid};
 use rustc_index::vec::IndexVec;
 use rustc_errors::Diagnostic;
@@ -31,17 +32,17 @@
 use crate::util::pretty;
 
 mod constraint_generation;
-pub mod explain_borrow;
 mod facts;
 mod invalidation;
-crate mod region_infer;
 mod renumber;
-crate mod type_check;
-mod universal_regions;
 
-mod constraints;
 mod member_constraints;
 
+crate mod constraints;
+crate mod universal_regions;
+crate mod type_check;
+crate mod region_infer;
+
 use self::facts::AllFacts;
 use self::region_infer::RegionInferenceContext;
 use self::universal_regions::UniversalRegions;
@@ -54,8 +55,8 @@
     infcx: &InferCtxt<'cx, 'tcx>,
     def_id: DefId,
     param_env: ty::ParamEnv<'tcx>,
-    body: &mut Body<'tcx>,
-    promoted: &mut IndexVec<Promoted, Body<'tcx>>,
+    body: &mut BodyCache<'tcx>,
+    promoted: &mut IndexVec<Promoted, BodyCache<'tcx>>,
 ) -> UniversalRegions<'tcx> {
     debug!("replace_regions_in_mir(def_id={:?})", def_id);
 
@@ -157,8 +158,8 @@
     infcx: &InferCtxt<'cx, 'tcx>,
     def_id: DefId,
     universal_regions: UniversalRegions<'tcx>,
-    body: &Body<'tcx>,
-    promoted: &IndexVec<Promoted, Body<'tcx>>,
+    body: ReadOnlyBodyCache<'_, 'tcx>,
+    promoted: &IndexVec<Promoted, ReadOnlyBodyCache<'_, 'tcx>>,
     local_names: &IndexVec<Local, Option<Symbol>>,
     upvars: &[Upvar],
     location_table: &LocationTable,
@@ -172,15 +173,12 @@
     Option<Rc<Output<RegionVid, BorrowIndex, LocationIndex, Local, MovePathIndex>>>,
     Option<ClosureRegionRequirements<'tcx>>,
 ) {
-    let mut all_facts = if AllFacts::enabled(infcx.tcx) {
-        Some(AllFacts::default())
-    } else {
-        None
-    };
+    let mut all_facts = AllFacts::enabled(infcx.tcx).then_some(AllFacts::default());
 
     let universal_regions = Rc::new(universal_regions);
 
-    let elements = &Rc::new(RegionValueElements::new(body));
+    let elements
+        = &Rc::new(RegionValueElements::new(&body));
 
     // Run the MIR type-checker.
     let MirTypeckResults {
@@ -205,7 +203,7 @@
         all_facts
             .universal_region
             .extend(universal_regions.universal_regions());
-        populate_polonius_move_facts(all_facts, move_data, location_table, body);
+        populate_polonius_move_facts(all_facts, move_data, location_table, &body);
     }
 
     // Create the region inference context, taking ownership of the
@@ -238,7 +236,6 @@
         universal_regions,
         placeholder_indices,
         universal_region_relations,
-        body,
         outlives_constraints,
         member_constraints,
         closure_bounds_mapping,
@@ -253,7 +250,7 @@
         param_env,
         &mut all_facts,
         location_table,
-        &body,
+        body,
         borrow_set,
     );
 
@@ -283,7 +280,7 @@
 
     // Solve the region constraints.
     let closure_region_requirements =
-        regioncx.solve(infcx, body, local_names, upvars, def_id, errors_buffer);
+        regioncx.solve(infcx, &body, local_names, upvars, def_id, errors_buffer);
 
     // Dump MIR results into a file, if that is enabled. This let us
     // write unit-tests, as well as helping with debugging.
@@ -297,7 +294,13 @@
 
     // We also have a `#[rustc_nll]` annotation that causes us to dump
     // information
-    dump_annotation(infcx, &body, def_id, &regioncx, &closure_region_requirements, errors_buffer);
+    dump_annotation(
+        infcx,
+        &body,
+        def_id,
+        &regioncx,
+        &closure_region_requirements,
+        errors_buffer);
 
     (regioncx, polonius_output, closure_region_requirements)
 }
diff --git a/src/librustc_mir/borrow_check/nll/region_infer/mod.rs b/src/librustc_mir/borrow_check/nll/region_infer/mod.rs
index d44e85f..0b3cb29 100644
--- a/src/librustc_mir/borrow_check/nll/region_infer/mod.rs
+++ b/src/librustc_mir/borrow_check/nll/region_infer/mod.rs
@@ -1,21 +1,5 @@
 use std::rc::Rc;
 
-use crate::borrow_check::nll::{
-    constraints::{
-        graph::NormalConstraintGraph,
-        ConstraintSccIndex,
-        OutlivesConstraint,
-        OutlivesConstraintSet,
-    },
-    member_constraints::{MemberConstraintSet, NllMemberConstraintIndex},
-    region_infer::values::{
-        PlaceholderIndices, RegionElement, ToElementIndex
-    },
-    region_infer::error_reporting::outlives_suggestion::OutlivesSuggestionBuilder,
-    type_check::{free_region_relations::UniversalRegionRelations, Locations},
-};
-use crate::borrow_check::Upvar;
-
 use rustc::hir::def_id::DefId;
 use rustc::infer::canonical::QueryOutlivesConstraint;
 use rustc::infer::opaque_types;
@@ -38,13 +22,31 @@
 use syntax_pos::Span;
 use syntax_pos::symbol::Symbol;
 
-crate use self::error_reporting::{RegionName, RegionNameSource, RegionErrorNamingCtx};
+use crate::borrow_check::{
+    nll::{
+        constraints::{
+            graph::NormalConstraintGraph,
+            ConstraintSccIndex,
+            OutlivesConstraint,
+            OutlivesConstraintSet,
+        },
+        member_constraints::{MemberConstraintSet, NllMemberConstraintIndex},
+        region_infer::values::{
+            PlaceholderIndices, RegionElement, ToElementIndex
+        },
+        type_check::{free_region_relations::UniversalRegionRelations, Locations},
+    },
+    diagnostics::{
+        OutlivesSuggestionBuilder, RegionErrorNamingCtx,
+    },
+    Upvar,
+};
+
 use self::values::{LivenessValues, RegionValueElements, RegionValues};
 use super::universal_regions::UniversalRegions;
 use super::ToRegionVid;
 
 mod dump_mir;
-mod error_reporting;
 mod graphviz;
 
 pub mod values;
@@ -54,48 +56,51 @@
     /// variables are identified by their index (`RegionVid`). The
     /// definition contains information about where the region came
     /// from as well as its final inferred value.
-    definitions: IndexVec<RegionVid, RegionDefinition<'tcx>>,
+    pub(in crate::borrow_check) definitions: IndexVec<RegionVid, RegionDefinition<'tcx>>,
 
     /// The liveness constraints added to each region. For most
     /// regions, these start out empty and steadily grow, though for
     /// each universally quantified region R they start out containing
     /// the entire CFG and `end(R)`.
-    liveness_constraints: LivenessValues<RegionVid>,
+    pub(in crate::borrow_check) liveness_constraints: LivenessValues<RegionVid>,
 
     /// The outlives constraints computed by the type-check.
-    constraints: Rc<OutlivesConstraintSet>,
+    pub(in crate::borrow_check) constraints: Rc<OutlivesConstraintSet>,
 
     /// The constraint-set, but in graph form, making it easy to traverse
     /// the constraints adjacent to a particular region. Used to construct
     /// the SCC (see `constraint_sccs`) and for error reporting.
-    constraint_graph: Rc<NormalConstraintGraph>,
+    pub(in crate::borrow_check) constraint_graph: Rc<NormalConstraintGraph>,
 
     /// The SCC computed from `constraints` and the constraint
     /// graph. We have an edge from SCC A to SCC B if `A: B`. Used to
     /// compute the values of each region.
-    constraint_sccs: Rc<Sccs<RegionVid, ConstraintSccIndex>>,
+    pub(in crate::borrow_check) constraint_sccs: Rc<Sccs<RegionVid, ConstraintSccIndex>>,
 
     /// Reverse of the SCC constraint graph -- i.e., an edge `A -> B`
     /// exists if `B: A`. Computed lazilly.
-    rev_constraint_graph: Option<Rc<VecGraph<ConstraintSccIndex>>>,
+    pub(in crate::borrow_check) rev_constraint_graph:
+        Option<Rc<VecGraph<ConstraintSccIndex>>>,
 
     /// The "R0 member of [R1..Rn]" constraints, indexed by SCC.
-    member_constraints: Rc<MemberConstraintSet<'tcx, ConstraintSccIndex>>,
+    pub(in crate::borrow_check) member_constraints:
+        Rc<MemberConstraintSet<'tcx, ConstraintSccIndex>>,
 
     /// Records the member constraints that we applied to each scc.
     /// This is useful for error reporting. Once constraint
     /// propagation is done, this vector is sorted according to
     /// `member_region_scc`.
-    member_constraints_applied: Vec<AppliedMemberConstraint>,
+    pub(in crate::borrow_check) member_constraints_applied: Vec<AppliedMemberConstraint>,
 
     /// Map closure bounds to a `Span` that should be used for error reporting.
-    closure_bounds_mapping:
+    pub(in crate::borrow_check) closure_bounds_mapping:
         FxHashMap<Location, FxHashMap<(RegionVid, RegionVid), (ConstraintCategory, Span)>>,
 
     /// Contains the minimum universe of any variable within the same
     /// SCC. We will ensure that no SCC contains values that are not
     /// visible from this index.
-    scc_universes: IndexVec<ConstraintSccIndex, ty::UniverseIndex>,
+    pub(in crate::borrow_check) scc_universes:
+        IndexVec<ConstraintSccIndex, ty::UniverseIndex>,
 
     /// Contains a "representative" from each SCC. This will be the
     /// minimal RegionVid belonging to that universe. It is used as a
@@ -104,23 +109,25 @@
     /// of its SCC and be sure that -- if they have the same repr --
     /// they *must* be equal (though not having the same repr does not
     /// mean they are unequal).
-    scc_representatives: IndexVec<ConstraintSccIndex, ty::RegionVid>,
+    pub(in crate::borrow_check) scc_representatives:
+        IndexVec<ConstraintSccIndex, ty::RegionVid>,
 
     /// The final inferred values of the region variables; we compute
     /// one value per SCC. To get the value for any given *region*,
     /// you first find which scc it is a part of.
-    scc_values: RegionValues<ConstraintSccIndex>,
+    pub(in crate::borrow_check) scc_values: RegionValues<ConstraintSccIndex>,
 
     /// Type constraints that we check after solving.
-    type_tests: Vec<TypeTest<'tcx>>,
+    pub(in crate::borrow_check) type_tests: Vec<TypeTest<'tcx>>,
 
     /// Information about the universally quantified regions in scope
     /// on this function.
-    universal_regions: Rc<UniversalRegions<'tcx>>,
+    pub (in crate::borrow_check) universal_regions: Rc<UniversalRegions<'tcx>>,
 
     /// Information about how the universally quantified regions in
     /// scope on this function relate to one another.
-    universal_region_relations: Rc<UniversalRegionRelations<'tcx>>,
+    pub(in crate::borrow_check) universal_region_relations:
+        Rc<UniversalRegionRelations<'tcx>>,
 }
 
 /// Each time that `apply_member_constraint` is successful, it appends
@@ -132,38 +139,38 @@
 /// with `'R: 'O` where `'R` is the pick-region and `'O` is the
 /// minimal viable option.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]
-struct AppliedMemberConstraint {
+pub(crate) struct AppliedMemberConstraint {
     /// The SCC that was affected. (The "member region".)
     ///
     /// The vector if `AppliedMemberConstraint` elements is kept sorted
     /// by this field.
-    member_region_scc: ConstraintSccIndex,
+    pub(in crate::borrow_check) member_region_scc: ConstraintSccIndex,
 
     /// The "best option" that `apply_member_constraint` found -- this was
     /// added as an "ad-hoc" lower-bound to `member_region_scc`.
-    min_choice: ty::RegionVid,
+    pub(in crate::borrow_check) min_choice: ty::RegionVid,
 
     /// The "member constraint index" -- we can find out details about
     /// the constraint from
     /// `set.member_constraints[member_constraint_index]`.
-    member_constraint_index: NllMemberConstraintIndex,
+    pub(in crate::borrow_check) member_constraint_index: NllMemberConstraintIndex,
 }
 
-struct RegionDefinition<'tcx> {
+pub(crate) struct RegionDefinition<'tcx> {
     /// What kind of variable is this -- a free region? existential
     /// variable? etc. (See the `NLLRegionVariableOrigin` for more
     /// info.)
-    origin: NLLRegionVariableOrigin,
+    pub(in crate::borrow_check) origin: NLLRegionVariableOrigin,
 
     /// Which universe is this region variable defined in? This is
     /// most often `ty::UniverseIndex::ROOT`, but when we encounter
     /// forall-quantifiers like `for<'a> { 'a = 'b }`, we would create
     /// the variable for `'a` in a fresh universe that extends ROOT.
-    universe: ty::UniverseIndex,
+    pub(in crate::borrow_check) universe: ty::UniverseIndex,
 
     /// If this is 'static or an early-bound region, then this is
     /// `Some(X)` where `X` is the name of the region.
-    external_name: Option<ty::Region<'tcx>>,
+    pub(in crate::borrow_check) external_name: Option<ty::Region<'tcx>>,
 }
 
 /// N.B., the variants in `Cause` are intentionally ordered. Lower
@@ -239,7 +246,6 @@
         universal_regions: Rc<UniversalRegions<'tcx>>,
         placeholder_indices: Rc<PlaceholderIndices>,
         universal_region_relations: Rc<UniversalRegionRelations<'tcx>>,
-        _body: &Body<'tcx>,
         outlives_constraints: OutlivesConstraintSet,
         member_constraints_in: MemberConstraintSet<'tcx, RegionVid>,
         closure_bounds_mapping: FxHashMap<
@@ -456,7 +462,9 @@
     /// Once region solving has completed, this function will return
     /// the member constraints that were applied to the value of a given
     /// region `r`. See `AppliedMemberConstraint`.
-    fn applied_member_constraints(&self, r: impl ToRegionVid) -> &[AppliedMemberConstraint] {
+    pub(in crate::borrow_check) fn applied_member_constraints(
+        &self, r: impl ToRegionVid
+    ) -> &[AppliedMemberConstraint] {
         let scc = self.constraint_sccs.scc(r.to_region_vid());
         binary_search_util::binary_search_slice(
             &self.member_constraints_applied,
@@ -485,7 +493,7 @@
         // functions below, which will trigger them to report errors
         // eagerly.
         let mut outlives_requirements =
-            if infcx.tcx.is_closure(mir_def_id) { Some(vec![]) } else { None };
+            infcx.tcx.is_closure(mir_def_id).then(|| vec![]);
 
         self.check_type_tests(
             infcx,
@@ -701,14 +709,11 @@
         let min = |r1: ty::RegionVid, r2: ty::RegionVid| -> Option<ty::RegionVid> {
             let r1_outlives_r2 = self.universal_region_relations.outlives(r1, r2);
             let r2_outlives_r1 = self.universal_region_relations.outlives(r2, r1);
-            if r1_outlives_r2 && r2_outlives_r1 {
-                Some(r1.min(r2))
-            } else if r1_outlives_r2 {
-                Some(r2)
-            } else if r2_outlives_r1 {
-                Some(r1)
-            } else {
-                None
+            match (r1_outlives_r2, r2_outlives_r1) {
+                (true, true) => Some(r1.min(r2)),
+                (true, false) => Some(r2),
+                (false, true) => Some(r1),
+                (false, false) => None,
             }
         };
         let mut min_choice = choice_regions[0];
diff --git a/src/librustc_mir/borrow_check/nll/region_infer/values.rs b/src/librustc_mir/borrow_check/nll/region_infer/values.rs
index 7a86536..b4414c5 100644
--- a/src/librustc_mir/borrow_check/nll/region_infer/values.rs
+++ b/src/librustc_mir/borrow_check/nll/region_infer/values.rs
@@ -1,4 +1,4 @@
-use rustc::mir::{BasicBlock, Location, Body};
+use rustc::mir::{BasicBlock, Location, Body, ReadOnlyBodyCache};
 use rustc::ty::{self, RegionVid};
 use rustc_index::bit_set::{HybridBitSet, SparseBitMatrix};
 use rustc_data_structures::fx::FxHashMap;
@@ -92,7 +92,7 @@
     /// Pushes all predecessors of `index` onto `stack`.
     crate fn push_predecessors(
         &self,
-        body: &Body<'_>,
+        body: ReadOnlyBodyCache<'_, '_>,
         index: PointIndex,
         stack: &mut Vec<PointIndex>,
     ) {
diff --git a/src/librustc_mir/borrow_check/nll/renumber.rs b/src/librustc_mir/borrow_check/nll/renumber.rs
index d949c7e..db15d2c 100644
--- a/src/librustc_mir/borrow_check/nll/renumber.rs
+++ b/src/librustc_mir/borrow_check/nll/renumber.rs
@@ -1,6 +1,6 @@
 use rustc::ty::subst::SubstsRef;
 use rustc::ty::{self, Ty, TyCtxt, TypeFoldable};
-use rustc::mir::{Body, Location, PlaceElem, Promoted};
+use rustc::mir::{BodyCache, Location, PlaceElem, Promoted};
 use rustc::mir::visit::{MutVisitor, TyContext};
 use rustc::infer::{InferCtxt, NLLRegionVariableOrigin};
 use rustc_index::vec::IndexVec;
@@ -9,8 +9,8 @@
 /// inference variables, returning the number of variables created.
 pub fn renumber_mir<'tcx>(
     infcx: &InferCtxt<'_, 'tcx>,
-    body: &mut Body<'tcx>,
-    promoted: &mut IndexVec<Promoted, Body<'tcx>>,
+    body: &mut BodyCache<'tcx>,
+    promoted: &mut IndexVec<Promoted, BodyCache<'tcx>>,
 ) {
     debug!("renumber_mir()");
     debug!("renumber_mir: body.arg_count={:?}", body.arg_count);
diff --git a/src/librustc_mir/borrow_check/nll/type_check/liveness/local_use_map.rs b/src/librustc_mir/borrow_check/nll/type_check/liveness/local_use_map.rs
index 7dee00b..d6aa314 100644
--- a/src/librustc_mir/borrow_check/nll/type_check/liveness/local_use_map.rs
+++ b/src/librustc_mir/borrow_check/nll/type_check/liveness/local_use_map.rs
@@ -1,7 +1,7 @@
 use crate::borrow_check::nll::region_infer::values::{PointIndex, RegionValueElements};
 use crate::util::liveness::{categorize, DefUse};
 use rustc::mir::visit::{PlaceContext, Visitor};
-use rustc::mir::{Body, Local, Location};
+use rustc::mir::{Local, Location, ReadOnlyBodyCache};
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_data_structures::vec_linked_list as vll;
 
@@ -60,7 +60,7 @@
     crate fn build(
         live_locals: &Vec<Local>,
         elements: &RegionValueElements,
-        body: &Body<'_>,
+        body: ReadOnlyBodyCache<'_, '_>,
     ) -> Self {
         let nones = IndexVec::from_elem_n(None, body.local_decls.len());
         let mut local_use_map = LocalUseMap {
diff --git a/src/librustc_mir/borrow_check/nll/type_check/liveness/mod.rs b/src/librustc_mir/borrow_check/nll/type_check/liveness/mod.rs
index a01b528..dfd505f 100644
--- a/src/librustc_mir/borrow_check/nll/type_check/liveness/mod.rs
+++ b/src/librustc_mir/borrow_check/nll/type_check/liveness/mod.rs
@@ -7,7 +7,7 @@
 use crate::dataflow::move_paths::MoveData;
 use crate::dataflow::FlowAtLocation;
 use crate::dataflow::MaybeInitializedPlaces;
-use rustc::mir::{Body, Local};
+use rustc::mir::{Body, Local, ReadOnlyBodyCache};
 use rustc::ty::{RegionVid, TyCtxt};
 use rustc_data_structures::fx::FxHashSet;
 use std::rc::Rc;
@@ -28,7 +28,7 @@
 /// performed before
 pub(super) fn generate<'tcx>(
     typeck: &mut TypeChecker<'_, 'tcx>,
-    body: &Body<'tcx>,
+    body: ReadOnlyBodyCache<'_, 'tcx>,
     elements: &Rc<RegionValueElements>,
     flow_inits: &mut FlowAtLocation<'tcx, MaybeInitializedPlaces<'_, 'tcx>>,
     move_data: &MoveData<'tcx>,
@@ -41,10 +41,9 @@
         &typeck.borrowck_context.universal_regions,
         &typeck.borrowck_context.constraints.outlives_constraints,
     );
-    let live_locals = compute_live_locals(typeck.tcx(), &free_regions, body);
+    let live_locals = compute_live_locals(typeck.tcx(), &free_regions, &body);
     let facts_enabled = AllFacts::enabled(typeck.tcx());
 
-
     let polonius_drop_used = if facts_enabled {
         let mut drop_used = Vec::new();
         polonius::populate_access_facts(
diff --git a/src/librustc_mir/borrow_check/nll/type_check/liveness/polonius.rs b/src/librustc_mir/borrow_check/nll/type_check/liveness/polonius.rs
index 526ad7f..e67de6c 100644
--- a/src/librustc_mir/borrow_check/nll/type_check/liveness/polonius.rs
+++ b/src/librustc_mir/borrow_check/nll/type_check/liveness/polonius.rs
@@ -3,7 +3,7 @@
 use crate::dataflow::move_paths::{LookupResult, MoveData};
 use crate::util::liveness::{categorize, DefUse};
 use rustc::mir::visit::{MutatingUseContext, PlaceContext, Visitor};
-use rustc::mir::{Body, Local, Location, Place};
+use rustc::mir::{Local, Location, Place, ReadOnlyBodyCache};
 use rustc::ty::subst::GenericArg;
 use rustc::ty::Ty;
 
@@ -97,7 +97,7 @@
 
 pub(super) fn populate_access_facts(
     typeck: &mut TypeChecker<'_, 'tcx>,
-    body: &Body<'tcx>,
+    body: ReadOnlyBodyCache<'_, 'tcx>,
     location_table: &LocationTable,
     move_data: &MoveData<'_>,
     drop_used: &mut Vec<(Local, Location)>,
diff --git a/src/librustc_mir/borrow_check/nll/type_check/liveness/trace.rs b/src/librustc_mir/borrow_check/nll/type_check/liveness/trace.rs
index eacc4d0..229cbed 100644
--- a/src/librustc_mir/borrow_check/nll/type_check/liveness/trace.rs
+++ b/src/librustc_mir/borrow_check/nll/type_check/liveness/trace.rs
@@ -7,7 +7,7 @@
 use crate::dataflow::move_paths::MoveData;
 use crate::dataflow::{FlowAtLocation, FlowsAtLocation, MaybeInitializedPlaces};
 use rustc::infer::canonical::QueryRegionConstraints;
-use rustc::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
+use rustc::mir::{BasicBlock, ConstraintCategory, Local, Location, ReadOnlyBodyCache};
 use rustc::traits::query::dropck_outlives::DropckOutlivesResult;
 use rustc::traits::query::type_op::outlives::DropckOutlives;
 use rustc::traits::query::type_op::TypeOp;
@@ -32,7 +32,7 @@
 /// this respects `#[may_dangle]` annotations).
 pub(super) fn trace(
     typeck: &mut TypeChecker<'_, 'tcx>,
-    body: &Body<'tcx>,
+    body: ReadOnlyBodyCache<'_, 'tcx>,
     elements: &Rc<RegionValueElements>,
     flow_inits: &mut FlowAtLocation<'tcx, MaybeInitializedPlaces<'_, 'tcx>>,
     move_data: &MoveData<'tcx>,
@@ -71,7 +71,7 @@
     elements: &'me RegionValueElements,
 
     /// MIR we are analyzing.
-    body: &'me Body<'tcx>,
+    body: ReadOnlyBodyCache<'me, 'tcx>,
 
     /// Mapping to/from the various indices used for initialization tracking.
     move_data: &'me MoveData<'tcx>,
@@ -302,7 +302,8 @@
             }
         }
 
-        for &pred_block in self.cx.body.predecessors_for(block).iter() {
+        let body = self.cx.body;
+        for &pred_block in body.predecessors_for(block).iter() {
             debug!("compute_drop_live_points_for_block: pred_block = {:?}", pred_block,);
 
             // Check whether the variable is (at least partially)
diff --git a/src/librustc_mir/borrow_check/nll/type_check/mod.rs b/src/librustc_mir/borrow_check/nll/type_check/mod.rs
index 99bcfa9..de30420 100644
--- a/src/librustc_mir/borrow_check/nll/type_check/mod.rs
+++ b/src/librustc_mir/borrow_check/nll/type_check/mod.rs
@@ -1,57 +1,59 @@
 //! This pass type-checks the MIR to ensure it is not broken.
 
+use std::{fmt, iter, mem};
+use std::rc::Rc;
+
+use either::Either;
+
+use rustc::hir;
+use rustc::hir::def_id::DefId;
+use rustc::infer::{InferCtxt, InferOk, LateBoundRegionConversionTime, NLLRegionVariableOrigin};
+use rustc::infer::canonical::QueryRegionConstraints;
+use rustc::infer::outlives::env::RegionBoundPairs;
+use rustc::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
+use rustc::mir::*;
+use rustc::mir::interpret::PanicInfo;
+use rustc::mir::tcx::PlaceTy;
+use rustc::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
+use rustc::traits::{self, ObligationCause, PredicateObligations};
+use rustc::traits::query::{Fallible, NoSolution};
+use rustc::traits::query::type_op;
+use rustc::traits::query::type_op::custom::CustomTypeOp;
+use rustc::ty::{
+    self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, RegionVid, ToPolyTraitRef, Ty,
+    TyCtxt, UserType,
+    UserTypeAnnotationIndex,
+};
+use rustc::ty::adjustment::PointerCast;
+use rustc::ty::cast::CastTy;
+use rustc::ty::fold::TypeFoldable;
+use rustc::ty::layout::VariantIdx;
+use rustc::ty::subst::{GenericArgKind, Subst, SubstsRef, UserSubsts};
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_error_codes::*;
+use rustc_index::vec::{Idx, IndexVec};
+use syntax_pos::{DUMMY_SP, Span};
+
 use crate::borrow_check::borrow_set::BorrowSet;
 use crate::borrow_check::location::LocationTable;
-use crate::borrow_check::nll::constraints::{OutlivesConstraintSet, OutlivesConstraint};
-use crate::borrow_check::nll::member_constraints::MemberConstraintSet;
+use crate::borrow_check::nll::constraints::{OutlivesConstraint, OutlivesConstraintSet};
 use crate::borrow_check::nll::facts::AllFacts;
+use crate::borrow_check::nll::member_constraints::MemberConstraintSet;
+use crate::borrow_check::nll::region_infer::{ClosureRegionRequirementsExt, TypeTest};
 use crate::borrow_check::nll::region_infer::values::LivenessValues;
 use crate::borrow_check::nll::region_infer::values::PlaceholderIndex;
 use crate::borrow_check::nll::region_infer::values::PlaceholderIndices;
 use crate::borrow_check::nll::region_infer::values::RegionValueElements;
-use crate::borrow_check::nll::region_infer::{ClosureRegionRequirementsExt, TypeTest};
 use crate::borrow_check::nll::renumber;
+use crate::borrow_check::nll::ToRegionVid;
 use crate::borrow_check::nll::type_check::free_region_relations::{
     CreateResult, UniversalRegionRelations,
 };
 use crate::borrow_check::nll::universal_regions::{DefiningTy, UniversalRegions};
-use crate::borrow_check::nll::ToRegionVid;
-use crate::transform::promote_consts::should_suggest_const_in_array_repeat_expressions_attribute;
-use crate::dataflow::move_paths::MoveData;
 use crate::dataflow::FlowAtLocation;
 use crate::dataflow::MaybeInitializedPlaces;
-use either::Either;
-use rustc::hir;
-use rustc::hir::def_id::DefId;
-use rustc::infer::canonical::QueryRegionConstraints;
-use rustc::infer::outlives::env::RegionBoundPairs;
-use rustc::infer::{InferCtxt, InferOk, LateBoundRegionConversionTime, NLLRegionVariableOrigin};
-use rustc::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use rustc::mir::interpret::PanicInfo;
-use rustc::mir::tcx::PlaceTy;
-use rustc::mir::visit::{PlaceContext, Visitor, NonMutatingUseContext};
-use rustc::mir::*;
-use rustc::traits::query::type_op;
-use rustc::traits::query::type_op::custom::CustomTypeOp;
-use rustc::traits::query::{Fallible, NoSolution};
-use rustc::traits::{self, ObligationCause, PredicateObligations};
-use rustc::ty::adjustment::{PointerCast};
-use rustc::ty::cast::CastTy;
-use rustc::ty::fold::TypeFoldable;
-use rustc::ty::subst::{Subst, SubstsRef, GenericArgKind, UserSubsts};
-use rustc::ty::{
-    self, RegionVid, ToPolyTraitRef, Ty, TyCtxt, UserType,
-    CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations,
-    UserTypeAnnotationIndex,
-};
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_index::vec::{IndexVec, Idx};
-use rustc::ty::layout::VariantIdx;
-use std::rc::Rc;
-use std::{fmt, iter, mem};
-use syntax_pos::{Span, DUMMY_SP};
-
-use rustc_error_codes::*;
+use crate::dataflow::move_paths::MoveData;
+use crate::transform::promote_consts::should_suggest_const_in_array_repeat_expressions_attribute;
 
 macro_rules! span_mirbug {
     ($context:expr, $elem:expr, $($message:tt)*) => ({
@@ -115,8 +117,8 @@
 pub(crate) fn type_check<'tcx>(
     infcx: &InferCtxt<'_, 'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    body: &Body<'tcx>,
-    promoted: &IndexVec<Promoted, Body<'tcx>>,
+    body: ReadOnlyBodyCache<'_, 'tcx>,
+    promoted: &IndexVec<Promoted, ReadOnlyBodyCache<'_, 'tcx>>,
     mir_def_id: DefId,
     universal_regions: &Rc<UniversalRegions<'tcx>>,
     location_table: &LocationTable,
@@ -168,8 +170,17 @@
         &mut borrowck_context,
         &universal_region_relations,
         |mut cx| {
-            cx.equate_inputs_and_outputs(body, universal_regions, &normalized_inputs_and_output);
-            liveness::generate(&mut cx, body, elements, flow_inits, move_data, location_table);
+            cx.equate_inputs_and_outputs(
+                &body,
+                universal_regions,
+                &normalized_inputs_and_output);
+            liveness::generate(
+                &mut cx,
+                body,
+                elements,
+                flow_inits,
+                move_data,
+                location_table);
 
             translate_outlives_facts(cx.borrowck_context);
         },
@@ -185,17 +196,17 @@
     infcx: &'a InferCtxt<'a, 'tcx>,
     mir_def_id: DefId,
     param_env: ty::ParamEnv<'tcx>,
-    body: &'a Body<'tcx>,
-    promoted: &'a IndexVec<Promoted, Body<'tcx>>,
+    body: ReadOnlyBodyCache<'a, 'tcx>,
+    promoted: &'a IndexVec<Promoted, ReadOnlyBodyCache<'_, 'tcx>>,
     region_bound_pairs: &'a RegionBoundPairs<'tcx>,
     implicit_region_bound: ty::Region<'tcx>,
     borrowck_context: &'a mut BorrowCheckContext<'a, 'tcx>,
     universal_region_relations: &'a UniversalRegionRelations<'tcx>,
     mut extra: impl FnMut(&mut TypeChecker<'a, 'tcx>) -> R,
-) -> R where {
+) -> R {
     let mut checker = TypeChecker::new(
         infcx,
-        body,
+        body.body(),
         mir_def_id,
         param_env,
         region_bound_pairs,
@@ -204,7 +215,7 @@
         universal_region_relations,
     );
     let errors_reported = {
-        let mut verifier = TypeVerifier::new(&mut checker, body, promoted);
+        let mut verifier = TypeVerifier::new(&mut checker, body.body(), promoted);
         verifier.visit_body(body);
         verifier.errors_reported
     };
@@ -261,7 +272,7 @@
 struct TypeVerifier<'a, 'b, 'tcx> {
     cx: &'a mut TypeChecker<'b, 'tcx>,
     body: &'b Body<'tcx>,
-    promoted: &'b IndexVec<Promoted, Body<'tcx>>,
+    promoted: &'b IndexVec<Promoted, ReadOnlyBodyCache<'b, 'tcx>>,
     last_span: Span,
     mir_def_id: DefId,
     errors_reported: bool,
@@ -385,7 +396,7 @@
         }
     }
 
-    fn visit_body(&mut self, body: &Body<'tcx>) {
+    fn visit_body(&mut self, body: ReadOnlyBodyCache<'_, 'tcx>) {
         self.sanitize_type(&"return type", body.return_ty());
         for local_decl in &body.local_decls {
             self.sanitize_type(local_decl, local_decl.ty);
@@ -401,7 +412,7 @@
     fn new(
         cx: &'a mut TypeChecker<'b, 'tcx>,
         body: &'b Body<'tcx>,
-        promoted: &'b IndexVec<Promoted, Body<'tcx>>,
+        promoted: &'b IndexVec<Promoted, ReadOnlyBodyCache<'b, 'tcx>>,
     ) -> Self {
         TypeVerifier {
             body,
@@ -464,10 +475,10 @@
                 match kind {
                     StaticKind::Promoted(promoted, _) => {
                         if !self.errors_reported {
-                            let promoted_body = &self.promoted[*promoted];
-                            self.sanitize_promoted(promoted_body, location);
+                            let promoted_body_cache = self.promoted[*promoted];
+                            self.sanitize_promoted(promoted_body_cache, location);
 
-                            let promoted_ty = promoted_body.return_ty();
+                            let promoted_ty = promoted_body_cache.return_ty();
                             check_err(self, place, promoted_ty, san_ty);
                         }
                     }
@@ -535,12 +546,16 @@
         place_ty
     }
 
-    fn sanitize_promoted(&mut self, promoted_body: &'b Body<'tcx>, location: Location) {
+    fn sanitize_promoted(
+        &mut self,
+        promoted_body: ReadOnlyBodyCache<'b, 'tcx>,
+        location: Location
+    ) {
         // Determine the constraints from the promoted MIR by running the type
         // checker on the promoted MIR, then transfer the constraints back to
         // the main MIR, changing the locations to the provided location.
 
-        let parent_body = mem::replace(&mut self.body, promoted_body);
+        let parent_body = mem::replace(&mut self.body, promoted_body.body());
 
         // Use new sets of constraints and closure bounds so that we can
         // modify their locations.
@@ -548,7 +563,7 @@
         let mut constraints = Default::default();
         let mut closure_bounds = Default::default();
         let mut liveness_constraints = LivenessValues::new(
-            Rc::new(RegionValueElements::new(promoted_body)),
+            Rc::new(RegionValueElements::new(&promoted_body)),
         );
         // Don't try to add borrow_region facts for the promoted MIR
 
@@ -1361,7 +1376,12 @@
         self.infcx.tcx
     }
 
-    fn check_stmt(&mut self, body: &Body<'tcx>, stmt: &Statement<'tcx>, location: Location) {
+    fn check_stmt(
+        &mut self,
+        body: ReadOnlyBodyCache<'_, 'tcx>,
+        stmt: &Statement<'tcx>,
+        location: Location)
+    {
         debug!("check_stmt: {:?}", stmt);
         let tcx = self.tcx();
         match stmt.kind {
@@ -1393,9 +1413,9 @@
                     _ => ConstraintCategory::Assignment,
                 };
 
-                let place_ty = place.ty(body, tcx).ty;
+                let place_ty = place.ty(*body, tcx).ty;
                 let place_ty = self.normalize(place_ty, location);
-                let rv_ty = rv.ty(body, tcx);
+                let rv_ty = rv.ty(*body, tcx);
                 let rv_ty = self.normalize(rv_ty, location);
                 if let Err(terr) =
                     self.sub_types_or_anon(rv_ty, place_ty, location.to_locations(), category)
@@ -1447,7 +1467,7 @@
                 ref place,
                 variant_index,
             } => {
-                let place_type = place.ty(body, tcx).ty;
+                let place_type = place.ty(*body, tcx).ty;
                 let adt = match place_type.kind {
                     ty::Adt(adt, _) if adt.is_enum() => adt,
                     _ => {
@@ -1469,7 +1489,7 @@
                 };
             }
             StatementKind::AscribeUserType(box(ref place, ref projection), variance) => {
-                let place_ty = place.ty(body, tcx).ty;
+                let place_ty = place.ty(*body, tcx).ty;
                 if let Err(terr) = self.relate_type_and_user_type(
                     place_ty,
                     variance,
@@ -1972,12 +1992,17 @@
         }
     }
 
-    fn check_rvalue(&mut self, body: &Body<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) {
+    fn check_rvalue(
+        &mut self,
+        body: ReadOnlyBodyCache<'_, 'tcx>,
+        rvalue: &Rvalue<'tcx>,
+        location: Location)
+    {
         let tcx = self.tcx();
 
         match rvalue {
             Rvalue::Aggregate(ak, ops) => {
-                self.check_aggregate_rvalue(body, rvalue, ak, ops, location)
+                self.check_aggregate_rvalue(&body, rvalue, ak, ops, location)
             }
 
             Rvalue::Repeat(operand, len) => if *len > 1 {
@@ -1985,7 +2010,7 @@
                     // While this is located in `nll::typeck` this error is not an NLL error, it's
                     // a required check to make sure that repeated elements implement `Copy`.
                     let span = body.source_info(location).span;
-                    let ty = operand.ty(body, tcx);
+                    let ty = operand.ty(*body, tcx);
                     if !self.infcx.type_is_copy_modulo_regions(self.param_env, ty, span) {
                         // To determine if `const_in_array_repeat_expressions` feature gate should
                         // be mentioned, need to check if the rvalue is promotable.
@@ -2039,7 +2064,7 @@
             Rvalue::Cast(cast_kind, op, ty) => {
                 match cast_kind {
                     CastKind::Pointer(PointerCast::ReifyFnPointer) => {
-                        let fn_sig = op.ty(body, tcx).fn_sig(tcx);
+                        let fn_sig = op.ty(*body, tcx).fn_sig(tcx);
 
                         // The type that we see in the fcx is like
                         // `foo::<'a, 'b>`, where `foo` is the path to a
@@ -2068,7 +2093,7 @@
                     }
 
                     CastKind::Pointer(PointerCast::ClosureFnPointer(unsafety)) => {
-                        let sig = match op.ty(body, tcx).kind {
+                        let sig = match op.ty(*body, tcx).kind {
                             ty::Closure(def_id, substs) => {
                                 substs.as_closure().sig_ty(def_id, tcx).fn_sig(tcx)
                             }
@@ -2094,7 +2119,7 @@
                     }
 
                     CastKind::Pointer(PointerCast::UnsafeFnPointer) => {
-                        let fn_sig = op.ty(body, tcx).fn_sig(tcx);
+                        let fn_sig = op.ty(*body, tcx).fn_sig(tcx);
 
                         // The type that we see in the fcx is like
                         // `foo::<'a, 'b>`, where `foo` is the path to a
@@ -2126,7 +2151,7 @@
                         let &ty = ty;
                         let trait_ref = ty::TraitRef {
                             def_id: tcx.lang_items().coerce_unsized_trait().unwrap(),
-                            substs: tcx.mk_substs_trait(op.ty(body, tcx), &[ty.into()]),
+                            substs: tcx.mk_substs_trait(op.ty(*body, tcx), &[ty.into()]),
                         };
 
                         self.prove_trait_ref(
@@ -2137,7 +2162,7 @@
                     }
 
                     CastKind::Pointer(PointerCast::MutToConstPointer) => {
-                        let ty_from = match op.ty(body, tcx).kind {
+                        let ty_from = match op.ty(*body, tcx).kind {
                             ty::RawPtr(ty::TypeAndMut {
                                 ty: ty_from,
                                 mutbl: hir::Mutability::Mutable,
@@ -2185,7 +2210,7 @@
                     }
 
                     CastKind::Pointer(PointerCast::ArrayToPointer)  => {
-                        let ty_from = op.ty(body, tcx);
+                        let ty_from = op.ty(*body, tcx);
 
                         let opt_ty_elem = match ty_from.kind {
                             ty::RawPtr(
@@ -2247,7 +2272,7 @@
                     }
 
                     CastKind::Misc => {
-                        let ty_from = op.ty(body, tcx);
+                        let ty_from = op.ty(*body, tcx);
                         let cast_ty_from = CastTy::from_ty(ty_from);
                         let cast_ty_to = CastTy::from_ty(ty);
                         match (cast_ty_from, cast_ty_to) {
@@ -2305,7 +2330,7 @@
             }
 
             Rvalue::Ref(region, _borrow_kind, borrowed_place) => {
-                self.add_reborrow_constraint(body, location, region, borrowed_place);
+                self.add_reborrow_constraint(&body, location, region, borrowed_place);
             }
 
             Rvalue::BinaryOp(BinOp::Eq, left, right)
@@ -2314,9 +2339,9 @@
             | Rvalue::BinaryOp(BinOp::Le, left, right)
             | Rvalue::BinaryOp(BinOp::Gt, left, right)
             | Rvalue::BinaryOp(BinOp::Ge, left, right) => {
-                let ty_left = left.ty(body, tcx);
+                let ty_left = left.ty(*body, tcx);
                 if let ty::RawPtr(_) | ty::FnPtr(_) = ty_left.kind {
-                    let ty_right = right.ty(body, tcx);
+                    let ty_right = right.ty(*body, tcx);
                     let common_ty = self.infcx.next_ty_var(
                         TypeVariableOrigin {
                             kind: TypeVariableOriginKind::MiscVariable,
@@ -2741,12 +2766,12 @@
         })
     }
 
-    fn typeck_mir(&mut self, body: &Body<'tcx>) {
+    fn typeck_mir(&mut self, body: ReadOnlyBodyCache<'_, 'tcx>) {
         self.last_span = body.span;
         debug!("run_on_mir: {:?}", body.span);
 
         for (local, local_decl) in body.local_decls.iter_enumerated() {
-            self.check_local(body, local, local_decl);
+            self.check_local(&body, local, local_decl);
         }
 
         for (block, block_data) in body.basic_blocks().iter_enumerated() {
@@ -2762,8 +2787,8 @@
                 location.statement_index += 1;
             }
 
-            self.check_terminator(body, block_data.terminator(), location);
-            self.check_iscleanup(body, block_data);
+            self.check_terminator(&body, block_data.terminator(), location);
+            self.check_iscleanup(&body, block_data);
         }
     }
 
diff --git a/src/librustc_mir/borrow_check/nll/universal_regions.rs b/src/librustc_mir/borrow_check/nll/universal_regions.rs
index fbedac4..9ad15fc 100644
--- a/src/librustc_mir/borrow_check/nll/universal_regions.rs
+++ b/src/librustc_mir/borrow_check/nll/universal_regions.rs
@@ -312,9 +312,9 @@
         match self.defining_ty {
             DefiningTy::Closure(def_id, substs) => {
                 err.note(&format!(
-                    "defining type: {:?} with closure substs {:#?}",
-                    def_id,
-                    &substs[..]
+                    "defining type: {} with closure substs {:#?}",
+                    tcx.def_path_str_with_substs(def_id, substs),
+                    &substs[tcx.generics_of(def_id).parent_count..],
                 ));
 
                 // FIXME: It'd be nice to print the late-bound regions
@@ -332,9 +332,9 @@
             }
             DefiningTy::Generator(def_id, substs, _) => {
                 err.note(&format!(
-                    "defining type: {:?} with generator substs {:#?}",
-                    def_id,
-                    &substs[..]
+                    "defining type: {} with generator substs {:#?}",
+                    tcx.def_path_str_with_substs(def_id, substs),
+                    &substs[tcx.generics_of(def_id).parent_count..],
                 ));
 
                 // FIXME: As above, we'd like to print out the region
@@ -350,16 +350,14 @@
             }
             DefiningTy::FnDef(def_id, substs) => {
                 err.note(&format!(
-                    "defining type: {:?} with substs {:#?}",
-                    def_id,
-                    &substs[..]
+                    "defining type: {}",
+                    tcx.def_path_str_with_substs(def_id, substs),
                 ));
             }
             DefiningTy::Const(def_id, substs) => {
                 err.note(&format!(
-                    "defining constant type: {:?} with substs {:#?}",
-                    def_id,
-                    &substs[..]
+                    "defining constant type: {}",
+                    tcx.def_path_str_with_substs(def_id, substs),
                 ));
             }
         }
diff --git a/src/librustc_mir/borrow_check/prefixes.rs b/src/librustc_mir/borrow_check/prefixes.rs
index 57833ca..b58bf73 100644
--- a/src/librustc_mir/borrow_check/prefixes.rs
+++ b/src/librustc_mir/borrow_check/prefixes.rs
@@ -11,7 +11,7 @@
 
 use rustc::hir;
 use rustc::ty::{self, TyCtxt};
-use rustc::mir::{Body, Place, PlaceBase, PlaceRef, ProjectionElem};
+use rustc::mir::{Place, PlaceBase, PlaceRef, ProjectionElem, ReadOnlyBodyCache};
 
 pub trait IsPrefixOf<'cx, 'tcx> {
     fn is_prefix_of(&self, other: PlaceRef<'cx, 'tcx>) -> bool;
@@ -26,7 +26,7 @@
 }
 
 pub(super) struct Prefixes<'cx, 'tcx> {
-    body: &'cx Body<'tcx>,
+    body: ReadOnlyBodyCache<'cx, 'tcx>,
     tcx: TyCtxt<'tcx>,
     kind: PrefixSet,
     next: Option<PlaceRef<'cx, 'tcx>>,
@@ -143,7 +143,7 @@
                     // derefs, except we stop at the deref of a shared
                     // reference.
 
-                    let ty = Place::ty_from(cursor.base, proj_base, self.body, self.tcx).ty;
+                    let ty = Place::ty_from(cursor.base, proj_base, *self.body, self.tcx).ty;
                     match ty.kind {
                         ty::RawPtr(_) |
                         ty::Ref(
diff --git a/src/librustc_mir/build/matches/test.rs b/src/librustc_mir/build/matches/test.rs
index c893d6f..e320811 100644
--- a/src/librustc_mir/build/matches/test.rs
+++ b/src/librustc_mir/build/matches/test.rs
@@ -680,7 +680,7 @@
                     }
                 })();
 
-                if no_overlap == Some(true) {
+                if let Some(true) = no_overlap {
                     // Testing range does not overlap with pattern range,
                     // so the pattern can be matched only if this test fails.
                     Some(1)
@@ -690,7 +690,7 @@
             }
 
             (&TestKind::Range(range), &PatKind::Constant { value }) => {
-                if self.const_range_contains(range, value) == Some(false) {
+                if let Some(false) = self.const_range_contains(range, value) {
                     // `value` is not contained in the testing range,
                     // so `value` can be matched only if this test fails.
                     Some(1)
diff --git a/src/librustc_mir/build/mod.rs b/src/librustc_mir/build/mod.rs
index 91ddd5a..b84461d 100644
--- a/src/librustc_mir/build/mod.rs
+++ b/src/librustc_mir/build/mod.rs
@@ -24,7 +24,7 @@
 use super::lints;
 
 /// Construct the MIR for a given `DefId`.
-pub fn mir_build(tcx: TyCtxt<'_>, def_id: DefId) -> Body<'_> {
+pub fn mir_build(tcx: TyCtxt<'_>, def_id: DefId) -> BodyCache<'_> {
     let id = tcx.hir().as_local_hir_id(def_id).unwrap();
 
     // Figure out what primary body this item has.
@@ -196,6 +196,8 @@
 
         lints::check(tcx, &body, def_id);
 
+        let mut body = BodyCache::new(body);
+        body.ensure_predecessors();
         body
     })
 }
@@ -309,7 +311,6 @@
     /// The vector of all scopes that we have created thus far;
     /// we track this for debuginfo later.
     source_scopes: IndexVec<SourceScope, SourceScopeData>,
-    source_scope_local_data: IndexVec<SourceScope, SourceScopeLocalData>,
     source_scope: SourceScope,
 
     /// The guard-context: each time we build the guard expression for
@@ -704,7 +705,6 @@
             block_context: BlockContext::new(),
             source_scopes: IndexVec::new(),
             source_scope: OUTERMOST_SOURCE_SCOPE,
-            source_scope_local_data: IndexVec::new(),
             guard_context: vec![],
             push_unsafe_count: 0,
             unpushed_unsafe: safety,
@@ -741,7 +741,6 @@
         Body::new(
             self.cfg.basic_blocks,
             self.source_scopes,
-            ClearCrossCrate::Set(self.source_scope_local_data),
             self.local_decls,
             self.canonical_user_type_annotations,
             self.arg_count,
@@ -942,7 +941,11 @@
             self.hir.root_lint_level
         );
         let parent_root = tcx.maybe_lint_level_root_bounded(
-            self.source_scope_local_data[original_source_scope].lint_root,
+            self.source_scopes[original_source_scope]
+                .local_data
+                .as_ref()
+                .assert_crate_local()
+                .lint_root,
             self.hir.root_lint_level,
         );
         if current_root != parent_root {
diff --git a/src/librustc_mir/build/scope.rs b/src/librustc_mir/build/scope.rs
index bb25b28..00a30af 100644
--- a/src/librustc_mir/build/scope.rs
+++ b/src/librustc_mir/build/scope.rs
@@ -436,7 +436,11 @@
             // We estimate the true lint roots here to avoid creating a lot of source scopes.
 
             let parent_root = tcx.maybe_lint_level_root_bounded(
-                self.source_scope_local_data[source_scope].lint_root,
+                self.source_scopes[source_scope]
+                    .local_data
+                    .as_ref()
+                    .assert_crate_local()
+                    .lint_root,
                 self.hir.root_lint_level,
             );
             let current_root = tcx.maybe_lint_level_root_bounded(
@@ -654,23 +658,22 @@
         let parent = self.source_scope;
         debug!("new_source_scope({:?}, {:?}, {:?}) - parent({:?})={:?}",
                span, lint_level, safety,
-               parent, self.source_scope_local_data.get(parent));
-        let scope = self.source_scopes.push(SourceScopeData {
-            span,
-            parent_scope: Some(parent),
-        });
+               parent, self.source_scopes.get(parent));
         let scope_local_data = SourceScopeLocalData {
             lint_root: if let LintLevel::Explicit(lint_root) = lint_level {
                 lint_root
             } else {
-                self.source_scope_local_data[parent].lint_root
+                self.source_scopes[parent].local_data.as_ref().assert_crate_local().lint_root
             },
             safety: safety.unwrap_or_else(|| {
-                self.source_scope_local_data[parent].safety
+                self.source_scopes[parent].local_data.as_ref().assert_crate_local().safety
             })
         };
-        self.source_scope_local_data.push(scope_local_data);
-        scope
+        self.source_scopes.push(SourceScopeData {
+            span,
+            parent_scope: Some(parent),
+            local_data: ClearCrossCrate::Set(scope_local_data),
+        })
     }
 
     /// Given a span and the current source scope, make a SourceInfo.
diff --git a/src/librustc_mir/const_eval.rs b/src/librustc_mir/const_eval.rs
index 0967b25..968a8a7 100644
--- a/src/librustc_mir/const_eval.rs
+++ b/src/librustc_mir/const_eval.rs
@@ -23,7 +23,7 @@
 use crate::interpret::{self,
     PlaceTy, MPlaceTy, OpTy, ImmTy, Immediate, Scalar, Pointer,
     RawConst, ConstValue, Machine,
-    InterpResult, InterpErrorInfo, GlobalId, InterpCx, StackPopCleanup,
+    InterpResult, InterpErrorInfo, GlobalId, InterpCx, StackPopCleanup, AssertMessage,
     Allocation, AllocId, MemoryKind, Memory,
     snapshot, RefTracking, intern_const_alloc_recursive,
 };
@@ -328,20 +328,32 @@
         false // for now, we don't enforce validity
     }
 
-    fn find_fn(
+    fn find_mir_or_eval_fn(
         ecx: &mut InterpCx<'mir, 'tcx, Self>,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx>],
         ret: Option<(PlaceTy<'tcx>, mir::BasicBlock)>,
         _unwind: Option<mir::BasicBlock> // unwinding is not supported in consts
     ) -> InterpResult<'tcx, Option<&'mir mir::Body<'tcx>>> {
-        debug!("eval_fn_call: {:?}", instance);
+        debug!("find_mir_or_eval_fn: {:?}", instance);
+
         // Only check non-glue functions
         if let ty::InstanceDef::Item(def_id) = instance.def {
             // Execution might have wandered off into other crates, so we cannot do a stability-
             // sensitive check here.  But we can at least rule out functions that are not const
             // at all.
-            if !ecx.tcx.is_const_fn_raw(def_id) {
+            if ecx.tcx.is_const_fn_raw(def_id) {
+                // If this function is a `const fn` then as an optimization we can query this
+                // evaluation immediately.
+                //
+                // For the moment we only do this for functions which take no arguments
+                // (or all arguments are ZSTs) so that we don't memoize too much.
+                if args.iter().all(|a| a.layout.is_zst()) {
+                    let gid = GlobalId { instance, promoted: None };
+                    ecx.eval_const_fn_call(gid, ret)?;
+                    return Ok(None);
+                }
+            } else {
                 // Some functions we support even if they are non-const -- but avoid testing
                 // that for const fn!  We certainly do *not* want to actually call the fn
                 // though, so be sure we return here.
@@ -354,7 +366,7 @@
         }
         // This is a const fn. Call it.
         Ok(Some(match ecx.load_mir(instance.def, None) {
-            Ok(body) => body,
+            Ok(body) => body.body(),
             Err(err) => {
                 if let err_unsup!(NoMirFor(ref path)) = err.kind {
                     return Err(
@@ -395,6 +407,40 @@
         )
     }
 
+    fn assert_panic(
+        ecx: &mut InterpCx<'mir, 'tcx, Self>,
+        _span: Span,
+        msg: &AssertMessage<'tcx>,
+        _unwind: Option<mir::BasicBlock>,
+    ) -> InterpResult<'tcx> {
+        use rustc::mir::interpret::PanicInfo::*;
+        Err(match msg {
+            BoundsCheck { ref len, ref index } => {
+                let len = ecx
+                    .read_immediate(ecx.eval_operand(len, None)?)
+                    .expect("can't eval len")
+                    .to_scalar()?
+                    .to_machine_usize(&*ecx)?;
+                let index = ecx
+                    .read_immediate(ecx.eval_operand(index, None)?)
+                    .expect("can't eval index")
+                    .to_scalar()?
+                    .to_machine_usize(&*ecx)?;
+                err_panic!(BoundsCheck { len, index })
+            }
+            Overflow(op) => err_panic!(Overflow(*op)),
+            OverflowNeg => err_panic!(OverflowNeg),
+            DivisionByZero => err_panic!(DivisionByZero),
+            RemainderByZero => err_panic!(RemainderByZero),
+            ResumedAfterReturn(generator_kind)
+                => err_panic!(ResumedAfterReturn(*generator_kind)),
+            ResumedAfterPanic(generator_kind)
+                => err_panic!(ResumedAfterPanic(*generator_kind)),
+            Panic { .. } => bug!("`Panic` variant cannot occur in MIR"),
+        }
+        .into())
+    }
+
     fn ptr_to_int(
         _mem: &Memory<'mir, 'tcx, Self>,
         _ptr: Pointer,
@@ -423,7 +469,7 @@
     }
 
     #[inline(always)]
-    fn tag_allocation<'b>(
+    fn init_allocation_extra<'b>(
         _memory_extra: &(),
         _id: AllocId,
         alloc: Cow<'b, Allocation>,
@@ -518,7 +564,7 @@
         tcx.type_of(tcx.require_lang_item(PanicLocationLangItem, None))
             .subst(tcx, tcx.mk_substs([tcx.lifetimes.re_static.into()].iter())),
     );
-    let loc_place = ecx.alloc_caller_location(file, line, col).unwrap();
+    let loc_place = ecx.alloc_caller_location(file, line, col);
     intern_const_alloc_recursive(&mut ecx, None, loc_place).unwrap();
     let loc_const = ty::Const {
         ty: loc_ty,
@@ -697,7 +743,7 @@
 
     let res = ecx.load_mir(cid.instance.def, cid.promoted);
     res.and_then(
-        |body| eval_body_using_ecx(&mut ecx, cid, body)
+        |body| eval_body_using_ecx(&mut ecx, cid, body.body())
     ).and_then(|place| {
         Ok(RawConst {
             alloc_id: place.ptr.assert_ptr().alloc_id,
diff --git a/src/librustc_mir/dataflow/impls/borrows.rs b/src/librustc_mir/dataflow/impls/borrows.rs
index 402e5ae..7e7652c 100644
--- a/src/librustc_mir/dataflow/impls/borrows.rs
+++ b/src/librustc_mir/dataflow/impls/borrows.rs
@@ -98,7 +98,7 @@
             // Add successor BBs to the work list, if necessary.
             let bb_data = &body[bb];
             assert!(hi == bb_data.statements.len());
-            for &succ_bb in bb_data.terminator.as_ref().unwrap().successors() {
+            for &succ_bb in bb_data.terminator().successors() {
                 visited.entry(succ_bb)
                     .and_modify(|lo| {
                         // `succ_bb` has been seen before. If it wasn't
@@ -153,8 +153,8 @@
         }
 
         Borrows {
-            tcx: tcx,
-            body: body,
+            tcx,
+            body,
             param_env,
             borrow_set: borrow_set.clone(),
             borrows_out_of_scope_at_location,
diff --git a/src/librustc_mir/dataflow/impls/mod.rs b/src/librustc_mir/dataflow/impls/mod.rs
index 6f860d0..0fb912b 100644
--- a/src/librustc_mir/dataflow/impls/mod.rs
+++ b/src/librustc_mir/dataflow/impls/mod.rs
@@ -71,7 +71,7 @@
 
 impl<'a, 'tcx> MaybeInitializedPlaces<'a, 'tcx> {
     pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, mdpe: &'a MoveDataParamEnv<'tcx>) -> Self {
-        MaybeInitializedPlaces { tcx: tcx, body: body, mdpe: mdpe }
+        MaybeInitializedPlaces { tcx, body, mdpe }
     }
 }
 
@@ -122,7 +122,7 @@
 
 impl<'a, 'tcx> MaybeUninitializedPlaces<'a, 'tcx> {
     pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, mdpe: &'a MoveDataParamEnv<'tcx>) -> Self {
-        MaybeUninitializedPlaces { tcx: tcx, body: body, mdpe: mdpe }
+        MaybeUninitializedPlaces { tcx, body, mdpe }
     }
 }
 
@@ -172,7 +172,7 @@
 
 impl<'a, 'tcx> DefinitelyInitializedPlaces<'a, 'tcx> {
     pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, mdpe: &'a MoveDataParamEnv<'tcx>) -> Self {
-        DefinitelyInitializedPlaces { tcx: tcx, body: body, mdpe: mdpe }
+        DefinitelyInitializedPlaces { tcx, body, mdpe }
     }
 }
 
@@ -217,7 +217,7 @@
 
 impl<'a, 'tcx> EverInitializedPlaces<'a, 'tcx> {
     pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, mdpe: &'a MoveDataParamEnv<'tcx>) -> Self {
-        EverInitializedPlaces { tcx: tcx, body: body, mdpe: mdpe }
+        EverInitializedPlaces { tcx, body, mdpe }
     }
 }
 
diff --git a/src/librustc_mir/dataflow/impls/storage_liveness.rs b/src/librustc_mir/dataflow/impls/storage_liveness.rs
index 1b81032..c4b97d1 100644
--- a/src/librustc_mir/dataflow/impls/storage_liveness.rs
+++ b/src/librustc_mir/dataflow/impls/storage_liveness.rs
@@ -75,24 +75,26 @@
 /// Dataflow analysis that determines whether each local requires storage at a
 /// given location; i.e. whether its storage can go away without being observed.
 pub struct RequiresStorage<'mir, 'tcx> {
-    body: &'mir Body<'tcx>,
+    body: ReadOnlyBodyCache<'mir, 'tcx>,
     borrowed_locals:
         RefCell<DataflowResultsRefCursor<'mir, 'tcx, HaveBeenBorrowedLocals<'mir, 'tcx>>>,
 }
 
 impl<'mir, 'tcx: 'mir> RequiresStorage<'mir, 'tcx> {
     pub fn new(
-        body: &'mir Body<'tcx>,
+        body: ReadOnlyBodyCache<'mir, 'tcx>,
         borrowed_locals: &'mir DataflowResults<'tcx, HaveBeenBorrowedLocals<'mir, 'tcx>>,
     ) -> Self {
         RequiresStorage {
             body,
-            borrowed_locals: RefCell::new(DataflowResultsCursor::new(borrowed_locals, body)),
+            borrowed_locals: RefCell::new(
+                DataflowResultsCursor::new(borrowed_locals, body.body())
+            ),
         }
     }
 
     pub fn body(&self) -> &Body<'tcx> {
-        self.body
+        &self.body
     }
 }
 
diff --git a/src/librustc_mir/hair/pattern/_match.rs b/src/librustc_mir/hair/pattern/_match.rs
index f2c5bf1..37a9381 100644
--- a/src/librustc_mir/hair/pattern/_match.rs
+++ b/src/librustc_mir/hair/pattern/_match.rs
@@ -425,16 +425,12 @@
     }
 
     /// This computes `S(constructor, self)`. See top of the file for explanations.
-    fn specialize_constructor<'a, 'q>(
+    fn specialize_constructor(
         &self,
-        cx: &mut MatchCheckCtxt<'a, 'tcx>,
+        cx: &mut MatchCheckCtxt<'p, 'tcx>,
         constructor: &Constructor<'tcx>,
-        ctor_wild_subpatterns: &[&'q Pat<'tcx>],
-    ) -> Option<PatStack<'q, 'tcx>>
-    where
-        'a: 'q,
-        'p: 'q,
-    {
+        ctor_wild_subpatterns: &'p [Pat<'tcx>],
+    ) -> Option<PatStack<'p, 'tcx>> {
         let new_heads = specialize_one_pattern(cx, self.head(), constructor, ctor_wild_subpatterns);
         new_heads.map(|mut new_head| {
             new_head.0.extend_from_slice(&self.0[1..]);
@@ -459,6 +455,7 @@
 }
 
 /// A 2D matrix.
+#[derive(Clone)]
 pub struct Matrix<'p, 'tcx>(Vec<PatStack<'p, 'tcx>>);
 
 impl<'p, 'tcx> Matrix<'p, 'tcx> {
@@ -486,16 +483,12 @@
     }
 
     /// This computes `S(constructor, self)`. See top of the file for explanations.
-    fn specialize_constructor<'a, 'q>(
+    fn specialize_constructor(
         &self,
-        cx: &mut MatchCheckCtxt<'a, 'tcx>,
+        cx: &mut MatchCheckCtxt<'p, 'tcx>,
         constructor: &Constructor<'tcx>,
-        ctor_wild_subpatterns: &[&'q Pat<'tcx>],
-    ) -> Matrix<'q, 'tcx>
-    where
-        'a: 'q,
-        'p: 'q,
-    {
+        ctor_wild_subpatterns: &'p [Pat<'tcx>],
+    ) -> Matrix<'p, 'tcx> {
         self.0
             .iter()
             .filter_map(|r| r.specialize_constructor(cx, constructor, ctor_wild_subpatterns))
@@ -1033,17 +1026,19 @@
 }
 
 #[derive(Clone, Debug)]
-pub enum Usefulness<'tcx> {
-    Useful,
+pub enum Usefulness<'tcx, 'p> {
+    /// Carries a list of unreachable subpatterns. Used only in the presence of or-patterns.
+    Useful(Vec<&'p Pat<'tcx>>),
+    /// Carries a list of witnesses of non-exhaustiveness.
     UsefulWithWitness(Vec<Witness<'tcx>>),
     NotUseful,
 }
 
-impl<'tcx> Usefulness<'tcx> {
+impl<'tcx, 'p> Usefulness<'tcx, 'p> {
     fn new_useful(preference: WitnessPreference) -> Self {
         match preference {
             ConstructWitness => UsefulWithWitness(vec![Witness(vec![])]),
-            LeaveOutWitness => Useful,
+            LeaveOutWitness => Useful(vec![]),
         }
     }
 
@@ -1604,13 +1599,13 @@
 /// relation to preceding patterns, it is not reachable) and exhaustiveness
 /// checking (if a wildcard pattern is useful in relation to a matrix, the
 /// matrix isn't exhaustive).
-pub fn is_useful<'p, 'a, 'tcx>(
-    cx: &mut MatchCheckCtxt<'a, 'tcx>,
+pub fn is_useful<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
     matrix: &Matrix<'p, 'tcx>,
-    v: &PatStack<'_, 'tcx>,
+    v: &PatStack<'p, 'tcx>,
     witness_preference: WitnessPreference,
     hir_id: HirId,
-) -> Usefulness<'tcx> {
+) -> Usefulness<'tcx, 'p> {
     let &Matrix(ref rows) = matrix;
     debug!("is_useful({:#?}, {:#?})", matrix, v);
 
@@ -1631,11 +1626,26 @@
 
     // If the first pattern is an or-pattern, expand it.
     if let Some(vs) = v.expand_or_pat() {
-        return vs
-            .into_iter()
-            .map(|v| is_useful(cx, matrix, &v, witness_preference, hir_id))
-            .find(|result| result.is_useful())
-            .unwrap_or(NotUseful);
+        // We need to push the already-seen patterns into the matrix in order to detect redundant
+        // branches like `Some(_) | Some(0)`. We also keep track of the unreachable subpatterns.
+        let mut matrix = matrix.clone();
+        let mut unreachable_pats = Vec::new();
+        let mut any_is_useful = false;
+        for v in vs {
+            let res = is_useful(cx, &matrix, &v, witness_preference, hir_id);
+            match res {
+                Useful(pats) => {
+                    any_is_useful = true;
+                    unreachable_pats.extend(pats);
+                }
+                NotUseful => unreachable_pats.push(v.head()),
+                UsefulWithWitness(_) => {
+                    bug!("Encountered or-pat in `v` during exhaustiveness checking")
+                }
+            }
+            matrix.push(v);
+        }
+        return if any_is_useful { Useful(unreachable_pats) } else { NotUseful };
     }
 
     let (ty, span) = matrix
@@ -1768,21 +1778,21 @@
 
 /// A shorthand for the `U(S(c, P), S(c, q))` operation from the paper. I.e., `is_useful` applied
 /// to the specialised version of both the pattern matrix `P` and the new pattern `q`.
-fn is_useful_specialized<'p, 'a, 'tcx>(
-    cx: &mut MatchCheckCtxt<'a, 'tcx>,
+fn is_useful_specialized<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
     matrix: &Matrix<'p, 'tcx>,
-    v: &PatStack<'_, 'tcx>,
+    v: &PatStack<'p, 'tcx>,
     ctor: Constructor<'tcx>,
     lty: Ty<'tcx>,
     witness_preference: WitnessPreference,
     hir_id: HirId,
-) -> Usefulness<'tcx> {
+) -> Usefulness<'tcx, 'p> {
     debug!("is_useful_specialized({:#?}, {:#?}, {:?})", v, ctor, lty);
 
-    let ctor_wild_subpatterns_owned: Vec<_> = ctor.wildcard_subpatterns(cx, lty);
-    let ctor_wild_subpatterns: Vec<_> = ctor_wild_subpatterns_owned.iter().collect();
-    let matrix = matrix.specialize_constructor(cx, &ctor, &ctor_wild_subpatterns);
-    v.specialize_constructor(cx, &ctor, &ctor_wild_subpatterns)
+    let ctor_wild_subpatterns =
+        cx.pattern_arena.alloc_from_iter(ctor.wildcard_subpatterns(cx, lty));
+    let matrix = matrix.specialize_constructor(cx, &ctor, ctor_wild_subpatterns);
+    v.specialize_constructor(cx, &ctor, ctor_wild_subpatterns)
         .map(|v| is_useful(cx, &matrix, &v, witness_preference, hir_id))
         .map(|u| u.apply_constructor(cx, &ctor, lty))
         .unwrap_or(NotUseful)
@@ -2250,13 +2260,13 @@
     if intersects { Some(()) } else { None }
 }
 
-fn patterns_for_variant<'p, 'a: 'p, 'tcx>(
-    cx: &mut MatchCheckCtxt<'a, 'tcx>,
+fn patterns_for_variant<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
     subpatterns: &'p [FieldPat<'tcx>],
-    ctor_wild_subpatterns: &[&'p Pat<'tcx>],
+    ctor_wild_subpatterns: &'p [Pat<'tcx>],
     is_non_exhaustive: bool,
 ) -> PatStack<'p, 'tcx> {
-    let mut result = SmallVec::from_slice(ctor_wild_subpatterns);
+    let mut result: SmallVec<_> = ctor_wild_subpatterns.iter().collect();
 
     for subpat in subpatterns {
         if !is_non_exhaustive || !cx.is_uninhabited(subpat.pattern.ty) {
@@ -2280,11 +2290,11 @@
 /// different patterns.
 /// Structure patterns with a partial wild pattern (Foo { a: 42, .. }) have their missing
 /// fields filled with wild patterns.
-fn specialize_one_pattern<'p, 'a: 'p, 'q: 'p, 'tcx>(
-    cx: &mut MatchCheckCtxt<'a, 'tcx>,
-    pat: &'q Pat<'tcx>,
+fn specialize_one_pattern<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
+    pat: &'p Pat<'tcx>,
     constructor: &Constructor<'tcx>,
-    ctor_wild_subpatterns: &[&'p Pat<'tcx>],
+    ctor_wild_subpatterns: &'p [Pat<'tcx>],
 ) -> Option<PatStack<'p, 'tcx>> {
     if let NonExhaustive = constructor {
         // Only a wildcard pattern can match the special extra constructor
@@ -2294,9 +2304,7 @@
     let result = match *pat.kind {
         PatKind::AscribeUserType { .. } => bug!(), // Handled by `expand_pattern`
 
-        PatKind::Binding { .. } | PatKind::Wild => {
-            Some(PatStack::from_slice(ctor_wild_subpatterns))
-        }
+        PatKind::Binding { .. } | PatKind::Wild => Some(ctor_wild_subpatterns.iter().collect()),
 
         PatKind::Variant { adt_def, variant_index, ref subpatterns, .. } => {
             let ref variant = adt_def.variants[variant_index];
@@ -2406,7 +2414,6 @@
                                 .chain(
                                     ctor_wild_subpatterns
                                         .iter()
-                                        .map(|p| *p)
                                         .skip(prefix.len())
                                         .take(slice_count)
                                         .chain(suffix.iter()),
diff --git a/src/librustc_mir/hair/pattern/check_match.rs b/src/librustc_mir/hair/pattern/check_match.rs
index 30586e9..737af3e 100644
--- a/src/librustc_mir/hair/pattern/check_match.rs
+++ b/src/librustc_mir/hair/pattern/check_match.rs
@@ -139,39 +139,22 @@
         MatchCheckCtxt::create_and_enter(self.tcx, self.param_env, module, |ref mut cx| {
             let mut have_errors = false;
 
-            let inlined_arms: Vec<(Vec<_>, _)> = arms
+            let inlined_arms: Vec<_> = arms
                 .iter()
                 .map(|arm| {
-                    (
-                        // HACK(or_patterns; Centril | dlrobertson): Remove this and
-                        // correctly handle exhaustiveness checking for nested or-patterns.
-                        match &arm.pat.kind {
-                            hir::PatKind::Or(pats) => pats,
-                            _ => std::slice::from_ref(&arm.pat),
-                        }
-                        .iter()
-                        .map(|pat| {
-                            let mut patcx = PatCtxt::new(
-                                self.tcx,
-                                self.param_env.and(self.identity_substs),
-                                self.tables,
-                            );
-                            patcx.include_lint_checks();
-                            let pattern = cx
-                                .pattern_arena
-                                .alloc(expand_pattern(cx, patcx.lower_pattern(&pat)))
-                                as &_;
-                            if !patcx.errors.is_empty() {
-                                patcx.report_inlining_errors(pat.span);
-                                have_errors = true;
-                            }
-                            (pattern, &**pat)
-                        })
-                        .collect(),
-                        arm.guard.as_ref().map(|g| match g {
-                            hir::Guard::If(ref e) => &**e,
-                        }),
-                    )
+                    let mut patcx = PatCtxt::new(
+                        self.tcx,
+                        self.param_env.and(self.identity_substs),
+                        self.tables,
+                    );
+                    patcx.include_lint_checks();
+                    let pattern: &_ =
+                        cx.pattern_arena.alloc(expand_pattern(cx, patcx.lower_pattern(&arm.pat)));
+                    if !patcx.errors.is_empty() {
+                        patcx.report_inlining_errors(arm.pat.span);
+                        have_errors = true;
+                    }
+                    (pattern, &*arm.pat, arm.guard.is_some())
                 })
                 .collect();
 
@@ -181,7 +164,7 @@
             }
 
             // Fourth, check for unreachable arms.
-            check_arms(cx, &inlined_arms, source);
+            let matrix = check_arms(cx, &inlined_arms, source);
 
             // Then, if the match has no arms, check whether the scrutinee
             // is uninhabited.
@@ -248,12 +231,6 @@
                 return;
             }
 
-            let matrix: Matrix<'_, '_> = inlined_arms
-                .iter()
-                .filter(|&&(_, guard)| guard.is_none())
-                .flat_map(|arm| &arm.0)
-                .map(|pat| PatStack::from_pattern(pat.0))
-                .collect();
             let scrut_ty = self.tables.node_type(scrut.hir_id);
             check_exhaustive(cx, scrut_ty, scrut.span, &matrix, scrut.hir_id);
         })
@@ -267,8 +244,8 @@
             patcx.include_lint_checks();
             let pattern = patcx.lower_pattern(pat);
             let pattern_ty = pattern.ty;
-            let pattern = expand_pattern(cx, pattern);
-            let pats: Matrix<'_, '_> = vec![PatStack::from_pattern(&pattern)].into_iter().collect();
+            let pattern = cx.pattern_arena.alloc(expand_pattern(cx, pattern));
+            let pats: Matrix<'_, '_> = vec![PatStack::from_pattern(pattern)].into_iter().collect();
 
             let witnesses = match check_not_useful(cx, pattern_ty, &pats, pat.hir_id) {
                 Ok(_) => return,
@@ -403,113 +380,120 @@
 }
 
 // Check for unreachable patterns
-fn check_arms<'tcx>(
-    cx: &mut MatchCheckCtxt<'_, 'tcx>,
-    arms: &[(Vec<(&super::Pat<'tcx>, &hir::Pat)>, Option<&hir::Expr>)],
+fn check_arms<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
+    arms: &[(&'p super::Pat<'tcx>, &hir::Pat, bool)],
     source: hir::MatchSource,
-) {
+) -> Matrix<'p, 'tcx> {
     let mut seen = Matrix::empty();
     let mut catchall = None;
-    for (arm_index, &(ref pats, guard)) in arms.iter().enumerate() {
-        for &(pat, hir_pat) in pats {
-            let v = PatStack::from_pattern(pat);
+    for (arm_index, (pat, hir_pat, has_guard)) in arms.iter().enumerate() {
+        let v = PatStack::from_pattern(pat);
 
-            match is_useful(cx, &seen, &v, LeaveOutWitness, hir_pat.hir_id) {
-                NotUseful => {
-                    match source {
-                        hir::MatchSource::IfDesugar { .. } | hir::MatchSource::WhileDesugar => {
-                            bug!()
-                        }
-                        hir::MatchSource::IfLetDesugar { .. } => {
-                            cx.tcx.lint_hir(
-                                lint::builtin::IRREFUTABLE_LET_PATTERNS,
-                                hir_pat.hir_id,
-                                pat.span,
-                                "irrefutable if-let pattern",
-                            );
-                        }
+        match is_useful(cx, &seen, &v, LeaveOutWitness, hir_pat.hir_id) {
+            NotUseful => {
+                match source {
+                    hir::MatchSource::IfDesugar { .. } | hir::MatchSource::WhileDesugar => bug!(),
 
-                        hir::MatchSource::WhileLetDesugar => {
-                            // check which arm we're on.
-                            match arm_index {
-                                // The arm with the user-specified pattern.
-                                0 => {
-                                    cx.tcx.lint_hir(
-                                        lint::builtin::UNREACHABLE_PATTERNS,
-                                        hir_pat.hir_id,
-                                        pat.span,
-                                        "unreachable pattern",
-                                    );
-                                }
-                                // The arm with the wildcard pattern.
-                                1 => {
-                                    cx.tcx.lint_hir(
-                                        lint::builtin::IRREFUTABLE_LET_PATTERNS,
-                                        hir_pat.hir_id,
-                                        pat.span,
-                                        "irrefutable while-let pattern",
-                                    );
-                                }
-                                _ => bug!(),
+                    hir::MatchSource::IfLetDesugar { .. } | hir::MatchSource::WhileLetDesugar => {
+                        // check which arm we're on.
+                        match arm_index {
+                            // The arm with the user-specified pattern.
+                            0 => {
+                                cx.tcx.lint_hir(
+                                    lint::builtin::UNREACHABLE_PATTERNS,
+                                    hir_pat.hir_id,
+                                    pat.span,
+                                    "unreachable pattern",
+                                );
                             }
-                        }
-
-                        hir::MatchSource::ForLoopDesugar | hir::MatchSource::Normal => {
-                            let mut err = cx.tcx.struct_span_lint_hir(
-                                lint::builtin::UNREACHABLE_PATTERNS,
-                                hir_pat.hir_id,
-                                pat.span,
-                                "unreachable pattern",
-                            );
-                            // if we had a catchall pattern, hint at that
-                            if let Some(catchall) = catchall {
-                                err.span_label(pat.span, "unreachable pattern");
-                                err.span_label(catchall, "matches any value");
+                            // The arm with the wildcard pattern.
+                            1 => {
+                                let msg = match source {
+                                    hir::MatchSource::IfLetDesugar { .. } => {
+                                        "irrefutable if-let pattern"
+                                    }
+                                    hir::MatchSource::WhileLetDesugar => {
+                                        "irrefutable while-let pattern"
+                                    }
+                                    _ => bug!(),
+                                };
+                                cx.tcx.lint_hir(
+                                    lint::builtin::IRREFUTABLE_LET_PATTERNS,
+                                    hir_pat.hir_id,
+                                    pat.span,
+                                    msg,
+                                );
                             }
-                            err.emit();
+                            _ => bug!(),
                         }
-
-                        // Unreachable patterns in try and await expressions occur when one of
-                        // the arms are an uninhabited type. Which is OK.
-                        hir::MatchSource::AwaitDesugar | hir::MatchSource::TryDesugar => {}
                     }
+
+                    hir::MatchSource::ForLoopDesugar | hir::MatchSource::Normal => {
+                        let mut err = cx.tcx.struct_span_lint_hir(
+                            lint::builtin::UNREACHABLE_PATTERNS,
+                            hir_pat.hir_id,
+                            pat.span,
+                            "unreachable pattern",
+                        );
+                        // if we had a catchall pattern, hint at that
+                        if let Some(catchall) = catchall {
+                            err.span_label(pat.span, "unreachable pattern");
+                            err.span_label(catchall, "matches any value");
+                        }
+                        err.emit();
+                    }
+
+                    // Unreachable patterns in try and await expressions occur when one of
+                    // the arms are an uninhabited type. Which is OK.
+                    hir::MatchSource::AwaitDesugar | hir::MatchSource::TryDesugar => {}
                 }
-                Useful => (),
-                UsefulWithWitness(_) => bug!(),
             }
-            if guard.is_none() {
-                seen.push(v);
-                if catchall.is_none() && pat_is_catchall(hir_pat) {
-                    catchall = Some(pat.span);
+            Useful(unreachable_subpatterns) => {
+                for pat in unreachable_subpatterns {
+                    cx.tcx.lint_hir(
+                        lint::builtin::UNREACHABLE_PATTERNS,
+                        hir_pat.hir_id,
+                        pat.span,
+                        "unreachable pattern",
+                    );
                 }
             }
+            UsefulWithWitness(_) => bug!(),
+        }
+        if !has_guard {
+            seen.push(v);
+            if catchall.is_none() && pat_is_catchall(hir_pat) {
+                catchall = Some(pat.span);
+            }
         }
     }
+    seen
 }
 
-fn check_not_useful(
-    cx: &mut MatchCheckCtxt<'_, 'tcx>,
+fn check_not_useful<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
     ty: Ty<'tcx>,
-    matrix: &Matrix<'_, 'tcx>,
+    matrix: &Matrix<'p, 'tcx>,
     hir_id: HirId,
 ) -> Result<(), Vec<super::Pat<'tcx>>> {
-    let wild_pattern = super::Pat::wildcard_from_ty(ty);
-    match is_useful(cx, matrix, &PatStack::from_pattern(&wild_pattern), ConstructWitness, hir_id) {
+    let wild_pattern = cx.pattern_arena.alloc(super::Pat::wildcard_from_ty(ty));
+    match is_useful(cx, matrix, &PatStack::from_pattern(wild_pattern), ConstructWitness, hir_id) {
         NotUseful => Ok(()), // This is good, wildcard pattern isn't reachable.
         UsefulWithWitness(pats) => Err(if pats.is_empty() {
-            vec![wild_pattern]
+            bug!("Exhaustiveness check returned no witnesses")
         } else {
             pats.into_iter().map(|w| w.single_pattern()).collect()
         }),
-        Useful => bug!(),
+        Useful(_) => bug!(),
     }
 }
 
-fn check_exhaustive<'tcx>(
-    cx: &mut MatchCheckCtxt<'_, 'tcx>,
+fn check_exhaustive<'p, 'tcx>(
+    cx: &mut MatchCheckCtxt<'p, 'tcx>,
     scrut_ty: Ty<'tcx>,
     sp: Span,
-    matrix: &Matrix<'_, 'tcx>,
+    matrix: &Matrix<'p, 'tcx>,
     hir_id: HirId,
 ) {
     let witnesses = match check_not_useful(cx, scrut_ty, matrix, hir_id) {
diff --git a/src/librustc_mir/hair/pattern/mod.rs b/src/librustc_mir/hair/pattern/mod.rs
index 92c7178..0086c3b 100644
--- a/src/librustc_mir/hair/pattern/mod.rs
+++ b/src/librustc_mir/hair/pattern/mod.rs
@@ -1154,13 +1154,7 @@
 ) -> Option<Ordering> {
     trace!("compare_const_vals: {:?}, {:?}", a, b);
 
-    let from_bool = |v: bool| {
-        if v {
-            Some(Ordering::Equal)
-        } else {
-            None
-        }
-    };
+    let from_bool = |v: bool| v.then_some(Ordering::Equal);
 
     let fallback = || from_bool(a == b);
 
diff --git a/src/librustc_mir/interpret/cast.rs b/src/librustc_mir/interpret/cast.rs
index 1fb8b3c..e9602ec 100644
--- a/src/librustc_mir/interpret/cast.rs
+++ b/src/librustc_mir/interpret/cast.rs
@@ -55,7 +55,7 @@
                         ).ok_or_else(|| err_inval!(TooGeneric))?;
 
                         let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance));
-                        self.write_scalar(Scalar::Ptr(fn_ptr.into()), dest)?;
+                        self.write_scalar(fn_ptr, dest)?;
                     }
                     _ => bug!("reify fn pointer on {:?}", src.layout.ty),
                 }
@@ -88,8 +88,7 @@
                             ty::ClosureKind::FnOnce,
                         );
                         let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance));
-                        let val = Immediate::Scalar(Scalar::Ptr(fn_ptr.into()).into());
-                        self.write_immediate(val, dest)?;
+                        self.write_scalar(fn_ptr, dest)?;
                     }
                     _ => bug!("closure fn pointer on {:?}", src.layout.ty),
                 }
diff --git a/src/librustc_mir/interpret/eval_context.rs b/src/librustc_mir/interpret/eval_context.rs
index 228e5ca..9af47f3 100644
--- a/src/librustc_mir/interpret/eval_context.rs
+++ b/src/librustc_mir/interpret/eval_context.rs
@@ -164,6 +164,20 @@
     }
 }
 
+impl<'mir, 'tcx, Tag, Extra> Frame<'mir, 'tcx, Tag, Extra> {
+    /// Return the `SourceInfo` of the current instruction.
+    pub fn current_source_info(&self) -> Option<mir::SourceInfo> {
+        self.block.map(|block| {
+            let block = &self.body.basic_blocks()[block];
+            if self.stmt < block.statements.len() {
+                block.statements[self.stmt].source_info
+            } else {
+                block.terminator().source_info
+            }
+        })
+    }
+}
+
 impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> HasDataLayout for InterpCx<'mir, 'tcx, M> {
     #[inline]
     fn data_layout(&self) -> &layout::TargetDataLayout {
@@ -236,6 +250,12 @@
         self.memory.force_bits(scalar, size)
     }
 
+    /// Call this to turn untagged "global" pointers (obtained via `tcx`) into
+    /// the *canonical* machine pointer to the allocation.  Must never be used
+    /// for any other pointers!
+    ///
+    /// This represents a *direct* access to that memory, as opposed to access
+    /// through a pointer that was created by the program.
     #[inline(always)]
     pub fn tag_static_base_pointer(&self, ptr: Pointer) -> Pointer<M::PointerTag> {
         self.memory.tag_static_base_pointer(ptr)
@@ -292,7 +312,7 @@
         &self,
         instance: ty::InstanceDef<'tcx>,
         promoted: Option<mir::Promoted>,
-    ) -> InterpResult<'tcx, &'tcx mir::Body<'tcx>> {
+    ) -> InterpResult<'tcx, mir::ReadOnlyBodyCache<'tcx, 'tcx>> {
         // do not continue if typeck errors occurred (can only occur in local crate)
         let did = instance.def_id();
         if did.is_local()
@@ -303,11 +323,11 @@
         }
         trace!("load mir(instance={:?}, promoted={:?})", instance, promoted);
         if let Some(promoted) = promoted {
-            return Ok(&self.tcx.promoted_mir(did)[promoted]);
+            return Ok(self.tcx.promoted_mir(did)[promoted].unwrap_read_only());
         }
         match instance {
             ty::InstanceDef::Item(def_id) => if self.tcx.is_mir_available(did) {
-                Ok(self.tcx.optimized_mir(did))
+                Ok(self.tcx.optimized_mir(did).unwrap_read_only())
             } else {
                 throw_unsup!(NoMirFor(self.tcx.def_path_str(def_id)))
             },
@@ -828,34 +848,28 @@
     pub fn generate_stacktrace(&self, explicit_span: Option<Span>) -> Vec<FrameInfo<'tcx>> {
         let mut last_span = None;
         let mut frames = Vec::new();
-        for &Frame { instance, span, body, block, stmt, .. } in self.stack().iter().rev() {
+        for frame in self.stack().iter().rev() {
             // make sure we don't emit frames that are duplicates of the previous
-            if explicit_span == Some(span) {
-                last_span = Some(span);
+            if explicit_span == Some(frame.span) {
+                last_span = Some(frame.span);
                 continue;
             }
             if let Some(last) = last_span {
-                if last == span {
+                if last == frame.span {
                     continue;
                 }
             } else {
-                last_span = Some(span);
+                last_span = Some(frame.span);
             }
 
-            let lint_root = block.and_then(|block| {
-                let block = &body.basic_blocks()[block];
-                let source_info = if stmt < block.statements.len() {
-                    block.statements[stmt].source_info
-                } else {
-                    block.terminator().source_info
-                };
-                match body.source_scope_local_data {
-                    mir::ClearCrossCrate::Set(ref ivs) => Some(ivs[source_info.scope].lint_root),
+            let lint_root = frame.current_source_info().and_then(|source_info| {
+                match &frame.body.source_scopes[source_info.scope].local_data {
+                    mir::ClearCrossCrate::Set(data) => Some(data.lint_root),
                     mir::ClearCrossCrate::Clear => None,
                 }
             });
 
-            frames.push(FrameInfo { call_site: span, instance, lint_root });
+            frames.push(FrameInfo { call_site: frame.span, instance: frame.instance, lint_root });
         }
         trace!("generate stacktrace: {:#?}, {:?}", frames, explicit_span);
         frames
diff --git a/src/librustc_mir/interpret/intrinsics.rs b/src/librustc_mir/interpret/intrinsics.rs
index 7bcf84a..ad5df5a 100644
--- a/src/librustc_mir/interpret/intrinsics.rs
+++ b/src/librustc_mir/interpret/intrinsics.rs
@@ -2,7 +2,7 @@
 //! looking at their MIR. Intrinsics/functions supported here are shared by CTFE
 //! and miri.
 
-use syntax::symbol::Symbol;
+use syntax_pos::symbol::{sym, Symbol};
 use syntax_pos::Span;
 use rustc::ty;
 use rustc::ty::layout::{LayoutOf, Primitive, Size};
@@ -22,7 +22,7 @@
 mod type_name;
 
 fn numeric_intrinsic<'tcx, Tag>(
-    name: &str,
+    name: Symbol,
     bits: u128,
     kind: Primitive,
 ) -> InterpResult<'tcx, Scalar<Tag>> {
@@ -32,11 +32,11 @@
     };
     let extra = 128 - size.bits() as u128;
     let bits_out = match name {
-        "ctpop" => bits.count_ones() as u128,
-        "ctlz" => bits.leading_zeros() as u128 - extra,
-        "cttz" => (bits << extra).trailing_zeros() as u128 - extra,
-        "bswap" => (bits << extra).swap_bytes(),
-        "bitreverse" => (bits << extra).reverse_bits(),
+        sym::ctpop => bits.count_ones() as u128,
+        sym::ctlz => bits.leading_zeros() as u128 - extra,
+        sym::cttz => (bits << extra).trailing_zeros() as u128 - extra,
+        sym::bswap => (bits << extra).swap_bytes(),
+        sym::bitreverse => (bits << extra).reverse_bits(),
         _ => bug!("not a numeric intrinsic: {}", name),
     };
     Ok(Scalar::from_uint(bits_out, size))
@@ -51,9 +51,9 @@
     substs: SubstsRef<'tcx>,
 ) -> InterpResult<'tcx, &'tcx ty::Const<'tcx>> {
     let tp_ty = substs.type_at(0);
-    let name = &*tcx.item_name(def_id).as_str();
+    let name = tcx.item_name(def_id);
     Ok(match name {
-        "type_name" => {
+        sym::type_name => {
             let alloc = type_name::alloc_type_name(tcx, tp_ty);
             tcx.mk_const(ty::Const {
                 val: ty::ConstKind::Value(ConstValue::Slice {
@@ -64,20 +64,20 @@
                 ty: tcx.mk_static_str(),
             })
         },
-        "needs_drop" => ty::Const::from_bool(tcx, tp_ty.needs_drop(tcx, param_env)),
-        "size_of" |
-        "min_align_of" |
-        "pref_align_of" => {
+        sym::needs_drop => ty::Const::from_bool(tcx, tp_ty.needs_drop(tcx, param_env)),
+        sym::size_of |
+        sym::min_align_of |
+        sym::pref_align_of => {
             let layout = tcx.layout_of(param_env.and(tp_ty)).map_err(|e| err_inval!(Layout(e)))?;
             let n = match name {
-                "pref_align_of" => layout.align.pref.bytes(),
-                "min_align_of" => layout.align.abi.bytes(),
-                "size_of" => layout.size.bytes(),
+                sym::pref_align_of => layout.align.pref.bytes(),
+                sym::min_align_of => layout.align.abi.bytes(),
+                sym::size_of => layout.size.bytes(),
                 _ => bug!(),
             };
             ty::Const::from_usize(tcx, n)
         },
-        "type_id" => ty::Const::from_bits(
+        sym::type_id => ty::Const::from_bits(
             tcx,
             tcx.type_id_hash(tp_ty).into(),
             param_env.and(tcx.types.u64),
@@ -96,36 +96,32 @@
         ret: Option<(PlaceTy<'tcx, M::PointerTag>, mir::BasicBlock)>,
     ) -> InterpResult<'tcx, bool> {
         let substs = instance.substs;
-        let intrinsic_name = &*self.tcx.item_name(instance.def_id()).as_str();
+        let intrinsic_name = self.tcx.item_name(instance.def_id());
 
         // We currently do not handle any intrinsics that are *allowed* to diverge,
         // but `transmute` could lack a return place in case of UB.
         let (dest, ret) = match ret {
             Some(p) => p,
             None => match intrinsic_name {
-                "transmute" => throw_ub!(Unreachable),
+                sym::transmute => throw_ub!(Unreachable),
                 _ => return Ok(false),
             }
         };
 
+        // Keep the patterns in this match ordered the same as the list in
+        // `src/librustc/ty/constness.rs`
         match intrinsic_name {
-            "caller_location" => {
-                let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
-                let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo());
-                let location = self.alloc_caller_location(
-                    Symbol::intern(&caller.file.name.to_string()),
-                    caller.line as u32,
-                    caller.col_display as u32 + 1,
-                )?;
+            sym::caller_location => {
+                let location = self.alloc_caller_location_for_span(span);
                 self.write_scalar(location.ptr, dest)?;
             }
 
-            "min_align_of" |
-            "pref_align_of" |
-            "needs_drop" |
-            "size_of" |
-            "type_id" |
-            "type_name" => {
+            sym::min_align_of |
+            sym::pref_align_of |
+            sym::needs_drop |
+            sym::size_of |
+            sym::type_id |
+            sym::type_name => {
                 let gid = GlobalId {
                     instance,
                     promoted: None,
@@ -135,13 +131,13 @@
                 self.copy_op(val, dest)?;
             }
 
-            | "ctpop"
-            | "cttz"
-            | "cttz_nonzero"
-            | "ctlz"
-            | "ctlz_nonzero"
-            | "bswap"
-            | "bitreverse" => {
+            | sym::ctpop
+            | sym::cttz
+            | sym::cttz_nonzero
+            | sym::ctlz
+            | sym::ctlz_nonzero
+            | sym::bswap
+            | sym::bitreverse => {
                 let ty = substs.type_at(0);
                 let layout_of = self.layout_of(ty)?;
                 let val = self.read_scalar(args[0])?.not_undef()?;
@@ -150,31 +146,32 @@
                     ty::layout::Abi::Scalar(ref scalar) => scalar.value,
                     _ => throw_unsup!(TypeNotPrimitive(ty)),
                 };
-                let out_val = if intrinsic_name.ends_with("_nonzero") {
-                    if bits == 0 {
-                        throw_ub_format!("`{}` called on 0", intrinsic_name);
-                    }
-                    numeric_intrinsic(intrinsic_name.trim_end_matches("_nonzero"), bits, kind)?
-                } else {
-                    numeric_intrinsic(intrinsic_name, bits, kind)?
+                let (nonzero, intrinsic_name) = match intrinsic_name {
+                    sym::cttz_nonzero => (true, sym::cttz),
+                    sym::ctlz_nonzero => (true, sym::ctlz),
+                    other => (false, other),
                 };
+                if nonzero && bits == 0 {
+                    throw_ub_format!("`{}_nonzero` called on 0", intrinsic_name);
+                }
+                let out_val = numeric_intrinsic(intrinsic_name, bits, kind)?;
                 self.write_scalar(out_val, dest)?;
             }
-            | "wrapping_add"
-            | "wrapping_sub"
-            | "wrapping_mul"
-            | "add_with_overflow"
-            | "sub_with_overflow"
-            | "mul_with_overflow" => {
+            | sym::wrapping_add
+            | sym::wrapping_sub
+            | sym::wrapping_mul
+            | sym::add_with_overflow
+            | sym::sub_with_overflow
+            | sym::mul_with_overflow => {
                 let lhs = self.read_immediate(args[0])?;
                 let rhs = self.read_immediate(args[1])?;
                 let (bin_op, ignore_overflow) = match intrinsic_name {
-                    "wrapping_add" => (BinOp::Add, true),
-                    "wrapping_sub" => (BinOp::Sub, true),
-                    "wrapping_mul" => (BinOp::Mul, true),
-                    "add_with_overflow" => (BinOp::Add, false),
-                    "sub_with_overflow" => (BinOp::Sub, false),
-                    "mul_with_overflow" => (BinOp::Mul, false),
+                    sym::wrapping_add => (BinOp::Add, true),
+                    sym::wrapping_sub => (BinOp::Sub, true),
+                    sym::wrapping_mul => (BinOp::Mul, true),
+                    sym::add_with_overflow => (BinOp::Add, false),
+                    sym::sub_with_overflow => (BinOp::Sub, false),
+                    sym::mul_with_overflow => (BinOp::Mul, false),
                     _ => bug!("Already checked for int ops")
                 };
                 if ignore_overflow {
@@ -183,10 +180,10 @@
                     self.binop_with_overflow(bin_op, lhs, rhs, dest)?;
                 }
             }
-            "saturating_add" | "saturating_sub" => {
+            sym::saturating_add | sym::saturating_sub => {
                 let l = self.read_immediate(args[0])?;
                 let r = self.read_immediate(args[1])?;
-                let is_add = intrinsic_name == "saturating_add";
+                let is_add = intrinsic_name == sym::saturating_add;
                 let (val, overflowed, _ty) = self.overflowing_binary_op(if is_add {
                     BinOp::Add
                 } else {
@@ -226,12 +223,12 @@
                 };
                 self.write_scalar(val, dest)?;
             }
-            "unchecked_shl" | "unchecked_shr" => {
+            sym::unchecked_shl | sym::unchecked_shr => {
                 let l = self.read_immediate(args[0])?;
                 let r = self.read_immediate(args[1])?;
                 let bin_op = match intrinsic_name {
-                    "unchecked_shl" => BinOp::Shl,
-                    "unchecked_shr" => BinOp::Shr,
+                    sym::unchecked_shl => BinOp::Shl,
+                    sym::unchecked_shr => BinOp::Shr,
                     _ => bug!("Already checked for int ops")
                 };
                 let (val, overflowed, _ty) = self.overflowing_binary_op(bin_op, l, r)?;
@@ -242,7 +239,7 @@
                 }
                 self.write_scalar(val, dest)?;
             }
-            "rotate_left" | "rotate_right" => {
+            sym::rotate_left | sym::rotate_right => {
                 // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
                 // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
                 let layout = self.layout_of(substs.type_at(0))?;
@@ -253,7 +250,7 @@
                 let width_bits = layout.size.bits() as u128;
                 let shift_bits = raw_shift_bits % width_bits;
                 let inv_shift_bits = (width_bits - shift_bits) % width_bits;
-                let result_bits = if intrinsic_name == "rotate_left" {
+                let result_bits = if intrinsic_name == sym::rotate_left {
                     (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
                 } else {
                     (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
@@ -263,7 +260,7 @@
                 self.write_scalar(result, dest)?;
             }
 
-            "ptr_offset_from" => {
+            sym::ptr_offset_from => {
                 let isize_layout = self.layout_of(self.tcx.types.isize)?;
                 let a = self.read_immediate(args[0])?.to_scalar()?;
                 let b = self.read_immediate(args[1])?.to_scalar()?;
@@ -309,10 +306,10 @@
                 }
             }
 
-            "transmute" => {
+            sym::transmute => {
                 self.copy_op_transmute(args[0], dest)?;
             }
-            "simd_insert" => {
+            sym::simd_insert => {
                 let index = u64::from(self.read_scalar(args[1])?.to_u32()?);
                 let elem = args[2];
                 let input = args[0];
@@ -343,7 +340,7 @@
                     self.copy_op(value, place)?;
                 }
             }
-            "simd_extract" => {
+            sym::simd_extract => {
                 let index = u64::from(self.read_scalar(args[1])?.to_u32()?);
                 let (len, e_ty) = args[0].layout.ty.simd_size_and_type(self.tcx.tcx);
                 assert!(
@@ -430,13 +427,13 @@
         if self.binary_op(BinOp::Rem, a, b)?.to_bits()? != 0 {
             // Then, check if `b` is -1, which is the "min_value / -1" case.
             let minus1 = Scalar::from_int(-1, dest.layout.size);
-            let b = b.to_scalar().unwrap();
-            if b == minus1 {
+            let b_scalar = b.to_scalar().unwrap();
+            if b_scalar == minus1 {
                 throw_ub_format!("exact_div: result of dividing MIN by -1 cannot be represented")
             } else {
                 throw_ub_format!(
                     "exact_div: {} cannot be divided by {} without remainder",
-                    a.to_scalar().unwrap(),
+                    a,
                     b,
                 )
             }
diff --git a/src/librustc_mir/interpret/intrinsics/caller_location.rs b/src/librustc_mir/interpret/intrinsics/caller_location.rs
index 9e07a3f..ecf4b7a 100644
--- a/src/librustc_mir/interpret/intrinsics/caller_location.rs
+++ b/src/librustc_mir/interpret/intrinsics/caller_location.rs
@@ -1,50 +1,48 @@
 use rustc::middle::lang_items::PanicLocationLangItem;
-use rustc::mir::interpret::{Pointer, PointerArithmetic, Scalar};
 use rustc::ty::subst::Subst;
-use rustc_target::abi::{LayoutOf, Size};
-use syntax_pos::Symbol;
+use rustc_target::abi::LayoutOf;
+use syntax_pos::{Symbol, Span};
 
-use crate::interpret::{MemoryKind, MPlaceTy, intrinsics::{InterpCx, InterpResult, Machine}};
+use crate::interpret::{Scalar, MemoryKind, MPlaceTy, intrinsics::{InterpCx, Machine}};
 
 impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
-    pub fn alloc_caller_location(
+    crate fn alloc_caller_location(
         &mut self,
         filename: Symbol,
         line: u32,
         col: u32,
-    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
+    ) -> MPlaceTy<'tcx, M::PointerTag> {
+        let file = self.allocate_str(&filename.as_str(), MemoryKind::CallerLocation);
         let line = Scalar::from_u32(line);
         let col = Scalar::from_u32(col);
 
-        let ptr_size = self.pointer_size();
-        let u32_size = Size::from_bits(32);
-
+        // Allocate memory for `CallerLocation` struct.
         let loc_ty = self.tcx.type_of(self.tcx.require_lang_item(PanicLocationLangItem, None))
             .subst(*self.tcx, self.tcx.mk_substs([self.tcx.lifetimes.re_static.into()].iter()));
-        let loc_layout = self.layout_of(loc_ty)?;
-
-        let file_alloc = self.tcx.allocate_bytes(filename.as_str().as_bytes());
-        let file_ptr = Pointer::new(file_alloc, Size::ZERO);
-        let file = Scalar::Ptr(self.tag_static_base_pointer(file_ptr));
-        let file_len = Scalar::from_uint(filename.as_str().len() as u128, ptr_size);
-
+        let loc_layout = self.layout_of(loc_ty).unwrap();
         let location = self.allocate(loc_layout, MemoryKind::CallerLocation);
 
-        let file_out = self.mplace_field(location, 0)?;
-        let file_ptr_out = self.force_ptr(self.mplace_field(file_out, 0)?.ptr)?;
-        let file_len_out = self.force_ptr(self.mplace_field(file_out, 1)?.ptr)?;
-        let line_out = self.force_ptr(self.mplace_field(location, 1)?.ptr)?;
-        let col_out = self.force_ptr(self.mplace_field(location, 2)?.ptr)?;
+        // Initialize fields.
+        self.write_immediate(file.to_ref(), self.mplace_field(location, 0).unwrap().into())
+            .expect("writing to memory we just allocated cannot fail");
+        self.write_scalar(line, self.mplace_field(location, 1).unwrap().into())
+            .expect("writing to memory we just allocated cannot fail");
+        self.write_scalar(col, self.mplace_field(location, 2).unwrap().into())
+            .expect("writing to memory we just allocated cannot fail");
 
-        let layout = &self.tcx.data_layout;
-        // We just allocated this, so we can skip the bounds checks.
-        let alloc = self.memory.get_raw_mut(file_ptr_out.alloc_id)?;
+        location
+    }
 
-        alloc.write_scalar(layout, file_ptr_out, file.into(), ptr_size)?;
-        alloc.write_scalar(layout, file_len_out, file_len.into(), ptr_size)?;
-        alloc.write_scalar(layout, line_out, line.into(), u32_size)?;
-        alloc.write_scalar(layout, col_out, col.into(), u32_size)?;
-
-        Ok(location)
+    pub fn alloc_caller_location_for_span(
+        &mut self,
+        span: Span,
+    ) -> MPlaceTy<'tcx, M::PointerTag> {
+        let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
+        let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo());
+        self.alloc_caller_location(
+            Symbol::intern(&caller.file.name.to_string()),
+            caller.line as u32,
+            caller.col_display as u32 + 1,
+        )
     }
 }
diff --git a/src/librustc_mir/interpret/machine.rs b/src/librustc_mir/interpret/machine.rs
index b7cde62..74206ad 100644
--- a/src/librustc_mir/interpret/machine.rs
+++ b/src/librustc_mir/interpret/machine.rs
@@ -11,7 +11,7 @@
 use syntax_pos::Span;
 
 use super::{
-    Allocation, AllocId, InterpResult, Scalar, AllocationExtra,
+    Allocation, AllocId, InterpResult, Scalar, AllocationExtra, AssertMessage,
     InterpCx, PlaceTy, OpTy, ImmTy, MemoryKind, Pointer, Memory,
     Frame, Operand,
 };
@@ -146,7 +146,7 @@
     /// nor just jump to `ret`, but instead push their own stack frame.)
     /// Passing `dest`and `ret` in the same `Option` proved very annoying when only one of them
     /// was used.
-    fn find_fn(
+    fn find_mir_or_eval_fn(
         ecx: &mut InterpCx<'mir, 'tcx, Self>,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx, Self::PointerTag>],
@@ -175,6 +175,14 @@
         unwind: Option<mir::BasicBlock>,
     ) -> InterpResult<'tcx>;
 
+    /// Called to evaluate `Assert` MIR terminators that trigger a panic.
+    fn assert_panic(
+        ecx: &mut InterpCx<'mir, 'tcx, Self>,
+        span: Span,
+        msg: &AssertMessage<'tcx>,
+        unwind: Option<mir::BasicBlock>,
+    ) -> InterpResult<'tcx>;
+
     /// Called for read access to a foreign static item.
     ///
     /// This will only be called once per static and machine; the result is cached in
@@ -233,20 +241,19 @@
     /// cache the result. (This relies on `AllocMap::get_or` being able to add the
     /// owned allocation to the map even when the map is shared.)
     ///
-    /// For static allocations, the tag returned must be the same as the one returned by
-    /// `tag_static_base_pointer`.
-    fn tag_allocation<'b>(
+    /// Also return the "base" tag to use for this allocation: the one that is used for direct
+    /// accesses to this allocation. If `kind == STATIC_KIND`, this tag must be consistent
+    /// with `tag_static_base_pointer`.
+    fn init_allocation_extra<'b>(
         memory_extra: &Self::MemoryExtra,
         id: AllocId,
         alloc: Cow<'b, Allocation>,
         kind: Option<MemoryKind<Self::MemoryKinds>>,
     ) -> (Cow<'b, Allocation<Self::PointerTag, Self::AllocExtra>>, Self::PointerTag);
 
-    /// Return the "base" tag for the given static allocation: the one that is used for direct
-    /// accesses to this static/const/fn allocation.
-    ///
-    /// Be aware that requesting the `Allocation` for that `id` will lead to cycles
-    /// for cyclic statics!
+    /// Return the "base" tag for the given *static* allocation: the one that is used for direct
+    /// accesses to this static/const/fn allocation. If `id` is not a static allocation,
+    /// this will return an unusable tag (i.e., accesses will be UB)!
     fn tag_static_base_pointer(
         memory_extra: &Self::MemoryExtra,
         id: AllocId,
diff --git a/src/librustc_mir/interpret/memory.rs b/src/librustc_mir/interpret/memory.rs
index a8011f7..8f177ad 100644
--- a/src/librustc_mir/interpret/memory.rs
+++ b/src/librustc_mir/interpret/memory.rs
@@ -143,6 +143,12 @@
         }
     }
 
+    /// Call this to turn untagged "global" pointers (obtained via `tcx`) into
+    /// the *canonical* machine pointer to the allocation.  Must never be used
+    /// for any other pointers!
+    ///
+    /// This represents a *direct* access to that memory, as opposed to access
+    /// through a pointer that was created by the program.
     #[inline]
     pub fn tag_static_base_pointer(&self, ptr: Pointer) -> Pointer<M::PointerTag> {
         ptr.with_tag(M::tag_static_base_pointer(&self.extra, ptr.alloc_id))
@@ -191,7 +197,9 @@
         kind: MemoryKind<M::MemoryKinds>,
     ) -> Pointer<M::PointerTag> {
         let id = self.tcx.alloc_map.lock().reserve();
-        let (alloc, tag) = M::tag_allocation(&self.extra, id, Cow::Owned(alloc), Some(kind));
+        debug_assert_ne!(Some(kind), M::STATIC_KIND.map(MemoryKind::Machine),
+            "dynamically allocating static memory");
+        let (alloc, tag) = M::init_allocation_extra(&self.extra, id, Cow::Owned(alloc), Some(kind));
         self.alloc_map.insert(id, (kind, alloc.into_owned()));
         Pointer::from(id).with_tag(tag)
     }
@@ -316,7 +324,7 @@
         size: Size,
         align: Align,
     ) -> InterpResult<'tcx, Option<Pointer<M::PointerTag>>> {
-        let align = if M::CHECK_ALIGN { Some(align) } else { None };
+        let align = M::CHECK_ALIGN.then_some(align);
         self.check_ptr_access_align(sptr, size, align, CheckInAllocMsg::MemoryAccessTest)
     }
 
@@ -350,7 +358,7 @@
             sptr
         } else {
             // A "real" access, we must get a pointer.
-            Scalar::Ptr(self.force_ptr(sptr)?)
+            Scalar::from(self.force_ptr(sptr)?)
         };
         Ok(match normalized.to_bits_or_ptr(self.pointer_size(), self) {
             Ok(bits) => {
@@ -473,14 +481,15 @@
                 }
             }
         };
-        // We got tcx memory. Let the machine figure out whether and how to
-        // turn that into memory with the right pointer tag.
-        Ok(M::tag_allocation(
+        // We got tcx memory. Let the machine initialize its "extra" stuff.
+        let (alloc, tag) = M::init_allocation_extra(
             memory_extra,
             id, // always use the ID we got as input, not the "hidden" one.
             alloc,
             M::STATIC_KIND.map(MemoryKind::Machine),
-        ).0)
+        );
+        debug_assert_eq!(tag, M::tag_static_base_pointer(memory_extra, id));
+        Ok(alloc)
     }
 
     /// Gives raw access to the `Allocation`, without bounds or alignment checks.
diff --git a/src/librustc_mir/interpret/operand.rs b/src/librustc_mir/interpret/operand.rs
index 8dd5807..48e7193 100644
--- a/src/librustc_mir/interpret/operand.rs
+++ b/src/librustc_mir/interpret/operand.rs
@@ -19,12 +19,13 @@
 };
 pub use rustc::mir::interpret::ScalarMaybeUndef;
 use rustc_macros::HashStable;
+use syntax::ast;
 
 /// An `Immediate` represents a single immediate self-contained Rust value.
 ///
 /// For optimization of a few very common cases, there is also a representation for a pair of
 /// primitive values (`ScalarPair`). It allows Miri to avoid making allocations for checked binary
-/// operations and fat pointers. This idea was taken from rustc's codegen.
+/// operations and wide pointers. This idea was taken from rustc's codegen.
 /// In particular, thanks to `ScalarPair`, arithmetic operations and casts can be entirely
 /// defined on `Immediate`, and do not have to work with a `Place`.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, HashStable, Hash)]
@@ -47,6 +48,13 @@
     }
 }
 
+impl<Tag> From<Pointer<Tag>> for Immediate<Tag> {
+    #[inline(always)]
+    fn from(val: Pointer<Tag>) -> Self {
+        Immediate::Scalar(Scalar::from(val).into())
+    }
+}
+
 impl<'tcx, Tag> Immediate<Tag> {
     pub fn new_slice(
         val: Scalar<Tag>,
@@ -60,14 +68,14 @@
     }
 
     pub fn new_dyn_trait(val: Scalar<Tag>, vtable: Pointer<Tag>) -> Self {
-        Immediate::ScalarPair(val.into(), Scalar::Ptr(vtable).into())
+        Immediate::ScalarPair(val.into(), vtable.into())
     }
 
     #[inline]
     pub fn to_scalar_or_undef(self) -> ScalarMaybeUndef<Tag> {
         match self {
             Immediate::Scalar(val) => val,
-            Immediate::ScalarPair(..) => bug!("Got a fat pointer where a scalar was expected"),
+            Immediate::ScalarPair(..) => bug!("Got a wide pointer where a scalar was expected"),
         }
     }
 
@@ -93,6 +101,42 @@
     pub layout: TyLayout<'tcx>,
 }
 
+// `Tag: Copy` because some methods on `Scalar` consume them by value
+impl<Tag: Copy> std::fmt::Display for ImmTy<'tcx, Tag> {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match &self.imm {
+            Immediate::Scalar(ScalarMaybeUndef::Scalar(s)) => match s.to_bits(self.layout.size) {
+                Ok(s) => {
+                    match self.layout.ty.kind {
+                        ty::Int(_) => return write!(
+                            fmt, "{}",
+                            super::sign_extend(s, self.layout.size) as i128,
+                        ),
+                        ty::Uint(_) => return write!(fmt, "{}", s),
+                        ty::Bool if s == 0 => return fmt.write_str("false"),
+                        ty::Bool if s == 1 => return fmt.write_str("true"),
+                        ty::Char => if let Some(c) =
+                            u32::try_from(s).ok().and_then(std::char::from_u32) {
+                            return write!(fmt, "{}", c);
+                        },
+                        ty::Float(ast::FloatTy::F32) => if let Ok(u) = u32::try_from(s) {
+                            return write!(fmt, "{}", f32::from_bits(u));
+                        },
+                        ty::Float(ast::FloatTy::F64) => if let Ok(u) = u64::try_from(s) {
+                            return write!(fmt, "{}", f64::from_bits(u));
+                        },
+                        _ => {},
+                    }
+                    write!(fmt, "{:x}", s)
+                },
+                Err(_) => fmt.write_str("{pointer}"),
+            },
+            Immediate::Scalar(ScalarMaybeUndef::Undef) => fmt.write_str("{undef}"),
+            Immediate::ScalarPair(..) => fmt.write_str("{wide pointer or tuple}"),
+        }
+    }
+}
+
 impl<'tcx, Tag> ::std::ops::Deref for ImmTy<'tcx, Tag> {
     type Target = Immediate<Tag>;
     #[inline(always)]
@@ -324,7 +368,7 @@
         Ok(self.read_immediate(op)?.to_scalar_or_undef())
     }
 
-    // Turn the MPlace into a string (must already be dereferenced!)
+    // Turn the wide MPlace into a string (must already be dereferenced!)
     pub fn read_str(
         &self,
         mplace: MPlaceTy<'tcx, M::PointerTag>,
diff --git a/src/librustc_mir/interpret/place.rs b/src/librustc_mir/interpret/place.rs
index f34c311..a600eb1 100644
--- a/src/librustc_mir/interpret/place.rs
+++ b/src/librustc_mir/interpret/place.rs
@@ -125,7 +125,7 @@
         Self::from_scalar_ptr(ptr.into(), align)
     }
 
-    /// Turn a mplace into a (thin or fat) pointer, as a reference, pointing to the same space.
+    /// Turn a mplace into a (thin or wide) pointer, as a reference, pointing to the same space.
     /// This is the inverse of `ref_to_mplace`.
     #[inline(always)]
     pub fn to_ref(self) -> Immediate<Tag> {
@@ -278,7 +278,7 @@
     M::MemoryMap: AllocMap<AllocId, (MemoryKind<M::MemoryKinds>, Allocation<Tag, M::AllocExtra>)>,
     M::AllocExtra: AllocationExtra<Tag>,
 {
-    /// Take a value, which represents a (thin or fat) reference, and make it a place.
+    /// Take a value, which represents a (thin or wide) reference, and make it a place.
     /// Alignment is just based on the type.  This is the inverse of `MemPlace::to_ref()`.
     ///
     /// Only call this if you are sure the place is "valid" (aligned and inbounds), or do not
@@ -649,20 +649,28 @@
         use rustc::mir::PlaceBase;
 
         let mut place_ty = match &place.base {
-            PlaceBase::Local(mir::RETURN_PLACE) => match self.frame().return_place {
-                Some(return_place) => {
-                    // We use our layout to verify our assumption; caller will validate
-                    // their layout on return.
-                    PlaceTy {
-                        place: *return_place,
-                        layout: self.layout_of(
-                            self.subst_from_frame_and_normalize_erasing_regions(
-                                self.frame().body.return_ty()
-                            )
-                        )?,
-                    }
+            PlaceBase::Local(mir::RETURN_PLACE) => {
+                // `return_place` has the *caller* layout, but we want to use our
+                // `layout to verify our assumption. The caller will validate
+                // their layout on return.
+                PlaceTy {
+                    place: match self.frame().return_place {
+                        Some(p) => *p,
+                        // Even if we don't have a return place, we sometimes need to
+                        // create this place, but any attempt to read from / write to it
+                        // (even a ZST read/write) needs to error, so let us make this
+                        // a NULL place.
+                        //
+                        // FIXME: Ideally we'd make sure that the place projections also
+                        // bail out.
+                        None => Place::null(&*self),
+                    },
+                    layout: self.layout_of(
+                        self.subst_from_frame_and_normalize_erasing_regions(
+                            self.frame().body.return_ty()
+                        )
+                    )?,
                 }
-                None => throw_unsup!(InvalidNullPointerUsage),
             },
             PlaceBase::Local(local) => PlaceTy {
                 // This works even for dead/uninitialized locals; we check further when writing
@@ -684,6 +692,7 @@
     }
 
     /// Write a scalar to a place
+    #[inline(always)]
     pub fn write_scalar(
         &mut self,
         val: impl Into<ScalarMaybeUndef<M::PointerTag>>,
@@ -789,8 +798,8 @@
         // to handle padding properly, which is only correct if we never look at this data with the
         // wrong type.
 
-        let ptr = match self.check_mplace_access(dest, None)
-            .expect("places should be checked on creation")
+        // Invalid places are a thing: the return place of a diverging function
+        let ptr = match self.check_mplace_access(dest, None)?
         {
             Some(ptr) => ptr,
             None => return Ok(()), // zero-sized access
@@ -1031,18 +1040,39 @@
         MPlaceTy::from_aligned_ptr(ptr, layout)
     }
 
+    /// Returns a wide MPlace.
+    pub fn allocate_str(
+        &mut self,
+        str: &str,
+        kind: MemoryKind<M::MemoryKinds>,
+    ) -> MPlaceTy<'tcx, M::PointerTag> {
+        let ptr = self.memory.allocate_static_bytes(str.as_bytes(), kind);
+        let meta = Scalar::from_uint(str.len() as u128, self.pointer_size());
+        let mplace = MemPlace {
+            ptr: ptr.into(),
+            align: Align::from_bytes(1).unwrap(),
+            meta: Some(meta),
+        };
+
+        let layout = self.layout_of(self.tcx.mk_static_str()).unwrap();
+        MPlaceTy { mplace, layout }
+    }
+
     pub fn write_discriminant_index(
         &mut self,
         variant_index: VariantIdx,
         dest: PlaceTy<'tcx, M::PointerTag>,
     ) -> InterpResult<'tcx> {
-        let variant_scalar = Scalar::from_u32(variant_index.as_u32()).into();
+
+        // Layout computation excludes uninhabited variants from consideration
+        // therefore there's no way to represent those variants in the given layout.
+        if dest.layout.for_variant(self, variant_index).abi.is_uninhabited() {
+            throw_ub!(Unreachable);
+        }
 
         match dest.layout.variants {
             layout::Variants::Single { index } => {
-                if index != variant_index {
-                    throw_ub!(InvalidDiscriminant(variant_scalar));
-                }
+                assert_eq!(index, variant_index);
             }
             layout::Variants::Multiple {
                 discr_kind: layout::DiscriminantKind::Tag,
@@ -1050,9 +1080,9 @@
                 discr_index,
                 ..
             } => {
-                if !dest.layout.ty.variant_range(*self.tcx).unwrap().contains(&variant_index) {
-                    throw_ub!(InvalidDiscriminant(variant_scalar));
-                }
+                // No need to validate that the discriminant here because the
+                // `TyLayout::for_variant()` call earlier already checks the variant is valid.
+
                 let discr_val =
                     dest.layout.ty.discriminant_for_variant(*self.tcx, variant_index).unwrap().val;
 
@@ -1075,9 +1105,9 @@
                 discr_index,
                 ..
             } => {
-                if !variant_index.as_usize() < dest.layout.ty.ty_adt_def().unwrap().variants.len() {
-                    throw_ub!(InvalidDiscriminant(variant_scalar));
-                }
+                // No need to validate that the discriminant here because the
+                // `TyLayout::for_variant()` call earlier already checks the variant is valid.
+
                 if variant_index != dataful_variant {
                     let variants_start = niche_variants.start().as_u32();
                     let variant_index_relative = variant_index.as_u32()
diff --git a/src/librustc_mir/interpret/step.rs b/src/librustc_mir/interpret/step.rs
index d749333..55b9427 100644
--- a/src/librustc_mir/interpret/step.rs
+++ b/src/librustc_mir/interpret/step.rs
@@ -157,9 +157,9 @@
             }
 
             BinaryOp(bin_op, ref left, ref right) => {
-                let layout = if binop_left_homogeneous(bin_op) { Some(dest.layout) } else { None };
+                let layout = binop_left_homogeneous(bin_op).then_some(dest.layout);
                 let left = self.read_immediate(self.eval_operand(left, layout)?)?;
-                let layout = if binop_right_homogeneous(bin_op) { Some(left.layout) } else { None };
+                let layout = binop_right_homogeneous(bin_op).then_some(left.layout);
                 let right = self.read_immediate(self.eval_operand(right, layout)?)?;
                 self.binop_ignore_overflow(
                     bin_op,
@@ -172,7 +172,7 @@
             CheckedBinaryOp(bin_op, ref left, ref right) => {
                 // Due to the extra boolean in the result, we can never reuse the `dest.layout`.
                 let left = self.read_immediate(self.eval_operand(left, None)?)?;
-                let layout = if binop_right_homogeneous(bin_op) { Some(left.layout) } else { None };
+                let layout = binop_right_homogeneous(bin_op).then_some(left.layout);
                 let right = self.read_immediate(self.eval_operand(right, layout)?)?;
                 self.binop_with_overflow(
                     bin_op,
diff --git a/src/librustc_mir/interpret/terminator.rs b/src/librustc_mir/interpret/terminator.rs
index 50cd318..b8dc15f 100644
--- a/src/librustc_mir/interpret/terminator.rs
+++ b/src/librustc_mir/interpret/terminator.rs
@@ -7,8 +7,8 @@
 use rustc_target::spec::abi::Abi;
 
 use super::{
-    GlobalId, InterpResult, PointerArithmetic,
-    InterpCx, Machine, OpTy, ImmTy, PlaceTy, MPlaceTy, StackPopCleanup, FnVal,
+    GlobalId, InterpResult, InterpCx, Machine,
+    OpTy, ImmTy, PlaceTy, MPlaceTy, StackPopCleanup, FnVal,
 };
 
 impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
@@ -115,40 +115,14 @@
                 expected,
                 ref msg,
                 target,
-                ..
+                cleanup,
             } => {
                 let cond_val = self.read_immediate(self.eval_operand(cond, None)?)?
                     .to_scalar()?.to_bool()?;
                 if expected == cond_val {
                     self.go_to_block(target);
                 } else {
-                    // Compute error message
-                    use rustc::mir::interpret::PanicInfo::*;
-                    return Err(match msg {
-                        BoundsCheck { ref len, ref index } => {
-                            let len = self
-                                .read_immediate(self.eval_operand(len, None)?)
-                                .expect("can't eval len")
-                                .to_scalar()?
-                                .to_bits(self.memory.pointer_size())? as u64;
-                            let index = self
-                                .read_immediate(self.eval_operand(index, None)?)
-                                .expect("can't eval index")
-                                .to_scalar()?
-                                .to_bits(self.memory.pointer_size())? as u64;
-                            err_panic!(BoundsCheck { len, index })
-                        }
-                        Overflow(op) => err_panic!(Overflow(*op)),
-                        OverflowNeg => err_panic!(OverflowNeg),
-                        DivisionByZero => err_panic!(DivisionByZero),
-                        RemainderByZero => err_panic!(RemainderByZero),
-                        ResumedAfterReturn(generator_kind)
-                            => err_panic!(ResumedAfterReturn(*generator_kind)),
-                        ResumedAfterPanic(generator_kind)
-                            => err_panic!(ResumedAfterPanic(*generator_kind)),
-                        Panic { .. } => bug!("`Panic` variant cannot occur in MIR"),
-                    }
-                    .into());
+                    M::assert_panic(self, terminator.source_info.span, msg, cleanup)?;
                 }
             }
 
@@ -164,15 +138,21 @@
                 return Ok(())
             },
 
+            // It is UB to ever encounter this.
+            Unreachable => throw_ub!(Unreachable),
+
+            // These should never occur for MIR we actually run.
+            DropAndReplace { .. } |
+            FalseEdges { .. } |
+            FalseUnwind { .. } =>
+                bug!("{:#?} should have been eliminated by MIR pass", terminator.kind),
+
+            // These are not (yet) supported. It is unclear if they even can occur in
+            // MIR that we actually run.
             Yield { .. } |
             GeneratorDrop |
-            DropAndReplace { .. } |
-            Abort => unimplemented!("{:#?}", terminator.kind),
-            FalseEdges { .. } => bug!("should have been eliminated by\
-                                      `simplify_branches` mir pass"),
-            FalseUnwind { .. } => bug!("should have been eliminated by\
-                                       `simplify_branches` mir pass"),
-            Unreachable => throw_ub!(Unreachable),
+            Abort =>
+                throw_unsup_format!("Unsupported terminator kind: {:#?}", terminator.kind),
         }
 
         Ok(())
@@ -286,20 +266,8 @@
             ty::InstanceDef::DropGlue(..) |
             ty::InstanceDef::CloneShim(..) |
             ty::InstanceDef::Item(_) => {
-                // If this function is a `const fn` then as an optimization we can query this
-                // evaluation immediately.
-                //
-                // For the moment we only do this for functions which take no arguments
-                // (or all arguments are ZSTs) so that we don't memoize too much.
-                if self.tcx.is_const_fn_raw(instance.def.def_id()) &&
-                   args.iter().all(|a| a.layout.is_zst())
-                {
-                    let gid = GlobalId { instance, promoted: None };
-                    return self.eval_const_fn_call(gid, ret);
-                }
-
                 // We need MIR for this fn
-                let body = match M::find_fn(self, instance, args, ret, unwind)? {
+                let body = match M::find_mir_or_eval_fn(self, instance, args, ret, unwind)? {
                     Some(body) => body,
                     None => return Ok(()),
                 };
@@ -465,7 +433,8 @@
 
     /// Evaluate a const function where all arguments (if any) are zero-sized types.
     /// The evaluation is memoized thanks to the query system.
-    fn eval_const_fn_call(
+    // FIXME: Consider moving this to `const_eval.rs`.
+    pub (crate) fn eval_const_fn_call(
         &mut self,
         gid: GlobalId<'tcx>,
         ret: Option<(PlaceTy<'tcx, M::PointerTag>, mir::BasicBlock)>,
diff --git a/src/librustc_mir/interpret/traits.rs b/src/librustc_mir/interpret/traits.rs
index 3a7f47a..efa0d26 100644
--- a/src/librustc_mir/interpret/traits.rs
+++ b/src/librustc_mir/interpret/traits.rs
@@ -67,7 +67,7 @@
         // allocation is correctly aligned as we created it above. Also we're only offsetting by
         // multiples of `ptr_align`, which means that it will stay aligned to `ptr_align`.
         let vtable_alloc = self.memory.get_raw_mut(vtable.alloc_id)?;
-        vtable_alloc.write_ptr_sized(tcx, vtable, Scalar::Ptr(drop).into())?;
+        vtable_alloc.write_ptr_sized(tcx, vtable, drop.into())?;
 
         let size_ptr = vtable.offset(ptr_size, tcx)?;
         vtable_alloc.write_ptr_sized(tcx, size_ptr, Scalar::from_uint(size, ptr_size).into())?;
@@ -87,7 +87,7 @@
                 // We cannot use `vtable_allic` as we are creating fn ptrs in this loop.
                 let method_ptr = vtable.offset(ptr_size * (3 + i as u64), tcx)?;
                 self.memory.get_raw_mut(vtable.alloc_id)?
-                    .write_ptr_sized(tcx, method_ptr, Scalar::Ptr(fn_ptr).into())?;
+                    .write_ptr_sized(tcx, method_ptr, fn_ptr.into())?;
             }
         }
 
diff --git a/src/librustc_mir/lib.rs b/src/librustc_mir/lib.rs
index cca7670..42b72b6 100644
--- a/src/librustc_mir/lib.rs
+++ b/src/librustc_mir/lib.rs
@@ -8,6 +8,7 @@
 #![feature(in_band_lifetimes)]
 #![feature(inner_deref)]
 #![feature(slice_patterns)]
+#![feature(bool_to_option)]
 #![feature(box_patterns)]
 #![feature(box_syntax)]
 #![feature(crate_visibility_modifier)]
@@ -27,8 +28,8 @@
 #![feature(associated_type_bounds)]
 #![feature(range_is_empty)]
 #![feature(stmt_expr_attributes)]
-#![feature(bool_to_option)]
 #![feature(trait_alias)]
+#![feature(matches_macro)]
 
 #![recursion_limit="256"]
 
diff --git a/src/librustc_mir/monomorphize/collector.rs b/src/librustc_mir/monomorphize/collector.rs
index a4c4c7f..93a7671 100644
--- a/src/librustc_mir/monomorphize/collector.rs
+++ b/src/librustc_mir/monomorphize/collector.rs
@@ -741,23 +741,21 @@
     }
 
     match instance.def {
-        ty::InstanceDef::Intrinsic(def_id) => {
+        ty::InstanceDef::Virtual(..) |
+        ty::InstanceDef::Intrinsic(_) => {
             if !is_direct_call {
-                bug!("intrinsic {:?} being reified", def_id);
+                bug!("{:?} being reified", instance);
             }
         }
-        ty::InstanceDef::VtableShim(..) |
-        ty::InstanceDef::ReifyShim(..) |
-        ty::InstanceDef::Virtual(..) |
         ty::InstanceDef::DropGlue(_, None) => {
-            // Don't need to emit shim if we are calling directly.
+            // Don't need to emit noop drop glue if we are calling directly.
             if !is_direct_call {
                 output.push(create_fn_mono_item(instance));
             }
         }
-        ty::InstanceDef::DropGlue(_, Some(_)) => {
-            output.push(create_fn_mono_item(instance));
-        }
+        ty::InstanceDef::DropGlue(_, Some(_)) |
+        ty::InstanceDef::VtableShim(..) |
+        ty::InstanceDef::ReifyShim(..) |
         ty::InstanceDef::ClosureOnceShim { .. } |
         ty::InstanceDef::Item(..) |
         ty::InstanceDef::FnPtrShim(..) |
@@ -1255,7 +1253,7 @@
         body: &body,
         output,
         param_substs: instance.substs,
-    }.visit_body(&body);
+    }.visit_body(body);
 }
 
 fn def_id_to_string(tcx: TyCtxt<'_>, def_id: DefId) -> String {
diff --git a/src/librustc_mir/monomorphize/partitioning.rs b/src/librustc_mir/monomorphize/partitioning.rs
index 42f0877..591f220 100644
--- a/src/librustc_mir/monomorphize/partitioning.rs
+++ b/src/librustc_mir/monomorphize/partitioning.rs
@@ -761,11 +761,7 @@
             .iter()
             .map(|part| part.data.as_symbol());
 
-        let volatile_suffix = if volatile {
-            Some("volatile")
-        } else {
-            None
-        };
+        let volatile_suffix = volatile.then_some("volatile");
 
         name_builder.build_cgu_name(def_path.krate, components, volatile_suffix)
     }).clone()
diff --git a/src/librustc_mir/shim.rs b/src/librustc_mir/shim.rs
index 08af271..5b208dd 100644
--- a/src/librustc_mir/shim.rs
+++ b/src/librustc_mir/shim.rs
@@ -26,7 +26,7 @@
     providers.mir_shims = make_shim;
 }
 
-fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> &'tcx Body<'tcx> {
+fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> &'tcx BodyCache<'tcx> {
     debug!("make_shim({:?})", instance);
 
     let mut result = match instance {
@@ -35,7 +35,7 @@
         ty::InstanceDef::VtableShim(def_id) => {
             build_call_shim(
                 tcx,
-                def_id,
+                instance,
                 Adjustment::DerefMove,
                 CallKind::Direct(def_id),
                 None,
@@ -60,27 +60,27 @@
 
             build_call_shim(
                 tcx,
-                def_id,
+                instance,
                 adjustment,
                 CallKind::Indirect,
                 Some(arg_tys)
             )
         }
         // We are generating a call back to our def-id, which the
-        // codegen backend knows to turn to an actual virtual call.
-        ty::InstanceDef::Virtual(def_id, _) |
-        // ...or we are generating a direct call to a function for which indirect calls must be
-        // codegen'd differently than direct ones (example: #[track_caller])
+        // codegen backend knows to turn to an actual call, be it
+        // a virtual call, or a direct call to a function for which
+        // indirect calls must be codegen'd differently than direct ones
+        // (such as `#[track_caller]`).
         ty::InstanceDef::ReifyShim(def_id) => {
             build_call_shim(
                 tcx,
-                def_id,
+                instance,
                 Adjustment::Identity,
                 CallKind::Direct(def_id),
                 None
             )
         }
-        ty::InstanceDef::ClosureOnceShim { call_once } => {
+        ty::InstanceDef::ClosureOnceShim { call_once: _ } => {
             let fn_mut = tcx.lang_items().fn_mut_trait().unwrap();
             let call_mut = tcx
                 .associated_items(fn_mut)
@@ -89,7 +89,7 @@
 
             build_call_shim(
                 tcx,
-                call_once,
+                instance,
                 Adjustment::RefMut,
                 CallKind::Direct(call_mut),
                 None
@@ -109,6 +109,9 @@
                 bug!("builtin clone shim {:?} not supported", instance)
             }
         }
+        ty::InstanceDef::Virtual(..) => {
+            bug!("InstanceDef::Virtual ({:?}) is for direct calls only", instance)
+        }
         ty::InstanceDef::Intrinsic(_) => {
             bug!("creating shims from intrinsics ({:?}) is unsupported", instance)
         }
@@ -125,6 +128,7 @@
 
     debug!("make_shim({:?}) = {:?}", instance, result);
 
+    result.ensure_predecessors();
     tcx.arena.alloc(result)
 }
 
@@ -164,7 +168,9 @@
         .collect()
 }
 
-fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) -> Body<'tcx> {
+fn build_drop_shim<'tcx>(
+    tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>
+) -> BodyCache<'tcx> {
     debug!("build_drop_shim(def_id={:?}, ty={:?})", def_id, ty);
 
     // Check if this is a generator, if so, return the drop glue for it
@@ -196,15 +202,14 @@
     block(&mut blocks, TerminatorKind::Goto { target: return_block });
     block(&mut blocks, TerminatorKind::Return);
 
-    let mut body = new_body(
+    let body = new_body(
         blocks,
-        IndexVec::from_elem_n(
-            SourceScopeData { span, parent_scope: None }, 1
-        ),
         local_decls_for_sig(&sig, span),
         sig.inputs().len(),
         span);
 
+    let mut body = BodyCache::new(body);
+
     if let Some(..) = ty {
         // The first argument (index 0), but add 1 for the return value.
         let dropee_ptr = Place::from(Local::new(1+0));
@@ -244,15 +249,16 @@
 
 fn new_body<'tcx>(
     basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
-    source_scopes: IndexVec<SourceScope, SourceScopeData>,
     local_decls: IndexVec<Local, LocalDecl<'tcx>>,
     arg_count: usize,
     span: Span,
 ) -> Body<'tcx> {
     Body::new(
         basic_blocks,
-        source_scopes,
-        ClearCrossCrate::Clear,
+        IndexVec::from_elem_n(
+            SourceScopeData { span, parent_scope: None, local_data: ClearCrossCrate::Clear },
+            1,
+        ),
         local_decls,
         IndexVec::new(),
         arg_count,
@@ -316,7 +322,7 @@
 }
 
 /// Builds a `Clone::clone` shim for `self_ty`. Here, `def_id` is `Clone::clone`.
-fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
+fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> BodyCache<'tcx> {
     debug!("build_clone_shim(def_id={:?})", def_id);
 
     let param_env = tcx.param_env(def_id);
@@ -345,7 +351,7 @@
         }
     };
 
-    builder.into_mir()
+    BodyCache::new(builder.into_mir())
 }
 
 struct CloneShimBuilder<'tcx> {
@@ -380,9 +386,6 @@
     fn into_mir(self) -> Body<'tcx> {
         new_body(
             self.blocks,
-            IndexVec::from_elem_n(
-                SourceScopeData { span: self.span, parent_scope: None }, 1
-            ),
             self.local_decls,
             self.sig.inputs().len(),
             self.span,
@@ -697,7 +700,7 @@
     }
 }
 
-/// Builds a "call" shim for `def_id`. The shim calls the
+/// Builds a "call" shim for `instance`. The shim calls the
 /// function specified by `call_kind`, first adjusting its first
 /// argument according to `rcvr_adjustment`.
 ///
@@ -705,17 +708,30 @@
 /// function will be untupled as these types.
 fn build_call_shim<'tcx>(
     tcx: TyCtxt<'tcx>,
-    def_id: DefId,
+    instance: ty::InstanceDef<'tcx>,
     rcvr_adjustment: Adjustment,
     call_kind: CallKind,
     untuple_args: Option<&[Ty<'tcx>]>,
-) -> Body<'tcx> {
-    debug!("build_call_shim(def_id={:?}, rcvr_adjustment={:?}, \
+) -> BodyCache<'tcx> {
+    debug!("build_call_shim(instance={:?}, rcvr_adjustment={:?}, \
             call_kind={:?}, untuple_args={:?})",
-           def_id, rcvr_adjustment, call_kind, untuple_args);
+           instance, rcvr_adjustment, call_kind, untuple_args);
 
+    let def_id = instance.def_id();
     let sig = tcx.fn_sig(def_id);
-    let sig = tcx.erase_late_bound_regions(&sig);
+    let mut sig = tcx.erase_late_bound_regions(&sig);
+
+    // FIXME(eddyb) avoid having this snippet both here and in
+    // `Instance::fn_sig` (introduce `InstanceDef::fn_sig`?).
+    if let ty::InstanceDef::VtableShim(..) = instance {
+        // Modify fn(self, ...) to fn(self: *mut Self, ...)
+        let mut inputs_and_output = sig.inputs_and_output.to_vec();
+        let self_arg = &mut inputs_and_output[0];
+        debug_assert!(tcx.generics_of(def_id).has_self && *self_arg == tcx.types.self_param);
+        *self_arg = tcx.mk_mut_ptr(*self_arg);
+        sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
+    }
+
     let span = tcx.def_span(def_id);
 
     debug!("build_call_shim: sig={:?}", sig);
@@ -730,14 +746,7 @@
     let rcvr = match rcvr_adjustment {
         Adjustment::Identity => Operand::Move(rcvr_l),
         Adjustment::Deref => Operand::Copy(tcx.mk_place_deref(rcvr_l)),
-        Adjustment::DerefMove => {
-            // fn(Self, ...) -> fn(*mut Self, ...)
-            let arg_ty = local_decls[rcvr_arg].ty;
-            debug_assert!(tcx.generics_of(def_id).has_self && arg_ty == tcx.types.self_param);
-            local_decls[rcvr_arg].ty = tcx.mk_mut_ptr(arg_ty);
-
-            Operand::Move(tcx.mk_place_deref(rcvr_l))
-        }
+        Adjustment::DerefMove => Operand::Move(tcx.mk_place_deref(rcvr_l)),
         Adjustment::RefMut => {
             // let rcvr = &mut rcvr;
             let ref_rcvr = local_decls.push(temp_decl(
@@ -836,9 +845,6 @@
 
     let mut body = new_body(
         blocks,
-        IndexVec::from_elem_n(
-            SourceScopeData { span, parent_scope: None }, 1
-        ),
         local_decls,
         sig.inputs().len(),
         span,
@@ -847,10 +853,10 @@
     if let Abi::RustCall = sig.abi {
         body.spread_arg = Some(Local::new(sig.inputs().len()));
     }
-    body
+    BodyCache::new(body)
 }
 
-pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> &Body<'_> {
+pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> &BodyCache<'_> {
     debug_assert!(tcx.is_constructor(ctor_id));
 
     let span = tcx.hir().span_if_local(ctor_id)
@@ -919,9 +925,6 @@
 
     let body = new_body(
         IndexVec::from_elem_n(start_block, 1),
-        IndexVec::from_elem_n(
-            SourceScopeData { span, parent_scope: None }, 1
-        ),
         local_decls,
         sig.inputs().len(),
         span,
@@ -937,5 +940,7 @@
         |_, _| Ok(()),
     );
 
+    let mut body = BodyCache::new(body);
+    body.ensure_predecessors();
     tcx.arena.alloc(body)
 }
diff --git a/src/librustc_mir/transform/add_call_guards.rs b/src/librustc_mir/transform/add_call_guards.rs
index bf3df1a..35238e2 100644
--- a/src/librustc_mir/transform/add_call_guards.rs
+++ b/src/librustc_mir/transform/add_call_guards.rs
@@ -31,15 +31,16 @@
  */
 
 impl<'tcx> MirPass<'tcx> for AddCallGuards {
-    fn run_pass(&self, _tcx: TyCtxt<'tcx>, _src: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(
+        &self, _tcx: TyCtxt<'tcx>, _src: MirSource<'tcx>, body: &mut BodyCache<'tcx>
+    ) {
         self.add_call_guards(body);
     }
 }
 
 impl AddCallGuards {
-    pub fn add_call_guards(&self, body: &mut Body<'_>) {
-        let pred_count: IndexVec<_, _> =
-            body.predecessors().iter().map(|ps| ps.len()).collect();
+    pub fn add_call_guards(&self, body: &mut BodyCache<'_>) {
+        let pred_count: IndexVec<_, _> = body.predecessors().iter().map(|ps| ps.len()).collect();
 
         // We need a place to store the new blocks generated
         let mut new_blocks = Vec::new();
diff --git a/src/librustc_mir/transform/add_moves_for_packed_drops.rs b/src/librustc_mir/transform/add_moves_for_packed_drops.rs
index 052631d..98c6a5e 100644
--- a/src/librustc_mir/transform/add_moves_for_packed_drops.rs
+++ b/src/librustc_mir/transform/add_moves_for_packed_drops.rs
@@ -40,13 +40,15 @@
 pub struct AddMovesForPackedDrops;
 
 impl<'tcx> MirPass<'tcx> for AddMovesForPackedDrops {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut BodyCache<'tcx>) {
         debug!("add_moves_for_packed_drops({:?} @ {:?})", src, body.span);
         add_moves_for_packed_drops(tcx, body, src.def_id());
     }
 }
 
-pub fn add_moves_for_packed_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, def_id: DefId) {
+pub fn add_moves_for_packed_drops<'tcx>(
+    tcx: TyCtxt<'tcx>, body: &mut BodyCache<'tcx>, def_id: DefId
+) {
     let patch = add_moves_for_packed_drops_patch(tcx, body, def_id);
     patch.apply(body);
 }
diff --git a/src/librustc_mir/transform/add_retag.rs b/src/librustc_mir/transform/add_retag.rs
index b56a1b2..e6ad37a 100644
--- a/src/librustc_mir/transform/add_retag.rs
+++ b/src/librustc_mir/transform/add_retag.rs
@@ -59,7 +59,7 @@
 }
 
 impl<'tcx> MirPass<'tcx> for AddRetag {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, _src: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, _src: MirSource<'tcx>, body: &mut BodyCache<'tcx>) {
         if !tcx.sess.opts.debugging_opts.mir_emit_retag {
             return;
         }
diff --git a/src/librustc_mir/transform/check_consts/mod.rs b/src/librustc_mir/transform/check_consts/mod.rs
index 4d00aaf..82ffafb 100644
--- a/src/librustc_mir/transform/check_consts/mod.rs
+++ b/src/librustc_mir/transform/check_consts/mod.rs
@@ -20,7 +20,7 @@
 /// Information about the item currently being const-checked, as well as a reference to the global
 /// context.
 pub struct Item<'mir, 'tcx> {
-    pub body: &'mir mir::Body<'tcx>,
+    pub body: mir::ReadOnlyBodyCache<'mir, 'tcx>,
     pub tcx: TyCtxt<'tcx>,
     pub def_id: DefId,
     pub param_env: ty::ParamEnv<'tcx>,
@@ -31,7 +31,7 @@
     pub fn new(
         tcx: TyCtxt<'tcx>,
         def_id: DefId,
-        body: &'mir mir::Body<'tcx>,
+        body: mir::ReadOnlyBodyCache<'mir, 'tcx>,
     ) -> Self {
         let param_env = tcx.param_env(def_id);
         let const_kind = ConstKind::for_item(tcx, def_id);
@@ -77,7 +77,12 @@
         let mode = match tcx.hir().body_owner_kind(hir_id) {
             HirKind::Closure => return None,
 
-            HirKind::Fn if tcx.is_const_fn(def_id) => ConstKind::ConstFn,
+            // Note: this is deliberately checking for `is_const_fn_raw`, as the `is_const_fn`
+            // checks take into account the `rustc_const_unstable` attribute combined with enabled
+            // feature gates. Otherwise, const qualification would _not check_ whether this
+            // function body follows the `const fn` rules, as an unstable `const fn` would
+            // be considered "not const". More details are available in issue #67053.
+            HirKind::Fn if tcx.is_const_fn_raw(def_id) => ConstKind::ConstFn,
             HirKind::Fn => return None,
 
             HirKind::Const => ConstKind::Const,
diff --git a/src/librustc_mir/transform/check_consts/ops.rs b/src/librustc_mir/transform/check_consts/ops.rs
index a4f12a4..393ae94 100644
--- a/src/librustc_mir/transform/check_consts/ops.rs
+++ b/src/librustc_mir/transform/check_consts/ops.rs
@@ -1,7 +1,6 @@
 //! Concrete error types for all operations which may be invalid in a certain const context.
 
 use rustc::hir::def_id::DefId;
-use rustc::mir::BorrowKind;
 use rustc::session::config::nightly_options;
 use rustc::ty::TyCtxt;
 use syntax::feature_gate::feature_err;
@@ -181,38 +180,53 @@
 }
 
 #[derive(Debug)]
-pub struct MutBorrow(pub BorrowKind);
-impl NonConstOp for MutBorrow {
+pub struct CellBorrow;
+impl NonConstOp for CellBorrow {
     fn emit_error(&self, item: &Item<'_, '_>, span: Span) {
-        let kind = self.0;
-        if let BorrowKind::Mut { .. } = kind {
-            let mut err = struct_span_err!(item.tcx.sess, span, E0017,
-                                           "references in {}s may only refer \
-                                            to immutable values", item.const_kind());
-            err.span_label(span, format!("{}s require immutable values",
-                                                item.const_kind()));
-            if item.tcx.sess.teach(&err.get_code().unwrap()) {
-                err.note("References in statics and constants may only refer \
-                          to immutable values.\n\n\
-                          Statics are shared everywhere, and if they refer to \
-                          mutable data one might violate memory safety since \
-                          holding multiple mutable references to shared data \
-                          is not allowed.\n\n\
-                          If you really want global mutable state, try using \
-                          static mut or a global UnsafeCell.");
-            }
-            err.emit();
-        } else {
-            span_err!(item.tcx.sess, span, E0492,
-                      "cannot borrow a constant which may contain \
-                       interior mutability, create a static instead");
+        span_err!(item.tcx.sess, span, E0492,
+            "cannot borrow a constant which may contain \
+            interior mutability, create a static instead");
+    }
+}
+
+#[derive(Debug)]
+pub struct MutBorrow;
+impl NonConstOp for MutBorrow {
+    fn feature_gate(tcx: TyCtxt<'_>) -> Option<bool> {
+        Some(tcx.features().const_mut_refs)
+    }
+
+    fn emit_error(&self, item: &Item<'_, '_>, span: Span) {
+        let mut err = feature_err(
+            &item.tcx.sess.parse_sess,
+            sym::const_mut_refs,
+            span,
+            &format!("references in {}s may only refer \
+                      to immutable values", item.const_kind())
+        );
+        err.span_label(span, format!("{}s require immutable values",
+                                            item.const_kind()));
+        if item.tcx.sess.teach(&err.get_code().unwrap()) {
+            err.note("References in statics and constants may only refer \
+                      to immutable values.\n\n\
+                      Statics are shared everywhere, and if they refer to \
+                      mutable data one might violate memory safety since \
+                      holding multiple mutable references to shared data \
+                      is not allowed.\n\n\
+                      If you really want global mutable state, try using \
+                      static mut or a global UnsafeCell.");
         }
+        err.emit();
     }
 }
 
 #[derive(Debug)]
 pub struct MutDeref;
-impl NonConstOp for MutDeref {}
+impl NonConstOp for MutDeref {
+    fn feature_gate(tcx: TyCtxt<'_>) -> Option<bool> {
+        Some(tcx.features().const_mut_refs)
+    }
+}
 
 #[derive(Debug)]
 pub struct Panic;
@@ -327,22 +341,6 @@
 }
 
 #[derive(Debug)]
-pub struct Transmute;
-impl NonConstOp for Transmute {
-    fn feature_gate(tcx: TyCtxt<'_>) -> Option<bool> {
-        Some(tcx.features().const_transmute)
-    }
-
-    fn emit_error(&self, item: &Item<'_, '_>, span: Span) {
-        feature_err(
-            &item.tcx.sess.parse_sess, sym::const_transmute, span,
-            &format!("The use of std::mem::transmute() is gated in {}s", item.const_kind())
-        )
-        .emit();
-    }
-}
-
-#[derive(Debug)]
 pub struct UnionAccess;
 impl NonConstOp for UnionAccess {
     fn is_allowed_in_item(&self, item: &Item<'_, '_>) -> bool {
diff --git a/src/librustc_mir/transform/check_consts/qualifs.rs b/src/librustc_mir/transform/check_consts/qualifs.rs
index 9ed1ca7..223a5f8 100644
--- a/src/librustc_mir/transform/check_consts/qualifs.rs
+++ b/src/librustc_mir/transform/check_consts/qualifs.rs
@@ -5,7 +5,7 @@
 use rustc::hir::def_id::DefId;
 use syntax_pos::DUMMY_SP;
 
-use super::{ConstKind, Item as ConstCx};
+use super::Item as ConstCx;
 
 pub fn in_any_value_of_ty(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> ConstQualifs {
     ConstQualifs {
@@ -33,9 +33,10 @@
     /// of the type.
     fn in_any_value_of_ty(_cx: &ConstCx<'_, 'tcx>, _ty: Ty<'tcx>) -> bool;
 
-    fn in_static(_cx: &ConstCx<'_, 'tcx>, _def_id: DefId) -> bool {
-        // FIXME(eddyb) should we do anything here for value properties?
-        false
+    fn in_static(cx: &ConstCx<'_, 'tcx>, def_id: DefId) -> bool {
+        // `mir_const_qualif` does return the qualifs in the final value of a `static`, so we could
+        // use value-based qualification here, but we shouldn't do this without a good reason.
+        Self::in_any_value_of_ty(cx, cx.tcx.type_of(def_id))
     }
 
     fn in_projection_structurally(
@@ -50,7 +51,7 @@
             });
             let qualif = base_qualif && Self::in_any_value_of_ty(
                 cx,
-                Place::ty_from(place.base, proj_base, cx.body, cx.tcx)
+                Place::ty_from(place.base, proj_base, *cx.body, cx.tcx)
                     .projection_ty(cx.tcx, elem)
                     .ty,
             );
@@ -154,7 +155,7 @@
                 // Special-case reborrows to be more like a copy of the reference.
                 if let &[ref proj_base @ .., elem] = place.projection.as_ref() {
                     if ProjectionElem::Deref == elem {
-                        let base_ty = Place::ty_from(&place.base, proj_base, cx.body, cx.tcx).ty;
+                        let base_ty = Place::ty_from(&place.base, proj_base, *cx.body, cx.tcx).ty;
                         if let ty::Ref(..) = base_ty.kind {
                             return Self::in_place(cx, per_local, PlaceRef {
                                 base: &place.base,
@@ -217,38 +218,10 @@
         rvalue: &Rvalue<'tcx>,
     ) -> bool {
         match *rvalue {
-            // Returning `true` for `Rvalue::Ref` indicates the borrow isn't
-            // allowed in constants (and the `Checker` will error), and/or it
-            // won't be promoted, due to `&mut ...` or interior mutability.
-            Rvalue::Ref(_, kind, ref place) => {
-                let ty = place.ty(cx.body, cx.tcx).ty;
-
-                if let BorrowKind::Mut { .. } = kind {
-                    // In theory, any zero-sized value could be borrowed
-                    // mutably without consequences.
-                    match ty.kind {
-                        // Inside a `static mut`, &mut [...] is also allowed.
-                        | ty::Array(..)
-                        | ty::Slice(_)
-                        if cx.const_kind == Some(ConstKind::StaticMut)
-                        => {},
-
-                        // FIXME(eddyb): We only return false for `&mut []` outside a const
-                        // context which seems unnecessary given that this is merely a ZST.
-                        | ty::Array(_, len)
-                        if len.try_eval_usize(cx.tcx, cx.param_env) == Some(0)
-                            && cx.const_kind == None
-                        => {},
-
-                        _ => return true,
-                    }
-                }
-            }
-
             Rvalue::Aggregate(ref kind, _) => {
                 if let AggregateKind::Adt(def, ..) = **kind {
                     if Some(def.did) == cx.tcx.lang_items().unsafe_cell_type() {
-                        let ty = rvalue.ty(cx.body, cx.tcx);
+                        let ty = rvalue.ty(*cx.body, cx.tcx);
                         assert_eq!(Self::in_any_value_of_ty(cx, ty), true);
                         return true;
                     }
diff --git a/src/librustc_mir/transform/check_consts/resolver.rs b/src/librustc_mir/transform/check_consts/resolver.rs
index 8909ef7..cb54248 100644
--- a/src/librustc_mir/transform/check_consts/resolver.rs
+++ b/src/librustc_mir/transform/check_consts/resolver.rs
@@ -77,7 +77,7 @@
         args: &[mir::Operand<'tcx>],
         return_place: &mir::Place<'tcx>,
     ) {
-        let return_ty = return_place.ty(self.item.body, self.item.tcx).ty;
+        let return_ty = return_place.ty(*self.item.body, self.item.tcx).ty;
         let qualif = Q::in_call(
             self.item,
             &|l| self.qualifs_per_local.contains(l),
diff --git a/src/librustc_mir/transform/check_consts/validation.rs b/src/librustc_mir/transform/check_consts/validation.rs
index 829d9ee..7170857 100644
--- a/src/librustc_mir/transform/check_consts/validation.rs
+++ b/src/librustc_mir/transform/check_consts/validation.rs
@@ -8,7 +8,6 @@
 use rustc::ty::cast::CastTy;
 use rustc::ty::{self, TyCtxt};
 use rustc_index::bit_set::BitSet;
-use rustc_target::spec::abi::Abi;
 use rustc_error_codes::*;
 use syntax::symbol::sym;
 use syntax_pos::Span;
@@ -23,13 +22,6 @@
 use super::resolver::FlowSensitiveAnalysis;
 use super::{ConstKind, Item, Qualif, is_lang_panic_fn};
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum CheckOpResult {
-    Forbidden,
-    Unleashed,
-    Allowed,
-}
-
 pub type IndirectlyMutableResults<'mir, 'tcx> =
     old_dataflow::DataflowResultsCursor<'mir, 'tcx, IndirectlyMutableLocals<'mir, 'tcx>>;
 
@@ -46,9 +38,9 @@
     ) -> Self {
         let analysis = FlowSensitiveAnalysis::new(q, item);
         let results =
-            dataflow::Engine::new(item.tcx, item.body, item.def_id, dead_unwinds, analysis)
+            dataflow::Engine::new(item.tcx, &item.body, item.def_id, dead_unwinds, analysis)
                 .iterate_to_fixpoint();
-        let cursor = dataflow::ResultsCursor::new(item.body, results);
+        let cursor = dataflow::ResultsCursor::new(item.body.body(), results);
 
         let mut in_any_value_of_ty = BitSet::new_empty(item.body.local_decls.len());
         for (local, decl) in item.body.local_decls.iter_enumerated() {
@@ -149,17 +141,6 @@
 
     /// The span of the current statement.
     span: Span,
-
-    /// True if the local was assigned the result of an illegal borrow (`ops::MutBorrow`).
-    ///
-    /// This is used to hide errors from {re,}borrowing the newly-assigned local, instead pointing
-    /// the user to the place where the illegal borrow occurred. This set is only populated once an
-    /// error has been emitted, so it will never cause an erroneous `mir::Body` to pass validation.
-    ///
-    /// FIXME(ecstaticmorse): assert at the end of checking that if `tcx.has_errors() == false`,
-    /// this set is empty. Note that if we start removing locals from
-    /// `derived_from_illegal_borrow`, just checking at the end won't be enough.
-    derived_from_illegal_borrow: BitSet<Local>,
 }
 
 impl Deref for Validator<'_, 'mir, 'tcx> {
@@ -190,17 +171,17 @@
 
         let indirectly_mutable = old_dataflow::do_dataflow(
             item.tcx,
-            item.body,
+            &*item.body,
             item.def_id,
             &item.tcx.get_attrs(item.def_id),
             &dead_unwinds,
-            old_dataflow::IndirectlyMutableLocals::new(item.tcx, item.body, item.param_env),
+            old_dataflow::IndirectlyMutableLocals::new(item.tcx, item.body.body(), item.param_env),
             |_, local| old_dataflow::DebugFormatted::new(&local),
         );
 
         let indirectly_mutable = old_dataflow::DataflowResultsCursor::new(
             indirectly_mutable,
-            item.body,
+            item.body.body(),
         );
 
         let qualifs = Qualifs {
@@ -213,7 +194,6 @@
             span: item.body.span,
             item,
             qualifs,
-            derived_from_illegal_borrow: BitSet::new_empty(item.body.local_decls.len()),
         }
     }
 
@@ -221,13 +201,13 @@
         let Item { tcx, body, def_id, const_kind, ..  } = *self.item;
 
         let use_min_const_fn_checks =
-            tcx.is_min_const_fn(def_id)
+            (const_kind == Some(ConstKind::ConstFn) && tcx.is_min_const_fn(def_id))
             && !tcx.sess.opts.debugging_opts.unleash_the_miri_inside_of_you;
 
         if use_min_const_fn_checks {
             // Enforce `min_const_fn` for stable `const fn`s.
             use crate::transform::qualify_min_const_fn::is_min_const_fn;
-            if let Err((span, err)) = is_min_const_fn(tcx, def_id, body) {
+            if let Err((span, err)) = is_min_const_fn(tcx, def_id, &body) {
                 error_min_const_fn_violation(tcx, span, err);
                 return;
             }
@@ -249,7 +229,7 @@
 
         if should_check_for_sync {
             let hir_id = tcx.hir().as_local_hir_id(def_id).unwrap();
-            check_return_ty_is_sync(tcx, body, hir_id);
+            check_return_ty_is_sync(tcx, &body, hir_id);
         }
     }
 
@@ -258,15 +238,15 @@
     }
 
     /// Emits an error at the given `span` if an expression cannot be evaluated in the current
-    /// context. Returns `Forbidden` if an error was emitted.
-    pub fn check_op_spanned<O>(&mut self, op: O, span: Span) -> CheckOpResult
+    /// context.
+    pub fn check_op_spanned<O>(&mut self, op: O, span: Span)
     where
         O: NonConstOp
     {
         trace!("check_op: op={:?}", op);
 
         if op.is_allowed_in_item(self) {
-            return CheckOpResult::Allowed;
+            return;
         }
 
         // If an operation is supported in miri (and is not already controlled by a feature gate) it
@@ -276,20 +256,19 @@
 
         if is_unleashable && self.tcx.sess.opts.debugging_opts.unleash_the_miri_inside_of_you {
             self.tcx.sess.span_warn(span, "skipping const checks");
-            return CheckOpResult::Unleashed;
+            return;
         }
 
         op.emit_error(self, span);
-        CheckOpResult::Forbidden
     }
 
     /// Emits an error if an expression cannot be evaluated in the current context.
-    pub fn check_op(&mut self, op: impl NonConstOp) -> CheckOpResult {
+    pub fn check_op(&mut self, op: impl NonConstOp) {
         let span = self.span;
         self.check_op_spanned(op, span)
     }
 
-    fn check_static(&mut self, def_id: DefId, span: Span) -> CheckOpResult {
+    fn check_static(&mut self, def_id: DefId, span: Span) {
         let is_thread_local = self.tcx.has_attr(def_id, sym::thread_local);
         if is_thread_local {
             self.check_op_spanned(ops::ThreadLocalAccess, span)
@@ -322,20 +301,9 @@
     fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) {
         trace!("visit_rvalue: rvalue={:?} location={:?}", rvalue, location);
 
-        // Check nested operands and places.
+        // Special-case reborrows to be more like a copy of a reference.
         if let Rvalue::Ref(_, kind, ref place) = *rvalue {
-            // Special-case reborrows to be more like a copy of a reference.
-            let mut reborrow_place = None;
-            if let &[ref proj_base @ .., elem] = place.projection.as_ref() {
-                if elem == ProjectionElem::Deref {
-                    let base_ty = Place::ty_from(&place.base, proj_base, self.body, self.tcx).ty;
-                    if let ty::Ref(..) = base_ty.kind {
-                        reborrow_place = Some(proj_base);
-                    }
-                }
-            }
-
-            if let Some(proj) = reborrow_place {
+            if let Some(reborrowed_proj) = place_as_reborrow(self.tcx, *self.body, place) {
                 let ctx = match kind {
                     BorrowKind::Shared => PlaceContext::NonMutatingUse(
                         NonMutatingUseContext::SharedBorrow,
@@ -351,14 +319,13 @@
                     ),
                 };
                 self.visit_place_base(&place.base, ctx, location);
-                self.visit_projection(&place.base, proj, ctx, location);
-            } else {
-                self.super_rvalue(rvalue, location);
+                self.visit_projection(&place.base, reborrowed_proj, ctx, location);
+                return;
             }
-        } else {
-            self.super_rvalue(rvalue, location);
         }
 
+        self.super_rvalue(rvalue, location);
+
         match *rvalue {
             Rvalue::Use(_) |
             Rvalue::Repeat(..) |
@@ -369,11 +336,68 @@
             Rvalue::Cast(CastKind::Pointer(_), ..) |
             Rvalue::Discriminant(..) |
             Rvalue::Len(_) |
-            Rvalue::Ref(..) |
             Rvalue::Aggregate(..) => {}
 
+            | Rvalue::Ref(_, kind @ BorrowKind::Mut { .. }, ref place)
+            | Rvalue::Ref(_, kind @ BorrowKind::Unique, ref place)
+            => {
+                let ty = place.ty(*self.body, self.tcx).ty;
+                let is_allowed = match ty.kind {
+                    // Inside a `static mut`, `&mut [...]` is allowed.
+                    ty::Array(..) | ty::Slice(_) if self.const_kind() == ConstKind::StaticMut
+                        => true,
+
+                    // FIXME(ecstaticmorse): We could allow `&mut []` inside a const context given
+                    // that this is merely a ZST and it is already eligible for promotion.
+                    // This may require an RFC?
+                    /*
+                    ty::Array(_, len) if len.try_eval_usize(cx.tcx, cx.param_env) == Some(0)
+                        => true,
+                    */
+
+                    _ => false,
+                };
+
+                if !is_allowed {
+                    if let BorrowKind::Mut{ .. } = kind {
+                        self.check_op(ops::MutBorrow);
+                    } else {
+                        self.check_op(ops::CellBorrow);
+                    }
+                }
+            }
+
+            // At the moment, `PlaceBase::Static` is only used for promoted MIR.
+            | Rvalue::Ref(_, BorrowKind::Shared, ref place)
+            | Rvalue::Ref(_, BorrowKind::Shallow, ref place)
+            if matches!(place.base, PlaceBase::Static(_))
+            => bug!("Saw a promoted during const-checking, which must run before promotion"),
+
+            | Rvalue::Ref(_, kind @ BorrowKind::Shared, ref place)
+            | Rvalue::Ref(_, kind @ BorrowKind::Shallow, ref place)
+            => {
+                // FIXME: Change the `in_*` methods to take a `FnMut` so we don't have to manually
+                // seek the cursors beforehand.
+                self.qualifs.has_mut_interior.cursor.seek_before(location);
+                self.qualifs.indirectly_mutable.seek(location);
+
+                let borrowed_place_has_mut_interior = HasMutInterior::in_place(
+                    &self.item,
+                    &|local| self.qualifs.has_mut_interior_eager_seek(local),
+                    place.as_ref(),
+                );
+
+                if borrowed_place_has_mut_interior {
+                    if let BorrowKind::Mut{ .. } = kind {
+                        self.check_op(ops::MutBorrow);
+                    } else {
+                        self.check_op(ops::CellBorrow);
+                    }
+                }
+            }
+
             Rvalue::Cast(CastKind::Misc, ref operand, cast_ty) => {
-                let operand_ty = operand.ty(self.body, self.tcx);
+                let operand_ty = operand.ty(*self.body, self.tcx);
                 let cast_in = CastTy::from_ty(operand_ty).expect("bad input type for cast");
                 let cast_out = CastTy::from_ty(cast_ty).expect("bad output type for cast");
 
@@ -384,7 +408,7 @@
             }
 
             Rvalue::BinaryOp(op, ref lhs, _) => {
-                if let ty::RawPtr(_) | ty::FnPtr(..) = lhs.ty(self.body, self.tcx).kind {
+                if let ty::RawPtr(_) | ty::FnPtr(..) = lhs.ty(*self.body, self.tcx).kind {
                     assert!(op == BinOp::Eq || op == BinOp::Ne ||
                             op == BinOp::Le || op == BinOp::Lt ||
                             op == BinOp::Ge || op == BinOp::Gt ||
@@ -435,59 +459,6 @@
             }
         }
     }
-
-    fn visit_assign(&mut self, dest: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) {
-        trace!("visit_assign: dest={:?} rvalue={:?} location={:?}", dest, rvalue, location);
-
-        // Error on mutable borrows or shared borrows of values with interior mutability.
-        //
-        // This replicates the logic at the start of `assign` in the old const checker.  Note that
-        // it depends on `HasMutInterior` being set for mutable borrows as well as values with
-        // interior mutability.
-        if let Rvalue::Ref(_, kind, ref borrowed_place) = *rvalue {
-            // FIXME: Change the `in_*` methods to take a `FnMut` so we don't have to manually seek
-            // the cursors beforehand.
-            self.qualifs.has_mut_interior.cursor.seek_before(location);
-            self.qualifs.indirectly_mutable.seek(location);
-
-            let rvalue_has_mut_interior = HasMutInterior::in_rvalue(
-                &self.item,
-                &|local| self.qualifs.has_mut_interior_eager_seek(local),
-                rvalue,
-            );
-
-            if rvalue_has_mut_interior {
-                let is_derived_from_illegal_borrow = match borrowed_place.as_local() {
-                    // If an unprojected local was borrowed and its value was the result of an
-                    // illegal borrow, suppress this error and mark the result of this borrow as
-                    // illegal as well.
-                    Some(borrowed_local)
-                        if self.derived_from_illegal_borrow.contains(borrowed_local) =>
-                    {
-                        true
-                    }
-
-                    // Otherwise proceed normally: check the legality of a mutable borrow in this
-                    // context.
-                    _ => self.check_op(ops::MutBorrow(kind)) == CheckOpResult::Forbidden,
-                };
-
-                // When the target of the assignment is a local with no projections, mark it as
-                // derived from an illegal borrow if necessary.
-                //
-                // FIXME: should we also clear `derived_from_illegal_borrow` when a local is
-                // assigned a new value?
-                if is_derived_from_illegal_borrow {
-                    if let Some(dest) = dest.as_local() {
-                        self.derived_from_illegal_borrow.insert(dest);
-                    }
-                }
-            }
-        }
-
-        self.super_assign(dest, rvalue, location);
-    }
-
     fn visit_projection_elem(
         &mut self,
         place_base: &PlaceBase<'tcx>,
@@ -510,7 +481,7 @@
 
         match elem {
             ProjectionElem::Deref => {
-                let base_ty = Place::ty_from(place_base, proj_base, self.body, self.tcx).ty;
+                let base_ty = Place::ty_from(place_base, proj_base, *self.body, self.tcx).ty;
                 if let ty::RawPtr(_) = base_ty.kind {
                     if proj_base.is_empty() {
                         if let (PlaceBase::Local(local), []) = (place_base, proj_base) {
@@ -534,7 +505,7 @@
             ProjectionElem::Subslice {..} |
             ProjectionElem::Field(..) |
             ProjectionElem::Index(_) => {
-                let base_ty = Place::ty_from(place_base, proj_base, self.body, self.tcx).ty;
+                let base_ty = Place::ty_from(place_base, proj_base, *self.body, self.tcx).ty;
                 match base_ty.ty_adt_def() {
                     Some(def) if def.is_union() => {
                         self.check_op(ops::UnionAccess);
@@ -583,7 +554,7 @@
 
         match kind {
             TerminatorKind::Call { func, .. } => {
-                let fn_ty = func.ty(self.body, self.tcx);
+                let fn_ty = func.ty(*self.body, self.tcx);
 
                 let def_id = match fn_ty.kind {
                     ty::FnDef(def_id, _) => def_id,
@@ -599,23 +570,6 @@
                 };
 
                 // At this point, we are calling a function whose `DefId` is known...
-
-                if let Abi::RustIntrinsic | Abi::PlatformIntrinsic = self.tcx.fn_sig(def_id).abi() {
-                    assert!(!self.tcx.is_const_fn(def_id));
-
-                    if self.tcx.item_name(def_id) == sym::transmute {
-                        self.check_op(ops::Transmute);
-                        return;
-                    }
-
-                    // To preserve the current semantics, we return early, allowing all
-                    // intrinsics (except `transmute`) to pass unchecked to miri.
-                    //
-                    // FIXME: We should keep a whitelist of allowed intrinsics (or at least a
-                    // blacklist of unimplemented ones) and fail here instead.
-                    return;
-                }
-
                 if self.tcx.is_const_fn(def_id) {
                     return;
                 }
@@ -644,7 +598,7 @@
                 // Check to see if the type of this place can ever have a drop impl. If not, this
                 // `Drop` terminator is frivolous.
                 let ty_needs_drop = dropped_place
-                    .ty(self.body, self.tcx)
+                    .ty(*self.body, self.tcx)
                     .ty
                     .needs_drop(self.tcx, self.param_env);
 
@@ -724,3 +678,36 @@
         }
     });
 }
+
+fn place_as_reborrow(
+    tcx: TyCtxt<'tcx>,
+    body: &Body<'tcx>,
+    place: &'a Place<'tcx>,
+) -> Option<&'a [PlaceElem<'tcx>]> {
+    place
+        .projection
+        .split_last()
+        .and_then(|(outermost, inner)| {
+            if outermost != &ProjectionElem::Deref {
+                return None;
+            }
+
+            // A borrow of a `static` also looks like `&(*_1)` in the MIR, but `_1` is a `const`
+            // that points to the allocation for the static. Don't treat these as reborrows.
+            if let PlaceBase::Local(local) = place.base {
+                if body.local_decls[local].is_ref_to_static() {
+                    return None;
+                }
+            }
+
+            // Ensure the type being derefed is a reference and not a raw pointer.
+            //
+            // This is sufficient to prevent an access to a `static mut` from being marked as a
+            // reborrow, even if the check above were to disappear.
+            let inner_ty = Place::ty_from(&place.base, inner, body, tcx).ty;
+            match inner_ty.kind {
+                ty::Ref(..) => Some(inner),
+                _ => None,
+            }
+        })
+}
diff --git a/src/librustc_mir/transform/check_unsafety.rs b/src/librustc_mir/transform/check_unsafety.rs
index b7cc4e9..284285c 100644
--- a/src/librustc_mir/transform/check_unsafety.rs
+++ b/src/librustc_mir/transform/check_unsafety.rs
@@ -1,6 +1,4 @@
 use rustc_data_structures::fx::FxHashSet;
-use rustc_index::vec::IndexVec;
-use rustc_data_structures::sync::Lrc;
 
 use rustc::ty::query::Providers;
 use rustc::ty::{self, TyCtxt};
@@ -24,7 +22,6 @@
     body: &'a Body<'tcx>,
     const_context: bool,
     min_const_fn: bool,
-    source_scope_local_data: &'a IndexVec<SourceScope, SourceScopeLocalData>,
     violations: Vec<UnsafetyViolation>,
     source_info: SourceInfo,
     tcx: TyCtxt<'tcx>,
@@ -39,7 +36,6 @@
         const_context: bool,
         min_const_fn: bool,
         body: &'a Body<'tcx>,
-        source_scope_local_data: &'a IndexVec<SourceScope, SourceScopeLocalData>,
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> Self {
@@ -51,7 +47,6 @@
             body,
             const_context,
             min_const_fn,
-            source_scope_local_data,
             violations: vec![],
             source_info: SourceInfo {
                 span: body.span,
@@ -219,8 +214,11 @@
             if context.is_borrow() {
                 if util::is_disaligned(self.tcx, self.body, self.param_env, place) {
                     let source_info = self.source_info;
-                    let lint_root =
-                        self.source_scope_local_data[source_info.scope].lint_root;
+                    let lint_root = self.body.source_scopes[source_info.scope]
+                        .local_data
+                        .as_ref()
+                        .assert_crate_local()
+                        .lint_root;
                     self.register_violations(&[UnsafetyViolation {
                         source_info,
                         description: Symbol::intern("borrow of packed field"),
@@ -346,7 +344,11 @@
     fn register_violations(&mut self,
                            violations: &[UnsafetyViolation],
                            unsafe_blocks: &[(hir::HirId, bool)]) {
-        let safety = self.source_scope_local_data[self.source_info.scope].safety;
+        let safety = self.body.source_scopes[self.source_info.scope]
+            .local_data
+            .as_ref()
+            .assert_crate_local()
+            .safety;
         let within_unsafe = match safety {
             // `unsafe` blocks are required in safe code
             Safety::Safe => {
@@ -516,17 +518,6 @@
     // `mir_built` force this.
     let body = &tcx.mir_built(def_id).borrow();
 
-    let source_scope_local_data = match body.source_scope_local_data {
-        ClearCrossCrate::Set(ref data) => data,
-        ClearCrossCrate::Clear => {
-            debug!("unsafety_violations: {:?} - remote, skipping", def_id);
-            return UnsafetyCheckResult {
-                violations: Lrc::new([]),
-                unsafe_blocks: Lrc::new([])
-            }
-        }
-    };
-
     let param_env = tcx.param_env(def_id);
 
     let id = tcx.hir().as_local_hir_id(def_id).unwrap();
@@ -536,9 +527,10 @@
         hir::BodyOwnerKind::Const |
         hir::BodyOwnerKind::Static(_) => (true, false),
     };
-    let mut checker = UnsafetyChecker::new(
-        const_context, min_const_fn,
-        body, source_scope_local_data, tcx, param_env);
+    let mut checker = UnsafetyChecker::new(const_context, min_const_fn, body, tcx, param_env);
+    // mir_built ensures that body has a computed cache, so we don't (and can't) attempt to
+    // recompute it here.
+    let body = body.unwrap_read_only();
     checker.visit_body(body);
 
     check_unused_unsafe(tcx, def_id, &checker.used_unsafe, &mut checker.inherited_blocks);
@@ -666,7 +658,7 @@
     let mut unsafe_blocks: Vec<_> = unsafe_blocks.into_iter().collect();
     unsafe_blocks.sort_by_cached_key(|(hir_id, _)| tcx.hir().hir_to_node_id(*hir_id));
     let used_unsafe: FxHashSet<_> = unsafe_blocks.iter()
-        .flat_map(|&&(id, used)| if used { Some(id) } else { None })
+        .flat_map(|&&(id, used)| used.then_some(id))
         .collect();
     for &(block_id, is_used) in unsafe_blocks {
         if !is_used {
diff --git a/src/librustc_mir/transform/cleanup_post_borrowck.rs b/src/librustc_mir/transform/cleanup_post_borrowck.rs
index 4fd4fe4..d9dd7c9 100644
--- a/src/librustc_mir/transform/cleanup_post_borrowck.rs
+++ b/src/librustc_mir/transform/cleanup_post_borrowck.rs
@@ -16,7 +16,7 @@
 //! [`FakeRead`]: rustc::mir::StatementKind::FakeRead
 //! [`Nop`]: rustc::mir::StatementKind::Nop
 
-use rustc::mir::{BorrowKind, Rvalue, Location, Body};
+use rustc::mir::{BodyCache, BorrowKind, Rvalue, Location};
 use rustc::mir::{Statement, StatementKind};
 use rustc::mir::visit::MutVisitor;
 use rustc::ty::TyCtxt;
@@ -29,7 +29,9 @@
 }
 
 impl<'tcx> MirPass<'tcx> for CleanupNonCodegenStatements {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, _source: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(
+        &self, tcx: TyCtxt<'tcx>, _source: MirSource<'tcx>, body: &mut BodyCache<'tcx>
+    ) {
         let mut delete = DeleteNonCodegenStatements { tcx };
         delete.visit_body(body);
     }
diff --git a/src/librustc_mir/transform/const_prop.rs b/src/librustc_mir/transform/const_prop.rs
index 0fe7530..95de635 100644
--- a/src/librustc_mir/transform/const_prop.rs
+++ b/src/librustc_mir/transform/const_prop.rs
@@ -7,9 +7,10 @@
 use rustc::hir::def::DefKind;
 use rustc::hir::def_id::DefId;
 use rustc::mir::{
-    AggregateKind, Constant, Location, Place, PlaceBase, Body, Operand, Rvalue, Local, UnOp,
-    StatementKind, Statement, LocalKind, TerminatorKind, Terminator,  ClearCrossCrate, SourceInfo,
-    BinOp, SourceScope, SourceScopeLocalData, LocalDecl, BasicBlock, RETURN_PLACE,
+    AggregateKind, Constant, Location, Place, PlaceBase, Body, BodyCache, Operand, Local, UnOp,
+    Rvalue, StatementKind, Statement, LocalKind, TerminatorKind, Terminator,  ClearCrossCrate,
+    SourceInfo, BinOp, SourceScope, SourceScopeData, LocalDecl, BasicBlock, ReadOnlyBodyCache,
+    read_only, RETURN_PLACE
 };
 use rustc::mir::visit::{
     Visitor, PlaceContext, MutatingUseContext, MutVisitor, NonMutatingUseContext,
@@ -41,7 +42,9 @@
 pub struct ConstProp;
 
 impl<'tcx> MirPass<'tcx> for ConstProp {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(
+        &self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut BodyCache<'tcx>
+    ) {
         // will be evaluated by miri and produce its errors there
         if source.promoted.is_some() {
             return;
@@ -74,17 +77,10 @@
 
         trace!("ConstProp starting for {:?}", source.def_id());
 
-        // Steal some data we need from `body`.
-        let source_scope_local_data = std::mem::replace(
-            &mut body.source_scope_local_data,
-            ClearCrossCrate::Clear
-        );
-
         let dummy_body =
             &Body::new(
                 body.basic_blocks().clone(),
-                Default::default(),
-                ClearCrossCrate::Clear,
+                body.source_scopes.clone(),
                 body.local_decls.clone(),
                 Default::default(),
                 body.arg_count,
@@ -99,21 +95,13 @@
         // That would require an uniform one-def no-mutation analysis
         // and RPO (or recursing when needing the value of a local).
         let mut optimization_finder = ConstPropagator::new(
-            body,
+            read_only!(body),
             dummy_body,
-            source_scope_local_data,
             tcx,
             source
         );
         optimization_finder.visit_body(body);
 
-        // put back the data we stole from `mir`
-        let source_scope_local_data = optimization_finder.release_stolen_data();
-        std::mem::replace(
-            &mut body.source_scope_local_data,
-            source_scope_local_data
-        );
-
         trace!("ConstProp done for {:?}", source.def_id());
     }
 }
@@ -140,7 +128,7 @@
         false
     }
 
-    fn find_fn(
+    fn find_mir_or_eval_fn(
         _ecx: &mut InterpCx<'mir, 'tcx, Self>,
         _instance: ty::Instance<'tcx>,
         _args: &[OpTy<'tcx>],
@@ -171,6 +159,15 @@
         throw_unsup!(ConstPropUnsupported("calling intrinsics isn't supported in ConstProp"));
     }
 
+    fn assert_panic(
+        _ecx: &mut InterpCx<'mir, 'tcx, Self>,
+        _span: Span,
+        _msg: &rustc::mir::interpret::AssertMessage<'tcx>,
+        _unwind: Option<rustc::mir::BasicBlock>,
+    ) -> InterpResult<'tcx> {
+        bug!("panics terminators are not evaluated in ConstProp");
+    }
+
     fn ptr_to_int(
         _mem: &Memory<'mir, 'tcx, Self>,
         _ptr: Pointer,
@@ -197,7 +194,7 @@
     }
 
     #[inline(always)]
-    fn tag_allocation<'b>(
+    fn init_allocation_extra<'b>(
         _memory_extra: &(),
         _id: AllocId,
         alloc: Cow<'b, Allocation>,
@@ -267,7 +264,9 @@
     source: MirSource<'tcx>,
     can_const_prop: IndexVec<Local, bool>,
     param_env: ParamEnv<'tcx>,
-    source_scope_local_data: ClearCrossCrate<IndexVec<SourceScope, SourceScopeLocalData>>,
+    // FIXME(eddyb) avoid cloning these two fields more than once,
+    // by accessing them through `ecx` instead.
+    source_scopes: IndexVec<SourceScope, SourceScopeData>,
     local_decls: IndexVec<Local, LocalDecl<'tcx>>,
     ret: Option<OpTy<'tcx, ()>>,
 }
@@ -297,9 +296,8 @@
 
 impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
     fn new(
-        body: &Body<'tcx>,
+        body: ReadOnlyBodyCache<'_, 'tcx>,
         dummy_body: &'mir Body<'tcx>,
-        source_scope_local_data: ClearCrossCrate<IndexVec<SourceScope, SourceScopeLocalData>>,
         tcx: TyCtxt<'tcx>,
         source: MirSource<'tcx>,
     ) -> ConstPropagator<'mir, 'tcx> {
@@ -337,17 +335,15 @@
             source,
             param_env,
             can_const_prop,
-            source_scope_local_data,
+            // FIXME(eddyb) avoid cloning these two fields more than once,
+            // by accessing them through `ecx` instead.
+            source_scopes: body.source_scopes.clone(),
             //FIXME(wesleywiser) we can't steal this because `Visitor::super_visit_body()` needs it
             local_decls: body.local_decls.clone(),
             ret: ret.map(Into::into),
         }
     }
 
-    fn release_stolen_data(self) -> ClearCrossCrate<IndexVec<SourceScope, SourceScopeLocalData>> {
-        self.source_scope_local_data
-    }
-
     fn get_const(&self, local: Local) -> Option<Const<'tcx>> {
         if local == RETURN_PLACE {
             // Try to read the return place as an immediate so that if it is representable as a
@@ -377,14 +373,11 @@
         F: FnOnce(&mut Self) -> InterpResult<'tcx, T>,
     {
         self.ecx.tcx.span = source_info.span;
-        let lint_root = match self.source_scope_local_data {
-            ClearCrossCrate::Set(ref ivs) => {
-                //FIXME(#51314): remove this check
-                if source_info.scope.index() >= ivs.len() {
-                    return None;
-                }
-                ivs[source_info.scope].lint_root
-            },
+        // FIXME(eddyb) move this to the `Panic(_)` error case, so that
+        // `f(self)` is always called, and that the only difference when the
+        // scope's `local_data` is missing, is that the lint isn't emitted.
+        let lint_root = match &self.source_scopes[source_info.scope].local_data {
+            ClearCrossCrate::Set(data) => data.lint_root,
             ClearCrossCrate::Clear => return None,
         };
         let r = match f(self) {
@@ -396,7 +389,7 @@
                     InterpError::*
                 };
                 match error.kind {
-                    Exit(_) => bug!("the CTFE program cannot exit"),
+                    MachineStop(_) => bug!("ConstProp does not stop"),
 
                     // Some error shouldn't come up because creating them causes
                     // an allocation, which we should avoid. When that happens,
@@ -525,8 +518,8 @@
                     let right_size = r.layout.size;
                     let r_bits = r.to_scalar().and_then(|r| r.to_bits(right_size));
                     if r_bits.ok().map_or(false, |b| b >= left_bits as u128) {
-                        let source_scope_local_data = match self.source_scope_local_data {
-                            ClearCrossCrate::Set(ref data) => data,
+                        let lint_root = match &self.source_scopes[source_info.scope].local_data {
+                            ClearCrossCrate::Set(data) => data.lint_root,
                             ClearCrossCrate::Clear => return None,
                         };
                         let dir = if *op == BinOp::Shr {
@@ -534,10 +527,9 @@
                         } else {
                             "left"
                         };
-                        let hir_id = source_scope_local_data[source_info.scope].lint_root;
                         self.tcx.lint_hir(
                             ::rustc::lint::builtin::EXCEEDING_BITSHIFTS,
-                            hir_id,
+                            lint_root,
                             span,
                             &format!("attempt to shift {} with overflow", dir));
                         return None;
@@ -698,7 +690,7 @@
 
 impl CanConstProp {
     /// returns true if `local` can be propagated
-    fn check(body: &Body<'_>) -> IndexVec<Local, bool> {
+    fn check(body: ReadOnlyBodyCache<'_, '_>) -> IndexVec<Local, bool> {
         let mut cpv = CanConstProp {
             can_const_prop: IndexVec::from_elem(true, &body.local_decls),
             found_assignment: IndexVec::from_elem(false, &body.local_decls),
diff --git a/src/librustc_mir/transform/copy_prop.rs b/src/librustc_mir/transform/copy_prop.rs
index 4c26fea..5e4caf2 100644
--- a/src/librustc_mir/transform/copy_prop.rs
+++ b/src/librustc_mir/transform/copy_prop.rs
@@ -19,7 +19,10 @@
 //! (non-mutating) use of `SRC`. These restrictions are conservative and may be relaxed in the
 //! future.
 
-use rustc::mir::{Constant, Local, LocalKind, Location, Place, Body, Operand, Rvalue, StatementKind};
+use rustc::mir::{
+    Constant, Local, LocalKind, Location, Place, Body, BodyCache, Operand, Rvalue,
+    StatementKind, read_only
+};
 use rustc::mir::visit::MutVisitor;
 use rustc::ty::TyCtxt;
 use crate::transform::{MirPass, MirSource};
@@ -28,7 +31,9 @@
 pub struct CopyPropagation;
 
 impl<'tcx> MirPass<'tcx> for CopyPropagation {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, _source: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(
+        &self, tcx: TyCtxt<'tcx>, _source: MirSource<'tcx>, body: &mut BodyCache<'tcx>
+    ) {
         // We only run when the MIR optimization level is > 1.
         // This avoids a slow pass, and messing up debug info.
         if tcx.sess.opts.debugging_opts.mir_opt_level <= 1 {
@@ -37,10 +42,10 @@
 
         let mut def_use_analysis = DefUseAnalysis::new(body);
         loop {
-            def_use_analysis.analyze(body);
+            def_use_analysis.analyze(read_only!(body));
 
             if eliminate_self_assignments(body, &def_use_analysis) {
-                def_use_analysis.analyze(body);
+                def_use_analysis.analyze(read_only!(body));
             }
 
             let mut changed = false;
@@ -97,7 +102,10 @@
                                     let maybe_action = match operand {
                                         Operand::Copy(ref src_place) |
                                         Operand::Move(ref src_place) => {
-                                            Action::local_copy(&body, &def_use_analysis, src_place)
+                                            Action::local_copy(
+                                                &body,
+                                                &def_use_analysis,
+                                                src_place)
                                         }
                                         Operand::Constant(ref src_constant) => {
                                             Action::constant(src_constant)
@@ -126,8 +134,8 @@
                     }
                 }
 
-                changed =
-                    action.perform(body, &def_use_analysis, dest_local, location, tcx) || changed;
+                changed = action.perform(body, &def_use_analysis, dest_local, location, tcx)
+                    || changed;
                 // FIXME(pcwalton): Update the use-def chains to delete the instructions instead of
                 // regenerating the chains.
                 break
@@ -242,7 +250,7 @@
     }
 
     fn perform(self,
-               body: &mut Body<'tcx>,
+               body: &mut BodyCache<'tcx>,
                def_use_analysis: &DefUseAnalysis,
                dest_local: Local,
                location: Location,
@@ -270,7 +278,8 @@
                 }
 
                 // Replace all uses of the destination local with the source local.
-                def_use_analysis.replace_all_defs_and_uses_with(dest_local, body, src_local, tcx);
+                def_use_analysis
+                    .replace_all_defs_and_uses_with(dest_local, body, src_local, tcx);
 
                 // Finally, zap the now-useless assignment instruction.
                 debug!("  Deleting assignment");
diff --git a/src/librustc_mir/transform/deaggregator.rs b/src/librustc_mir/transform/deaggregator.rs
index cdde9e1..933936e 100644
--- a/src/librustc_mir/transform/deaggregator.rs
+++ b/src/librustc_mir/transform/deaggregator.rs
@@ -6,7 +6,9 @@
 pub struct Deaggregator;
 
 impl<'tcx> MirPass<'tcx> for Deaggregator {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, _source: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(
+        &self, tcx: TyCtxt<'tcx>, _source: MirSource<'tcx>, body: &mut BodyCache<'tcx>
+    ) {
         let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut();
         let local_decls = &*local_decls;
         for bb in basic_blocks {
diff --git a/src/librustc_mir/transform/dump_mir.rs b/src/librustc_mir/transform/dump_mir.rs
index ed0eff9..13b3bb6 100644
--- a/src/librustc_mir/transform/dump_mir.rs
+++ b/src/librustc_mir/transform/dump_mir.rs
@@ -5,7 +5,7 @@
 use std::fs::File;
 use std::io;
 
-use rustc::mir::Body;
+use rustc::mir::{Body, BodyCache};
 use rustc::session::config::{OutputFilenames, OutputType};
 use rustc::ty::TyCtxt;
 use crate::transform::{MirPass, MirSource};
@@ -18,8 +18,9 @@
         Cow::Borrowed(self.0)
     }
 
-    fn run_pass(&self, _tcx: TyCtxt<'tcx>, _source: MirSource<'tcx>, _body: &mut Body<'tcx>) {
-    }
+    fn run_pass(
+        &self, _tcx: TyCtxt<'tcx>, _source: MirSource<'tcx>, _body: &mut BodyCache<'tcx>
+    ) {}
 }
 
 pub struct Disambiguator {
diff --git a/src/librustc_mir/transform/elaborate_drops.rs b/src/librustc_mir/transform/elaborate_drops.rs
index f91a08b..42daba9 100644
--- a/src/librustc_mir/transform/elaborate_drops.rs
+++ b/src/librustc_mir/transform/elaborate_drops.rs
@@ -21,7 +21,7 @@
 pub struct ElaborateDrops;
 
 impl<'tcx> MirPass<'tcx> for ElaborateDrops {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut BodyCache<'tcx>) {
         debug!("elaborate_drops({:?} @ {:?})", src, body.span);
 
         let def_id = src.def_id();
diff --git a/src/librustc_mir/transform/erase_regions.rs b/src/librustc_mir/transform/erase_regions.rs
index b30e2de..882e674 100644
--- a/src/librustc_mir/transform/erase_regions.rs
+++ b/src/librustc_mir/transform/erase_regions.rs
@@ -62,7 +62,7 @@
 pub struct EraseRegions;
 
 impl<'tcx> MirPass<'tcx> for EraseRegions {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, _: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, _: MirSource<'tcx>, body: &mut BodyCache<'tcx>) {
         EraseRegionsVisitor::new(tcx).visit_body(body);
     }
 }
diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs
index 5d88629..e55737e 100644
--- a/src/librustc_mir/transform/generator.rs
+++ b/src/librustc_mir/transform/generator.rs
@@ -368,7 +368,7 @@
                 VariantIdx::new(RETURNED) // state for returned
             };
             data.statements.push(self.set_discr(state, source_info));
-            data.terminator.as_mut().unwrap().kind = TerminatorKind::Return;
+            data.terminator_mut().kind = TerminatorKind::Return;
         }
 
         self.super_basic_block_data(block, data);
@@ -378,7 +378,7 @@
 fn make_generator_state_argument_indirect<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: DefId,
-    body: &mut Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
 ) {
     let gen_ty = body.local_decls.raw[1].ty;
 
@@ -401,7 +401,7 @@
     DerefArgVisitor { tcx }.visit_body(body);
 }
 
-fn make_generator_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+fn make_generator_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut BodyCache<'tcx>) {
     let ref_gen_ty = body.local_decls.raw[1].ty;
 
     let pin_did = tcx.lang_items().pin_type().unwrap();
@@ -418,7 +418,7 @@
 
 fn replace_result_variable<'tcx>(
     ret_ty: Ty<'tcx>,
-    body: &mut Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
     tcx: TyCtxt<'tcx>,
 ) -> Local {
     let source_info = source_info(body);
@@ -481,20 +481,21 @@
 
 fn locals_live_across_suspend_points(
     tcx: TyCtxt<'tcx>,
-    body: &Body<'tcx>,
+    body: ReadOnlyBodyCache<'_, 'tcx>,
     source: MirSource<'tcx>,
     movable: bool,
 ) -> LivenessInfo {
     let dead_unwinds = BitSet::new_empty(body.basic_blocks().len());
     let def_id = source.def_id();
+    let body_ref: &Body<'_> = &body;
 
     // Calculate when MIR locals have live storage. This gives us an upper bound of their
     // lifetimes.
-    let storage_live_analysis = MaybeStorageLive::new(body);
+    let storage_live_analysis = MaybeStorageLive::new(body_ref);
     let storage_live_results =
-        do_dataflow(tcx, body, def_id, &[], &dead_unwinds, storage_live_analysis,
+        do_dataflow(tcx, body_ref, def_id, &[], &dead_unwinds, storage_live_analysis,
                     |bd, p| DebugFormatted::new(&bd.body().local_decls[p]));
-    let mut storage_live_cursor = DataflowResultsCursor::new(&storage_live_results, body);
+    let mut storage_live_cursor = DataflowResultsCursor::new(&storage_live_results, body_ref);
 
     // Find the MIR locals which do not use StorageLive/StorageDead statements.
     // The storage of these locals are always live.
@@ -503,19 +504,20 @@
 
     // Calculate the MIR locals which have been previously
     // borrowed (even if they are still active).
-    let borrowed_locals_analysis = HaveBeenBorrowedLocals::new(body);
+    let borrowed_locals_analysis = HaveBeenBorrowedLocals::new(body_ref);
     let borrowed_locals_results =
-        do_dataflow(tcx, body, def_id, &[], &dead_unwinds, borrowed_locals_analysis,
+        do_dataflow(tcx, body_ref, def_id, &[], &dead_unwinds, borrowed_locals_analysis,
                     |bd, p| DebugFormatted::new(&bd.body().local_decls[p]));
-    let mut borrowed_locals_cursor = DataflowResultsCursor::new(&borrowed_locals_results, body);
+    let mut borrowed_locals_cursor = DataflowResultsCursor::new(&borrowed_locals_results, body_ref);
 
     // Calculate the MIR locals that we actually need to keep storage around
     // for.
     let requires_storage_analysis = RequiresStorage::new(body, &borrowed_locals_results);
     let requires_storage_results =
-        do_dataflow(tcx, body, def_id, &[], &dead_unwinds, requires_storage_analysis,
+        do_dataflow(tcx, body_ref, def_id, &[], &dead_unwinds, requires_storage_analysis,
                     |bd, p| DebugFormatted::new(&bd.body().local_decls[p]));
-    let mut requires_storage_cursor = DataflowResultsCursor::new(&requires_storage_results, body);
+    let mut requires_storage_cursor
+        = DataflowResultsCursor::new(&requires_storage_results, body_ref);
 
     // Calculate the liveness of MIR locals ignoring borrows.
     let mut live_locals = liveness::LiveVarSet::new_empty(body.local_decls.len());
@@ -526,7 +528,7 @@
         tcx,
         "generator_liveness",
         source,
-        body,
+        body_ref,
         &liveness,
     );
 
@@ -593,7 +595,7 @@
         .collect();
 
     let storage_conflicts = compute_storage_conflicts(
-        body,
+        body_ref,
         &live_locals,
         &ignored,
         requires_storage_results);
@@ -749,7 +751,7 @@
     upvars: &Vec<Ty<'tcx>>,
     interior: Ty<'tcx>,
     movable: bool,
-    body: &mut Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
 ) -> (
     FxHashMap<Local, (Ty<'tcx>, VariantIdx, usize)>,
     GeneratorLayout<'tcx>,
@@ -758,7 +760,7 @@
     // Use a liveness analysis to compute locals which are live across a suspension point
     let LivenessInfo {
         live_locals, live_locals_at_suspension_points, storage_conflicts, storage_liveness
-    } = locals_live_across_suspend_points(tcx, body, source, movable);
+    } = locals_live_across_suspend_points(tcx, read_only!(body), source, movable);
 
     // Erase regions from the types passed in from typeck so we can compare them with
     // MIR types
@@ -828,7 +830,7 @@
 }
 
 fn insert_switch<'tcx>(
-    body: &mut Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
     cases: Vec<(usize, BasicBlock)>,
     transform: &TransformVisitor<'tcx>,
     default: TerminatorKind<'tcx>,
@@ -859,7 +861,9 @@
     }
 }
 
-fn elaborate_generator_drops<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, body: &mut Body<'tcx>) {
+fn elaborate_generator_drops<'tcx>(
+    tcx: TyCtxt<'tcx>, def_id: DefId, body: &mut BodyCache<'tcx>
+) {
     use crate::util::elaborate_drops::{elaborate_drop, Unwind};
     use crate::util::patch::MirPatch;
     use crate::shim::DropShimElaborator;
@@ -872,7 +876,7 @@
     let gen = self_arg();
 
     let mut elaborator = DropShimElaborator {
-        body: body,
+        body,
         patch: MirPatch::new(body),
         tcx,
         param_env
@@ -924,9 +928,9 @@
     def_id: DefId,
     source: MirSource<'tcx>,
     gen_ty: Ty<'tcx>,
-    body: &Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
     drop_clean: BasicBlock,
-) -> Body<'tcx> {
+) -> BodyCache<'tcx> {
     let mut body = body.clone();
 
     let source_info = source_info(&body);
@@ -992,7 +996,9 @@
     body
 }
 
-fn insert_term_block<'tcx>(body: &mut Body<'tcx>, kind: TerminatorKind<'tcx>) -> BasicBlock {
+fn insert_term_block<'tcx>(
+    body: &mut BodyCache<'tcx>, kind: TerminatorKind<'tcx>
+) -> BasicBlock {
     let term_block = BasicBlock::new(body.basic_blocks().len());
     let source_info = source_info(body);
     body.basic_blocks_mut().push(BasicBlockData {
@@ -1008,7 +1014,7 @@
 
 fn insert_panic_block<'tcx>(
     tcx: TyCtxt<'tcx>,
-    body: &mut Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
     message: AssertMessage<'tcx>,
 ) -> BasicBlock {
     let assert_block = BasicBlock::new(body.basic_blocks().len());
@@ -1042,7 +1048,7 @@
     transform: TransformVisitor<'tcx>,
     def_id: DefId,
     source: MirSource<'tcx>,
-    body: &mut Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
 ) {
     // Poison the generator when it unwinds
     for block in body.basic_blocks_mut() {
@@ -1095,7 +1101,7 @@
     }
 }
 
-fn insert_clean_drop(body: &mut Body<'_>) -> BasicBlock {
+fn insert_clean_drop(body: &mut BodyCache<'_>) -> BasicBlock {
     let return_block = insert_term_block(body, TerminatorKind::Return);
 
     // Create a block to destroy an unresumed generators. This can only destroy upvars.
@@ -1119,7 +1125,7 @@
 }
 
 fn create_cases<'tcx, F>(
-    body: &mut Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
     transform: &TransformVisitor<'tcx>,
     target: F,
 ) -> Vec<(usize, BasicBlock)>
@@ -1163,7 +1169,9 @@
 }
 
 impl<'tcx> MirPass<'tcx> for StateTransform {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(
+        &self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut BodyCache<'tcx>
+    ) {
         let yield_ty = if let Some(yield_ty) = body.yield_ty {
             yield_ty
         } else {
@@ -1252,12 +1260,12 @@
 
         // Create a copy of our MIR and use it to create the drop shim for the generator
         let drop_shim = create_generator_drop_shim(tcx,
-            &transform,
-            def_id,
-            source,
-            gen_ty,
-            &body,
-            drop_clean);
+                                                   &transform,
+                                                   def_id,
+                                                   source,
+                                                   gen_ty,
+                                                   body,
+                                                   drop_clean);
 
         body.generator_drop = Some(box drop_shim);
 
diff --git a/src/librustc_mir/transform/inline.rs b/src/librustc_mir/transform/inline.rs
index 867673b..79cb7fb 100644
--- a/src/librustc_mir/transform/inline.rs
+++ b/src/librustc_mir/transform/inline.rs
@@ -38,7 +38,9 @@
 }
 
 impl<'tcx> MirPass<'tcx> for Inline {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(
+        &self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut BodyCache<'tcx>
+    ) {
         if tcx.sess.opts.debugging_opts.mir_opt_level >= 2 {
             Inliner { tcx, source }.run_pass(body);
         }
@@ -51,7 +53,7 @@
 }
 
 impl Inliner<'tcx> {
-    fn run_pass(&self, caller_body: &mut Body<'tcx>) {
+    fn run_pass(&self, caller_body: &mut BodyCache<'tcx>) {
         // Keep a queue of callsites to try inlining on. We take
         // advantage of the fact that queries detect cycles here to
         // allow us to try and fetch the fully optimized MIR of a
@@ -75,9 +77,9 @@
         {
             for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated() {
                 if let Some(callsite) = self.get_valid_function_call(bb,
-                                                                    bb_data,
-                                                                    caller_body,
-                                                                    param_env) {
+                                                                     bb_data,
+                                                                     caller_body,
+                                                                     param_env) {
                     callsites.push_back(callsite);
                 }
             }
@@ -136,7 +138,8 @@
                 debug!("attempting to inline callsite {:?} - success", callsite);
 
                 // Add callsites from inlined function
-                for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated().skip(start) {
+                for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated().skip(start)
+                {
                     if let Some(new_callsite) = self.get_valid_function_call(bb,
                                                                              bb_data,
                                                                              caller_body,
@@ -377,8 +380,8 @@
 
     fn inline_call(&self,
                    callsite: CallSite<'tcx>,
-                   caller_body: &mut Body<'tcx>,
-                   mut callee_body: Body<'tcx>) -> bool {
+                   caller_body: &mut BodyCache<'tcx>,
+                   mut callee_body: BodyCache<'tcx>) -> bool {
         let terminator = caller_body[callsite.bb].terminator.take().unwrap();
         match terminator.kind {
             // FIXME: Handle inlining of diverging calls
@@ -391,9 +394,14 @@
                 for mut scope in callee_body.source_scopes.iter().cloned() {
                     if scope.parent_scope.is_none() {
                         scope.parent_scope = Some(callsite.location.scope);
+                        // FIXME(eddyb) is this really needed?
+                        // (also note that it's always overwritten below)
                         scope.span = callee_body.span;
                     }
 
+                    // FIXME(eddyb) this doesn't seem right at all.
+                    // The inlined source scopes should probably be annotated as
+                    // such, but also contain all of the original information.
                     scope.span = callsite.location.span;
 
                     let idx = caller_body.source_scopes.push(scope);
@@ -440,7 +448,7 @@
                         BorrowKind::Mut { allow_two_phase_borrow: false },
                         destination.0);
 
-                    let ty = dest.ty(caller_body, self.tcx);
+                    let ty = dest.ty(&**caller_body, self.tcx);
 
                     let temp = LocalDecl::new_temp(ty, callsite.location.span);
 
@@ -509,7 +517,7 @@
         &self,
         args: Vec<Operand<'tcx>>,
         callsite: &CallSite<'tcx>,
-        caller_body: &mut Body<'tcx>,
+        caller_body: &mut BodyCache<'tcx>,
     ) -> Vec<Local> {
         let tcx = self.tcx;
 
@@ -538,12 +546,14 @@
         // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`.
         if tcx.is_closure(callsite.callee) {
             let mut args = args.into_iter();
-            let self_ = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
-            let tuple = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
+            let self_
+                = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
+            let tuple
+                = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
             assert!(args.next().is_none());
 
             let tuple = Place::from(tuple);
-            let tuple_tys = if let ty::Tuple(s) = tuple.ty(caller_body, tcx).ty.kind {
+            let tuple_tys = if let ty::Tuple(s) = tuple.ty(&**caller_body, tcx).ty.kind {
                 s
             } else {
                 bug!("Closure arguments are not passed as a tuple");
@@ -580,7 +590,7 @@
         &self,
         arg: Operand<'tcx>,
         callsite: &CallSite<'tcx>,
-        caller_body: &mut Body<'tcx>,
+        caller_body: &mut BodyCache<'tcx>,
     ) -> Local {
         // FIXME: Analysis of the usage of the arguments to avoid
         // unnecessary temporaries.
@@ -598,7 +608,7 @@
         // Otherwise, create a temporary for the arg
         let arg = Rvalue::Use(arg);
 
-        let ty = arg.ty(caller_body, self.tcx);
+        let ty = arg.ty(&**caller_body, self.tcx);
 
         let arg_tmp = LocalDecl::new_temp(ty, callsite.location.span);
         let arg_tmp = caller_body.local_decls.push(arg_tmp);
diff --git a/src/librustc_mir/transform/instcombine.rs b/src/librustc_mir/transform/instcombine.rs
index a567ed6..bd237b5 100644
--- a/src/librustc_mir/transform/instcombine.rs
+++ b/src/librustc_mir/transform/instcombine.rs
@@ -1,7 +1,8 @@
 //! Performs various peephole optimizations.
 
 use rustc::mir::{
-    Constant, Location, Place, PlaceBase, PlaceRef, Body, Operand, ProjectionElem, Rvalue, Local
+    Constant, Location, Place, PlaceBase, PlaceRef, Body, BodyCache, Operand, ProjectionElem,
+    Rvalue, Local, read_only
 };
 use rustc::mir::visit::{MutVisitor, Visitor};
 use rustc::ty::{self, TyCtxt};
@@ -13,7 +14,7 @@
 pub struct InstCombine;
 
 impl<'tcx> MirPass<'tcx> for InstCombine {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, _: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, _: MirSource<'tcx>, body: &mut BodyCache<'tcx>) {
         // We only run when optimizing MIR (at any level).
         if tcx.sess.opts.debugging_opts.mir_opt_level == 0 {
             return
@@ -23,8 +24,9 @@
         // read-only so that we can do global analyses on the MIR in the process (e.g.
         // `Place::ty()`).
         let optimizations = {
+            let read_only_cache = read_only!(body);
             let mut optimization_finder = OptimizationFinder::new(body, tcx);
-            optimization_finder.visit_body(body);
+            optimization_finder.visit_body(read_only_cache);
             optimization_finder.optimizations
         };
 
diff --git a/src/librustc_mir/transform/mod.rs b/src/librustc_mir/transform/mod.rs
index 2b2b529..df4cb76 100644
--- a/src/librustc_mir/transform/mod.rs
+++ b/src/librustc_mir/transform/mod.rs
@@ -1,7 +1,7 @@
 use crate::{build, shim};
 use rustc_index::vec::IndexVec;
 use rustc::hir::def_id::{CrateNum, DefId, LOCAL_CRATE};
-use rustc::mir::{Body, MirPhase, Promoted, ConstQualifs};
+use rustc::mir::{BodyCache, MirPhase, Promoted, ConstQualifs};
 use rustc::ty::{TyCtxt, InstanceDef, TypeFoldable};
 use rustc::ty::query::Providers;
 use rustc::ty::steal::Steal;
@@ -97,7 +97,7 @@
     tcx.arena.alloc(set)
 }
 
-fn mir_built(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal<Body<'_>> {
+fn mir_built(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal<BodyCache<'_>> {
     let mir = build::mir_build(tcx, def_id);
     tcx.alloc_steal_mir(mir)
 }
@@ -144,12 +144,12 @@
         default_name::<Self>()
     }
 
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut Body<'tcx>);
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut BodyCache<'tcx>);
 }
 
 pub fn run_passes(
     tcx: TyCtxt<'tcx>,
-    body: &mut Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
     instance: InstanceDef<'tcx>,
     promoted: Option<Promoted>,
     mir_phase: MirPhase,
@@ -205,7 +205,7 @@
     }
 
     let item = check_consts::Item {
-        body,
+        body: body.unwrap_read_only(),
         tcx,
         def_id,
         const_kind,
@@ -220,7 +220,7 @@
     validator.qualifs_in_return_place().into()
 }
 
-fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal<Body<'_>> {
+fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal<BodyCache<'_>> {
     // Unsafety check uses the raw mir, so make sure it is run
     let _ = tcx.unsafety_check_result(def_id);
 
@@ -231,13 +231,14 @@
         &rustc_peek::SanityCheck,
         &uniform_array_move_out::UniformArrayMoveOut,
     ]);
+    body.ensure_predecessors();
     tcx.alloc_steal_mir(body)
 }
 
 fn mir_validated(
     tcx: TyCtxt<'tcx>,
     def_id: DefId,
-) -> (&'tcx Steal<Body<'tcx>>, &'tcx Steal<IndexVec<Promoted, Body<'tcx>>>) {
+) -> (&'tcx Steal<BodyCache<'tcx>>, &'tcx Steal<IndexVec<Promoted, BodyCache<'tcx>>>) {
     // Ensure that we compute the `mir_const_qualif` for constants at
     // this point, before we steal the mir-const result.
     let _ = tcx.mir_const_qualif(def_id);
@@ -249,13 +250,14 @@
         &promote_pass,
         &simplify::SimplifyCfg::new("qualify-consts"),
     ]);
+
     let promoted = promote_pass.promoted_fragments.into_inner();
     (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted))
 }
 
 fn run_optimization_passes<'tcx>(
     tcx: TyCtxt<'tcx>,
-    body: &mut Body<'tcx>,
+    body: &mut BodyCache<'tcx>,
     def_id: DefId,
     promoted: Option<Promoted>,
 ) {
@@ -317,7 +319,7 @@
     ]);
 }
 
-fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> {
+fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &BodyCache<'_> {
     if tcx.is_constructor(def_id) {
         // There's no reason to run all of the MIR passes on constructors when
         // we can just output the MIR we want directly. This also saves const
@@ -333,10 +335,11 @@
     let (body, _) = tcx.mir_validated(def_id);
     let mut body = body.steal();
     run_optimization_passes(tcx, &mut body, def_id, None);
+    body.ensure_predecessors();
     tcx.arena.alloc(body)
 }
 
-fn promoted_mir<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx IndexVec<Promoted, Body<'tcx>> {
+fn promoted_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &IndexVec<Promoted, BodyCache<'_>> {
     if tcx.is_constructor(def_id) {
         return tcx.intern_promoted(IndexVec::new());
     }
@@ -347,6 +350,7 @@
 
     for (p, mut body) in promoted.iter_enumerated_mut() {
         run_optimization_passes(tcx, &mut body, def_id, Some(p));
+        body.ensure_predecessors();
     }
 
     tcx.intern_promoted(promoted)
diff --git a/src/librustc_mir/transform/no_landing_pads.rs b/src/librustc_mir/transform/no_landing_pads.rs
index fbd14d9..5e1d29d 100644
--- a/src/librustc_mir/transform/no_landing_pads.rs
+++ b/src/librustc_mir/transform/no_landing_pads.rs
@@ -17,12 +17,12 @@
 }
 
 impl<'tcx> MirPass<'tcx> for NoLandingPads<'tcx> {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, _: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, _: MirSource<'tcx>, body: &mut BodyCache<'tcx>) {
         no_landing_pads(tcx, body)
     }
 }
 
-pub fn no_landing_pads<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+pub fn no_landing_pads<'tcx>(tcx: TyCtxt<'tcx>, body: &mut BodyCache<'tcx>) {
     if tcx.sess.no_landing_pads() {
         NoLandingPads::new(tcx).visit_body(body);
     }
diff --git a/src/librustc_mir/transform/promote_consts.rs b/src/librustc_mir/transform/promote_consts.rs
index cc6108c..c758ccf 100644
--- a/src/librustc_mir/transform/promote_consts.rs
+++ b/src/librustc_mir/transform/promote_consts.rs
@@ -41,11 +41,11 @@
 /// newly created `StaticKind::Promoted`.
 #[derive(Default)]
 pub struct PromoteTemps<'tcx> {
-    pub promoted_fragments: Cell<IndexVec<Promoted, Body<'tcx>>>,
+    pub promoted_fragments: Cell<IndexVec<Promoted, BodyCache<'tcx>>>,
 }
 
 impl<'tcx> MirPass<'tcx> for PromoteTemps<'tcx> {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, body: &mut BodyCache<'tcx>) {
         // There's not really any point in promoting errorful MIR.
         //
         // This does not include MIR that failed const-checking, which we still try to promo