Importing rustc-1.76.0

Test: ./build.py --lto=thin
Change-Id: I0c36470ee5c44609436c910ed99532cf70ef8817
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index dfaa70b..eeff563 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -15,7 +15,7 @@
 ## About the [rustc-dev-guide]
 
 The [rustc-dev-guide] is meant to help document how rustc –the Rust compiler– works,
-as well as to help new contributors get involved in rustc development. It is recommend
+as well as to help new contributors get involved in rustc development. It is recommended
 to read and understand the [rustc-dev-guide] before making a contribution. This guide
 talks about the different bots in the Rust ecosystem, the Rust development tools,
 bootstrapping, the compiler architecture, source code representation, and more.
diff --git a/Cargo.lock b/Cargo.lock
index d3ecadf..117d8c2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -37,13 +37,14 @@
 
 [[package]]
 name = "ahash"
-version = "0.8.3"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
+checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
 dependencies = [
  "cfg-if",
  "once_cell",
  "version_check",
+ "zerocopy",
 ]
 
 [[package]]
@@ -194,12 +195,6 @@
 ]
 
 [[package]]
-name = "array_tool"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f8cb5d814eb646a863c4f24978cff2880c4be96ad8cde2c0f0678732902e271"
-
-[[package]]
 name = "arrayvec"
 version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -228,7 +223,7 @@
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -238,17 +233,6 @@
 checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
 
 [[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi 0.1.19",
- "libc",
- "winapi",
-]
-
-[[package]]
 name = "autocfg"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -356,7 +340,7 @@
 dependencies = [
  "anyhow",
  "curl",
- "indexmap 2.0.0",
+ "indexmap",
  "serde",
  "serde_json",
  "toml 0.5.11",
@@ -526,9 +510,9 @@
 
 [[package]]
 name = "clap_complete"
-version = "4.3.1"
+version = "4.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f6b5c519bab3ea61843a7923d074b04245624bb84a64a8c150f5deb014e388b"
+checksum = "bffe91f06a11b4b9420f62103854e90867812cd5d01557f853c5ee8e791b12ae"
 dependencies = [
  "clap",
 ]
@@ -542,7 +526,7 @@
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -553,7 +537,7 @@
 
 [[package]]
 name = "clippy"
-version = "0.1.75"
+version = "0.1.76"
 dependencies = [
  "anstream",
  "clippy_config",
@@ -564,12 +548,12 @@
  "futures",
  "if_chain",
  "itertools",
- "parking_lot 0.12.1",
+ "parking_lot",
  "quote",
  "regex",
  "rustc_tools_util",
  "serde",
- "syn 2.0.29",
+ "syn 2.0.32",
  "tempfile",
  "termize",
  "tester",
@@ -581,7 +565,7 @@
 
 [[package]]
 name = "clippy_config"
-version = "0.1.75"
+version = "0.1.76"
 dependencies = [
  "rustc-semver",
  "serde",
@@ -597,25 +581,24 @@
  "clap",
  "indoc",
  "itertools",
- "opener",
+ "opener 0.5.2",
  "shell-escape",
  "walkdir",
 ]
 
 [[package]]
 name = "clippy_lints"
-version = "0.1.75"
+version = "0.1.76"
 dependencies = [
  "arrayvec",
  "cargo_metadata 0.15.4",
  "clippy_config",
  "clippy_utils",
  "declare_clippy_lint",
- "if_chain",
  "itertools",
  "quine-mc_cluskey",
  "regex",
- "regex-syntax 0.7.2",
+ "regex-syntax 0.8.2",
  "rustc-semver",
  "semver",
  "serde",
@@ -630,11 +613,10 @@
 
 [[package]]
 name = "clippy_utils"
-version = "0.1.75"
+version = "0.1.76"
 dependencies = [
  "arrayvec",
  "clippy_config",
- "if_chain",
  "itertools",
  "rustc-semver",
 ]
@@ -741,7 +723,7 @@
  "getopts",
  "glob",
  "home",
- "indexmap 2.0.0",
+ "indexmap",
  "lazycell",
  "libc",
  "miow",
@@ -814,10 +796,6 @@
 ]
 
 [[package]]
-name = "coverage_test_macros"
-version = "0.0.0"
-
-[[package]]
 name = "cpufeatures"
 version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -889,16 +867,6 @@
 ]
 
 [[package]]
-name = "cstr"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c11a39d776a3b35896711da8a04dc1835169dcd36f710878187637314e47941b"
-dependencies = [
- "proc-macro2",
- "quote",
-]
-
-[[package]]
 name = "ctrlc"
 version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -983,7 +951,7 @@
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -1005,7 +973,7 @@
 dependencies = [
  "darling_core 0.20.3",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -1016,11 +984,11 @@
 
 [[package]]
 name = "declare_clippy_lint"
-version = "0.1.75"
+version = "0.1.76"
 dependencies = [
  "itertools",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -1087,7 +1055,7 @@
  "darling 0.20.3",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -1176,7 +1144,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -1249,24 +1217,11 @@
 
 [[package]]
 name = "env_logger"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
-dependencies = [
- "atty",
- "humantime 1.3.0",
- "log",
- "regex",
- "termcolor",
-]
-
-[[package]]
-name = "env_logger"
 version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
 dependencies = [
- "humantime 2.1.0",
+ "humantime",
  "is-terminal",
  "log",
  "regex",
@@ -1380,7 +1335,7 @@
  "intl-memoizer",
  "intl_pluralrules",
  "rustc-hash",
- "self_cell",
+ "self_cell 0.10.3",
  "smallvec",
  "unic-langid",
 ]
@@ -1521,7 +1476,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -1610,7 +1565,7 @@
 dependencies = [
  "compiler_builtins",
  "fallible-iterator",
- "indexmap 2.0.0",
+ "indexmap",
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
  "stable_deref_trait",
@@ -1646,9 +1601,9 @@
 
 [[package]]
 name = "h2"
-version = "0.3.19"
+version = "0.3.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782"
+checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178"
 dependencies = [
  "bytes",
  "fnv",
@@ -1656,7 +1611,7 @@
  "futures-sink",
  "futures-util",
  "http",
- "indexmap 1.9.3",
+ "indexmap",
  "slab",
  "tokio",
  "tokio-util",
@@ -1679,15 +1634,9 @@
 
 [[package]]
 name = "hashbrown"
-version = "0.12.3"
+version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-
-[[package]]
-name = "hashbrown"
-version = "0.14.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
 dependencies = [
  "ahash",
  "allocator-api2",
@@ -1704,15 +1653,6 @@
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "hermit-abi"
 version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
@@ -1804,15 +1744,6 @@
 
 [[package]]
 name = "humantime"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
-dependencies = [
- "quick-error",
-]
-
-[[package]]
-name = "humantime"
 version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
@@ -1968,7 +1899,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -2018,22 +1949,12 @@
 
 [[package]]
 name = "indexmap"
-version = "1.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
-dependencies = [
- "autocfg",
- "hashbrown 0.12.3",
-]
-
-[[package]]
-name = "indexmap"
 version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
 dependencies = [
  "equivalent",
- "hashbrown 0.14.2",
+ "hashbrown",
  "rustc-rayon",
  "serde",
 ]
@@ -2119,16 +2040,16 @@
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb"
 dependencies = [
- "hermit-abi 0.3.2",
+ "hermit-abi",
  "rustix",
  "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "itertools"
-version = "0.10.5"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
 dependencies = [
  "either",
 ]
@@ -2151,9 +2072,9 @@
 
 [[package]]
 name = "jobserver"
-version = "0.1.26"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
+checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
 dependencies = [
  "libc",
 ]
@@ -2195,12 +2116,10 @@
 
 [[package]]
 name = "jsonpath_lib"
-version = "0.2.6"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61352ec23883402b7d30b3313c16cbabefb8907361c4eb669d990cbb87ceee5a"
+checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f"
 dependencies = [
- "array_tool",
- "env_logger 0.7.1",
  "log",
  "serde",
  "serde_json",
@@ -2408,9 +2327,9 @@
 
 [[package]]
 name = "mdbook"
-version = "0.4.31"
+version = "0.4.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b67ee4a744f36e6280792016c17e69921b51df357181d1eb17d620fcc3609f3"
+checksum = "80992cb0e05f22cc052c99f8e883f1593b891014b96a8b4637fd274d7030c85e"
 dependencies = [
  "ammonia",
  "anyhow",
@@ -2418,12 +2337,13 @@
  "clap",
  "clap_complete",
  "elasticlunr-rs",
- "env_logger 0.10.0",
+ "env_logger",
  "handlebars",
  "log",
  "memchr",
  "once_cell",
- "opener",
+ "opener 0.6.1",
+ "pathdiff",
  "pulldown-cmark",
  "regex",
  "serde",
@@ -2436,13 +2356,13 @@
 
 [[package]]
 name = "measureme"
-version = "10.1.1"
+version = "10.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1930d162935fecd56fc4e0f6729eb3483bac1264542eb4ea31570b86a434b6bc"
+checksum = "45e381dcdad44c3c435f8052b08c5c4a1449c48ab56f312345eae12d7a693dbe"
 dependencies = [
  "log",
  "memmap2",
- "parking_lot 0.11.2",
+ "parking_lot",
  "perf-event-open-sys",
  "rustc-hash",
  "smallvec",
@@ -2543,7 +2463,7 @@
  "aes",
  "colored",
  "ctrlc",
- "env_logger 0.10.0",
+ "env_logger",
  "getrandom",
  "lazy_static",
  "libc",
@@ -2613,6 +2533,15 @@
 ]
 
 [[package]]
+name = "normpath"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec60c60a693226186f5d6edf073232bfb6464ed97eb22cf3b01c1e8198fd97f5"
+dependencies = [
+ "windows-sys 0.48.0",
+]
+
+[[package]]
 name = "ntapi"
 version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2647,7 +2576,7 @@
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.3.2",
+ "hermit-abi",
  "libc",
 ]
 
@@ -2659,15 +2588,15 @@
 
 [[package]]
 name = "object"
-version = "0.32.0"
+version = "0.32.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe"
+checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
 dependencies = [
  "compiler_builtins",
  "crc32fast",
  "flate2",
- "hashbrown 0.14.2",
- "indexmap 2.0.0",
+ "hashbrown",
+ "indexmap",
  "memchr",
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
@@ -2700,6 +2629,17 @@
 ]
 
 [[package]]
+name = "opener"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c62dcb6174f9cb326eac248f07e955d5d559c272730b6c03e396b443b562788"
+dependencies = [
+ "bstr",
+ "normpath",
+ "winapi",
+]
+
+[[package]]
 name = "openssl"
 version = "0.10.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2722,7 +2662,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -2752,11 +2692,11 @@
  "camino",
  "clap",
  "derive_builder",
- "env_logger 0.10.0",
+ "env_logger",
  "fs_extra",
  "glob",
  "humansize",
- "humantime 2.1.0",
+ "humantime",
  "log",
  "reqwest",
  "serde",
@@ -2842,37 +2782,12 @@
 
 [[package]]
 name = "parking_lot"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
-dependencies = [
- "instant",
- "lock_api",
- "parking_lot_core 0.8.6",
-]
-
-[[package]]
-name = "parking_lot"
 version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
 dependencies = [
  "lock_api",
- "parking_lot_core 0.9.8",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
-dependencies = [
- "cfg-if",
- "instant",
- "libc",
- "redox_syscall 0.2.16",
- "smallvec",
- "winapi",
+ "parking_lot_core",
 ]
 
 [[package]]
@@ -2939,7 +2854,7 @@
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -3107,12 +3022,6 @@
 checksum = "e9e1dcb320d6839f6edb64f7a4a59d39b30480d4d1765b56873f7c858538a5fe"
 
 [[package]]
-name = "quick-error"
-version = "1.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
-
-[[package]]
 name = "quine-mc_cluskey"
 version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3287,6 +3196,12 @@
 checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
 
 [[package]]
+name = "regex-syntax"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
+
+[[package]]
 name = "remote-test-client"
 version = "0.1.0"
 
@@ -3359,7 +3274,7 @@
 version = "0.1.0"
 dependencies = [
  "clap",
- "env_logger 0.10.0",
+ "env_logger",
  "mdbook",
 ]
 
@@ -3545,6 +3460,7 @@
 name = "rustc_ast_pretty"
 version = "0.0.0"
 dependencies = [
+ "itertools",
  "rustc_ast",
  "rustc_span",
  "thin-vec",
@@ -3597,7 +3513,6 @@
  "rustc_macros",
  "rustc_middle",
  "rustc_mir_dataflow",
- "rustc_serialize",
  "rustc_session",
  "rustc_span",
  "rustc_target",
@@ -3638,7 +3553,6 @@
 version = "0.0.0"
 dependencies = [
  "bitflags 1.3.2",
- "cstr",
  "itertools",
  "libc",
  "measureme",
@@ -3743,13 +3657,13 @@
  "bitflags 1.3.2",
  "elsa",
  "ena",
- "indexmap 2.0.0",
+ "indexmap",
  "itertools",
  "jobserver",
  "libc",
  "measureme",
  "memmap2",
- "parking_lot 0.12.1",
+ "parking_lot",
  "portable-atomic",
  "rustc-hash",
  "rustc-rayon",
@@ -3790,7 +3704,6 @@
  "rustc_const_eval",
  "rustc_data_structures",
  "rustc_error_codes",
- "rustc_error_messages",
  "rustc_errors",
  "rustc_expand",
  "rustc_feature",
@@ -3813,10 +3726,12 @@
  "rustc_monomorphize",
  "rustc_parse",
  "rustc_passes",
+ "rustc_pattern_analysis",
  "rustc_privacy",
  "rustc_query_system",
  "rustc_resolve",
  "rustc_session",
+ "rustc_smir",
  "rustc_span",
  "rustc_symbol_mangling",
  "rustc_target",
@@ -3844,7 +3759,6 @@
  "intl-memoizer",
  "rustc_baked_icu_data",
  "rustc_data_structures",
- "rustc_fluent_macro",
  "rustc_macros",
  "rustc_serialize",
  "rustc_span",
@@ -3922,7 +3836,7 @@
  "fluent-syntax",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
  "unic-langid",
 ]
 
@@ -3942,7 +3856,6 @@
  "rustc_arena",
  "rustc_ast",
  "rustc_data_structures",
- "rustc_error_messages",
  "rustc_index",
  "rustc_macros",
  "rustc_serialize",
@@ -4007,7 +3920,6 @@
  "rustc_lint",
  "rustc_macros",
  "rustc_middle",
- "rustc_serialize",
  "rustc_session",
  "rustc_span",
  "rustc_target",
@@ -4043,12 +3955,23 @@
 version = "0.0.0"
 dependencies = [
  "arrayvec",
+ "rustc_index_macros",
  "rustc_macros",
  "rustc_serialize",
  "smallvec",
 ]
 
 [[package]]
+name = "rustc_index_macros"
+version = "0.0.0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.32",
+ "synstructure",
+]
+
+[[package]]
 name = "rustc_infer"
 version = "0.0.0"
 dependencies = [
@@ -4059,7 +3982,6 @@
  "rustc_index",
  "rustc_macros",
  "rustc_middle",
- "rustc_serialize",
  "rustc_span",
  "rustc_target",
  "smallvec",
@@ -4105,6 +4027,7 @@
  "rustc_query_impl",
  "rustc_query_system",
  "rustc_resolve",
+ "rustc_serialize",
  "rustc_session",
  "rustc_span",
  "rustc_symbol_mangling",
@@ -4190,7 +4113,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
  "synstructure",
 ]
 
@@ -4277,7 +4200,7 @@
  "rustc_infer",
  "rustc_macros",
  "rustc_middle",
- "rustc_serialize",
+ "rustc_pattern_analysis",
  "rustc_session",
  "rustc_span",
  "rustc_target",
@@ -4301,7 +4224,6 @@
  "rustc_index",
  "rustc_macros",
  "rustc_middle",
- "rustc_serialize",
  "rustc_span",
  "rustc_target",
  "smallvec",
@@ -4312,7 +4234,6 @@
 name = "rustc_mir_transform"
 version = "0.0.0"
 dependencies = [
- "coverage_test_macros",
  "either",
  "itertools",
  "rustc_arena",
@@ -4328,7 +4249,6 @@
  "rustc_middle",
  "rustc_mir_build",
  "rustc_mir_dataflow",
- "rustc_serialize",
  "rustc_session",
  "rustc_span",
  "rustc_target",
@@ -4356,6 +4276,13 @@
 ]
 
 [[package]]
+name = "rustc_next_trait_solver"
+version = "0.0.0"
+dependencies = [
+ "rustc_type_ir",
+]
+
+[[package]]
 name = "rustc_parse"
 version = "0.0.0"
 dependencies = [
@@ -4402,7 +4329,6 @@
  "rustc_lexer",
  "rustc_macros",
  "rustc_middle",
- "rustc_serialize",
  "rustc_session",
  "rustc_span",
  "rustc_target",
@@ -4411,6 +4337,27 @@
 ]
 
 [[package]]
+name = "rustc_pattern_analysis"
+version = "0.0.0"
+dependencies = [
+ "rustc_apfloat",
+ "rustc_arena",
+ "rustc_data_structures",
+ "rustc_errors",
+ "rustc_fluent_macro",
+ "rustc_hir",
+ "rustc_index",
+ "rustc_macros",
+ "rustc_middle",
+ "rustc_session",
+ "rustc_span",
+ "rustc_target",
+ "smallvec",
+ "tracing",
+ "typed-arena",
+]
+
+[[package]]
 name = "rustc_privacy"
 version = "0.0.0"
 dependencies = [
@@ -4452,7 +4399,7 @@
 name = "rustc_query_system"
 version = "0.0.0"
 dependencies = [
- "parking_lot 0.12.1",
+ "parking_lot",
  "rustc-rayon-core",
  "rustc_ast",
  "rustc_data_structures",
@@ -4504,7 +4451,7 @@
 name = "rustc_serialize"
 version = "0.0.0"
 dependencies = [
- "indexmap 2.0.0",
+ "indexmap",
  "rustc_macros",
  "smallvec",
  "tempfile",
@@ -4540,6 +4487,7 @@
 name = "rustc_smir"
 version = "0.0.0"
 dependencies = [
+ "rustc_abi",
  "rustc_data_structures",
  "rustc_hir",
  "rustc_middle",
@@ -4554,7 +4502,7 @@
 name = "rustc_span"
 version = "0.0.0"
 dependencies = [
- "indexmap 2.0.0",
+ "indexmap",
  "md-5",
  "rustc_arena",
  "rustc_data_structures",
@@ -4624,9 +4572,9 @@
  "rustc_infer",
  "rustc_macros",
  "rustc_middle",
+ "rustc_next_trait_solver",
  "rustc_parse_format",
  "rustc_query_system",
- "rustc_serialize",
  "rustc_session",
  "rustc_span",
  "rustc_target",
@@ -4713,7 +4661,7 @@
  "arrayvec",
  "askama",
  "expect-test",
- "indexmap 2.0.0",
+ "indexmap",
  "itertools",
  "minifier",
  "once_cell",
@@ -4779,7 +4727,7 @@
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -4903,9 +4851,18 @@
 
 [[package]]
 name = "self_cell"
-version = "0.10.2"
+version = "0.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ef965a420fe14fdac7dd018862966a4c14094f900e1650bbc71ddd7d580c8af"
+checksum = "e14e4d63b804dc0c7ec4a1e52bcb63f02c7ac94476755aa579edac21e01f915d"
+dependencies = [
+ "self_cell 1.0.2",
+]
+
+[[package]]
+name = "self_cell"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e388332cd64eb80cd595a00941baf513caffae8dce9cfd0467fc9c66397dade6"
 
 [[package]]
 name = "semver"
@@ -4933,7 +4890,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -4942,7 +4899,7 @@
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
 dependencies = [
- "indexmap 2.0.0",
+ "indexmap",
  "itoa",
  "ryu",
  "serde",
@@ -5122,8 +5079,8 @@
  "core",
  "dlmalloc",
  "fortanix-sgx-abi",
- "hashbrown 0.14.2",
- "hermit-abi 0.3.2",
+ "hashbrown",
+ "hermit-abi",
  "libc",
  "miniz_oxide",
  "object",
@@ -5159,7 +5116,7 @@
 dependencies = [
  "new_debug_unreachable",
  "once_cell",
- "parking_lot 0.12.1",
+ "parking_lot",
  "phf_shared",
  "precomputed-hash",
  "serde",
@@ -5224,9 +5181,9 @@
 
 [[package]]
 name = "syn"
-version = "2.0.29"
+version = "2.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
+checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5241,7 +5198,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
  "unicode-xid",
 ]
 
@@ -5420,7 +5377,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -5430,7 +5387,7 @@
 checksum = "4db52ee8fec06e119b692ef3dd2c4cf621a99204c1b8c47407870ed050305b9b"
 dependencies = [
  "gimli",
- "hashbrown 0.14.2",
+ "hashbrown",
  "object",
  "tracing",
 ]
@@ -5602,7 +5559,7 @@
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "266f016b7f039eec8a1a80dfe6156b633d208b9fccca5e4db1d6775b0c4e34a7"
 dependencies = [
- "indexmap 2.0.0",
+ "indexmap",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -5641,7 +5598,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
@@ -5684,7 +5641,7 @@
  "matchers",
  "nu-ansi-term",
  "once_cell",
- "parking_lot 0.12.1",
+ "parking_lot",
  "regex",
  "sharded-slab",
  "smallvec",
@@ -5734,6 +5691,12 @@
 ]
 
 [[package]]
+name = "typed-arena"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
+
+[[package]]
 name = "typenum"
 version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5827,9 +5790,9 @@
 
 [[package]]
 name = "unicase"
-version = "2.6.0"
+version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
+checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
 dependencies = [
  "version_check",
 ]
@@ -5932,6 +5895,18 @@
  "compiler_builtins",
  "core",
  "libc",
+ "unwinding",
+]
+
+[[package]]
+name = "unwinding"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37a19a21a537f635c16c7576f22d0f2f7d63353c1337ad4ce0d8001c7952a25b"
+dependencies = [
+ "compiler_builtins",
+ "gimli",
+ "rustc-std-workspace-core",
 ]
 
 [[package]]
@@ -6035,7 +6010,7 @@
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
  "wasm-bindgen-shared",
 ]
 
@@ -6069,7 +6044,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -6132,21 +6107,21 @@
 
 [[package]]
 name = "windows-bindgen"
-version = "0.51.1"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc1f16b778125675feee0d15d6dd9f6af0e3ac52b3233d63a10aa39230c1cd75"
+checksum = "970efb0b6849eb8a87a898f586af7cc167567b070014c7434514c0bde0ca341c"
 dependencies = [
  "proc-macro2",
  "rayon",
- "syn 2.0.29",
+ "syn 2.0.32",
  "windows-metadata",
 ]
 
 [[package]]
 name = "windows-metadata"
-version = "0.51.1"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "753135d996f9da437c0b31dbde3032489a61708361929bcc07d4fba0b161000e"
+checksum = "218fd59201e26acdbb894fa2b302d1de84bf3eec7d0eb894ac8e9c5a854ee4ef"
 
 [[package]]
 name = "windows-sys"
@@ -6380,11 +6355,31 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
  "synstructure",
 ]
 
 [[package]]
+name = "zerocopy"
+version = "0.7.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.32",
+]
+
+[[package]]
 name = "zerofrom"
 version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -6401,7 +6396,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
  "synstructure",
 ]
 
@@ -6424,7 +6419,7 @@
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.32",
 ]
 
 [[package]]
diff --git a/README.md b/README.md
index a88ee4b..5d5beaf 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 [CONTRIBUTING.md](CONTRIBUTING.md) instead.
 
 <details>
-<summary>Table of content</summary>
+<summary>Table of Contents</summary>
 
 - [Quick Start](#quick-start)
 - [Installing from Source](#installing-from-source)
diff --git a/RELEASES.md b/RELEASES.md
index 3fb74b5..038a83c 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -1,3 +1,96 @@
+Version 1.76.0 (2024-02-08)
+==========================
+
+<a id="1.76.0-Language"></a>
+
+Language
+--------
+- [Document Rust ABI compatibility between various types](https://github.com/rust-lang/rust/pull/115476/)
+- [Also: guarantee that char and u32 are ABI-compatible](https://github.com/rust-lang/rust/pull/118032/)
+- [Warn against ambiguous wide pointer comparisons](https://github.com/rust-lang/rust/pull/117758/)
+
+<a id="1.76.0-Compiler"></a>
+
+Compiler
+--------
+- [Lint pinned `#[must_use]` pointers (in particular, `Box<T>` where `T` is `#[must_use]`) in `unused_must_use`.](https://github.com/rust-lang/rust/pull/118054/)
+- [Soundness fix: fix computing the offset of an unsized field in a packed struct](https://github.com/rust-lang/rust/pull/118540/)
+- [Soundness fix: fix dynamic size/align computation logic for packed types with dyn Trait tail](https://github.com/rust-lang/rust/pull/118538/)
+- [Add `$message_type` field to distinguish json diagnostic outputs](https://github.com/rust-lang/rust/pull/115691/)
+- [Enable Rust to use the EHCont security feature of Windows](https://github.com/rust-lang/rust/pull/118013/)
+- [Add tier 3 {x86_64,i686}-win7-windows-msvc targets](https://github.com/rust-lang/rust/pull/118150/)
+- [Add tier 3 aarch64-apple-watchos target](https://github.com/rust-lang/rust/pull/119074/)
+- [Add tier 3 arm64e-apple-ios & arm64e-apple-darwin targets](https://github.com/rust-lang/rust/pull/115526/)
+
+Refer to Rust's [platform support page][platform-support-doc]
+for more information on Rust's tiered platform support.
+
+<a id="1.76.0-Libraries"></a>
+
+Libraries
+---------
+- [Add a column number to `dbg!()`](https://github.com/rust-lang/rust/pull/114962/)
+- [Add `std::hash::{DefaultHasher, RandomState}` exports](https://github.com/rust-lang/rust/pull/115694/)
+- [Fix rounding issue with exponents in fmt](https://github.com/rust-lang/rust/pull/116301/)
+- [Add T: ?Sized to `RwLockReadGuard` and `RwLockWriteGuard`'s Debug impls.](https://github.com/rust-lang/rust/pull/117138/)
+- [Windows: Allow `File::create` to work on hidden files](https://github.com/rust-lang/rust/pull/116438/)
+
+<a id="1.76.0-Stabilized-APIs"></a>
+
+Stabilized APIs
+---------------
+
+- [`Arc::unwrap_or_clone`](https://doc.rust-lang.org/stable/std/sync/struct.Arc.html#method.unwrap_or_clone)
+- [`Rc::unwrap_or_clone`](https://doc.rust-lang.org/stable/std/rc/struct.Rc.html#method.unwrap_or_clone)
+- [`Result::inspect`](https://doc.rust-lang.org/stable/std/result/enum.Result.html#method.inspect)
+- [`Result::inspect_err`](https://doc.rust-lang.org/stable/std/result/enum.Result.html#method.inspect_err)
+- [`Option::inspect`](https://doc.rust-lang.org/stable/std/option/enum.Option.html#method.inspect)
+- [`type_name_of_val`](https://doc.rust-lang.org/stable/std/any/fn.type_name_of_val.html)
+- [`std::hash::{DefaultHasher, RandomState}`](https://doc.rust-lang.org/stable/std/hash/index.html#structs)
+  These were previously available only through `std::collections::hash_map`.
+- [`ptr::{from_ref, from_mut}`](https://doc.rust-lang.org/stable/std/ptr/fn.from_ref.html)
+- [`ptr::addr_eq`](https://doc.rust-lang.org/stable/std/ptr/fn.addr_eq.html)
+
+<a id="1.76.0-Cargo"></a>
+
+Cargo
+-----
+
+See [Cargo release notes](https://github.com/rust-lang/cargo/blob/master/CHANGELOG.md#cargo-176-2024-02-08).
+
+<a id="1.76.0-Rustdoc"></a>
+
+Rustdoc
+-------
+
+- [Don't merge cfg and doc(cfg) attributes for re-exports](https://github.com/rust-lang/rust/pull/113091/)
+- [rustdoc: allow resizing the sidebar / hiding the top bar](https://github.com/rust-lang/rust/pull/115660/)
+- [rustdoc-search: add support for traits and associated types](https://github.com/rust-lang/rust/pull/116085/)
+- [rustdoc: Add highlighting for comments in items declaration](https://github.com/rust-lang/rust/pull/117869/)
+
+<a id="1.76.0-Compatibility-Notes"></a>
+
+Compatibility Notes
+-------------------
+- [Add allow-by-default lint for unit bindings](https://github.com/rust-lang/rust/pull/112380/)
+  This is expected to be upgraded to a warning by default in a future Rust
+  release. Some macros emit bindings with type `()` with user-provided spans,
+  which means that this lint will warn for user code.
+- [Remove x86_64-sun-solaris target.](https://github.com/rust-lang/rust/pull/118091/)
+- [Remove asmjs-unknown-emscripten target](https://github.com/rust-lang/rust/pull/117338/)
+- [Report errors in jobserver inherited through environment variables](https://github.com/rust-lang/rust/pull/113730/)
+  This [may warn](https://github.com/rust-lang/rust/issues/120515) on benign problems too.
+- [Update the minimum external LLVM to 16.](https://github.com/rust-lang/rust/pull/117947/)
+- [Improve `print_tts`](https://github.com/rust-lang/rust/pull/114571/)
+  This change can break some naive manual parsing of token trees in proc macro
+  code which expect a particular structure after `.to_string()`, rather than just arbitrary Rust code.
+- [Make `IMPLIED_BOUNDS_ENTAILMENT` into a hard error from a lint](https://github.com/rust-lang/rust/pull/117984/)
+- [Vec's allocation behavior was changed when collecting some iterators](https://github.com/rust-lang/rust/pull/110353)
+  Allocation behavior is currently not specified, nevertheless changes can be surprising.
+  See [`impl FromIterator for Vec`](https://doc.rust-lang.org/nightly/std/vec/struct.Vec.html#impl-FromIterator%3CT%3E-for-Vec%3CT%3E)
+  for more details.
+- [Properly reject `default` on free const items](https://github.com/rust-lang/rust/pull/117818/)
+
 Version 1.75.0 (2023-12-28)
 ==========================
 
diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs
index 996fd5b..5252472 100644
--- a/compiler/rustc_abi/src/layout.rs
+++ b/compiler/rustc_abi/src/layout.rs
@@ -14,7 +14,7 @@
 pub trait LayoutCalculator {
     type TargetDataLayoutRef: Borrow<TargetDataLayout>;
 
-    fn delay_bug(&self, txt: String);
+    fn delayed_bug(&self, txt: String);
     fn current_data_layout(&self) -> Self::TargetDataLayoutRef;
 
     fn scalar_pair<FieldIdx: Idx, VariantIdx: Idx>(
@@ -111,8 +111,8 @@
                             alt_tail_space,
                             layout.fields.count(),
                             prefer_alt_layout,
-                            format_field_niches(&layout, &fields, &dl),
-                            format_field_niches(&alt_layout, &fields, &dl),
+                            format_field_niches(layout, fields, dl),
+                            format_field_niches(&alt_layout, fields, dl),
                         );
 
                         if prefer_alt_layout {
@@ -382,7 +382,7 @@
                             *offset += this_offset;
                         }
                     }
-                    _ => {
+                    FieldsShape::Primitive | FieldsShape::Array { .. } | FieldsShape::Union(..) => {
                         panic!("Layout of fields should be Arbitrary for variants")
                     }
                 }
@@ -600,7 +600,9 @@
                             variant.size = new_ity_size;
                         }
                     }
-                    _ => panic!(),
+                    FieldsShape::Primitive | FieldsShape::Array { .. } | FieldsShape::Union(..) => {
+                        panic!("encountered a non-arbitrary layout during enum layout")
+                    }
                 }
             }
         }
@@ -628,7 +630,7 @@
             let mut common_prim_initialized_in_all_variants = true;
             for (field_layouts, layout_variant) in iter::zip(variants, &layout_variants) {
                 let FieldsShape::Arbitrary { ref offsets, .. } = layout_variant.fields else {
-                    panic!();
+                    panic!("encountered a non-arbitrary layout during enum layout");
                 };
                 // We skip *all* ZST here and later check if we are good in terms of alignment.
                 // This lets us handle some cases involving aligned ZST.
@@ -681,7 +683,7 @@
                         assert_eq!(memory_index.raw, [0, 1]);
                         offsets
                     }
-                    _ => panic!(),
+                    _ => panic!("encountered a non-arbitrary layout during enum layout"),
                 };
                 if pair_offsets[FieldIdx::new(0)] == Size::ZERO
                     && pair_offsets[FieldIdx::new(1)] == *offset
@@ -758,7 +760,9 @@
             Variants::Multiple { tag, tag_encoding, tag_field, .. } => {
                 Variants::Multiple { tag, tag_encoding, tag_field, variants: best_layout.variants }
             }
-            _ => panic!(),
+            Variants::Single { .. } => {
+                panic!("encountered a single-variant enum during multi-variant layout")
+            }
         };
         Some(best_layout.layout)
     }
@@ -792,7 +796,7 @@
         let only_variant = &variants[VariantIdx::new(0)];
         for field in only_variant {
             if field.is_unsized() {
-                self.delay_bug("unsized field in union".to_string());
+                self.delayed_bug("unsized field in union".to_string());
             }
 
             align = align.max(field.align);
@@ -1025,7 +1029,7 @@
     // At the bottom of this function, we invert `inverse_memory_index` to
     // produce `memory_index` (see `invert_mapping`).
     let mut sized = true;
-    let mut offsets = IndexVec::from_elem(Size::ZERO, &fields);
+    let mut offsets = IndexVec::from_elem(Size::ZERO, fields);
     let mut offset = Size::ZERO;
     let mut largest_niche = None;
     let mut largest_niche_available = 0;
@@ -1038,7 +1042,7 @@
     for &i in &inverse_memory_index {
         let field = &fields[i];
         if !sized {
-            this.delay_bug(format!(
+            this.delayed_bug(format!(
                 "univariant: field #{} comes after unsized field",
                 offsets.len(),
             ));
@@ -1154,7 +1158,11 @@
                                 assert_eq!(memory_index.raw, [0, 1]);
                                 offsets
                             }
-                            _ => panic!(),
+                            FieldsShape::Primitive
+                            | FieldsShape::Array { .. }
+                            | FieldsShape::Union(..) => {
+                                panic!("encountered a non-arbitrary layout during enum layout")
+                            }
                         };
                         if offsets[i] == pair_offsets[FieldIdx::new(0)]
                             && offsets[j] == pair_offsets[FieldIdx::new(1)]
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 09a87cf..eb42803 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1,7 +1,7 @@
 #![cfg_attr(feature = "nightly", feature(step_trait))]
 #![cfg_attr(feature = "nightly", allow(internal_features))]
-#![cfg_attr(all(not(bootstrap), feature = "nightly"), doc(rust_logo))]
-#![cfg_attr(all(not(bootstrap), feature = "nightly"), feature(rustdoc_internals))]
+#![cfg_attr(feature = "nightly", doc(rust_logo))]
+#![cfg_attr(feature = "nightly", feature(rustdoc_internals))]
 
 use std::fmt;
 use std::num::{NonZeroUsize, ParseIntError};
diff --git a/compiler/rustc_arena/src/lib.rs b/compiler/rustc_arena/src/lib.rs
index a53fd4a..621516a 100644
--- a/compiler/rustc_arena/src/lib.rs
+++ b/compiler/rustc_arena/src/lib.rs
@@ -11,8 +11,8 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(no_crate_inject, attr(deny(warnings)))
 )]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(core_intrinsics)]
 #![feature(dropck_eyepatch)]
 #![feature(new_uninit)]
@@ -197,23 +197,24 @@
         start_ptr
     }
 
+    /// Allocates the elements of this iterator into a contiguous slice in the `TypedArena`.
+    ///
+    /// Note: for reasons of reentrancy and panic safety we collect into a `SmallVec<[_; 8]>` before
+    /// storing the elements in the arena.
     #[inline]
     pub fn alloc_from_iter<I: IntoIterator<Item = T>>(&self, iter: I) -> &mut [T] {
-        // This implementation is entirely separate to
-        // `DroplessIterator::alloc_from_iter`, even though conceptually they
-        // are the same.
+        // Despite the similarlty with `DroplessArena`, we cannot reuse their fast case. The reason
+        // is subtle: these arenas are reentrant. In other words, `iter` may very well be holding a
+        // reference to `self` and adding elements to the arena during iteration.
         //
-        // `DroplessIterator` (in the fast case) writes elements from the
-        // iterator one at a time into the allocated memory. That's easy
-        // because the elements don't implement `Drop`. But for `TypedArena`
-        // they do implement `Drop`, which means that if the iterator panics we
-        // could end up with some allocated-but-uninitialized elements, which
-        // will then cause UB in `TypedArena::drop`.
+        // For this reason, if we pre-allocated any space for the elements of this iterator, we'd
+        // have to track that some uninitialized elements are followed by some initialized elements,
+        // else we might accidentally drop uninitialized memory if something panics or if the
+        // iterator doesn't fill all the length we expected.
         //
-        // Instead we use an approach where any iterator panic will occur
-        // before the memory is allocated. This function is much less hot than
-        // `DroplessArena::alloc_from_iter`, so it doesn't need to be
-        // hyper-optimized.
+        // So we collect all the elements beforehand, which takes care of reentrancy and panic
+        // safety. This function is much less hot than `DroplessArena::alloc_from_iter`, so it
+        // doesn't need to be hyper-optimized.
         assert!(mem::size_of::<T>() != 0);
 
         let mut vec: SmallVec<[_; 8]> = iter.into_iter().collect();
@@ -483,10 +484,25 @@
         }
     }
 
+    /// Allocates a string slice that is copied into the `DroplessArena`, returning a
+    /// reference to it. Will panic if passed an empty string.
+    ///
+    /// Panics:
+    ///
+    ///  - Zero-length string
+    #[inline]
+    pub fn alloc_str(&self, string: &str) -> &str {
+        let slice = self.alloc_slice(string.as_bytes());
+
+        // SAFETY: the result has a copy of the same valid UTF-8 bytes.
+        unsafe { std::str::from_utf8_unchecked(slice) }
+    }
+
     /// # Safety
     ///
-    /// The caller must ensure that `mem` is valid for writes up to
-    /// `size_of::<T>() * len`.
+    /// The caller must ensure that `mem` is valid for writes up to `size_of::<T>() * len`, and that
+    /// that memory stays allocated and not shared for the lifetime of `self`. This must hold even
+    /// if `iter.next()` allocates onto `self`.
     #[inline]
     unsafe fn write_from_iter<T, I: Iterator<Item = T>>(
         &self,
@@ -516,6 +532,8 @@
 
     #[inline]
     pub fn alloc_from_iter<T, I: IntoIterator<Item = T>>(&self, iter: I) -> &mut [T] {
+        // Warning: this function is reentrant: `iter` could hold a reference to `&self` and
+        // allocate additional elements while we're iterating.
         let iter = iter.into_iter();
         assert!(mem::size_of::<T>() != 0);
         assert!(!mem::needs_drop::<T>());
@@ -524,7 +542,7 @@
 
         match size_hint {
             (min, Some(max)) if min == max => {
-                // We know the exact number of elements the iterator will produce here
+                // We know the exact number of elements the iterator expects to produce here.
                 let len = min;
 
                 if len == 0 {
@@ -532,10 +550,15 @@
                 }
 
                 let mem = self.alloc_raw(Layout::array::<T>(len).unwrap()) as *mut T;
+                // SAFETY: `write_from_iter` doesn't touch `self`. It only touches the slice we just
+                // reserved. If the iterator panics or doesn't output `len` elements, this will
+                // leave some unallocated slots in the arena, which is fine because we do not call
+                // `drop`.
                 unsafe { self.write_from_iter(iter, len, mem) }
             }
             (_, _) => {
                 outline(move || -> &mut [T] {
+                    // Takes care of reentrancy.
                     let mut vec: SmallVec<[_; 8]> = iter.collect();
                     if vec.is_empty() {
                         return &mut [];
@@ -646,6 +669,14 @@
             self.dropless.alloc_slice(value)
         }
 
+        #[inline]
+        pub fn alloc_str(&self, string: &str) -> &str {
+            if string.is_empty() {
+                return "";
+            }
+            self.dropless.alloc_str(string)
+        }
+
         #[allow(clippy::mut_from_ref)]
         pub fn alloc_from_iter<T: ArenaAllocatable<'tcx, C>, C>(
             &self,
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index c85ff6f..a121b5a 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -301,7 +301,7 @@
     Maybe,
 
     /// `~const Trait`
-    MaybeConst,
+    MaybeConst(Span),
 
     /// `~const !Trait`
     //
@@ -317,8 +317,7 @@
 impl TraitBoundModifier {
     pub fn to_constness(self) -> Const {
         match self {
-            // FIXME(effects) span
-            Self::MaybeConst => Const::Yes(DUMMY_SP),
+            Self::MaybeConst(span) => Const::Yes(span),
             _ => Const::No,
         }
     }
@@ -646,6 +645,7 @@
             // These patterns do not contain subpatterns, skip.
             PatKind::Wild
             | PatKind::Rest
+            | PatKind::Never
             | PatKind::Lit(_)
             | PatKind::Range(..)
             | PatKind::Ident(..)
@@ -658,6 +658,37 @@
     pub fn is_rest(&self) -> bool {
         matches!(self.kind, PatKind::Rest)
     }
+
+    /// Whether this could be a never pattern, taking into account that a macro invocation can
+    /// return a never pattern. Used to inform errors during parsing.
+    pub fn could_be_never_pattern(&self) -> bool {
+        let mut could_be_never_pattern = false;
+        self.walk(&mut |pat| match &pat.kind {
+            PatKind::Never | PatKind::MacCall(_) => {
+                could_be_never_pattern = true;
+                false
+            }
+            PatKind::Or(s) => {
+                could_be_never_pattern = s.iter().all(|p| p.could_be_never_pattern());
+                false
+            }
+            _ => true,
+        });
+        could_be_never_pattern
+    }
+
+    /// Whether this contains a `!` pattern. This in particular means that a feature gate error will
+    /// be raised if the feature is off. Used to avoid gating the feature twice.
+    pub fn contains_never_pattern(&self) -> bool {
+        let mut contains_never_pattern = false;
+        self.walk(&mut |pat| {
+            if matches!(pat.kind, PatKind::Never) {
+                contains_never_pattern = true;
+            }
+            true
+        });
+        contains_never_pattern
+    }
 }
 
 /// A single field in a struct pattern.
@@ -796,6 +827,9 @@
     /// only one rest pattern may occur in the pattern sequences.
     Rest,
 
+    // A never pattern `!`
+    Never,
+
     /// Parentheses in patterns used for grouping (i.e., `(PAT)`).
     Paren(P<Pat>),
 
@@ -818,7 +852,7 @@
     Raw,
 }
 
-#[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy)]
+#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum BinOpKind {
     /// The `+` operator (addition)
     Add,
@@ -859,9 +893,9 @@
 }
 
 impl BinOpKind {
-    pub fn to_string(&self) -> &'static str {
+    pub fn as_str(&self) -> &'static str {
         use BinOpKind::*;
-        match *self {
+        match self {
             Add => "+",
             Sub => "-",
             Mul => "*",
@@ -882,19 +916,25 @@
             Gt => ">",
         }
     }
-    pub fn lazy(&self) -> bool {
+
+    pub fn is_lazy(&self) -> bool {
         matches!(self, BinOpKind::And | BinOpKind::Or)
     }
 
     pub fn is_comparison(&self) -> bool {
         use BinOpKind::*;
-        // Note for developers: please keep this as is;
+        // Note for developers: please keep this match exhaustive;
         // we want compilation to fail if another variant is added.
         match *self {
             Eq | Lt | Le | Ne | Gt | Ge => true,
             And | Or | Add | Sub | Mul | Div | Rem | BitXor | BitAnd | BitOr | Shl | Shr => false,
         }
     }
+
+    /// Returns `true` if the binary operator takes its arguments by value.
+    pub fn is_by_value(self) -> bool {
+        !self.is_comparison()
+    }
 }
 
 pub type BinOp = Spanned<BinOpKind>;
@@ -902,7 +942,7 @@
 /// Unary operator.
 ///
 /// Note that `&data` is not an operator, it's an `AddrOf` expression.
-#[derive(Clone, Encodable, Decodable, Debug, Copy)]
+#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum UnOp {
     /// The `*` operator for dereferencing
     Deref,
@@ -913,13 +953,18 @@
 }
 
 impl UnOp {
-    pub fn to_string(op: UnOp) -> &'static str {
-        match op {
+    pub fn as_str(&self) -> &'static str {
+        match self {
             UnOp::Deref => "*",
             UnOp::Not => "!",
             UnOp::Neg => "-",
         }
     }
+
+    /// Returns `true` if the unary operator takes its argument by value.
+    pub fn is_by_value(self) -> bool {
+        matches!(self, Self::Neg | Self::Not)
+    }
 }
 
 /// A statement
@@ -1066,8 +1111,8 @@
     pub pat: P<Pat>,
     /// Match arm guard, e.g. `n > 10` in `match foo { n if n > 10 => {}, _ => {} }`
     pub guard: Option<P<Expr>>,
-    /// Match arm body.
-    pub body: P<Expr>,
+    /// Match arm body. Omitted if the pattern is a never pattern.
+    pub body: Option<P<Expr>>,
     pub span: Span,
     pub id: NodeId,
     pub is_placeholder: bool,
@@ -1297,7 +1342,7 @@
     pub binder: ClosureBinder,
     pub capture_clause: CaptureBy,
     pub constness: Const,
-    pub asyncness: Async,
+    pub coroutine_kind: Option<CoroutineKind>,
     pub movability: Movability,
     pub fn_decl: P<FnDecl>,
     pub body: P<Expr>,
@@ -1502,6 +1547,7 @@
 pub enum GenBlockKind {
     Async,
     Gen,
+    AsyncGen,
 }
 
 impl fmt::Display for GenBlockKind {
@@ -1515,6 +1561,7 @@
         match self {
             GenBlockKind::Async => "async",
             GenBlockKind::Gen => "gen",
+            GenBlockKind::AsyncGen => "async gen",
         }
     }
 }
@@ -2237,6 +2284,18 @@
     },
 }
 
+impl InlineAsmOperand {
+    pub fn reg(&self) -> Option<&InlineAsmRegOrRegClass> {
+        match self {
+            Self::In { reg, .. }
+            | Self::Out { reg, .. }
+            | Self::InOut { reg, .. }
+            | Self::SplitInOut { reg, .. } => Some(reg),
+            Self::Const { .. } | Self::Sym { .. } => None,
+        }
+    }
+}
+
 /// Inline assembly.
 ///
 /// E.g., `asm!("NOP");`.
@@ -2380,28 +2439,47 @@
     No,
 }
 
+/// Describes what kind of coroutine markers, if any, a function has.
+///
+/// Coroutine markers are things that cause the function to generate a coroutine, such as `async`,
+/// which makes the function return `impl Future`, or `gen`, which makes the function return `impl
+/// Iterator`.
 #[derive(Copy, Clone, Encodable, Decodable, Debug)]
-pub enum Async {
-    Yes { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
-    No,
+pub enum CoroutineKind {
+    /// `async`, which returns an `impl Future`
+    Async { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
+    /// `gen`, which returns an `impl Iterator`
+    Gen { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
+    /// `async gen`, which returns an `impl AsyncIterator`
+    AsyncGen { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
 }
 
-#[derive(Copy, Clone, Encodable, Decodable, Debug)]
-pub enum Gen {
-    Yes { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
-    No,
-}
-
-impl Async {
+impl CoroutineKind {
     pub fn is_async(self) -> bool {
-        matches!(self, Async::Yes { .. })
+        matches!(self, CoroutineKind::Async { .. })
     }
 
-    /// In this case this is an `async` return, the `NodeId` for the generated `impl Trait` item.
-    pub fn opt_return_id(self) -> Option<(NodeId, Span)> {
+    pub fn is_gen(self) -> bool {
+        matches!(self, CoroutineKind::Gen { .. })
+    }
+
+    pub fn closure_id(self) -> NodeId {
         match self {
-            Async::Yes { return_impl_trait_id, span, .. } => Some((return_impl_trait_id, span)),
-            Async::No => None,
+            CoroutineKind::Async { closure_id, .. }
+            | CoroutineKind::Gen { closure_id, .. }
+            | CoroutineKind::AsyncGen { closure_id, .. } => closure_id,
+        }
+    }
+
+    /// In this case this is an `async` or `gen` return, the `NodeId` for the generated `impl Trait`
+    /// item.
+    pub fn return_id(self) -> (NodeId, Span) {
+        match self {
+            CoroutineKind::Async { return_impl_trait_id, span, .. }
+            | CoroutineKind::Gen { return_impl_trait_id, span, .. }
+            | CoroutineKind::AsyncGen { return_impl_trait_id, span, .. } => {
+                (return_impl_trait_id, span)
+            }
         }
     }
 }
@@ -2574,7 +2652,7 @@
 }
 
 rustc_index::newtype_index! {
-    #[custom_encodable]
+    #[orderable]
     #[debug_format = "AttrId({})"]
     pub struct AttrId {}
 }
@@ -2710,7 +2788,11 @@
     /// Struct variant.
     ///
     /// E.g., `Bar { .. }` as in `enum Foo { Bar { .. } }`.
-    Struct(ThinVec<FieldDef>, bool),
+    Struct {
+        fields: ThinVec<FieldDef>,
+        // FIXME: investigate making this a `Option<ErrorGuaranteed>`
+        recovered: bool,
+    },
     /// Tuple variant.
     ///
     /// E.g., `Bar(..)` as in `enum Foo { Bar(..) }`.
@@ -2725,7 +2807,7 @@
     /// Return the fields of this variant.
     pub fn fields(&self) -> &[FieldDef] {
         match self {
-            VariantData::Struct(fields, ..) | VariantData::Tuple(fields, _) => fields,
+            VariantData::Struct { fields, .. } | VariantData::Tuple(fields, _) => fields,
             _ => &[],
         }
     }
@@ -2733,7 +2815,7 @@
     /// Return the `NodeId` of this variant's constructor, if it has one.
     pub fn ctor_node_id(&self) -> Option<NodeId> {
         match *self {
-            VariantData::Struct(..) => None,
+            VariantData::Struct { .. } => None,
             VariantData::Tuple(_, id) | VariantData::Unit(id) => Some(id),
         }
     }
@@ -2767,6 +2849,28 @@
     pub fn span_with_attributes(&self) -> Span {
         self.attrs.iter().fold(self.span, |acc, attr| acc.to(attr.span))
     }
+
+    pub fn opt_generics(&self) -> Option<&Generics> {
+        match &self.kind {
+            ItemKind::ExternCrate(_)
+            | ItemKind::Use(_)
+            | ItemKind::Mod(_, _)
+            | ItemKind::ForeignMod(_)
+            | ItemKind::GlobalAsm(_)
+            | ItemKind::MacCall(_)
+            | ItemKind::MacroDef(_) => None,
+            ItemKind::Static(_) => None,
+            ItemKind::Const(i) => Some(&i.generics),
+            ItemKind::Fn(i) => Some(&i.generics),
+            ItemKind::TyAlias(i) => Some(&i.generics),
+            ItemKind::TraitAlias(generics, _)
+            | ItemKind::Enum(_, generics)
+            | ItemKind::Struct(_, generics)
+            | ItemKind::Union(_, generics) => Some(&generics),
+            ItemKind::Trait(i) => Some(&i.generics),
+            ItemKind::Impl(i) => Some(&i.generics),
+        }
+    }
 }
 
 /// `extern` qualifier on a function item or function type.
@@ -2805,8 +2909,8 @@
 pub struct FnHeader {
     /// The `unsafe` keyword, if any
     pub unsafety: Unsafe,
-    /// The `async` keyword, if any
-    pub asyncness: Async,
+    /// Whether this is `async`, `gen`, or nothing.
+    pub coroutine_kind: Option<CoroutineKind>,
     /// The `const` keyword, if any
     pub constness: Const,
     /// The `extern` keyword and corresponding ABI string, if any
@@ -2816,9 +2920,9 @@
 impl FnHeader {
     /// Does this function header have any qualifiers or is it empty?
     pub fn has_qualifiers(&self) -> bool {
-        let Self { unsafety, asyncness, constness, ext } = self;
+        let Self { unsafety, coroutine_kind, constness, ext } = self;
         matches!(unsafety, Unsafe::Yes(_))
-            || asyncness.is_async()
+            || coroutine_kind.is_some()
             || matches!(constness, Const::Yes(_))
             || !matches!(ext, Extern::None)
     }
@@ -2828,7 +2932,7 @@
     fn default() -> FnHeader {
         FnHeader {
             unsafety: Unsafe::No,
-            asyncness: Async::No,
+            coroutine_kind: None,
             constness: Const::No,
             ext: Extern::None,
         }
@@ -3151,11 +3255,11 @@
     static_assert_size!(Block, 32);
     static_assert_size!(Expr, 72);
     static_assert_size!(ExprKind, 40);
-    static_assert_size!(Fn, 152);
+    static_assert_size!(Fn, 160);
     static_assert_size!(ForeignItem, 96);
     static_assert_size!(ForeignItemKind, 24);
     static_assert_size!(GenericArg, 24);
-    static_assert_size!(GenericBound, 56);
+    static_assert_size!(GenericBound, 64);
     static_assert_size!(Generics, 40);
     static_assert_size!(Impl, 136);
     static_assert_size!(Item, 136);
diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
index be7d1b2..98138ce 100644
--- a/compiler/rustc_ast/src/attr/mod.rs
+++ b/compiler/rustc_ast/src/attr/mod.rs
@@ -342,7 +342,7 @@
                 let span = span.with_hi(segments.last().unwrap().ident.span.hi());
                 Path { span, segments, tokens: None }
             }
-            Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &**nt {
+            Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &nt.0 {
                 token::Nonterminal::NtMeta(item) => return item.meta(item.path.span),
                 token::Nonterminal::NtPath(path) => (**path).clone(),
                 _ => return None,
@@ -387,11 +387,11 @@
         tokens: &mut impl Iterator<Item = &'a TokenTree>,
     ) -> Option<MetaItemKind> {
         match tokens.next() {
-            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
             }
             Some(TokenTree::Token(token, _)) => {
-                MetaItemLit::from_token(&token).map(MetaItemKind::NameValue)
+                MetaItemLit::from_token(token).map(MetaItemKind::NameValue)
             }
             _ => None,
         }
@@ -401,7 +401,7 @@
         tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
     ) -> Option<MetaItemKind> {
         match tokens.peek() {
-            Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
                 let inner_tokens = inner_tokens.clone();
                 tokens.next();
                 MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
@@ -524,7 +524,7 @@
                 tokens.next();
                 return Some(NestedMetaItem::Lit(lit));
             }
-            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
                 tokens.next();
                 return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
             }
diff --git a/compiler/rustc_ast/src/lib.rs b/compiler/rustc_ast/src/lib.rs
index c1f6ad6..7e713a4 100644
--- a/compiler/rustc_ast/src/lib.rs
+++ b/compiler/rustc_ast/src/lib.rs
@@ -8,9 +8,9 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(attr(deny(warnings)))
 )]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(const_trait_impl)]
@@ -59,9 +59,7 @@
 /// Requirements for a `StableHashingContext` to be used in this crate.
 /// This is a hack to allow using the `HashStable_Generic` derive macro
 /// instead of implementing everything in `rustc_middle`.
-pub trait HashStableContext:
-    rustc_type_ir::HashStableContext + rustc_span::HashStableContext
-{
+pub trait HashStableContext: rustc_span::HashStableContext {
     fn hash_attr(&mut self, _: &ast::Attribute, hasher: &mut StableHasher);
 }
 
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index 7c0a782..557ae02 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -7,10 +7,10 @@
 //! a `MutVisitor` renaming item names in a module will miss all of those
 //! that are created by the expansion of a macro.
 
+use crate::ast::*;
 use crate::ptr::P;
 use crate::token::{self, Token};
 use crate::tokenstream::*;
-use crate::{ast::*, StaticItem};
 
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -121,8 +121,8 @@
         noop_visit_fn_decl(d, self);
     }
 
-    fn visit_asyncness(&mut self, a: &mut Async) {
-        noop_visit_asyncness(a, self);
+    fn visit_coroutine_kind(&mut self, a: &mut CoroutineKind) {
+        noop_visit_coroutine_kind(a, self);
     }
 
     fn visit_closure_binder(&mut self, b: &mut ClosureBinder) {
@@ -453,7 +453,7 @@
     vis.visit_id(id);
     vis.visit_pat(pat);
     visit_opt(guard, |guard| vis.visit_expr(guard));
-    vis.visit_expr(body);
+    visit_opt(body, |body| vis.visit_expr(body));
     vis.visit_span(span);
     smallvec![arm]
 }
@@ -682,7 +682,7 @@
         AttrTokenTree::Token(token, _) => {
             visit_token(token, vis);
         }
-        AttrTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
+        AttrTokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
             vis.visit_span(open);
             vis.visit_span(close);
             visit_attr_tts(tts, vis);
@@ -709,7 +709,7 @@
         TokenTree::Token(token, _) => {
             visit_token(token, vis);
         }
-        TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
+        TokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
             vis.visit_span(open);
             vis.visit_span(close);
             visit_tts(tts, vis);
@@ -764,7 +764,10 @@
             return; // Avoid visiting the span for the second time.
         }
         token::Interpolated(nt) => {
-            visit_nonterminal(Lrc::make_mut(nt), vis);
+            let nt = Lrc::make_mut(nt);
+            let (nt, sp) = (&mut nt.0, &mut nt.1);
+            vis.visit_span(sp);
+            visit_nonterminal(nt, vis);
         }
         _ => {}
     }
@@ -868,13 +871,15 @@
     }
 }
 
-pub fn noop_visit_asyncness<T: MutVisitor>(asyncness: &mut Async, vis: &mut T) {
-    match asyncness {
-        Async::Yes { span: _, closure_id, return_impl_trait_id } => {
+pub fn noop_visit_coroutine_kind<T: MutVisitor>(coroutine_kind: &mut CoroutineKind, vis: &mut T) {
+    match coroutine_kind {
+        CoroutineKind::Async { span, closure_id, return_impl_trait_id }
+        | CoroutineKind::Gen { span, closure_id, return_impl_trait_id }
+        | CoroutineKind::AsyncGen { span, closure_id, return_impl_trait_id } => {
+            vis.visit_span(span);
             vis.visit_id(closure_id);
             vis.visit_id(return_impl_trait_id);
         }
-        Async::No => {}
     }
 }
 
@@ -971,7 +976,7 @@
 
 pub fn noop_visit_variant_data<T: MutVisitor>(vdata: &mut VariantData, vis: &mut T) {
     match vdata {
-        VariantData::Struct(fields, ..) => {
+        VariantData::Struct { fields, .. } => {
             fields.flat_map_in_place(|field| vis.flat_map_field_def(field));
         }
         VariantData::Tuple(fields, id) => {
@@ -1167,9 +1172,9 @@
 }
 
 pub fn noop_visit_fn_header<T: MutVisitor>(header: &mut FnHeader, vis: &mut T) {
-    let FnHeader { unsafety, asyncness, constness, ext: _ } = header;
+    let FnHeader { unsafety, coroutine_kind, constness, ext: _ } = header;
     visit_constness(constness, vis);
-    vis.visit_asyncness(asyncness);
+    coroutine_kind.as_mut().map(|coroutine_kind| vis.visit_coroutine_kind(coroutine_kind));
     visit_unsafety(unsafety, vis);
 }
 
@@ -1246,7 +1251,7 @@
     let Pat { id, kind, span, tokens } = pat.deref_mut();
     vis.visit_id(id);
     match kind {
-        PatKind::Wild | PatKind::Rest => {}
+        PatKind::Wild | PatKind::Rest | PatKind::Never => {}
         PatKind::Ident(_binding_mode, ident, sub) => {
             vis.visit_ident(ident);
             visit_opt(sub, |sub| vis.visit_pat(sub));
@@ -1403,7 +1408,7 @@
             binder,
             capture_clause,
             constness,
-            asyncness,
+            coroutine_kind,
             movability: _,
             fn_decl,
             body,
@@ -1412,7 +1417,7 @@
         }) => {
             vis.visit_closure_binder(binder);
             visit_constness(constness, vis);
-            vis.visit_asyncness(asyncness);
+            coroutine_kind.as_mut().map(|coroutine_kind| vis.visit_coroutine_kind(coroutine_kind));
             vis.visit_capture_by(capture_clause);
             vis.visit_fn_decl(fn_decl);
             vis.visit_expr(body);
diff --git a/compiler/rustc_ast/src/node_id.rs b/compiler/rustc_ast/src/node_id.rs
index d167417..1cd2449 100644
--- a/compiler/rustc_ast/src/node_id.rs
+++ b/compiler/rustc_ast/src/node_id.rs
@@ -8,6 +8,8 @@
     /// This is later turned into [`DefId`] and `HirId` for the HIR.
     ///
     /// [`DefId`]: rustc_span::def_id::DefId
+    #[encodable]
+    #[orderable]
     #[debug_format = "NodeId({})"]
     pub struct NodeId {
         /// The [`NodeId`] used to represent the root of the crate.
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
index 914c97a..b0cd2ec 100644
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@@ -13,7 +13,7 @@
 use rustc_span::symbol::{kw, sym};
 #[allow(hidden_glob_reexports)]
 use rustc_span::symbol::{Ident, Symbol};
-use rustc_span::{self, edition::Edition, Span, DUMMY_SP};
+use rustc_span::{edition::Edition, Span, DUMMY_SP};
 use std::borrow::Cow;
 use std::fmt;
 
@@ -110,7 +110,7 @@
             Ident(name, false) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
             Literal(token_lit) => Some(token_lit),
             Interpolated(ref nt)
-                if let NtExpr(expr) | NtLiteral(expr) = &**nt
+                if let NtExpr(expr) | NtLiteral(expr) = &nt.0
                     && let ast::ExprKind::Lit(token_lit) = expr.kind =>
             {
                 Some(token_lit)
@@ -238,9 +238,9 @@
     EqEq,
     /// `!=`
     Ne,
-    /// `>`
-    Ge,
     /// `>=`
+    Ge,
+    /// `>`
     Gt,
     /// `&&`
     AndAnd,
@@ -314,7 +314,7 @@
     /// - It prevents `Token` from implementing `Copy`.
     /// It adds complexity and likely slows things down. Please don't add new
     /// occurrences of this token kind!
-    Interpolated(Lrc<Nonterminal>),
+    Interpolated(Lrc<(Nonterminal, Span)>),
 
     /// A doc comment token.
     /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
@@ -388,7 +388,8 @@
         match *self {
             Comma => Some(vec![Dot, Lt, Semi]),
             Semi => Some(vec![Colon, Comma]),
-            FatArrow => Some(vec![Eq, RArrow]),
+            Colon => Some(vec![Semi]),
+            FatArrow => Some(vec![Eq, RArrow, Ge, Gt]),
             _ => None,
         }
     }
@@ -421,7 +422,7 @@
     /// if they keep spans or perform edition checks.
     pub fn uninterpolated_span(&self) -> Span {
         match &self.kind {
-            Interpolated(nt) => nt.span(),
+            Interpolated(nt) => nt.0.use_span(),
             _ => self.span,
         }
     }
@@ -464,7 +465,7 @@
             ModSep                            | // global path
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
-            Interpolated(ref nt) => matches!(**nt, NtLiteral(..) |
+            Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) |
                 NtExpr(..)    |
                 NtBlock(..)   |
                 NtPath(..)),
@@ -488,7 +489,7 @@
             | DotDot | DotDotDot | DotDotEq      // ranges
             | Lt | BinOp(Shl)                    // associated path
             | ModSep                    => true, // global path
-            Interpolated(ref nt) => matches!(**nt, NtLiteral(..) |
+            Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) |
                 NtPat(..)     |
                 NtBlock(..)   |
                 NtPath(..)),
@@ -511,7 +512,7 @@
             Lifetime(..)                | // lifetime bound in trait object
             Lt | BinOp(Shl)             | // associated path
             ModSep                      => true, // global path
-            Interpolated(ref nt) => matches!(**nt, NtTy(..) | NtPath(..)),
+            Interpolated(ref nt) => matches!(&nt.0, NtTy(..) | NtPath(..)),
             // For anonymous structs or unions, which only appear in specific positions
             // (type of struct fields or union fields), we don't consider them as regular types
             _ => false,
@@ -522,7 +523,7 @@
     pub fn can_begin_const_arg(&self) -> bool {
         match self.kind {
             OpenDelim(Delimiter::Brace) => true,
-            Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
+            Interpolated(ref nt) => matches!(&nt.0, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
             _ => self.can_begin_literal_maybe_minus(),
         }
     }
@@ -576,7 +577,7 @@
         match self.uninterpolate().kind {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, false) if name.is_bool_lit() => true,
-            Interpolated(ref nt) => match &**nt {
+            Interpolated(ref nt) => match &nt.0 {
                 NtLiteral(_) => true,
                 NtExpr(e) => match &e.kind {
                     ast::ExprKind::Lit(_) => true,
@@ -597,9 +598,9 @@
     /// otherwise returns the original token.
     pub fn uninterpolate(&self) -> Cow<'_, Token> {
         match &self.kind {
-            Interpolated(nt) => match **nt {
+            Interpolated(nt) => match &nt.0 {
                 NtIdent(ident, is_raw) => {
-                    Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
+                    Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span))
                 }
                 NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
                 _ => Cow::Borrowed(self),
@@ -614,8 +615,8 @@
         // We avoid using `Token::uninterpolate` here because it's slow.
         match &self.kind {
             &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
-            Interpolated(nt) => match **nt {
-                NtIdent(ident, is_raw) => Some((ident, is_raw)),
+            Interpolated(nt) => match &nt.0 {
+                NtIdent(ident, is_raw) => Some((*ident, *is_raw)),
                 _ => None,
             },
             _ => None,
@@ -628,8 +629,8 @@
         // We avoid using `Token::uninterpolate` here because it's slow.
         match &self.kind {
             &Lifetime(name) => Some(Ident::new(name, self.span)),
-            Interpolated(nt) => match **nt {
-                NtLifetime(ident) => Some(ident),
+            Interpolated(nt) => match &nt.0 {
+                NtLifetime(ident) => Some(*ident),
                 _ => None,
             },
             _ => None,
@@ -655,7 +656,7 @@
     /// Returns `true` if the token is an interpolated path.
     fn is_path(&self) -> bool {
         if let Interpolated(nt) = &self.kind
-            && let NtPath(..) = **nt
+            && let NtPath(..) = &nt.0
         {
             return true;
         }
@@ -668,7 +669,7 @@
     /// (which happens while parsing the result of macro expansion)?
     pub fn is_whole_expr(&self) -> bool {
         if let Interpolated(nt) = &self.kind
-            && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt
+            && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = &nt.0
         {
             return true;
         }
@@ -679,7 +680,7 @@
     /// Is the token an interpolated block (`$b:block`)?
     pub fn is_whole_block(&self) -> bool {
         if let Interpolated(nt) = &self.kind
-            && let NtBlock(..) = **nt
+            && let NtBlock(..) = &nt.0
         {
             return true;
         }
@@ -755,6 +756,11 @@
         )
     }
 
+    /// Returns `true` if the token is the integer literal.
+    pub fn is_integer_lit(&self) -> bool {
+        matches!(self.kind, Literal(Lit { kind: LitKind::Integer, .. }))
+    }
+
     /// Returns `true` if the token is a non-raw identifier for which `pred` holds.
     pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
         match self.ident() {
@@ -927,7 +933,7 @@
 }
 
 impl Nonterminal {
-    pub fn span(&self) -> Span {
+    pub fn use_span(&self) -> Span {
         match self {
             NtItem(item) => item.span,
             NtBlock(block) => block.span,
@@ -941,6 +947,23 @@
             NtVis(vis) => vis.span,
         }
     }
+
+    pub fn descr(&self) -> &'static str {
+        match self {
+            NtItem(..) => "item",
+            NtBlock(..) => "block",
+            NtStmt(..) => "statement",
+            NtPat(..) => "pattern",
+            NtExpr(..) => "expression",
+            NtLiteral(..) => "literal",
+            NtTy(..) => "type",
+            NtIdent(..) => "identifier",
+            NtLifetime(..) => "lifetime",
+            NtMeta(..) => "attribute",
+            NtPath(..) => "path",
+            NtVis(..) => "visibility",
+        }
+    }
 }
 
 impl PartialEq for Nonterminal {
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index 23b8f9c..4c0c496 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -46,7 +46,7 @@
     /// delimiters are implicitly represented by `Delimited`.
     Token(Token, Spacing),
     /// A delimited sequence of token trees.
-    Delimited(DelimSpan, Delimiter, TokenStream),
+    Delimited(DelimSpan, DelimSpacing, Delimiter, TokenStream),
 }
 
 // Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
@@ -62,11 +62,11 @@
 }
 
 impl TokenTree {
-    /// Checks if this `TokenTree` is equal to the other, regardless of span information.
+    /// Checks if this `TokenTree` is equal to the other, regardless of span/spacing information.
     pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
         match (self, other) {
             (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
-            (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
+            (TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
                 delim == delim2 && tts.eq_unspanned(tts2)
             }
             _ => false,
@@ -99,6 +99,11 @@
         TokenTree::Token(Token::new(kind, span), Spacing::Joint)
     }
 
+    /// Create a `TokenTree::Token` with joint-hidden spacing.
+    pub fn token_joint_hidden(kind: TokenKind, span: Span) -> TokenTree {
+        TokenTree::Token(Token::new(kind, span), Spacing::JointHidden)
+    }
+
     pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
         match self {
             TokenTree::Token(token, spacing) => match token.uninterpolate() {
@@ -183,7 +188,7 @@
 #[derive(Clone, Debug, Encodable, Decodable)]
 pub enum AttrTokenTree {
     Token(Token, Spacing),
-    Delimited(DelimSpan, Delimiter, AttrTokenStream),
+    Delimited(DelimSpan, DelimSpacing, Delimiter, AttrTokenStream),
     /// Stores the attributes for an attribute target,
     /// along with the tokens for that attribute target.
     /// See `AttributesData` for more information
@@ -208,9 +213,14 @@
                 AttrTokenTree::Token(inner, spacing) => {
                     smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
                 }
-                AttrTokenTree::Delimited(span, delim, stream) => {
-                    smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
-                        .into_iter()
+                AttrTokenTree::Delimited(span, spacing, delim, stream) => {
+                    smallvec![TokenTree::Delimited(
+                        *span,
+                        *spacing,
+                        *delim,
+                        stream.to_tokenstream()
+                    ),]
+                    .into_iter()
                 }
                 AttrTokenTree::Attributes(data) => {
                     let idx = data
@@ -230,7 +240,7 @@
                         let mut found = false;
                         // Check the last two trees (to account for a trailing semi)
                         for tree in target_tokens.iter_mut().rev().take(2) {
-                            if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
+                            if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
                                 // Inner attributes are only supported on extern blocks, functions,
                                 // impls, and modules. All of these have their inner attributes
                                 // placed at the beginning of the rightmost outermost braced group:
@@ -250,7 +260,7 @@
                                     stream.push_stream(inner_attr.tokens());
                                 }
                                 stream.push_stream(delim_tokens.clone());
-                                *tree = TokenTree::Delimited(*span, *delim, stream);
+                                *tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
                                 found = true;
                                 break;
                             }
@@ -303,21 +313,64 @@
 #[derive(Clone, Debug, Default, Encodable, Decodable)]
 pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);
 
-/// Similar to `proc_macro::Spacing`, but for tokens.
-///
-/// Note that all `ast::TokenTree::Token` instances have a `Spacing`, but when
-/// we convert to `proc_macro::TokenTree` for proc macros only `Punct`
-/// `TokenTree`s have a `proc_macro::Spacing`.
+/// Indicates whether a token can join with the following token to form a
+/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
+/// guide pretty-printing, which is where the `JointHidden` value (which isn't
+/// part of `proc_macro::Spacing`) comes in useful.
 #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum Spacing {
-    /// The token is not immediately followed by an operator token (as
-    /// determined by `Token::is_op`). E.g. a `+` token is `Alone` in `+ =`,
-    /// `+/*foo*/=`, `+ident`, and `+()`.
+    /// The token cannot join with the following token to form a compound
+    /// token.
+    ///
+    /// In token streams parsed from source code, the compiler will use `Alone`
+    /// for any token immediately followed by whitespace, a non-doc comment, or
+    /// EOF.
+    ///
+    /// When constructing token streams within the compiler, use this for each
+    /// token that (a) should be pretty-printed with a space after it, or (b)
+    /// is the last token in the stream. (In the latter case the choice of
+    /// spacing doesn't matter because it is never used for the last token. We
+    /// arbitrarily use `Alone`.)
+    ///
+    /// Converts to `proc_macro::Spacing::Alone`, and
+    /// `proc_macro::Spacing::Alone` converts back to this.
     Alone,
 
-    /// The token is immediately followed by an operator token. E.g. a `+`
-    /// token is `Joint` in `+=` and `++`.
+    /// The token can join with the following token to form a compound token.
+    ///
+    /// In token streams parsed from source code, the compiler will use `Joint`
+    /// for any token immediately followed by punctuation (as determined by
+    /// `Token::is_punct`).
+    ///
+    /// When constructing token streams within the compiler, use this for each
+    /// token that (a) should be pretty-printed without a space after it, and
+    /// (b) is followed by a punctuation token.
+    ///
+    /// Converts to `proc_macro::Spacing::Joint`, and
+    /// `proc_macro::Spacing::Joint` converts back to this.
     Joint,
+
+    /// The token can join with the following token to form a compound token,
+    /// but this will not be visible at the proc macro level. (This is what the
+    /// `Hidden` means; see below.)
+    ///
+    /// In token streams parsed from source code, the compiler will use
+    /// `JointHidden` for any token immediately followed by anything not
+    /// covered by the `Alone` and `Joint` cases: an identifier, lifetime,
+    /// literal, delimiter, doc comment.
+    ///
+    /// When constructing token streams, use this for each token that (a)
+    /// should be pretty-printed without a space after it, and (b) is followed
+    /// by a non-punctuation token.
+    ///
+    /// Converts to `proc_macro::Spacing::Alone`, but
+    /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`.
+    /// Because of that, pretty-printing of `TokenStream`s produced by proc
+    /// macros is unavoidably uglier (with more whitespace between tokens) than
+    /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed
+    /// source code, internally constructed token streams, and token streams
+    /// produced by declarative macros).
+    JointHidden,
 }
 
 impl TokenStream {
@@ -421,21 +474,14 @@
         self
     }
 
-    /// Create a token stream containing a single token with alone spacing.
+    /// Create a token stream containing a single token with alone spacing. The
+    /// spacing used for the final token in a constructed stream doesn't matter
+    /// because it's never used. In practice we arbitrarily use
+    /// `Spacing::Alone`.
     pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
         TokenStream::new(vec![TokenTree::token_alone(kind, span)])
     }
 
-    /// Create a token stream containing a single token with joint spacing.
-    pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream {
-        TokenStream::new(vec![TokenTree::token_joint(kind, span)])
-    }
-
-    /// Create a token stream containing a single `Delimited`.
-    pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
-        TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
-    }
-
     pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
         let Some(tokens) = node.tokens() else {
             panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
@@ -477,13 +523,14 @@
 
     fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
         match &token.kind {
-            token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = **nt => {
+            token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = nt.0 => {
                 TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
             }
             token::Interpolated(nt) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
+                DelimSpacing::new(Spacing::JointHidden, spacing),
                 Delimiter::Invisible,
-                TokenStream::from_nonterminal_ast(nt).flattened(),
+                TokenStream::from_nonterminal_ast(&nt.0).flattened(),
             ),
             _ => TokenTree::Token(token.clone(), spacing),
         }
@@ -492,8 +539,8 @@
     fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
         match tree {
             TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
-            TokenTree::Delimited(span, delim, tts) => {
-                TokenTree::Delimited(*span, *delim, tts.flattened())
+            TokenTree::Delimited(span, spacing, delim, tts) => {
+                TokenTree::Delimited(*span, *spacing, *delim, tts.flattened())
             }
         }
     }
@@ -503,7 +550,7 @@
         fn can_skip(stream: &TokenStream) -> bool {
             stream.trees().all(|tree| match tree {
                 TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
-                TokenTree::Delimited(_, _, inner) => can_skip(inner),
+                TokenTree::Delimited(.., inner) => can_skip(inner),
             })
         }
 
@@ -517,7 +564,7 @@
     // If `vec` is not empty, try to glue `tt` onto its last token. The return
     // value indicates if gluing took place.
     fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
-        if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last()
+        if let Some(TokenTree::Token(last_tok, Spacing::Joint | Spacing::JointHidden)) = vec.last()
             && let TokenTree::Token(tok, spacing) = tt
             && let Some(glued_tok) = last_tok.glue(tok)
         {
@@ -592,9 +639,10 @@
 
                     &TokenTree::Token(..) => i += 1,
 
-                    &TokenTree::Delimited(sp, delim, ref delim_stream) => {
+                    &TokenTree::Delimited(sp, spacing, delim, ref delim_stream) => {
                         if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
-                            let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
+                            let new_tt =
+                                TokenTree::Delimited(sp, spacing, delim, desugared_delim_stream);
                             Lrc::make_mut(&mut stream.0)[i] = new_tt;
                             modified = true;
                         }
@@ -622,10 +670,11 @@
                 num_of_hashes = cmp::max(num_of_hashes, count);
             }
 
-            // `/// foo` becomes `doc = r"foo"`.
+            // `/// foo` becomes `[doc = r"foo"]`.
             let delim_span = DelimSpan::from_single(span);
             let body = TokenTree::Delimited(
                 delim_span,
+                DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
                 Delimiter::Bracket,
                 [
                     TokenTree::token_alone(token::Ident(sym::doc, false), span),
@@ -641,7 +690,7 @@
 
             if attr_style == AttrStyle::Inner {
                 vec![
-                    TokenTree::token_alone(token::Pound, span),
+                    TokenTree::token_joint(token::Pound, span),
                     TokenTree::token_alone(token::Not, span),
                     body,
                 ]
@@ -738,6 +787,18 @@
     }
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
+pub struct DelimSpacing {
+    pub open: Spacing,
+    pub close: Spacing,
+}
+
+impl DelimSpacing {
+    pub fn new(open: Spacing, close: Spacing) -> DelimSpacing {
+        DelimSpacing { open, close }
+    }
+}
+
 // Some types are used a lot. Make sure they don't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 mod size_asserts {
diff --git a/compiler/rustc_ast/src/util/classify.rs b/compiler/rustc_ast/src/util/classify.rs
index 821fca6..4dece07 100644
--- a/compiler/rustc_ast/src/util/classify.rs
+++ b/compiler/rustc_ast/src/util/classify.rs
@@ -40,15 +40,44 @@
             | Range(_, Some(e), _)
             | Ret(Some(e))
             | Unary(_, e)
-            | Yield(Some(e)) => {
+            | Yield(Some(e))
+            | Yeet(Some(e))
+            | Become(e) => {
                 expr = e;
             }
             Closure(closure) => {
                 expr = &closure.body;
             }
             Gen(..) | Block(..) | ForLoop(..) | If(..) | Loop(..) | Match(..) | Struct(..)
-            | TryBlock(..) | While(..) => break Some(expr),
-            _ => break None,
+            | TryBlock(..) | While(..) | ConstBlock(_) => break Some(expr),
+
+            // FIXME: These can end in `}`, but changing these would break stable code.
+            InlineAsm(_) | OffsetOf(_, _) | MacCall(_) | IncludedBytes(_) | FormatArgs(_) => {
+                break None;
+            }
+
+            Break(_, None)
+            | Range(_, None, _)
+            | Ret(None)
+            | Yield(None)
+            | Array(_)
+            | Call(_, _)
+            | MethodCall(_)
+            | Tup(_)
+            | Lit(_)
+            | Cast(_, _)
+            | Type(_, _)
+            | Await(_, _)
+            | Field(_, _)
+            | Index(_, _, _)
+            | Underscore
+            | Path(_, _)
+            | Continue(_)
+            | Repeat(_, _)
+            | Paren(_)
+            | Try(_)
+            | Yeet(None)
+            | Err => break None,
         }
     }
 }
diff --git a/compiler/rustc_ast/src/util/literal.rs b/compiler/rustc_ast/src/util/literal.rs
index 50eb921..92b9adf 100644
--- a/compiler/rustc_ast/src/util/literal.rs
+++ b/compiler/rustc_ast/src/util/literal.rs
@@ -77,6 +77,8 @@
                 // new symbol because the string in the LitKind is different to the
                 // string in the token.
                 let s = symbol.as_str();
+                // Vanilla strings are so common we optimize for the common case where no chars
+                // requiring special behaviour are present.
                 let symbol = if s.contains(['\\', '\r']) {
                     let mut buf = String::with_capacity(s.len());
                     let mut error = Ok(());
@@ -104,27 +106,20 @@
                 LitKind::Str(symbol, ast::StrStyle::Cooked)
             }
             token::StrRaw(n) => {
-                // Ditto.
-                let s = symbol.as_str();
-                let symbol =
-                    if s.contains('\r') {
-                        let mut buf = String::with_capacity(s.len());
-                        let mut error = Ok(());
-                        unescape_literal(s, Mode::RawStr, &mut |_, unescaped_char| {
-                            match unescaped_char {
-                                Ok(c) => buf.push(c),
-                                Err(err) => {
-                                    if err.is_fatal() {
-                                        error = Err(LitError::LexerError);
-                                    }
-                                }
+                // Raw strings have no escapes, so we only need to check for invalid chars, and we
+                // can reuse the symbol on success.
+                let mut error = Ok(());
+                unescape_literal(symbol.as_str(), Mode::RawStr, &mut |_, unescaped_char| {
+                    match unescaped_char {
+                        Ok(_) => {}
+                        Err(err) => {
+                            if err.is_fatal() {
+                                error = Err(LitError::LexerError);
                             }
-                        });
-                        error?;
-                        Symbol::intern(&buf)
-                    } else {
-                        symbol
-                    };
+                        }
+                    }
+                });
+                error?;
                 LitKind::Str(symbol, ast::StrStyle::Raw(n))
             }
             token::ByteStr => {
@@ -143,25 +138,19 @@
                 LitKind::ByteStr(buf.into(), StrStyle::Cooked)
             }
             token::ByteStrRaw(n) => {
+                // Raw strings have no escapes, so we only need to check for invalid chars, and we
+                // can convert the symbol directly to a `Lrc<u8>` on success.
                 let s = symbol.as_str();
-                let bytes = if s.contains('\r') {
-                    let mut buf = Vec::with_capacity(s.len());
-                    let mut error = Ok(());
-                    unescape_literal(s, Mode::RawByteStr, &mut |_, c| match c {
-                        Ok(c) => buf.push(byte_from_char(c)),
-                        Err(err) => {
-                            if err.is_fatal() {
-                                error = Err(LitError::LexerError);
-                            }
+                let mut error = Ok(());
+                unescape_literal(s, Mode::RawByteStr, &mut |_, c| match c {
+                    Ok(_) => {}
+                    Err(err) => {
+                        if err.is_fatal() {
+                            error = Err(LitError::LexerError);
                         }
-                    });
-                    error?;
-                    buf
-                } else {
-                    symbol.to_string().into_bytes()
-                };
-
-                LitKind::ByteStr(bytes.into(), StrStyle::Raw(n))
+                    }
+                });
+                LitKind::ByteStr(s.to_owned().into_bytes().into(), StrStyle::Raw(n))
             }
             token::CStr => {
                 let s = symbol.as_str();
@@ -172,7 +161,6 @@
                         error = Err(LitError::NulInCStr(span));
                     }
                     Ok(CStrUnit::Byte(b)) => buf.push(b),
-                    Ok(CStrUnit::Char(c)) if c.len_utf8() == 1 => buf.push(c as u8),
                     Ok(CStrUnit::Char(c)) => {
                         buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes())
                     }
@@ -187,18 +175,15 @@
                 LitKind::CStr(buf.into(), StrStyle::Cooked)
             }
             token::CStrRaw(n) => {
+                // Raw strings have no escapes, so we only need to check for invalid chars, and we
+                // can convert the symbol directly to a `Lrc<u8>` on success.
                 let s = symbol.as_str();
-                let mut buf = Vec::with_capacity(s.len());
                 let mut error = Ok(());
                 unescape_c_string(s, Mode::RawCStr, &mut |span, c| match c {
                     Ok(CStrUnit::Byte(0) | CStrUnit::Char('\0')) => {
                         error = Err(LitError::NulInCStr(span));
                     }
-                    Ok(CStrUnit::Byte(b)) => buf.push(b),
-                    Ok(CStrUnit::Char(c)) if c.len_utf8() == 1 => buf.push(c as u8),
-                    Ok(CStrUnit::Char(c)) => {
-                        buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes())
-                    }
+                    Ok(_) => {}
                     Err(err) => {
                         if err.is_fatal() {
                             error = Err(LitError::LexerError);
@@ -206,6 +191,7 @@
                     }
                 });
                 error?;
+                let mut buf = s.to_owned().into_bytes();
                 buf.push(0);
                 LitKind::CStr(buf.into(), StrStyle::Raw(n))
             }
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index 1caa39e..27f1b84 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -13,7 +13,7 @@
 //! instance, a walker looking for item names in a module will miss all of
 //! those that are created by the expansion of a macro.
 
-use crate::{ast::*, StaticItem};
+use crate::ast::*;
 
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
@@ -559,7 +559,7 @@
             walk_list!(visitor, visit_expr, lower_bound);
             walk_list!(visitor, visit_expr, upper_bound);
         }
-        PatKind::Wild | PatKind::Rest => {}
+        PatKind::Wild | PatKind::Rest | PatKind::Never => {}
         PatKind::Tuple(elems) | PatKind::Slice(elems) | PatKind::Or(elems) => {
             walk_list!(visitor, visit_pat, elems);
         }
@@ -861,7 +861,7 @@
         ExprKind::Closure(box Closure {
             binder,
             capture_clause,
-            asyncness: _,
+            coroutine_kind: _,
             constness: _,
             movability: _,
             fn_decl,
@@ -951,7 +951,7 @@
 pub fn walk_arm<'a, V: Visitor<'a>>(visitor: &mut V, arm: &'a Arm) {
     visitor.visit_pat(&arm.pat);
     walk_list!(visitor, visit_expr, &arm.guard);
-    visitor.visit_expr(&arm.body);
+    walk_list!(visitor, visit_expr, &arm.body);
     walk_list!(visitor, visit_attribute, &arm.attrs);
 }
 
diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl
index 91591a7..6bde4f2 100644
--- a/compiler/rustc_ast_lowering/messages.ftl
+++ b/compiler/rustc_ast_lowering/messages.ftl
@@ -91,6 +91,10 @@
 ast_lowering_invalid_register_class =
     invalid register class `{$reg_class}`: {$error}
 
+ast_lowering_match_arm_with_no_body =
+    `match` arm with no body
+    .suggestion = add a body after the pattern
+
 ast_lowering_misplaced_assoc_ty_binding =
     associated type bounds are only allowed in where clauses and function signatures, not in {$position}
 
@@ -104,6 +108,15 @@
 ast_lowering_misplaced_relax_trait_bound =
     `?Trait` bounds are only permitted at the point where a type parameter is declared
 
+ast_lowering_never_pattern_with_body =
+    a never pattern is always unreachable
+    .label = this will never be executed
+    .suggestion = remove this expression
+
+ast_lowering_never_pattern_with_guard =
+    a guard on a never pattern will never be run
+    .suggestion = remove this guard
+
 ast_lowering_not_supported_for_lifetime_binder_async_closure =
     `for<...>` binders on `async` closures are not currently supported
 
diff --git a/compiler/rustc_ast_lowering/src/asm.rs b/compiler/rustc_ast_lowering/src/asm.rs
index a1e6269..4c81983 100644
--- a/compiler/rustc_ast_lowering/src/asm.rs
+++ b/compiler/rustc_ast_lowering/src/asm.rs
@@ -14,8 +14,8 @@
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::definitions::DefPathData;
 use rustc_session::parse::feature_err;
+use rustc_span::symbol::kw;
 use rustc_span::{sym, Span};
 use rustc_target::asm;
 use std::collections::hash_map::Entry;
@@ -227,7 +227,8 @@
                             self.create_def(
                                 parent_def_id.def_id,
                                 node_id,
-                                DefPathData::AnonConst,
+                                kw::Empty,
+                                DefKind::AnonConst,
                                 *op_sp,
                             );
                             let anon_const = AnonConst { id: node_id, value: P(expr) };
@@ -335,67 +336,81 @@
                         hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
                         | hir::InlineAsmOperand::SymStatic { .. } => {
-                            unreachable!()
+                            unreachable!("{op:?} is not a register operand");
                         }
                     };
 
                     // Flag to output the error only once per operand
                     let mut skip = false;
-                    reg.overlapping_regs(|r| {
-                        let mut check = |used_regs: &mut FxHashMap<asm::InlineAsmReg, usize>,
-                                         input| {
-                            match used_regs.entry(r) {
-                                Entry::Occupied(o) => {
-                                    if skip {
-                                        return;
-                                    }
-                                    skip = true;
 
-                                    let idx2 = *o.get();
-                                    let (ref op2, op_sp2) = operands[idx2];
-                                    let Some(asm::InlineAsmRegOrRegClass::Reg(reg2)) = op2.reg()
-                                    else {
-                                        unreachable!();
-                                    };
-
-                                    let in_out = match (op, op2) {
-                                        (
-                                            hir::InlineAsmOperand::In { .. },
-                                            hir::InlineAsmOperand::Out { late, .. },
-                                        )
-                                        | (
-                                            hir::InlineAsmOperand::Out { late, .. },
-                                            hir::InlineAsmOperand::In { .. },
-                                        ) => {
-                                            assert!(!*late);
-                                            let out_op_sp = if input { op_sp2 } else { op_sp };
-                                            Some(out_op_sp)
-                                        }
-                                        _ => None,
-                                    };
-
-                                    sess.emit_err(RegisterConflict {
-                                        op_span1: op_sp,
-                                        op_span2: op_sp2,
-                                        reg1_name: reg.name(),
-                                        reg2_name: reg2.name(),
-                                        in_out,
-                                    });
+                    let mut check = |used_regs: &mut FxHashMap<asm::InlineAsmReg, usize>,
+                                     input,
+                                     r: asm::InlineAsmReg| {
+                        match used_regs.entry(r) {
+                            Entry::Occupied(o) => {
+                                if skip {
+                                    return;
                                 }
-                                Entry::Vacant(v) => {
-                                    if r == reg {
-                                        v.insert(idx);
+                                skip = true;
+
+                                let idx2 = *o.get();
+                                let (ref op2, op_sp2) = operands[idx2];
+
+                                let in_out = match (op, op2) {
+                                    (
+                                        hir::InlineAsmOperand::In { .. },
+                                        hir::InlineAsmOperand::Out { late, .. },
+                                    )
+                                    | (
+                                        hir::InlineAsmOperand::Out { late, .. },
+                                        hir::InlineAsmOperand::In { .. },
+                                    ) => {
+                                        assert!(!*late);
+                                        let out_op_sp = if input { op_sp2 } else { op_sp };
+                                        Some(out_op_sp)
                                     }
+                                    _ => None,
+                                };
+                                let reg_str = |idx| -> &str {
+                                    // HIR asm doesn't preserve the original alias string of the explicit register,
+                                    // so we have to retrieve it from AST
+                                    let (op, _): &(InlineAsmOperand, Span) = &asm.operands[idx];
+                                    if let Some(ast::InlineAsmRegOrRegClass::Reg(reg_sym)) =
+                                        op.reg()
+                                    {
+                                        reg_sym.as_str()
+                                    } else {
+                                        unreachable!("{op:?} is not a register operand");
+                                    }
+                                };
+
+                                sess.emit_err(RegisterConflict {
+                                    op_span1: op_sp,
+                                    op_span2: op_sp2,
+                                    reg1_name: reg_str(idx),
+                                    reg2_name: reg_str(idx2),
+                                    in_out,
+                                });
+                            }
+                            Entry::Vacant(v) => {
+                                if r == reg {
+                                    v.insert(idx);
                                 }
                             }
-                        };
+                        }
+                    };
+                    let mut overlapping_with = vec![];
+                    reg.overlapping_regs(|r| {
+                        overlapping_with.push(r);
+                    });
+                    for r in overlapping_with {
                         if input {
-                            check(&mut used_input_regs, true);
+                            check(&mut used_input_regs, true, r);
                         }
                         if output {
-                            check(&mut used_output_regs, false);
+                            check(&mut used_output_regs, false, r);
                         }
-                    });
+                    }
                 }
             }
         }
@@ -410,12 +425,12 @@
                     continue;
                 }
 
-                let mut output_used = false;
+                let mut overlapping_with = vec![];
                 clobber.overlapping_regs(|reg| {
-                    if used_output_regs.contains_key(&reg) {
-                        output_used = true;
-                    }
+                    overlapping_with.push(reg);
                 });
+                let output_used =
+                    overlapping_with.iter().any(|reg| used_output_regs.contains_key(&reg));
 
                 if !output_used {
                     operands.push((
diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs
index 6e1a9ef..11bb559 100644
--- a/compiler/rustc_ast_lowering/src/errors.rs
+++ b/compiler/rustc_ast_lowering/src/errors.rs
@@ -340,6 +340,32 @@
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(ast_lowering_match_arm_with_no_body)]
+pub struct MatchArmWithNoBody {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(code = " => todo!(),", applicability = "has-placeholders")]
+    pub suggestion: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_lowering_never_pattern_with_body)]
+pub struct NeverPatternWithBody {
+    #[primary_span]
+    #[label]
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_lowering_never_pattern_with_guard)]
+pub struct NeverPatternWithGuard {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub span: Span,
+}
+
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_arbitrary_expression_in_pattern)]
 pub struct ArbitraryExpressionInPattern {
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 0fff9a6..704f124 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -1,22 +1,22 @@
 use super::errors::{
     AsyncCoroutinesNotSupported, AsyncNonMoveClosureNotSupported, AwaitOnlyInAsyncFnAndBlocks,
     BaseExpressionDoubleDot, ClosureCannotBeStatic, CoroutineTooManyParameters,
-    FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd,
-    NotSupportedForLifetimeBinderAsyncClosure, UnderscoreExprLhsAssign,
+    FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd, MatchArmWithNoBody,
+    NeverPatternWithBody, NeverPatternWithGuard, NotSupportedForLifetimeBinderAsyncClosure,
+    UnderscoreExprLhsAssign,
 };
 use super::ResolverAstLoweringExt;
 use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericArgs};
 use crate::{FnDeclKind, ImplTraitPosition};
-use rustc_ast::attr;
 use rustc_ast::ptr::P as AstP;
 use rustc_ast::*;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
-use rustc_hir::def::Res;
-use rustc_hir::definitions::DefPathData;
+use rustc_hir::def::{DefKind, Res};
+use rustc_middle::span_bug;
 use rustc_session::errors::report_lit_error;
 use rustc_span::source_map::{respan, Spanned};
-use rustc_span::symbol::{sym, Ident, Symbol};
+use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::DUMMY_SP;
 use rustc_span::{DesugaringKind, Span};
 use thin_vec::{thin_vec, ThinVec};
@@ -42,8 +42,7 @@
                     }
                     // Merge attributes into the inner expression.
                     if !e.attrs.is_empty() {
-                        let old_attrs =
-                            self.attrs.get(&ex.hir_id.local_id).map(|la| *la).unwrap_or(&[]);
+                        let old_attrs = self.attrs.get(&ex.hir_id.local_id).copied().unwrap_or(&[]);
                         self.attrs.insert(
                             ex.hir_id.local_id,
                             &*self.arena.alloc_from_iter(
@@ -73,7 +72,7 @@
             let kind = match &e.kind {
                 ExprKind::Array(exprs) => hir::ExprKind::Array(self.lower_exprs(exprs)),
                 ExprKind::ConstBlock(c) => {
-                    let c = self.with_new_scopes(|this| hir::ConstBlock {
+                    let c = self.with_new_scopes(c.value.span, |this| hir::ConstBlock {
                         def_id: this.local_def_id(c.id),
                         hir_id: this.lower_node_id(c.id),
                         body: this.lower_const_body(c.value.span, Some(&c.value)),
@@ -190,46 +189,43 @@
                     None,
                     e.span,
                     hir::CoroutineSource::Block,
-                    |this| this.with_new_scopes(|this| this.lower_block_expr(block)),
+                    |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
                 ),
                 ExprKind::Await(expr, await_kw_span) => self.lower_expr_await(*await_kw_span, expr),
                 ExprKind::Closure(box Closure {
                     binder,
                     capture_clause,
                     constness,
-                    asyncness,
+                    coroutine_kind,
                     movability,
                     fn_decl,
                     body,
                     fn_decl_span,
                     fn_arg_span,
-                }) => {
-                    if let Async::Yes { closure_id, .. } = asyncness {
-                        self.lower_expr_async_closure(
-                            binder,
-                            *capture_clause,
-                            e.id,
-                            hir_id,
-                            *closure_id,
-                            fn_decl,
-                            body,
-                            *fn_decl_span,
-                            *fn_arg_span,
-                        )
-                    } else {
-                        self.lower_expr_closure(
-                            binder,
-                            *capture_clause,
-                            e.id,
-                            *constness,
-                            *movability,
-                            fn_decl,
-                            body,
-                            *fn_decl_span,
-                            *fn_arg_span,
-                        )
-                    }
-                }
+                }) => match coroutine_kind {
+                    Some(coroutine_kind) => self.lower_expr_coroutine_closure(
+                        binder,
+                        *capture_clause,
+                        e.id,
+                        hir_id,
+                        *coroutine_kind,
+                        fn_decl,
+                        body,
+                        *fn_decl_span,
+                        *fn_arg_span,
+                    ),
+                    None => self.lower_expr_closure(
+                        binder,
+                        *capture_clause,
+                        e.id,
+                        *constness,
+                        *movability,
+                        fn_decl,
+                        body,
+                        *fn_decl_span,
+                        *fn_arg_span,
+                    ),
+                },
                 ExprKind::Block(blk, opt_label) => {
                     let opt_label = self.lower_label(*opt_label);
                     hir::ExprKind::Block(self.lower_block(blk, opt_label.is_some()), opt_label)
@@ -324,11 +320,20 @@
                     None,
                     e.span,
                     hir::CoroutineSource::Block,
-                    |this| this.with_new_scopes(|this| this.lower_block_expr(block)),
+                    |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
                 ),
+                ExprKind::Gen(capture_clause, block, GenBlockKind::AsyncGen) => self
+                    .make_async_gen_expr(
+                        *capture_clause,
+                        e.id,
+                        None,
+                        e.span,
+                        hir::CoroutineSource::Block,
+                        |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
+                    ),
                 ExprKind::Yield(opt_expr) => self.lower_expr_yield(e.span, opt_expr.as_deref()),
                 ExprKind::Err => hir::ExprKind::Err(
-                    self.tcx.sess.delay_span_bug(e.span, "lowered ExprKind::Err"),
+                    self.tcx.sess.span_delayed_bug(e.span, "lowered ExprKind::Err"),
                 ),
                 ExprKind::Try(sub_expr) => self.lower_expr_try(e.span, sub_expr),
 
@@ -351,30 +356,8 @@
         }
     }
 
-    fn lower_binop(&mut self, b: BinOp) -> hir::BinOp {
-        Spanned {
-            node: match b.node {
-                BinOpKind::Add => hir::BinOpKind::Add,
-                BinOpKind::Sub => hir::BinOpKind::Sub,
-                BinOpKind::Mul => hir::BinOpKind::Mul,
-                BinOpKind::Div => hir::BinOpKind::Div,
-                BinOpKind::Rem => hir::BinOpKind::Rem,
-                BinOpKind::And => hir::BinOpKind::And,
-                BinOpKind::Or => hir::BinOpKind::Or,
-                BinOpKind::BitXor => hir::BinOpKind::BitXor,
-                BinOpKind::BitAnd => hir::BinOpKind::BitAnd,
-                BinOpKind::BitOr => hir::BinOpKind::BitOr,
-                BinOpKind::Shl => hir::BinOpKind::Shl,
-                BinOpKind::Shr => hir::BinOpKind::Shr,
-                BinOpKind::Eq => hir::BinOpKind::Eq,
-                BinOpKind::Lt => hir::BinOpKind::Lt,
-                BinOpKind::Le => hir::BinOpKind::Le,
-                BinOpKind::Ne => hir::BinOpKind::Ne,
-                BinOpKind::Ge => hir::BinOpKind::Ge,
-                BinOpKind::Gt => hir::BinOpKind::Gt,
-            },
-            span: self.lower_span(b.span),
-        }
+    fn lower_binop(&mut self, b: BinOp) -> BinOp {
+        Spanned { node: b.node, span: self.lower_span(b.span) }
     }
 
     fn lower_legacy_const_generics(
@@ -396,7 +379,13 @@
                 let node_id = self.next_node_id();
 
                 // Add a definition for the in-band const def.
-                self.create_def(parent_def_id.def_id, node_id, DefPathData::AnonConst, f.span);
+                self.create_def(
+                    parent_def_id.def_id,
+                    node_id,
+                    kw::Empty,
+                    DefKind::AnonConst,
+                    f.span,
+                );
 
                 let anon_const = AnonConst { id: node_id, value: arg };
                 generic_args.push(AngleBracketedArg::Arg(GenericArg::Const(anon_const)));
@@ -525,7 +514,7 @@
                     this.mark_span_with_reason(
                         DesugaringKind::TryBlock,
                         expr.span,
-                        this.allow_try_trait.clone(),
+                        Some(this.allow_try_trait.clone()),
                     ),
                     expr,
                 )
@@ -533,7 +522,7 @@
                 let try_span = this.mark_span_with_reason(
                     DesugaringKind::TryBlock,
                     this.tcx.sess.source_map().end_point(body.span),
-                    this.allow_try_trait.clone(),
+                    Some(this.allow_try_trait.clone()),
                 );
 
                 (try_span, this.expr_unit(try_span))
@@ -561,13 +550,13 @@
         expr: &'hir hir::Expr<'hir>,
         overall_span: Span,
     ) -> &'hir hir::Expr<'hir> {
-        let constructor = self.arena.alloc(self.expr_lang_item_path(method_span, lang_item, None));
+        let constructor = self.arena.alloc(self.expr_lang_item_path(method_span, lang_item));
         self.expr_call(overall_span, constructor, std::slice::from_ref(expr))
     }
 
     fn lower_arm(&mut self, arm: &Arm) -> hir::Arm<'hir> {
         let pat = self.lower_pat(&arm.pat);
-        let guard = arm.guard.as_ref().map(|cond| {
+        let mut guard = arm.guard.as_ref().map(|cond| {
             if let ExprKind::Let(pat, scrutinee, span, is_recovered) = &cond.kind {
                 hir::Guard::IfLet(self.arena.alloc(hir::Let {
                     hir_id: self.next_id(),
@@ -582,14 +571,46 @@
             }
         });
         let hir_id = self.next_id();
+        let span = self.lower_span(arm.span);
         self.lower_attrs(hir_id, &arm.attrs);
-        hir::Arm {
-            hir_id,
-            pat,
-            guard,
-            body: self.lower_expr(&arm.body),
-            span: self.lower_span(arm.span),
-        }
+        let is_never_pattern = pat.is_never_pattern();
+        let body = if let Some(body) = &arm.body
+            && !is_never_pattern
+        {
+            self.lower_expr(body)
+        } else {
+            // Either `body.is_none()` or `is_never_pattern` here.
+            if !is_never_pattern {
+                if self.tcx.features().never_patterns {
+                    // If the feature is off we already emitted the error after parsing.
+                    let suggestion = span.shrink_to_hi();
+                    self.tcx.sess.emit_err(MatchArmWithNoBody { span, suggestion });
+                }
+            } else if let Some(body) = &arm.body {
+                self.tcx.sess.emit_err(NeverPatternWithBody { span: body.span });
+                guard = None;
+            } else if let Some(g) = &arm.guard {
+                self.tcx.sess.emit_err(NeverPatternWithGuard { span: g.span });
+                guard = None;
+            }
+
+            // We add a fake `loop {}` arm body so that it typecks to `!`.
+            // FIXME(never_patterns): Desugar into a call to `unreachable_unchecked`.
+            let block = self.arena.alloc(hir::Block {
+                stmts: &[],
+                expr: None,
+                hir_id: self.next_id(),
+                rules: hir::BlockCheckMode::DefaultBlock,
+                span,
+                targeted_by_break: false,
+            });
+            self.arena.alloc(hir::Expr {
+                hir_id: self.next_id(),
+                kind: hir::ExprKind::Loop(block, None, hir::LoopSource::Loop, span),
+                span,
+            })
+        };
+        hir::Arm { hir_id, pat, guard, body, span }
     }
 
     /// Lower an `async` construct to a coroutine that implements `Future`.
@@ -613,9 +634,12 @@
         let output = ret_ty.unwrap_or_else(|| hir::FnRetTy::DefaultReturn(self.lower_span(span)));
 
         // Resume argument type: `ResumeTy`
-        let unstable_span =
-            self.mark_span_with_reason(DesugaringKind::Async, span, self.allow_gen_future.clone());
-        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span, None);
+        let unstable_span = self.mark_span_with_reason(
+            DesugaringKind::Async,
+            self.lower_span(span),
+            Some(self.allow_gen_future.clone()),
+        );
+        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span);
         let input_ty = hir::Ty {
             hir_id: self.next_id(),
             kind: hir::TyKind::Path(resume_ty),
@@ -721,6 +745,87 @@
         }))
     }
 
+    /// Lower a `async gen` construct to a generator that implements `AsyncIterator`.
+    ///
+    /// This results in:
+    ///
+    /// ```text
+    /// static move? |_task_context| -> () {
+    ///     <body>
+    /// }
+    /// ```
+    pub(super) fn make_async_gen_expr(
+        &mut self,
+        capture_clause: CaptureBy,
+        closure_node_id: NodeId,
+        _yield_ty: Option<hir::FnRetTy<'hir>>,
+        span: Span,
+        async_coroutine_source: hir::CoroutineSource,
+        body: impl FnOnce(&mut Self) -> hir::Expr<'hir>,
+    ) -> hir::ExprKind<'hir> {
+        let output = hir::FnRetTy::DefaultReturn(self.lower_span(span));
+
+        // Resume argument type: `ResumeTy`
+        let unstable_span = self.mark_span_with_reason(
+            DesugaringKind::Async,
+            self.lower_span(span),
+            Some(self.allow_gen_future.clone()),
+        );
+        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span);
+        let input_ty = hir::Ty {
+            hir_id: self.next_id(),
+            kind: hir::TyKind::Path(resume_ty),
+            span: unstable_span,
+        };
+
+        // The closure/coroutine `FnDecl` takes a single (resume) argument of type `input_ty`.
+        let fn_decl = self.arena.alloc(hir::FnDecl {
+            inputs: arena_vec![self; input_ty],
+            output,
+            c_variadic: false,
+            implicit_self: hir::ImplicitSelfKind::None,
+            lifetime_elision_allowed: false,
+        });
+
+        // Lower the argument pattern/ident. The ident is used again in the `.await` lowering.
+        let (pat, task_context_hid) = self.pat_ident_binding_mode(
+            span,
+            Ident::with_dummy_span(sym::_task_context),
+            hir::BindingAnnotation::MUT,
+        );
+        let param = hir::Param {
+            hir_id: self.next_id(),
+            pat,
+            ty_span: self.lower_span(span),
+            span: self.lower_span(span),
+        };
+        let params = arena_vec![self; param];
+
+        let body = self.lower_body(move |this| {
+            this.coroutine_kind = Some(hir::CoroutineKind::AsyncGen(async_coroutine_source));
+
+            let old_ctx = this.task_context;
+            this.task_context = Some(task_context_hid);
+            let res = body(this);
+            this.task_context = old_ctx;
+            (params, res)
+        });
+
+        // `static |_task_context| -> <ret_ty> { body }`:
+        hir::ExprKind::Closure(self.arena.alloc(hir::Closure {
+            def_id: self.local_def_id(closure_node_id),
+            binder: hir::ClosureBinder::Default,
+            capture_clause,
+            bound_generic_params: &[],
+            fn_decl,
+            body,
+            fn_decl_span: self.lower_span(span),
+            fn_arg_span: None,
+            movability: Some(hir::Movability::Static),
+            constness: hir::Constness::NotConst,
+        }))
+    }
+
     /// Forwards a possible `#[track_caller]` annotation from `outer_hir_id` to
     /// `inner_hir_id` in case the `async_fn_track_caller` feature is enabled.
     pub(super) fn maybe_forward_track_caller(
@@ -736,7 +841,7 @@
             let unstable_span = self.mark_span_with_reason(
                 DesugaringKind::Async,
                 span,
-                self.allow_gen_future.clone(),
+                Some(self.allow_gen_future.clone()),
             );
             self.lower_attrs(
                 inner_hir_id,
@@ -770,20 +875,23 @@
     /// ```
     fn lower_expr_await(&mut self, await_kw_span: Span, expr: &Expr) -> hir::ExprKind<'hir> {
         let full_span = expr.span.to(await_kw_span);
-        match self.coroutine_kind {
-            Some(hir::CoroutineKind::Async(_)) => {}
+
+        let is_async_gen = match self.coroutine_kind {
+            Some(hir::CoroutineKind::Async(_)) => false,
+            Some(hir::CoroutineKind::AsyncGen(_)) => true,
             Some(hir::CoroutineKind::Coroutine) | Some(hir::CoroutineKind::Gen(_)) | None => {
-                self.tcx.sess.emit_err(AwaitOnlyInAsyncFnAndBlocks {
+                return hir::ExprKind::Err(self.tcx.sess.emit_err(AwaitOnlyInAsyncFnAndBlocks {
                     await_kw_span,
                     item_span: self.current_item,
-                });
+                }));
             }
-        }
+        };
+
         let span = self.mark_span_with_reason(DesugaringKind::Await, await_kw_span, None);
         let gen_future_span = self.mark_span_with_reason(
             DesugaringKind::Await,
             full_span,
-            self.allow_gen_future.clone(),
+            Some(self.allow_gen_future.clone()),
         );
         let expr = self.lower_expr_mut(expr);
         let expr_hir_id = expr.hir_id;
@@ -792,8 +900,11 @@
         // debuggers and debugger extensions expect it to be called `__awaitee`. They use
         // this name to identify what is being awaited by a suspended async functions.
         let awaitee_ident = Ident::with_dummy_span(sym::__awaitee);
-        let (awaitee_pat, awaitee_pat_hid) =
-            self.pat_ident_binding_mode(span, awaitee_ident, hir::BindingAnnotation::MUT);
+        let (awaitee_pat, awaitee_pat_hid) = self.pat_ident_binding_mode(
+            gen_future_span,
+            awaitee_ident,
+            hir::BindingAnnotation::MUT,
+        );
 
         let task_context_ident = Ident::with_dummy_span(sym::_task_context);
 
@@ -806,29 +917,27 @@
         let poll_expr = {
             let awaitee = self.expr_ident(span, awaitee_ident, awaitee_pat_hid);
             let ref_mut_awaitee = self.expr_mut_addr_of(span, awaitee);
-            let task_context = if let Some(task_context_hid) = self.task_context {
-                self.expr_ident_mut(span, task_context_ident, task_context_hid)
-            } else {
-                // Use of `await` outside of an async context, we cannot use `task_context` here.
-                self.expr_err(span, self.tcx.sess.delay_span_bug(span, "no task_context hir id"))
+
+            let Some(task_context_hid) = self.task_context else {
+                unreachable!("use of `await` outside of an async context.");
             };
+
+            let task_context = self.expr_ident_mut(span, task_context_ident, task_context_hid);
+
             let new_unchecked = self.expr_call_lang_item_fn_mut(
                 span,
                 hir::LangItem::PinNewUnchecked,
                 arena_vec![self; ref_mut_awaitee],
-                Some(expr_hir_id),
             );
             let get_context = self.expr_call_lang_item_fn_mut(
                 gen_future_span,
                 hir::LangItem::GetContext,
                 arena_vec![self; task_context],
-                Some(expr_hir_id),
             );
             let call = self.expr_call_lang_item_fn(
                 span,
                 hir::LangItem::FuturePoll,
                 arena_vec![self; new_unchecked, get_context],
-                Some(expr_hir_id),
             );
             self.arena.alloc(self.expr_unsafe(call))
         };
@@ -841,12 +950,7 @@
             let (x_pat, x_pat_hid) = self.pat_ident(gen_future_span, x_ident);
             let x_expr = self.expr_ident(gen_future_span, x_ident, x_pat_hid);
             let ready_field = self.single_pat_field(gen_future_span, x_pat);
-            let ready_pat = self.pat_lang_item_variant(
-                span,
-                hir::LangItem::PollReady,
-                ready_field,
-                Some(expr_hir_id),
-            );
+            let ready_pat = self.pat_lang_item_variant(span, hir::LangItem::PollReady, ready_field);
             let break_x = self.with_loop_scope(loop_node_id, move |this| {
                 let expr_break =
                     hir::ExprKind::Break(this.lower_loop_destination(None), Some(x_expr));
@@ -857,12 +961,7 @@
 
         // `::std::task::Poll::Pending => {}`
         let pending_arm = {
-            let pending_pat = self.pat_lang_item_variant(
-                span,
-                hir::LangItem::PollPending,
-                &[],
-                Some(expr_hir_id),
-            );
+            let pending_pat = self.pat_lang_item_variant(span, hir::LangItem::PollPending, &[]);
             let empty_block = self.expr_block_empty(span);
             self.arm(pending_pat, empty_block)
         };
@@ -877,25 +976,30 @@
             self.stmt_expr(span, match_expr)
         };
 
-        // task_context = yield ();
+        // Depending on `async` of `async gen`:
+        // async     - task_context = yield ();
+        // async gen - task_context = yield ASYNC_GEN_PENDING;
         let yield_stmt = {
-            let unit = self.expr_unit(span);
+            let yielded = if is_async_gen {
+                self.arena.alloc(self.expr_lang_item_path(span, hir::LangItem::AsyncGenPending))
+            } else {
+                self.expr_unit(span)
+            };
+
             let yield_expr = self.expr(
                 span,
-                hir::ExprKind::Yield(unit, hir::YieldSource::Await { expr: Some(expr_hir_id) }),
+                hir::ExprKind::Yield(yielded, hir::YieldSource::Await { expr: Some(expr_hir_id) }),
             );
             let yield_expr = self.arena.alloc(yield_expr);
 
-            if let Some(task_context_hid) = self.task_context {
-                let lhs = self.expr_ident(span, task_context_ident, task_context_hid);
-                let assign =
-                    self.expr(span, hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span)));
-                self.stmt_expr(span, assign)
-            } else {
-                // Use of `await` outside of an async context. Return `yield_expr` so that we can
-                // proceed with type checking.
-                self.stmt(span, hir::StmtKind::Semi(yield_expr))
-            }
+            let Some(task_context_hid) = self.task_context else {
+                unreachable!("use of `await` outside of an async context.");
+            };
+
+            let lhs = self.expr_ident(span, task_context_ident, task_context_hid);
+            let assign =
+                self.expr(span, hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span)));
+            self.stmt_expr(span, assign)
         };
 
         let loop_block = self.block_all(span, arena_vec![self; inner_match_stmt, yield_stmt], None);
@@ -920,7 +1024,6 @@
             span,
             hir::LangItem::IntoFutureIntoFuture,
             arena_vec![self; expr],
-            Some(expr_hir_id),
         );
 
         // match <into_future_expr> {
@@ -947,9 +1050,7 @@
     ) -> hir::ExprKind<'hir> {
         let (binder_clause, generic_params) = self.lower_closure_binder(binder);
 
-        let (body_id, coroutine_option) = self.with_new_scopes(move |this| {
-            let prev = this.current_item;
-            this.current_item = Some(fn_decl_span);
+        let (body_id, coroutine_option) = self.with_new_scopes(fn_decl_span, move |this| {
             let mut coroutine_kind = None;
             let body_id = this.lower_fn_body(decl, |this| {
                 let e = this.lower_expr_mut(body);
@@ -957,8 +1058,7 @@
                 e
             });
             let coroutine_option =
-                this.coroutine_movability_for_fn(&decl, fn_decl_span, coroutine_kind, movability);
-            this.current_item = prev;
+                this.coroutine_movability_for_fn(decl, fn_decl_span, coroutine_kind, movability);
             (body_id, coroutine_option)
         });
 
@@ -996,7 +1096,11 @@
                 }
                 Some(movability)
             }
-            Some(hir::CoroutineKind::Gen(_)) | Some(hir::CoroutineKind::Async(_)) => {
+            Some(
+                hir::CoroutineKind::Gen(_)
+                | hir::CoroutineKind::Async(_)
+                | hir::CoroutineKind::AsyncGen(_),
+            ) => {
                 panic!("non-`async`/`gen` closure body turned `async`/`gen` during lowering");
             }
             None => {
@@ -1023,18 +1127,22 @@
         (binder, params)
     }
 
-    fn lower_expr_async_closure(
+    fn lower_expr_coroutine_closure(
         &mut self,
         binder: &ClosureBinder,
         capture_clause: CaptureBy,
         closure_id: NodeId,
         closure_hir_id: hir::HirId,
-        inner_closure_id: NodeId,
+        coroutine_kind: CoroutineKind,
         decl: &FnDecl,
         body: &Expr,
         fn_decl_span: Span,
         fn_arg_span: Span,
     ) -> hir::ExprKind<'hir> {
+        let CoroutineKind::Async { closure_id: inner_closure_id, .. } = coroutine_kind else {
+            span_bug!(fn_decl_span, "`async gen` and `gen` closures are not supported, yet");
+        };
+
         if let &ClosureBinder::For { span, .. } = binder {
             self.tcx.sess.emit_err(NotSupportedForLifetimeBinderAsyncClosure { span });
         }
@@ -1044,7 +1152,7 @@
         let outer_decl =
             FnDecl { inputs: decl.inputs.clone(), output: FnRetTy::Default(fn_decl_span) };
 
-        let body = self.with_new_scopes(|this| {
+        let body = self.with_new_scopes(fn_decl_span, |this| {
             // FIXME(cramertj): allow `async` non-`move` closures with arguments.
             if capture_clause == CaptureBy::Ref && !decl.inputs.is_empty() {
                 this.tcx.sess.emit_err(AsyncNonMoveClosureNotSupported { fn_decl_span });
@@ -1055,7 +1163,7 @@
             let body_id = this.lower_fn_body(&outer_decl, |this| {
                 let async_ret_ty = if let FnRetTy::Ty(ty) = &decl.output {
                     let itctx = ImplTraitContext::Disallowed(ImplTraitPosition::AsyncBlock);
-                    Some(hir::FnRetTy::Return(this.lower_ty(&ty, &itctx)))
+                    Some(hir::FnRetTy::Return(this.lower_ty(ty, &itctx)))
                 } else {
                     None
                 };
@@ -1066,7 +1174,7 @@
                     async_ret_ty,
                     body.span,
                     hir::CoroutineSource::Closure,
-                    |this| this.with_new_scopes(|this| this.lower_expr_mut(body)),
+                    |this| this.with_new_scopes(fn_decl_span, |this| this.lower_expr_mut(body)),
                 );
                 let hir_id = this.lower_node_id(inner_closure_id);
                 this.maybe_forward_track_caller(body.span, closure_hir_id, hir_id);
@@ -1113,6 +1221,8 @@
                 | ExprKind::Struct(..)
                 | ExprKind::Tup(..)
                 | ExprKind::Underscore => false,
+                // Check for unit struct constructor.
+                ExprKind::Path(..) => lower_ctx.extract_unit_struct_path(lhs).is_none(),
                 // Check for tuple struct constructor.
                 ExprKind::Call(callee, ..) => lower_ctx.extract_tuple_struct_path(callee).is_none(),
                 ExprKind::Paren(e) => {
@@ -1149,12 +1259,10 @@
         );
 
         // `a = lhs1; b = lhs2;`.
-        let stmts = self
-            .arena
-            .alloc_from_iter(std::iter::once(destructure_let).chain(assignments.into_iter()));
+        let stmts = self.arena.alloc_from_iter(std::iter::once(destructure_let).chain(assignments));
 
         // Wrap everything in a block.
-        hir::ExprKind::Block(&self.block_all(whole_span, stmts, None), None)
+        hir::ExprKind::Block(self.block_all(whole_span, stmts, None), None)
     }
 
     /// If the given expression is a path to a tuple struct, returns that path.
@@ -1377,8 +1485,7 @@
     fn lower_expr_range_closed(&mut self, span: Span, e1: &Expr, e2: &Expr) -> hir::ExprKind<'hir> {
         let e1 = self.lower_expr_mut(e1);
         let e2 = self.lower_expr_mut(e2);
-        let fn_path =
-            hir::QPath::LangItem(hir::LangItem::RangeInclusiveNew, self.lower_span(span), None);
+        let fn_path = hir::QPath::LangItem(hir::LangItem::RangeInclusiveNew, self.lower_span(span));
         let fn_expr = self.arena.alloc(self.expr(span, hir::ExprKind::Path(fn_path)));
         hir::ExprKind::Call(fn_expr, arena_vec![self; e1, e2])
     }
@@ -1411,7 +1518,7 @@
         let fields = self.arena.alloc_from_iter(
             e1.iter().map(|e| (sym::start, e)).chain(e2.iter().map(|e| (sym::end, e))).map(
                 |(s, e)| {
-                    let expr = self.lower_expr(&e);
+                    let expr = self.lower_expr(e);
                     let ident = Ident::new(s, self.lower_span(e.span));
                     self.expr_field(ident, expr, e.span)
                 },
@@ -1419,7 +1526,7 @@
         );
 
         hir::ExprKind::Struct(
-            self.arena.alloc(hir::QPath::LangItem(lang_item, self.lower_span(span), None)),
+            self.arena.alloc(hir::QPath::LangItem(lang_item, self.lower_span(span))),
             fields,
             None,
         )
@@ -1504,10 +1611,13 @@
     }
 
     fn lower_expr_yield(&mut self, span: Span, opt_expr: Option<&Expr>) -> hir::ExprKind<'hir> {
-        match self.coroutine_kind {
-            Some(hir::CoroutineKind::Gen(_)) => {}
+        let is_async_gen = match self.coroutine_kind {
+            Some(hir::CoroutineKind::Gen(_)) => false,
+            Some(hir::CoroutineKind::AsyncGen(_)) => true,
             Some(hir::CoroutineKind::Async(_)) => {
-                self.tcx.sess.emit_err(AsyncCoroutinesNotSupported { span });
+                return hir::ExprKind::Err(
+                    self.tcx.sess.emit_err(AsyncCoroutinesNotSupported { span }),
+                );
             }
             Some(hir::CoroutineKind::Coroutine) | None => {
                 if !self.tcx.features().coroutines {
@@ -1519,14 +1629,37 @@
                     )
                     .emit();
                 }
-                self.coroutine_kind = Some(hir::CoroutineKind::Coroutine)
+                self.coroutine_kind = Some(hir::CoroutineKind::Coroutine);
+                false
             }
-        }
+        };
 
-        let expr =
+        let yielded =
             opt_expr.as_ref().map(|x| self.lower_expr(x)).unwrap_or_else(|| self.expr_unit(span));
 
-        hir::ExprKind::Yield(expr, hir::YieldSource::Yield)
+        if is_async_gen {
+            // `yield $expr` is transformed into `task_context = yield async_gen_ready($expr)`.
+            // This ensures that we store our resumed `ResumeContext` correctly, and also that
+            // the apparent value of the `yield` expression is `()`.
+            let wrapped_yielded = self.expr_call_lang_item_fn(
+                span,
+                hir::LangItem::AsyncGenReady,
+                std::slice::from_ref(yielded),
+            );
+            let yield_expr = self.arena.alloc(
+                self.expr(span, hir::ExprKind::Yield(wrapped_yielded, hir::YieldSource::Yield)),
+            );
+
+            let Some(task_context_hid) = self.task_context else {
+                unreachable!("use of `await` outside of an async context.");
+            };
+            let task_context_ident = Ident::with_dummy_span(sym::_task_context);
+            let lhs = self.expr_ident(span, task_context_ident, task_context_hid);
+
+            hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span))
+        } else {
+            hir::ExprKind::Yield(yielded, hir::YieldSource::Yield)
+        }
     }
 
     /// Desugar `ExprForLoop` from: `[opt_ident]: for <pat> in <head> <body>` into:
@@ -1588,7 +1721,6 @@
                 head_span,
                 hir::LangItem::IteratorNext,
                 arena_vec![self; ref_mut_iter],
-                None,
             );
             let arms = arena_vec![self; none_arm, some_arm];
 
@@ -1617,7 +1749,6 @@
                 head_span,
                 hir::LangItem::IntoIterIntoIter,
                 arena_vec![self; head],
-                None,
             )
         };
 
@@ -1655,13 +1786,13 @@
         let unstable_span = self.mark_span_with_reason(
             DesugaringKind::QuestionMark,
             span,
-            self.allow_try_trait.clone(),
+            Some(self.allow_try_trait.clone()),
         );
         let try_span = self.tcx.sess.source_map().end_point(span);
         let try_span = self.mark_span_with_reason(
             DesugaringKind::QuestionMark,
             try_span,
-            self.allow_try_trait.clone(),
+            Some(self.allow_try_trait.clone()),
         );
 
         // `Try::branch(<expr>)`
@@ -1673,7 +1804,6 @@
                 unstable_span,
                 hir::LangItem::TryTraitBranch,
                 arena_vec![self; sub_expr],
-                None,
             )
         };
 
@@ -1755,7 +1885,7 @@
         let unstable_span = self.mark_span_with_reason(
             DesugaringKind::YeetExpr,
             span,
-            self.allow_try_trait.clone(),
+            Some(self.allow_try_trait.clone()),
         );
 
         let from_yeet_expr = self.wrap_in_try_constructor(
@@ -1878,9 +2008,8 @@
         span: Span,
         lang_item: hir::LangItem,
         args: &'hir [hir::Expr<'hir>],
-        hir_id: Option<hir::HirId>,
     ) -> hir::Expr<'hir> {
-        let path = self.arena.alloc(self.expr_lang_item_path(span, lang_item, hir_id));
+        let path = self.arena.alloc(self.expr_lang_item_path(span, lang_item));
         self.expr_call_mut(span, path, args)
     }
 
@@ -1889,21 +2018,12 @@
         span: Span,
         lang_item: hir::LangItem,
         args: &'hir [hir::Expr<'hir>],
-        hir_id: Option<hir::HirId>,
     ) -> &'hir hir::Expr<'hir> {
-        self.arena.alloc(self.expr_call_lang_item_fn_mut(span, lang_item, args, hir_id))
+        self.arena.alloc(self.expr_call_lang_item_fn_mut(span, lang_item, args))
     }
 
-    fn expr_lang_item_path(
-        &mut self,
-        span: Span,
-        lang_item: hir::LangItem,
-        hir_id: Option<hir::HirId>,
-    ) -> hir::Expr<'hir> {
-        self.expr(
-            span,
-            hir::ExprKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span), hir_id)),
-        )
+    fn expr_lang_item_path(&mut self, span: Span, lang_item: hir::LangItem) -> hir::Expr<'hir> {
+        self.expr(span, hir::ExprKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span))))
     }
 
     /// `<LangItem>::name`
@@ -1916,7 +2036,7 @@
         let path = hir::ExprKind::Path(hir::QPath::TypeRelative(
             self.arena.alloc(self.ty(
                 span,
-                hir::TyKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span), None)),
+                hir::TyKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span))),
             )),
             self.arena.alloc(hir::PathSegment::new(
                 Ident::new(name, span),
diff --git a/compiler/rustc_ast_lowering/src/format.rs b/compiler/rustc_ast_lowering/src/format.rs
index c7d0719..6a82005 100644
--- a/compiler/rustc_ast_lowering/src/format.rs
+++ b/compiler/rustc_ast_lowering/src/format.rs
@@ -1,6 +1,6 @@
 use super::LoweringContext;
 use rustc_ast as ast;
-use rustc_ast::visit::{self, Visitor};
+use rustc_ast::visit::Visitor;
 use rustc_ast::*;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir as hir;
@@ -267,7 +267,7 @@
                 ctx.expr(
                     sp,
                     hir::ExprKind::Err(
-                        ctx.tcx.sess.delay_span_bug(sp, "lowered bad format_args count"),
+                        ctx.tcx.sess.span_delayed_bug(sp, "lowered bad format_args count"),
                     ),
                 )
             }
@@ -306,7 +306,7 @@
         }
         Err(_) => ctx.expr(
             sp,
-            hir::ExprKind::Err(ctx.tcx.sess.delay_span_bug(sp, "lowered bad format_args count")),
+            hir::ExprKind::Err(ctx.tcx.sess.span_delayed_bug(sp, "lowered bad format_args count")),
         ),
     };
     let &FormatOptions {
@@ -338,8 +338,8 @@
         | ((debug_hex == Some(FormatDebugHex::Lower)) as u32) << 4
         | ((debug_hex == Some(FormatDebugHex::Upper)) as u32) << 5;
     let flags = ctx.expr_u32(sp, flags);
-    let precision = make_count(ctx, sp, &precision, argmap);
-    let width = make_count(ctx, sp, &width, argmap);
+    let precision = make_count(ctx, sp, precision, argmap);
+    let width = make_count(ctx, sp, width, argmap);
     let format_placeholder_new = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
         sp,
         hir::LangItem::FormatPlaceholder,
diff --git a/compiler/rustc_ast_lowering/src/index.rs b/compiler/rustc_ast_lowering/src/index.rs
index eff362f..993ddf0 100644
--- a/compiler/rustc_ast_lowering/src/index.rs
+++ b/compiler/rustc_ast_lowering/src/index.rs
@@ -1,8 +1,7 @@
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sorted_map::SortedMap;
 use rustc_hir as hir;
-use rustc_hir::def_id::LocalDefId;
-use rustc_hir::intravisit::{self, Visitor};
+use rustc_hir::def_id::{LocalDefId, LocalDefIdMap};
+use rustc_hir::intravisit::Visitor;
 use rustc_hir::*;
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::span_bug;
@@ -10,14 +9,14 @@
 use rustc_span::{Span, DUMMY_SP};
 
 /// A visitor that walks over the HIR and collects `Node`s into a HIR map.
-pub(super) struct NodeCollector<'a, 'hir> {
+struct NodeCollector<'a, 'hir> {
     tcx: TyCtxt<'hir>,
 
     bodies: &'a SortedMap<ItemLocalId, &'hir Body<'hir>>,
 
     /// Outputs
     nodes: IndexVec<ItemLocalId, Option<ParentedNode<'hir>>>,
-    parenting: FxHashMap<LocalDefId, ItemLocalId>,
+    parenting: LocalDefIdMap<ItemLocalId>,
 
     /// The parent of this node
     parent_node: hir::ItemLocalId,
@@ -30,7 +29,7 @@
     tcx: TyCtxt<'hir>,
     item: hir::OwnerNode<'hir>,
     bodies: &SortedMap<ItemLocalId, &'hir Body<'hir>>,
-) -> (IndexVec<ItemLocalId, Option<ParentedNode<'hir>>>, FxHashMap<LocalDefId, ItemLocalId>) {
+) -> (IndexVec<ItemLocalId, Option<ParentedNode<'hir>>>, LocalDefIdMap<ItemLocalId>) {
     let mut nodes = IndexVec::new();
     // This node's parent should never be accessed: the owner's parent is computed by the
     // hir_owner_parent query. Make it invalid (= ItemLocalId::MAX) to force an ICE whenever it is
@@ -42,12 +41,12 @@
         parent_node: ItemLocalId::new(0),
         nodes,
         bodies,
-        parenting: FxHashMap::default(),
+        parenting: Default::default(),
     };
 
     match item {
         OwnerNode::Crate(citem) => {
-            collector.visit_mod(&citem, citem.spans.inner_span, hir::CRATE_HIR_ID)
+            collector.visit_mod(citem, citem.spans.inner_span, hir::CRATE_HIR_ID)
         }
         OwnerNode::Item(item) => collector.visit_item(item),
         OwnerNode::TraitItem(item) => collector.visit_trait_item(item),
@@ -89,7 +88,7 @@
             }
         }
 
-        self.nodes.insert(hir_id.local_id, ParentedNode { parent: self.parent_node, node: node });
+        self.nodes.insert(hir_id.local_id, ParentedNode { parent: self.parent_node, node });
     }
 
     fn with_parent<F: FnOnce(&mut Self)>(&mut self, parent_node_id: HirId, f: F) {
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 9a70e6d..5bddbe5 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -3,11 +3,9 @@
 use super::{AstOwner, ImplTraitContext, ImplTraitPosition};
 use super::{FnDeclKind, LoweringContext, ParamMode};
 
-use hir::definitions::DefPathData;
 use rustc_ast::ptr::P;
 use rustc_ast::visit::AssocCtxt;
 use rustc_ast::*;
-use rustc_data_structures::sorted_map::SortedMap;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
@@ -55,42 +53,7 @@
         owner: NodeId,
         f: impl FnOnce(&mut LoweringContext<'_, 'hir>) -> hir::OwnerNode<'hir>,
     ) {
-        let allow_gen_future = Some(if self.tcx.features().async_fn_track_caller {
-            [sym::gen_future, sym::closure_track_caller][..].into()
-        } else {
-            [sym::gen_future][..].into()
-        });
-        let mut lctx = LoweringContext {
-            // Pseudo-globals.
-            tcx: self.tcx,
-            resolver: self.resolver,
-            arena: self.tcx.hir_arena,
-
-            // HirId handling.
-            bodies: Vec::new(),
-            attrs: SortedMap::default(),
-            children: Vec::default(),
-            current_hir_id_owner: hir::CRATE_OWNER_ID,
-            item_local_id_counter: hir::ItemLocalId::new(0),
-            node_id_to_local_id: Default::default(),
-            trait_map: Default::default(),
-
-            // Lowering state.
-            catch_scope: None,
-            loop_scope: None,
-            is_in_loop_condition: false,
-            is_in_trait_impl: false,
-            is_in_dyn_type: false,
-            coroutine_kind: None,
-            task_context: None,
-            current_item: None,
-            impl_trait_defs: Vec::new(),
-            impl_trait_bounds: Vec::new(),
-            allow_try_trait: Some([sym::try_trait_v2, sym::yeet_desugar_details][..].into()),
-            allow_gen_future,
-            generics_def_id_map: Default::default(),
-            host_param_id: None,
-        };
+        let mut lctx = LoweringContext::new(self.tcx, self.resolver);
         lctx.with_hir_id_owner(owner, |lctx| f(lctx));
 
         for (def_id, info) in lctx.children {
@@ -136,39 +99,9 @@
 
     fn lower_assoc_item(&mut self, item: &AssocItem, ctxt: AssocCtxt) {
         let def_id = self.resolver.node_id_to_def_id[&item.id];
-
         let parent_id = self.tcx.local_parent(def_id);
         let parent_hir = self.lower_node(parent_id).unwrap();
-        self.with_lctx(item.id, |lctx| {
-            // Evaluate with the lifetimes in `params` in-scope.
-            // This is used to track which lifetimes have already been defined,
-            // and which need to be replicated when lowering an async fn.
-
-            match parent_hir.node().expect_item().kind {
-                hir::ItemKind::Impl(impl_) => {
-                    lctx.is_in_trait_impl = impl_.of_trait.is_some();
-                }
-                hir::ItemKind::Trait(_, _, generics, _, _) if lctx.tcx.features().effects => {
-                    lctx.host_param_id = generics
-                        .params
-                        .iter()
-                        .find(|param| {
-                            parent_hir
-                                .attrs
-                                .get(param.hir_id.local_id)
-                                .iter()
-                                .any(|attr| attr.has_name(sym::rustc_host))
-                        })
-                        .map(|param| param.def_id);
-                }
-                _ => {}
-            }
-
-            match ctxt {
-                AssocCtxt::Trait => hir::OwnerNode::TraitItem(lctx.lower_trait_item(item)),
-                AssocCtxt::Impl => hir::OwnerNode::ImplItem(lctx.lower_impl_item(item)),
-            }
-        })
+        self.with_lctx(item.id, |lctx| lctx.lower_assoc_item(item, ctxt, parent_hir))
     }
 
     fn lower_foreign_item(&mut self, item: &ForeignItem) {
@@ -268,27 +201,30 @@
                 body,
                 ..
             }) => {
-                self.with_new_scopes(|this| {
-                    this.current_item = Some(ident.span);
-
+                self.with_new_scopes(ident.span, |this| {
                     // Note: we don't need to change the return type from `T` to
                     // `impl Future<Output = T>` here because lower_body
                     // only cares about the input argument patterns in the function
                     // declaration (decl), not the return types.
-                    let asyncness = header.asyncness;
-                    let body_id = this.lower_maybe_async_body(
+                    let coroutine_kind = header.coroutine_kind;
+                    let body_id = this.lower_maybe_coroutine_body(
                         span,
                         hir_id,
-                        &decl,
-                        asyncness,
+                        decl,
+                        coroutine_kind,
                         body.as_deref(),
                     );
 
                     let itctx = ImplTraitContext::Universal;
                     let (generics, decl) =
                         this.lower_generics(generics, header.constness, id, &itctx, |this| {
-                            let ret_id = asyncness.opt_return_id();
-                            this.lower_fn_decl(&decl, id, *fn_sig_span, FnDeclKind::Fn, ret_id)
+                            this.lower_fn_decl(
+                                decl,
+                                id,
+                                *fn_sig_span,
+                                FnDeclKind::Fn,
+                                coroutine_kind,
+                            )
                         });
                     let sig = hir::FnSig {
                         decl,
@@ -329,7 +265,7 @@
                     &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
                     |this| match ty {
                         None => {
-                            let guar = this.tcx.sess.delay_span_bug(
+                            let guar = this.tcx.sess.span_delayed_bug(
                                 span,
                                 "expected to lower type alias type, but it was missing",
                             );
@@ -485,7 +421,14 @@
             }
             ItemKind::MacroDef(MacroDef { body, macro_rules }) => {
                 let body = P(self.lower_delim_args(body));
-                let macro_kind = self.resolver.decl_macro_kind(self.local_def_id(id));
+                let def_id = self.local_def_id(id);
+                let def_kind = self.tcx.def_kind(def_id);
+                let DefKind::Macro(macro_kind) = def_kind else {
+                    unreachable!(
+                        "expected DefKind::Macro for macro item, found {}",
+                        def_kind.descr(def_id.to_def_id())
+                    );
+                };
                 let macro_def = self.arena.alloc(ast::MacroDef { body, macro_rules: *macro_rules });
                 hir::ItemKind::Macro(macro_def, macro_kind)
             }
@@ -614,6 +557,41 @@
         }
     }
 
+    fn lower_assoc_item(
+        &mut self,
+        item: &AssocItem,
+        ctxt: AssocCtxt,
+        parent_hir: &'hir hir::OwnerInfo<'hir>,
+    ) -> hir::OwnerNode<'hir> {
+        // Evaluate with the lifetimes in `params` in-scope.
+        // This is used to track which lifetimes have already been defined,
+        // and which need to be replicated when lowering an async fn.
+
+        match parent_hir.node().expect_item().kind {
+            hir::ItemKind::Impl(impl_) => {
+                self.is_in_trait_impl = impl_.of_trait.is_some();
+            }
+            hir::ItemKind::Trait(_, _, generics, _, _) if self.tcx.features().effects => {
+                self.host_param_id = generics
+                    .params
+                    .iter()
+                    .find(|param| {
+                        matches!(
+                            param.kind,
+                            hir::GenericParamKind::Const { is_host_effect: true, .. }
+                        )
+                    })
+                    .map(|param| param.def_id);
+            }
+            _ => {}
+        }
+
+        match ctxt {
+            AssocCtxt::Trait => hir::OwnerNode::TraitItem(self.lower_trait_item(item)),
+            AssocCtxt::Impl => hir::OwnerNode::ImplItem(self.lower_impl_item(item)),
+        }
+    }
+
     fn lower_foreign_item(&mut self, i: &ForeignItem) -> &'hir hir::ForeignItem<'hir> {
         let hir_id = self.lower_node_id(i.id);
         let owner_id = hir_id.expect_owner();
@@ -683,11 +661,12 @@
         vdata: &VariantData,
     ) -> hir::VariantData<'hir> {
         match vdata {
-            VariantData::Struct(fields, recovered) => hir::VariantData::Struct(
-                self.arena
+            VariantData::Struct { fields, recovered } => hir::VariantData::Struct {
+                fields: self
+                    .arena
                     .alloc_from_iter(fields.iter().enumerate().map(|f| self.lower_field_def(f))),
-                *recovered,
-            ),
+                recovered: *recovered,
+            },
             VariantData::Tuple(fields, id) => {
                 let ctor_id = self.lower_node_id(*id);
                 self.alias_attrs(ctor_id, parent_id);
@@ -744,7 +723,7 @@
         let (generics, kind, has_default) = match &i.kind {
             AssocItemKind::Const(box ConstItem { generics, ty, expr, .. }) => {
                 let (generics, kind) = self.lower_generics(
-                    &generics,
+                    generics,
                     Const::No,
                     i.id,
                     &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
@@ -761,27 +740,30 @@
                 (generics, kind, expr.is_some())
             }
             AssocItemKind::Fn(box Fn { sig, generics, body: None, .. }) => {
-                let asyncness = sig.header.asyncness;
                 let names = self.lower_fn_params_to_names(&sig.decl);
                 let (generics, sig) = self.lower_method_sig(
                     generics,
                     sig,
                     i.id,
                     FnDeclKind::Trait,
-                    asyncness.opt_return_id(),
+                    sig.header.coroutine_kind,
                 );
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Required(names)), false)
             }
             AssocItemKind::Fn(box Fn { sig, generics, body: Some(body), .. }) => {
-                let asyncness = sig.header.asyncness;
-                let body_id =
-                    self.lower_maybe_async_body(i.span, hir_id, &sig.decl, asyncness, Some(&body));
+                let body_id = self.lower_maybe_coroutine_body(
+                    i.span,
+                    hir_id,
+                    &sig.decl,
+                    sig.header.coroutine_kind,
+                    Some(body),
+                );
                 let (generics, sig) = self.lower_method_sig(
                     generics,
                     sig,
                     i.id,
                     FnDeclKind::Trait,
-                    asyncness.opt_return_id(),
+                    sig.header.coroutine_kind,
                 );
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Provided(body_id)), true)
             }
@@ -857,7 +839,7 @@
 
         let (generics, kind) = match &i.kind {
             AssocItemKind::Const(box ConstItem { generics, ty, expr, .. }) => self.lower_generics(
-                &generics,
+                generics,
                 Const::No,
                 i.id,
                 &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
@@ -870,13 +852,11 @@
                 },
             ),
             AssocItemKind::Fn(box Fn { sig, generics, body, .. }) => {
-                self.current_item = Some(i.span);
-                let asyncness = sig.header.asyncness;
-                let body_id = self.lower_maybe_async_body(
+                let body_id = self.lower_maybe_coroutine_body(
                     i.span,
                     hir_id,
                     &sig.decl,
-                    asyncness,
+                    sig.header.coroutine_kind,
                     body.as_deref(),
                 );
                 let (generics, sig) = self.lower_method_sig(
@@ -884,7 +864,7 @@
                     sig,
                     i.id,
                     if self.is_in_trait_impl { FnDeclKind::Impl } else { FnDeclKind::Inherent },
-                    asyncness.opt_return_id(),
+                    sig.header.coroutine_kind,
                 );
 
                 (generics, hir::ImplItemKind::Fn(sig, body_id))
@@ -899,7 +879,7 @@
                     &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
                     |this| match ty {
                         None => {
-                            let guar = this.tcx.sess.delay_span_bug(
+                            let guar = this.tcx.sess.span_delayed_bug(
                                 i.span,
                                 "expected to lower associated type, but it was missing",
                             );
@@ -1032,7 +1012,7 @@
     fn lower_block_expr_opt(&mut self, span: Span, block: Option<&Block>) -> hir::Expr<'hir> {
         match block {
             Some(block) => self.lower_block_expr(block),
-            None => self.expr_err(span, self.tcx.sess.delay_span_bug(span, "no block")),
+            None => self.expr_err(span, self.tcx.sess.span_delayed_bug(span, "no block")),
         }
     }
 
@@ -1042,24 +1022,26 @@
                 &[],
                 match expr {
                     Some(expr) => this.lower_expr_mut(expr),
-                    None => this.expr_err(span, this.tcx.sess.delay_span_bug(span, "no block")),
+                    None => this.expr_err(span, this.tcx.sess.span_delayed_bug(span, "no block")),
                 },
             )
         })
     }
 
-    fn lower_maybe_async_body(
+    /// Takes what may be the body of an `async fn` or a `gen fn` and wraps it in an `async {}` or
+    /// `gen {}` block as appropriate.
+    fn lower_maybe_coroutine_body(
         &mut self,
         span: Span,
         fn_id: hir::HirId,
         decl: &FnDecl,
-        asyncness: Async,
+        coroutine_kind: Option<CoroutineKind>,
         body: Option<&Block>,
     ) -> hir::BodyId {
-        let (closure_id, body) = match (asyncness, body) {
-            (Async::Yes { closure_id, .. }, Some(body)) => (closure_id, body),
-            _ => return self.lower_fn_body_block(span, decl, body),
+        let (Some(coroutine_kind), Some(body)) = (coroutine_kind, body) else {
+            return self.lower_fn_body_block(span, decl, body);
         };
+        let closure_id = coroutine_kind.closure_id();
 
         self.lower_body(|this| {
             let mut parameters: Vec<hir::Param<'_>> = Vec::new();
@@ -1200,44 +1182,63 @@
                 parameters.push(new_parameter);
             }
 
-            let async_expr = this.make_async_expr(
-                CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
-                closure_id,
-                None,
-                body.span,
-                hir::CoroutineSource::Fn,
-                |this| {
-                    // Create a block from the user's function body:
-                    let user_body = this.lower_block_expr(body);
+            let mkbody = |this: &mut LoweringContext<'_, 'hir>| {
+                // Create a block from the user's function body:
+                let user_body = this.lower_block_expr(body);
 
-                    // Transform into `drop-temps { <user-body> }`, an expression:
-                    let desugared_span =
-                        this.mark_span_with_reason(DesugaringKind::Async, user_body.span, None);
-                    let user_body =
-                        this.expr_drop_temps(desugared_span, this.arena.alloc(user_body));
+                // Transform into `drop-temps { <user-body> }`, an expression:
+                let desugared_span =
+                    this.mark_span_with_reason(DesugaringKind::Async, user_body.span, None);
+                let user_body = this.expr_drop_temps(desugared_span, this.arena.alloc(user_body));
 
-                    // As noted above, create the final block like
-                    //
-                    // ```
-                    // {
-                    //   let $param_pattern = $raw_param;
-                    //   ...
-                    //   drop-temps { <user-body> }
-                    // }
-                    // ```
-                    let body = this.block_all(
-                        desugared_span,
-                        this.arena.alloc_from_iter(statements),
-                        Some(user_body),
-                    );
+                // As noted above, create the final block like
+                //
+                // ```
+                // {
+                //   let $param_pattern = $raw_param;
+                //   ...
+                //   drop-temps { <user-body> }
+                // }
+                // ```
+                let body = this.block_all(
+                    desugared_span,
+                    this.arena.alloc_from_iter(statements),
+                    Some(user_body),
+                );
 
-                    this.expr_block(body)
-                },
-            );
+                this.expr_block(body)
+            };
+            // FIXME(gen_blocks): Consider unifying the `make_*_expr` functions.
+            let coroutine_expr = match coroutine_kind {
+                CoroutineKind::Async { .. } => this.make_async_expr(
+                    CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
+                    closure_id,
+                    None,
+                    body.span,
+                    hir::CoroutineSource::Fn,
+                    mkbody,
+                ),
+                CoroutineKind::Gen { .. } => this.make_gen_expr(
+                    CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
+                    closure_id,
+                    None,
+                    body.span,
+                    hir::CoroutineSource::Fn,
+                    mkbody,
+                ),
+                CoroutineKind::AsyncGen { .. } => this.make_async_gen_expr(
+                    CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
+                    closure_id,
+                    None,
+                    body.span,
+                    hir::CoroutineSource::Fn,
+                    mkbody,
+                ),
+            };
 
             let hir_id = this.lower_node_id(closure_id);
             this.maybe_forward_track_caller(body.span, fn_id, hir_id);
-            let expr = hir::Expr { hir_id, kind: async_expr, span: this.lower_span(body.span) };
+            let expr = hir::Expr { hir_id, kind: coroutine_expr, span: this.lower_span(body.span) };
 
             (this.arena.alloc_from_iter(parameters), expr)
         })
@@ -1249,21 +1250,26 @@
         sig: &FnSig,
         id: NodeId,
         kind: FnDeclKind,
-        is_async: Option<(NodeId, Span)>,
+        coroutine_kind: Option<CoroutineKind>,
     ) -> (&'hir hir::Generics<'hir>, hir::FnSig<'hir>) {
         let header = self.lower_fn_header(sig.header);
         let itctx = ImplTraitContext::Universal;
         let (generics, decl) =
             self.lower_generics(generics, sig.header.constness, id, &itctx, |this| {
-                this.lower_fn_decl(&sig.decl, id, sig.span, kind, is_async)
+                this.lower_fn_decl(&sig.decl, id, sig.span, kind, coroutine_kind)
             });
         (generics, hir::FnSig { header, decl, span: self.lower_span(sig.span) })
     }
 
     fn lower_fn_header(&mut self, h: FnHeader) -> hir::FnHeader {
+        let asyncness = if let Some(CoroutineKind::Async { span, .. }) = h.coroutine_kind {
+            hir::IsAsync::Async(span)
+        } else {
+            hir::IsAsync::NotAsync
+        };
         hir::FnHeader {
             unsafety: self.lower_unsafety(h.unsafety),
-            asyncness: self.lower_asyncness(h.asyncness),
+            asyncness: asyncness,
             constness: self.lower_constness(h.constness),
             abi: self.lower_extern(h.ext),
         }
@@ -1305,13 +1311,6 @@
         });
     }
 
-    fn lower_asyncness(&mut self, a: Async) -> hir::IsAsync {
-        match a {
-            Async::Yes { span, .. } => hir::IsAsync::Async(span),
-            Async::No => hir::IsAsync::NotAsync,
-        }
-    }
-
     pub(super) fn lower_constness(&mut self, c: Const) -> hir::Constness {
         match c {
             Const::Yes(_) => hir::Constness::Const,
@@ -1389,26 +1388,18 @@
         let host_param_parts = if let Const::Yes(span) = constness
             && self.tcx.features().effects
         {
-            if let Some(param) =
-                generics.params.iter().find(|x| x.attrs.iter().any(|x| x.has_name(sym::rustc_host)))
-            {
-                // user has manually specified a `rustc_host` param, in this case, we set
-                // the param id so that lowering logic can use that. But we don't create
-                // another host param, so this gives `None`.
-                self.host_param_id = Some(self.local_def_id(param.id));
-                None
-            } else {
-                let param_node_id = self.next_node_id();
-                let hir_id = self.next_id();
-                let def_id = self.create_def(
-                    self.local_def_id(parent_node_id),
-                    param_node_id,
-                    DefPathData::TypeNs(sym::host),
-                    span,
-                );
-                self.host_param_id = Some(def_id);
-                Some((span, hir_id, def_id))
-            }
+            let span = self.lower_span(span);
+            let param_node_id = self.next_node_id();
+            let hir_id = self.next_id();
+            let def_id = self.create_def(
+                self.local_def_id(parent_node_id),
+                param_node_id,
+                sym::host,
+                DefKind::ConstParam,
+                span,
+            );
+            self.host_param_id = Some(def_id);
+            Some((span, hir_id, def_id))
         } else {
             None
         };
@@ -1462,8 +1453,8 @@
 
         if let Some((span, hir_id, def_id)) = host_param_parts {
             let const_node_id = self.next_node_id();
-            let anon_const: LocalDefId =
-                self.create_def(def_id, const_node_id, DefPathData::AnonConst, span);
+            let anon_const =
+                self.create_def(def_id, const_node_id, kw::Empty, DefKind::AnonConst, span);
 
             let const_id = self.next_id();
             let const_expr_id = self.next_id();
@@ -1472,19 +1463,6 @@
             self.children.push((def_id, hir::MaybeOwner::NonOwner(hir_id)));
             self.children.push((anon_const, hir::MaybeOwner::NonOwner(const_id)));
 
-            let attr_id = self.tcx.sess.parse_sess.attr_id_generator.mk_attr_id();
-
-            let attrs = self.arena.alloc_from_iter([Attribute {
-                kind: AttrKind::Normal(P(NormalAttr::from_ident(Ident::new(
-                    sym::rustc_host,
-                    span,
-                )))),
-                span,
-                id: attr_id,
-                style: AttrStyle::Outer,
-            }]);
-            self.attrs.insert(hir_id.local_id, attrs);
-
             let const_body = self.lower_body(|this| {
                 (
                     &[],
@@ -1526,6 +1504,7 @@
                         hir_id: const_id,
                         body: const_body,
                     }),
+                    is_host_effect: true,
                 },
                 colon_span: None,
                 pure_wrt_drop: false,
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index a88493a..47b9298 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -30,12 +30,11 @@
 //! get confused if the spans from leaf AST nodes occur in multiple places
 //! in the HIR, especially for multiple identifiers.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![feature(box_patterns)]
 #![feature(let_chains)]
-#![feature(never_type)]
 #![recursion_limit = "256"]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -45,32 +44,24 @@
 
 use crate::errors::{AssocTyParentheses, AssocTyParenthesesSub, MisplacedImplTrait};
 
+use rustc_ast::node_id::NodeMap;
 use rustc_ast::ptr::P;
-use rustc_ast::visit;
 use rustc_ast::{self as ast, *};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sorted_map::SortedMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
-use rustc_errors::{
-    DiagnosticArgFromDisplay, DiagnosticMessage, Handler, StashKey, SubdiagnosticMessage,
-};
-use rustc_fluent_macro::fluent_messages;
+use rustc_errors::{DiagnosticArgFromDisplay, StashKey};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, LifetimeRes, Namespace, PartialRes, PerNS, Res};
-use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
-use rustc_hir::definitions::DefPathData;
-use rustc_hir::{ConstArg, GenericArg, ItemLocalId, ParamName, TraitCandidate};
+use rustc_hir::def_id::{LocalDefId, LocalDefIdMap, CRATE_DEF_ID, LOCAL_CRATE};
+use rustc_hir::{ConstArg, GenericArg, ItemLocalMap, ParamName, TraitCandidate};
 use rustc_index::{Idx, IndexSlice, IndexVec};
-use rustc_middle::{
-    span_bug,
-    ty::{ResolverAstLowering, TyCtxt},
-};
+use rustc_middle::span_bug;
+use rustc_middle::ty::{ResolverAstLowering, TyCtxt};
 use rustc_session::parse::{add_feature_diagnostics, feature_err};
-use rustc_span::hygiene::MacroKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{DesugaringKind, Span, DUMMY_SP};
 use smallvec::SmallVec;
@@ -94,7 +85,7 @@
 mod pat;
 mod path;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 struct LoweringContext<'a, 'hir> {
     tcx: TyCtxt<'hir>,
@@ -128,26 +119,70 @@
 
     current_hir_id_owner: hir::OwnerId,
     item_local_id_counter: hir::ItemLocalId,
-    trait_map: FxHashMap<ItemLocalId, Box<[TraitCandidate]>>,
+    trait_map: ItemLocalMap<Box<[TraitCandidate]>>,
 
     impl_trait_defs: Vec<hir::GenericParam<'hir>>,
     impl_trait_bounds: Vec<hir::WherePredicate<'hir>>,
 
     /// NodeIds that are lowered inside the current HIR owner.
-    node_id_to_local_id: FxHashMap<NodeId, hir::ItemLocalId>,
+    node_id_to_local_id: NodeMap<hir::ItemLocalId>,
 
-    allow_try_trait: Option<Lrc<[Symbol]>>,
-    allow_gen_future: Option<Lrc<[Symbol]>>,
+    allow_try_trait: Lrc<[Symbol]>,
+    allow_gen_future: Lrc<[Symbol]>,
+    allow_async_iterator: Lrc<[Symbol]>,
 
     /// Mapping from generics `def_id`s to TAIT generics `def_id`s.
     /// For each captured lifetime (e.g., 'a), we create a new lifetime parameter that is a generic
     /// defined on the TAIT, so we have type Foo<'a1> = ... and we establish a mapping in this
     /// field from the original parameter 'a to the new parameter 'a1.
-    generics_def_id_map: Vec<FxHashMap<LocalDefId, LocalDefId>>,
+    generics_def_id_map: Vec<LocalDefIdMap<LocalDefId>>,
 
     host_param_id: Option<LocalDefId>,
 }
 
+impl<'a, 'hir> LoweringContext<'a, 'hir> {
+    fn new(tcx: TyCtxt<'hir>, resolver: &'a mut ResolverAstLowering) -> Self {
+        Self {
+            // Pseudo-globals.
+            tcx,
+            resolver: resolver,
+            arena: tcx.hir_arena,
+
+            // HirId handling.
+            bodies: Vec::new(),
+            attrs: SortedMap::default(),
+            children: Vec::default(),
+            current_hir_id_owner: hir::CRATE_OWNER_ID,
+            item_local_id_counter: hir::ItemLocalId::new(0),
+            node_id_to_local_id: Default::default(),
+            trait_map: Default::default(),
+
+            // Lowering state.
+            catch_scope: None,
+            loop_scope: None,
+            is_in_loop_condition: false,
+            is_in_trait_impl: false,
+            is_in_dyn_type: false,
+            coroutine_kind: None,
+            task_context: None,
+            current_item: None,
+            impl_trait_defs: Vec::new(),
+            impl_trait_bounds: Vec::new(),
+            allow_try_trait: [sym::try_trait_v2, sym::yeet_desugar_details].into(),
+            allow_gen_future: if tcx.features().async_fn_track_caller {
+                [sym::gen_future, sym::closure_track_caller].into()
+            } else {
+                [sym::gen_future].into()
+            },
+            // FIXME(gen_blocks): how does `closure_track_caller`/`async_fn_track_caller`
+            // interact with `gen`/`async gen` blocks
+            allow_async_iterator: [sym::gen_future, sym::async_iterator].into(),
+            generics_def_id_map: Default::default(),
+            host_param_id: None,
+        }
+    }
+}
+
 trait ResolverAstLoweringExt {
     fn legacy_const_generic_args(&self, expr: &Expr) -> Option<Vec<usize>>;
     fn get_partial_res(&self, id: NodeId) -> Option<PartialRes>;
@@ -156,7 +191,6 @@
     fn get_lifetime_res(&self, id: NodeId) -> Option<LifetimeRes>;
     fn take_extra_lifetime_params(&mut self, id: NodeId) -> Vec<(Ident, NodeId, LifetimeRes)>;
     fn remap_extra_lifetime_params(&mut self, from: NodeId, to: NodeId);
-    fn decl_macro_kind(&self, def_id: LocalDefId) -> MacroKind;
 }
 
 impl ResolverAstLoweringExt for ResolverAstLowering {
@@ -220,10 +254,6 @@
         let lifetimes = self.extra_lifetime_params_map.remove(&from).unwrap_or_default();
         self.extra_lifetime_params_map.insert(to, lifetimes);
     }
-
-    fn decl_macro_kind(&self, def_id: LocalDefId) -> MacroKind {
-        self.builtin_macro_kinds.get(&def_id).copied().unwrap_or(MacroKind::Bang)
-    }
 }
 
 /// Context of `impl Trait` in code, which determines whether it is allowed in an HIR subtree,
@@ -350,7 +380,7 @@
 }
 
 fn index_crate<'a>(
-    node_id_to_def_id: &FxHashMap<NodeId, LocalDefId>,
+    node_id_to_def_id: &NodeMap<LocalDefId>,
     krate: &'a Crate,
 ) -> IndexVec<LocalDefId, AstOwner<'a>> {
     let mut indexer = Indexer { node_id_to_def_id, index: IndexVec::new() };
@@ -360,7 +390,7 @@
     return indexer.index;
 
     struct Indexer<'s, 'a> {
-        node_id_to_def_id: &'s FxHashMap<NodeId, LocalDefId>,
+        node_id_to_def_id: &'s NodeMap<LocalDefId>,
         index: IndexVec<LocalDefId, AstOwner<'a>>,
     }
 
@@ -421,6 +451,7 @@
     tcx.ensure_with_value().output_filenames(());
     tcx.ensure_with_value().early_lint_checks(());
     tcx.ensure_with_value().debugger_visualizers(LOCAL_CRATE);
+    tcx.ensure_with_value().get_lang_items(());
     let (mut resolver, krate) = tcx.resolver_for_lowering(()).steal();
 
     let ast_index = index_crate(&resolver.node_id_to_def_id, &krate);
@@ -443,11 +474,6 @@
     drop(ast_index);
     sess.time("drop_ast", || drop(krate));
 
-    // Discard hygiene data, which isn't required after lowering to HIR.
-    if !sess.opts.unstable_opts.keep_hygiene_data {
-        rustc_span::hygiene::clear_syntax_context_map();
-    }
-
     // Don't hash unless necessary, because it's expensive.
     let opt_hir_hash =
         if tcx.needs_crate_hash() { Some(compute_hir_hash(tcx, &owners)) } else { None };
@@ -474,19 +500,20 @@
         &mut self,
         parent: LocalDefId,
         node_id: ast::NodeId,
-        data: DefPathData,
+        name: Symbol,
+        def_kind: DefKind,
         span: Span,
     ) -> LocalDefId {
         debug_assert_ne!(node_id, ast::DUMMY_NODE_ID);
         assert!(
             self.opt_local_def_id(node_id).is_none(),
-            "adding a def'n for node-id {:?} and data {:?} but a previous def'n exists: {:?}",
+            "adding a def'n for node-id {:?} and def kind {:?} but a previous def'n exists: {:?}",
             node_id,
-            data,
+            def_kind,
             self.tcx.hir().def_key(self.local_def_id(node_id)),
         );
 
-        let def_id = self.tcx.at(span).create_def(parent, data).def_id();
+        let def_id = self.tcx.at(span).create_def(parent, name, def_kind).def_id();
 
         debug!("create_def: def_id_to_node_id[{:?}] <-> {:?}", def_id, node_id);
         self.resolver.node_id_to_def_id.insert(node_id, def_id);
@@ -504,7 +531,7 @@
     /// Given the id of some node in the AST, finds the `LocalDefId` associated with it by the name
     /// resolver (if any).
     fn orig_opt_local_def_id(&self, node: NodeId) -> Option<LocalDefId> {
-        self.resolver.node_id_to_def_id.get(&node).map(|local_def_id| *local_def_id)
+        self.resolver.node_id_to_def_id.get(&node).copied()
     }
 
     /// Given the id of some node in the AST, finds the `LocalDefId` associated with it by the name
@@ -547,7 +574,7 @@
         self.generics_def_id_map
             .iter()
             .rev()
-            .find_map(|map| map.get(&local_def_id).map(|local_def_id| *local_def_id))
+            .find_map(|map| map.get(&local_def_id).copied())
             .unwrap_or(local_def_id)
     }
 
@@ -615,7 +642,7 @@
     /// `'a` declared on the TAIT, instead of the function.
     fn with_remapping<R>(
         &mut self,
-        remap: FxHashMap<LocalDefId, LocalDefId>,
+        remap: LocalDefIdMap<LocalDefId>,
         f: impl FnOnce(&mut Self) -> R,
     ) -> R {
         self.generics_def_id_map.push(remap);
@@ -737,8 +764,26 @@
         self.resolver.get_import_res(id).present_items()
     }
 
-    fn diagnostic(&self) -> &Handler {
-        self.tcx.sess.diagnostic()
+    fn make_lang_item_path(
+        &mut self,
+        lang_item: hir::LangItem,
+        span: Span,
+        args: Option<&'hir hir::GenericArgs<'hir>>,
+    ) -> &'hir hir::Path<'hir> {
+        let def_id = self.tcx.require_lang_item(lang_item, Some(span));
+        let def_kind = self.tcx.def_kind(def_id);
+        let res = Res::Def(def_kind, def_id);
+        self.arena.alloc(hir::Path {
+            span,
+            res,
+            segments: self.arena.alloc_from_iter([hir::PathSegment {
+                ident: Ident::new(lang_item.name(), span),
+                hir_id: self.next_id(),
+                res,
+                args,
+                infer_args: false,
+            }]),
+        })
     }
 
     /// Reuses the span but adds information like the kind of the desugaring and features that are
@@ -787,7 +832,8 @@
                 let _def_id = self.create_def(
                     self.current_hir_id_owner.def_id,
                     param,
-                    DefPathData::LifetimeNs(kw::UnderscoreLifetime),
+                    kw::UnderscoreLifetime,
+                    DefKind::LifetimeParam,
                     ident.span,
                 );
                 debug!(?_def_id);
@@ -851,7 +897,10 @@
         result
     }
 
-    fn with_new_scopes<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
+    fn with_new_scopes<T>(&mut self, scope_span: Span, f: impl FnOnce(&mut Self) -> T) -> T {
+        let current_item = self.current_item;
+        self.current_item = Some(scope_span);
+
         let was_in_loop_condition = self.is_in_loop_condition;
         self.is_in_loop_condition = false;
 
@@ -863,6 +912,8 @@
 
         self.is_in_loop_condition = was_in_loop_condition;
 
+        self.current_item = current_item;
+
         ret
     }
 
@@ -1162,7 +1213,7 @@
         itctx: &ImplTraitContext,
     ) -> hir::GenericArg<'hir> {
         match arg {
-            ast::GenericArg::Lifetime(lt) => GenericArg::Lifetime(self.lower_lifetime(&lt)),
+            ast::GenericArg::Lifetime(lt) => GenericArg::Lifetime(self.lower_lifetime(lt)),
             ast::GenericArg::Type(ty) => {
                 match &ty.kind {
                     TyKind::Infer if self.tcx.features().generic_arg_infer => {
@@ -1199,7 +1250,8 @@
                                 let def_id = self.create_def(
                                     parent_def_id.def_id,
                                     node_id,
-                                    DefPathData::AnonConst,
+                                    kw::Empty,
+                                    DefKind::AnonConst,
                                     span,
                                 );
 
@@ -1211,7 +1263,7 @@
                                     tokens: None,
                                 };
 
-                                let ct = self.with_new_scopes(|this| hir::AnonConst {
+                                let ct = self.with_new_scopes(span, |this| hir::AnonConst {
                                     def_id,
                                     hir_id: this.lower_node_id(node_id),
                                     body: this.lower_const_body(path_expr.span, Some(&path_expr)),
@@ -1226,10 +1278,10 @@
                     }
                     _ => {}
                 }
-                GenericArg::Type(self.lower_ty(&ty, itctx))
+                GenericArg::Type(self.lower_ty(ty, itctx))
             }
             ast::GenericArg::Const(ct) => GenericArg::Const(ConstArg {
-                value: self.lower_anon_const(&ct),
+                value: self.lower_anon_const(ct),
                 span: self.lower_span(ct.value.span),
                 is_desugared_from_effects: false,
             }),
@@ -1272,7 +1324,7 @@
                 let lifetime_bound = this.elided_dyn_bound(t.span);
                 (bounds, lifetime_bound)
             });
-            let kind = hir::TyKind::TraitObject(bounds, &lifetime_bound, TraitObjectSyntax::None);
+            let kind = hir::TyKind::TraitObject(bounds, lifetime_bound, TraitObjectSyntax::None);
             return hir::Ty { kind, span: self.lower_span(t.span), hir_id: self.next_id() };
         }
 
@@ -1293,7 +1345,7 @@
         let kind = match &t.kind {
             TyKind::Infer => hir::TyKind::Infer,
             TyKind::Err => {
-                hir::TyKind::Err(self.tcx.sess.delay_span_bug(t.span, "TyKind::Err lowered"))
+                hir::TyKind::Err(self.tcx.sess.span_delayed_bug(t.span, "TyKind::Err lowered"))
             }
             // FIXME(unnamed_fields): IMPLEMENTATION IN PROGRESS
             #[allow(rustc::untranslatable_diagnostic)]
@@ -1374,7 +1426,7 @@
                             GenericBound::Trait(
                                 ty,
                                 modifier @ (TraitBoundModifier::None
-                                | TraitBoundModifier::MaybeConst
+                                | TraitBoundModifier::MaybeConst(_)
                                 | TraitBoundModifier::Negative),
                             ) => {
                                 Some(this.lower_poly_trait_ref(ty, itctx, modifier.to_constness()))
@@ -1436,7 +1488,8 @@
                         self.create_def(
                             self.current_hir_id_owner.def_id,
                             *def_node_id,
-                            DefPathData::TypeNs(ident.name),
+                            ident.name,
+                            DefKind::TyParam,
                             span,
                         );
                         let (param, bounds, path) = self.lower_universal_param_and_bounds(
@@ -1476,7 +1529,7 @@
             }
             TyKind::MacCall(_) => panic!("`TyKind::MacCall` should have been expanded by now"),
             TyKind::CVarArgs => {
-                let guar = self.tcx.sess.delay_span_bug(
+                let guar = self.tcx.sess.span_delayed_bug(
                     t.span,
                     "`TyKind::CVarArgs` should have been handled elsewhere",
                 );
@@ -1542,8 +1595,11 @@
                 Vec::new()
             }
             hir::OpaqueTyOrigin::FnReturn(..) => {
-                if let FnDeclKind::Impl | FnDeclKind::Trait =
-                    fn_kind.expect("expected RPITs to be lowered with a FnKind")
+                if matches!(
+                    fn_kind.expect("expected RPITs to be lowered with a FnKind"),
+                    FnDeclKind::Impl | FnDeclKind::Trait
+                ) || self.tcx.features().lifetime_capture_rules_2024
+                    || span.at_least_rust_2024()
                 {
                     // return-position impl trait in trait was decided to capture all
                     // in-scope lifetimes, which we collect for all opaques during resolution.
@@ -1556,7 +1612,7 @@
                     // in fn return position, like the `fn test<'a>() -> impl Debug + 'a`
                     // example, we only need to duplicate lifetimes that appear in the
                     // bounds, since those are the only ones that are captured by the opaque.
-                    lifetime_collector::lifetimes_in_bounds(&self.resolver, bounds)
+                    lifetime_collector::lifetimes_in_bounds(self.resolver, bounds)
                 }
             }
             hir::OpaqueTyOrigin::AsyncFn(..) => {
@@ -1589,14 +1645,15 @@
         let opaque_ty_def_id = self.create_def(
             self.current_hir_id_owner.def_id,
             opaque_ty_node_id,
-            DefPathData::ImplTrait,
+            kw::Empty,
+            DefKind::OpaqueTy,
             opaque_ty_span,
         );
         debug!(?opaque_ty_def_id);
 
         // Map from captured (old) lifetime to synthetic (new) lifetime.
         // Used to resolve lifetimes in the bounds of the opaque.
-        let mut captured_to_synthesized_mapping = FxHashMap::default();
+        let mut captured_to_synthesized_mapping = LocalDefIdMap::default();
         // List of (early-bound) synthetic lifetimes that are owned by the opaque.
         // This is used to create the `hir::Generics` owned by the opaque.
         let mut synthesized_lifetime_definitions = vec![];
@@ -1618,7 +1675,7 @@
                     } else {
                         self.tcx
                             .sess
-                            .delay_span_bug(lifetime.ident.span, "no def-id for fresh lifetime");
+                            .span_delayed_bug(lifetime.ident.span, "no def-id for fresh lifetime");
                         continue;
                     }
                 }
@@ -1643,8 +1700,9 @@
                 let duplicated_lifetime_def_id = self.create_def(
                     opaque_ty_def_id,
                     duplicated_lifetime_node_id,
-                    DefPathData::LifetimeNs(lifetime.ident.name),
-                    lifetime.ident.span,
+                    lifetime.ident.name,
+                    DefKind::LifetimeParam,
+                    self.lower_span(lifetime.ident.span),
                 );
                 captured_to_synthesized_mapping.insert(old_def_id, duplicated_lifetime_def_id);
                 // FIXME: Instead of doing this, we could move this whole loop
@@ -1653,7 +1711,7 @@
                 synthesized_lifetime_definitions.push((
                     duplicated_lifetime_node_id,
                     duplicated_lifetime_def_id,
-                    lifetime.ident,
+                    self.lower_ident(lifetime.ident),
                 ));
 
                 // Now make an arg that we can use for the generic params of the opaque tykind.
@@ -1747,13 +1805,15 @@
         }))
     }
 
-    // Lowers a function declaration.
-    //
-    // `decl`: the unlowered (AST) function declaration.
-    // `fn_node_id`: `impl Trait` arguments are lowered into generic parameters on the given `NodeId`.
-    // `make_ret_async`: if `Some`, converts `-> T` into `-> impl Future<Output = T>` in the
-    //      return type. This is used for `async fn` declarations. The `NodeId` is the ID of the
-    //      return type `impl Trait` item, and the `Span` points to the `async` keyword.
+    /// Lowers a function declaration.
+    ///
+    /// `decl`: the unlowered (AST) function declaration.
+    ///
+    /// `fn_node_id`: `impl Trait` arguments are lowered into generic parameters on the given
+    /// `NodeId`.
+    ///
+    /// `transform_return_type`: if `Some`, applies some conversion to the return type, such as is
+    /// needed for `async fn` and `gen fn`. See [`CoroutineKind`] for more details.
     #[instrument(level = "debug", skip(self))]
     fn lower_fn_decl(
         &mut self,
@@ -1761,7 +1821,7 @@
         fn_node_id: NodeId,
         fn_span: Span,
         kind: FnDeclKind,
-        make_ret_async: Option<(NodeId, Span)>,
+        coro: Option<CoroutineKind>,
     ) -> &'hir hir::FnDecl<'hir> {
         let c_variadic = decl.c_variadic();
 
@@ -1790,11 +1850,12 @@
             self.lower_ty_direct(&param.ty, &itctx)
         }));
 
-        let output = if let Some((ret_id, _span)) = make_ret_async {
-            let fn_def_id = self.local_def_id(fn_node_id);
-            self.lower_async_fn_ret_ty(&decl.output, fn_def_id, ret_id, kind, fn_span)
-        } else {
-            match &decl.output {
+        let output = match coro {
+            Some(coro) => {
+                let fn_def_id = self.local_def_id(fn_node_id);
+                self.lower_coroutine_fn_ret_ty(&decl.output, fn_def_id, coro, kind, fn_span)
+            }
+            None => match &decl.output {
                 FnRetTy::Ty(ty) => {
                     let context = if kind.return_impl_trait_allowed() {
                         let fn_def_id = self.local_def_id(fn_node_id);
@@ -1818,7 +1879,7 @@
                     hir::FnRetTy::Return(self.lower_ty(ty, &context))
                 }
                 FnRetTy::Default(span) => hir::FnRetTy::DefaultReturn(self.lower_span(*span)),
-            }
+            },
         };
 
         self.arena.alloc(hir::FnDecl {
@@ -1857,16 +1918,26 @@
     // `fn_node_id`: `NodeId` of the parent function (used to create child impl trait definition)
     // `opaque_ty_node_id`: `NodeId` of the opaque `impl Trait` type that should be created
     #[instrument(level = "debug", skip(self))]
-    fn lower_async_fn_ret_ty(
+    fn lower_coroutine_fn_ret_ty(
         &mut self,
         output: &FnRetTy,
         fn_def_id: LocalDefId,
-        opaque_ty_node_id: NodeId,
+        coro: CoroutineKind,
         fn_kind: FnDeclKind,
         fn_span: Span,
     ) -> hir::FnRetTy<'hir> {
         let span = self.lower_span(fn_span);
-        let opaque_ty_span = self.mark_span_with_reason(DesugaringKind::Async, span, None);
+
+        let (opaque_ty_node_id, allowed_features) = match coro {
+            CoroutineKind::Async { return_impl_trait_id, .. } => (return_impl_trait_id, None),
+            CoroutineKind::Gen { return_impl_trait_id, .. } => (return_impl_trait_id, None),
+            CoroutineKind::AsyncGen { return_impl_trait_id, .. } => {
+                (return_impl_trait_id, Some(self.allow_async_iterator.clone()))
+            }
+        };
+
+        let opaque_ty_span =
+            self.mark_span_with_reason(DesugaringKind::Async, span, allowed_features);
 
         let captured_lifetimes: Vec<_> = self
             .resolver
@@ -1883,15 +1954,16 @@
             span,
             opaque_ty_span,
             |this| {
-                let future_bound = this.lower_async_fn_output_type_to_future_bound(
+                let bound = this.lower_coroutine_fn_output_type_to_bound(
                     output,
-                    span,
+                    coro,
+                    opaque_ty_span,
                     ImplTraitContext::ReturnPositionOpaqueTy {
                         origin: hir::OpaqueTyOrigin::FnReturn(fn_def_id),
                         fn_kind,
                     },
                 );
-                arena_vec![this; future_bound]
+                arena_vec![this; bound]
             },
         );
 
@@ -1900,10 +1972,11 @@
     }
 
     /// Transforms `-> T` into `Future<Output = T>`.
-    fn lower_async_fn_output_type_to_future_bound(
+    fn lower_coroutine_fn_output_type_to_bound(
         &mut self,
         output: &FnRetTy,
-        span: Span,
+        coro: CoroutineKind,
+        opaque_ty_span: Span,
         nested_impl_trait_context: ImplTraitContext,
     ) -> hir::GenericBound<'hir> {
         // Compute the `T` in `Future<Output = T>` from the return type.
@@ -1917,20 +1990,34 @@
             FnRetTy::Default(ret_ty_span) => self.arena.alloc(self.ty_tup(*ret_ty_span, &[])),
         };
 
-        // "<Output = T>"
-        let future_args = self.arena.alloc(hir::GenericArgs {
+        // "<$assoc_ty_name = T>"
+        let (assoc_ty_name, trait_lang_item) = match coro {
+            CoroutineKind::Async { .. } => (sym::Output, hir::LangItem::Future),
+            CoroutineKind::Gen { .. } => (sym::Item, hir::LangItem::Iterator),
+            CoroutineKind::AsyncGen { .. } => (sym::Item, hir::LangItem::AsyncIterator),
+        };
+
+        let bound_args = self.arena.alloc(hir::GenericArgs {
             args: &[],
-            bindings: arena_vec![self; self.output_ty_binding(span, output_ty)],
+            bindings: arena_vec![self; self.assoc_ty_binding(assoc_ty_name, opaque_ty_span, output_ty)],
             parenthesized: hir::GenericArgsParentheses::No,
             span_ext: DUMMY_SP,
         });
 
-        hir::GenericBound::LangItemTrait(
-            // ::std::future::Future<future_params>
-            hir::LangItem::Future,
-            self.lower_span(span),
-            self.next_id(),
-            future_args,
+        hir::GenericBound::Trait(
+            hir::PolyTraitRef {
+                bound_generic_params: &[],
+                trait_ref: hir::TraitRef {
+                    path: self.make_lang_item_path(
+                        trait_lang_item,
+                        opaque_ty_span,
+                        Some(bound_args),
+                    ),
+                    hir_ref_id: self.next_id(),
+                },
+                span: opaque_ty_span,
+            },
+            hir::TraitBoundModifier::None,
         )
     }
 
@@ -2072,14 +2159,12 @@
                 (hir::ParamName::Plain(self.lower_ident(param.ident)), kind)
             }
             GenericParamKind::Const { ty, kw_span: _, default } => {
-                let ty = self.lower_ty(
-                    &ty,
-                    &ImplTraitContext::Disallowed(ImplTraitPosition::GenericDefault),
-                );
+                let ty = self
+                    .lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::GenericDefault));
                 let default = default.as_ref().map(|def| self.lower_anon_const(def));
                 (
                     hir::ParamName::Plain(self.lower_ident(param.ident)),
-                    hir::GenericParamKind::Const { ty, default },
+                    hir::GenericParamKind::Const { ty, default, is_host_effect: false },
                 )
             }
         }
@@ -2200,7 +2285,7 @@
         match c.value.kind {
             ExprKind::Underscore => {
                 if self.tcx.features().generic_arg_infer {
-                    hir::ArrayLen::Infer(self.lower_node_id(c.id), c.value.span)
+                    hir::ArrayLen::Infer(self.lower_node_id(c.id), self.lower_span(c.value.span))
                 } else {
                     feature_err(
                         &self.tcx.sess.parse_sess,
@@ -2217,7 +2302,7 @@
     }
 
     fn lower_anon_const(&mut self, c: &AnonConst) -> hir::AnonConst {
-        self.with_new_scopes(|this| hir::AnonConst {
+        self.with_new_scopes(c.value.span, |this| hir::AnonConst {
             def_id: this.local_def_id(c.id),
             hir_id: this.lower_node_id(c.id),
             body: this.lower_const_body(c.value.span, Some(&c.value)),
@@ -2234,7 +2319,7 @@
     fn lower_trait_bound_modifier(&mut self, f: TraitBoundModifier) -> hir::TraitBoundModifier {
         match f {
             TraitBoundModifier::None => hir::TraitBoundModifier::None,
-            TraitBoundModifier::MaybeConst => hir::TraitBoundModifier::MaybeConst,
+            TraitBoundModifier::MaybeConst(_) => hir::TraitBoundModifier::MaybeConst,
 
             TraitBoundModifier::Negative => {
                 if self.tcx.features().negative_bounds {
@@ -2311,21 +2396,21 @@
 
     fn pat_cf_continue(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
         let field = self.single_pat_field(span, pat);
-        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowContinue, field, None)
+        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowContinue, field)
     }
 
     fn pat_cf_break(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
         let field = self.single_pat_field(span, pat);
-        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowBreak, field, None)
+        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowBreak, field)
     }
 
     fn pat_some(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
         let field = self.single_pat_field(span, pat);
-        self.pat_lang_item_variant(span, hir::LangItem::OptionSome, field, None)
+        self.pat_lang_item_variant(span, hir::LangItem::OptionSome, field)
     }
 
     fn pat_none(&mut self, span: Span) -> &'hir hir::Pat<'hir> {
-        self.pat_lang_item_variant(span, hir::LangItem::OptionNone, &[], None)
+        self.pat_lang_item_variant(span, hir::LangItem::OptionNone, &[])
     }
 
     fn single_pat_field(
@@ -2348,9 +2433,8 @@
         span: Span,
         lang_item: hir::LangItem,
         fields: &'hir [hir::PatField<'hir>],
-        hir_id: Option<hir::HirId>,
     ) -> &'hir hir::Pat<'hir> {
-        let qpath = hir::QPath::LangItem(lang_item, self.lower_span(span), hir_id);
+        let qpath = hir::QPath::LangItem(lang_item, self.lower_span(span));
         self.pat(span, hir::PatKind::Struct(qpath, fields, false))
     }
 
@@ -2482,9 +2566,10 @@
         let hir_id = lcx.next_id();
 
         let Some(host_param_id) = lcx.host_param_id else {
-            lcx.tcx
-                .sess
-                .delay_span_bug(span, "no host param id for call in const yet no errors reported");
+            lcx.tcx.sess.span_delayed_bug(
+                span,
+                "no host param id for call in const yet no errors reported",
+            );
             return;
         };
 
@@ -2507,17 +2592,14 @@
             })
         });
 
-        let attr_id = lcx.tcx.sess.parse_sess.attr_id_generator.mk_attr_id();
-        let attr = lcx.arena.alloc(Attribute {
-            kind: AttrKind::Normal(P(NormalAttr::from_ident(Ident::new(sym::rustc_host, span)))),
+        let def_id = lcx.create_def(
+            lcx.current_hir_id_owner.def_id,
+            id,
+            kw::Empty,
+            DefKind::AnonConst,
             span,
-            id: attr_id,
-            style: AttrStyle::Outer,
-        });
-        lcx.attrs.insert(hir_id.local_id, std::slice::from_ref(attr));
+        );
 
-        let def_id =
-            lcx.create_def(lcx.current_hir_id_owner.def_id, id, DefPathData::AnonConst, span);
         lcx.children.push((def_id, hir::MaybeOwner::NonOwner(hir_id)));
         self.args.push(hir::GenericArg::Const(hir::ConstArg {
             value: hir::AnonConst { def_id, hir_id, body },
diff --git a/compiler/rustc_ast_lowering/src/lifetime_collector.rs b/compiler/rustc_ast_lowering/src/lifetime_collector.rs
index d66bba5..4b1c057 100644
--- a/compiler/rustc_ast_lowering/src/lifetime_collector.rs
+++ b/compiler/rustc_ast_lowering/src/lifetime_collector.rs
@@ -108,7 +108,7 @@
     }
 }
 
-pub fn lifetimes_in_bounds(
+pub(crate) fn lifetimes_in_bounds(
     resolver: &ResolverAstLowering,
     bounds: &GenericBounds,
 ) -> Vec<Lifetime> {
diff --git a/compiler/rustc_ast_lowering/src/pat.rs b/compiler/rustc_ast_lowering/src/pat.rs
index a30f264..017314e 100644
--- a/compiler/rustc_ast_lowering/src/pat.rs
+++ b/compiler/rustc_ast_lowering/src/pat.rs
@@ -18,12 +18,13 @@
         self.arena.alloc(self.lower_pat_mut(pattern))
     }
 
-    pub(crate) fn lower_pat_mut(&mut self, mut pattern: &Pat) -> hir::Pat<'hir> {
+    fn lower_pat_mut(&mut self, mut pattern: &Pat) -> hir::Pat<'hir> {
         ensure_sufficient_stack(|| {
             // loop here to avoid recursion
             let node = loop {
                 match &pattern.kind {
                     PatKind::Wild => break hir::PatKind::Wild,
+                    PatKind::Never => break hir::PatKind::Never,
                     PatKind::Ident(binding_mode, ident, sub) => {
                         let lower_sub = |this: &mut Self| sub.as_ref().map(|s| this.lower_pat(s));
                         break self.lower_pat_ident(pattern, *binding_mode, *ident, lower_sub);
diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs
index 899f92a..5ceeb72 100644
--- a/compiler/rustc_ast_lowering/src/path.rs
+++ b/compiler/rustc_ast_lowering/src/path.rs
@@ -9,6 +9,7 @@
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, PartialRes, Res};
 use rustc_hir::GenericArg;
+use rustc_middle::span_bug;
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::{BytePos, Span, DUMMY_SP};
 
@@ -139,7 +140,7 @@
 
         // We should've returned in the for loop above.
 
-        self.diagnostic().span_bug(
+        self.tcx.sess.dcx().span_bug(
             p.span,
             format!(
                 "lower_qpath: no final extension segment in {}..{}",
@@ -285,7 +286,9 @@
         let (start, end) = match self.resolver.get_lifetime_res(segment_id) {
             Some(LifetimeRes::ElidedAnchor { start, end }) => (start, end),
             None => return,
-            Some(_) => panic!(),
+            Some(res) => {
+                span_bug!(path_span, "expected an elided lifetime to insert. found {res:?}")
+            }
         };
         let expected_lifetimes = end.as_usize() - start.as_usize();
         debug!(expected_lifetimes);
@@ -372,10 +375,10 @@
             // ```
             FnRetTy::Ty(ty) if matches!(itctx, ImplTraitContext::ReturnPositionOpaqueTy { .. }) => {
                 if self.tcx.features().impl_trait_in_fn_trait_return {
-                    self.lower_ty(&ty, itctx)
+                    self.lower_ty(ty, itctx)
                 } else {
                     self.lower_ty(
-                        &ty,
+                        ty,
                         &ImplTraitContext::FeatureGated(
                             ImplTraitPosition::FnTraitReturn,
                             sym::impl_trait_in_fn_trait_return,
@@ -384,12 +387,12 @@
                 }
             }
             FnRetTy::Ty(ty) => {
-                self.lower_ty(&ty, &ImplTraitContext::Disallowed(ImplTraitPosition::FnTraitReturn))
+                self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::FnTraitReturn))
             }
             FnRetTy::Default(_) => self.arena.alloc(self.ty_tup(*span, &[])),
         };
         let args = smallvec![GenericArg::Type(self.arena.alloc(self.ty_tup(*inputs_span, inputs)))];
-        let binding = self.output_ty_binding(output_ty.span, output_ty);
+        let binding = self.assoc_ty_binding(sym::Output, output_ty.span, output_ty);
         (
             GenericArgsCtor {
                 args,
@@ -401,13 +404,14 @@
         )
     }
 
-    /// An associated type binding `Output = $ty`.
-    pub(crate) fn output_ty_binding(
+    /// An associated type binding `$assoc_ty_name = $ty`.
+    pub(crate) fn assoc_ty_binding(
         &mut self,
+        assoc_ty_name: rustc_span::Symbol,
         span: Span,
         ty: &'hir hir::Ty<'hir>,
     ) -> hir::TypeBinding<'hir> {
-        let ident = Ident::with_dummy_span(hir::FN_OUTPUT_NAME);
+        let ident = Ident::with_dummy_span(assoc_ty_name);
         let kind = hir::TypeBindingKind::Equality { term: ty.into() };
         let args = arena_vec![self;];
         let bindings = arena_vec![self;];
diff --git a/compiler/rustc_ast_passes/Cargo.toml b/compiler/rustc_ast_passes/Cargo.toml
index 0001394..99e79f6 100644
--- a/compiler/rustc_ast_passes/Cargo.toml
+++ b/compiler/rustc_ast_passes/Cargo.toml
@@ -5,7 +5,7 @@
 
 [dependencies]
 # tidy-alphabetical-start
-itertools = "0.10.1"
+itertools = "0.11"
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr = { path = "../rustc_attr" }
diff --git a/compiler/rustc_ast_passes/messages.ftl b/compiler/rustc_ast_passes/messages.ftl
index d22bae8..790b583 100644
--- a/compiler/rustc_ast_passes/messages.ftl
+++ b/compiler/rustc_ast_passes/messages.ftl
@@ -174,6 +174,10 @@
 ast_passes_keyword_lifetime =
     lifetimes cannot use keyword names
 
+ast_passes_match_arm_with_no_body =
+    `match` arm with no body
+    .suggestion = add a body after the pattern
+
 ast_passes_module_nonascii = trying to load file for module `{$name}` with non-ascii identifier name
     .help = consider using the `#[path]` attribute to specify filesystem path
 
@@ -218,9 +222,13 @@
     .suggestion = provide a definition for the static
 
 ast_passes_tilde_const_disallowed = `~const` is not allowed here
-    .trait = trait objects cannot have `~const` trait bounds
     .closure = closures cannot have `~const` trait bounds
     .function = this function is not `const`, so it cannot have `~const` trait bounds
+    .trait = this trait is not a `#[const_trait]`, so it cannot have `~const` trait bounds
+    .trait_impl = this impl is not `const`, so it cannot have `~const` trait bounds
+    .impl = inherent impls cannot have `~const` trait bounds
+    .object = trait objects cannot have `~const` trait bounds
+    .item = this item cannot have `~const` trait bounds
 
 ast_passes_trait_fn_const =
     functions in traits cannot be declared const
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 3d0513c..887cb43 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -8,9 +8,9 @@
 
 use itertools::{Either, Itertools};
 use rustc_ast::ptr::P;
-use rustc_ast::visit::{self, AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor};
+use rustc_ast::visit::{AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor};
+use rustc_ast::walk_list;
 use rustc_ast::*;
-use rustc_ast::{walk_list, StaticItem};
 use rustc_ast_pretty::pprust::{self, State};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_feature::Features;
@@ -40,6 +40,10 @@
 enum DisallowTildeConstContext<'a> {
     TraitObject,
     Fn(FnKind<'a>),
+    Trait(Span),
+    TraitImpl(Span),
+    Impl(Span),
+    Item,
 }
 
 struct AstValidator<'a> {
@@ -110,18 +114,6 @@
         self.disallow_tilde_const = old;
     }
 
-    fn with_tilde_const_allowed(&mut self, f: impl FnOnce(&mut Self)) {
-        self.with_tilde_const(None, f)
-    }
-
-    fn with_banned_tilde_const(
-        &mut self,
-        ctx: DisallowTildeConstContext<'a>,
-        f: impl FnOnce(&mut Self),
-    ) {
-        self.with_tilde_const(Some(ctx), f)
-    }
-
     fn check_type_alias_where_clause_location(
         &mut self,
         ty_alias: &TyAlias,
@@ -173,7 +165,7 @@
                 self.with_impl_trait(Some(t.span), |this| visit::walk_ty(this, t))
             }
             TyKind::TraitObject(..) => self
-                .with_banned_tilde_const(DisallowTildeConstContext::TraitObject, |this| {
+                .with_tilde_const(Some(DisallowTildeConstContext::TraitObject), |this| {
                     visit::walk_ty(this, t)
                 }),
             TyKind::Path(qself, path) => {
@@ -229,8 +221,8 @@
         }
     }
 
-    fn err_handler(&self) -> &rustc_errors::Handler {
-        &self.session.diagnostic()
+    fn dcx(&self) -> &rustc_errors::DiagCtxt {
+        self.session.dcx()
     }
 
     fn check_lifetime(&self, ident: Ident) {
@@ -278,7 +270,7 @@
         ) {
             return;
         }
-        self.err_handler().emit_err(errors::InvalidUnnamedFieldTy { span, ty_span: ty.span });
+        self.dcx().emit_err(errors::InvalidUnnamedFieldTy { span, ty_span: ty.span });
     }
 
     fn deny_anon_struct_or_union(&self, ty: &Ty) {
@@ -287,15 +279,14 @@
             TyKind::AnonUnion(..) => "union",
             _ => return,
         };
-        self.err_handler()
-            .emit_err(errors::AnonStructOrUnionNotAllowed { struct_or_union, span: ty.span });
+        self.dcx().emit_err(errors::AnonStructOrUnionNotAllowed { struct_or_union, span: ty.span });
     }
 
     fn deny_unnamed_field(&self, field: &FieldDef) {
         if let Some(ident) = field.ident
             && ident.name == kw::Underscore
         {
-            self.err_handler()
+            self.dcx()
                 .emit_err(errors::InvalidUnnamedField { span: field.span, ident_span: ident.span });
         }
     }
@@ -401,7 +392,7 @@
             [b0] => b0.span(),
             [b0, .., bl] => b0.span().to(bl.span()),
         };
-        self.err_handler().emit_err(errors::BoundInContext { span, ctx });
+        self.dcx().emit_err(errors::BoundInContext { span, ctx });
     }
 
     fn check_foreign_ty_genericless(
@@ -411,7 +402,7 @@
         after_where_clause: &TyAliasWhereClause,
     ) {
         let cannot_have = |span, descr, remove_descr| {
-            self.err_handler().emit_err(errors::ExternTypesCannotHave {
+            self.dcx().emit_err(errors::ExternTypesCannotHave {
                 span,
                 descr,
                 remove_descr,
@@ -437,7 +428,7 @@
         let Some(body) = body else {
             return;
         };
-        self.err_handler().emit_err(errors::BodyInExtern {
+        self.dcx().emit_err(errors::BodyInExtern {
             span: ident.span,
             body,
             block: self.current_extern_span(),
@@ -450,7 +441,7 @@
         let Some(body) = body else {
             return;
         };
-        self.err_handler().emit_err(errors::FnBodyInExtern {
+        self.dcx().emit_err(errors::FnBodyInExtern {
             span: ident.span,
             body: body.span,
             block: self.current_extern_span(),
@@ -464,7 +455,7 @@
     /// An `fn` in `extern { ... }` cannot have qualifiers, e.g. `async fn`.
     fn check_foreign_fn_headerless(&self, ident: Ident, span: Span, header: FnHeader) {
         if header.has_qualifiers() {
-            self.err_handler().emit_err(errors::FnQualifierInExtern {
+            self.dcx().emit_err(errors::FnQualifierInExtern {
                 span: ident.span,
                 block: self.current_extern_span(),
                 sugg_span: span.until(ident.span.shrink_to_lo()),
@@ -475,7 +466,7 @@
     /// An item in `extern { ... }` cannot use non-ascii identifier.
     fn check_foreign_item_ascii_only(&self, ident: Ident) {
         if !ident.as_str().is_ascii() {
-            self.err_handler().emit_err(errors::ExternItemAscii {
+            self.dcx().emit_err(errors::ExternItemAscii {
                 span: ident.span,
                 block: self.current_extern_span(),
             });
@@ -504,7 +495,7 @@
             if let Const::Yes(const_span) = header.constness {
                 let mut spans = variadic_spans.clone();
                 spans.push(const_span);
-                self.err_handler().emit_err(errors::ConstAndCVariadic {
+                self.dcx().emit_err(errors::ConstAndCVariadic {
                     spans,
                     const_span,
                     variadic_spans: variadic_spans.clone(),
@@ -526,14 +517,14 @@
             _ => {}
         };
 
-        self.err_handler().emit_err(errors::BadCVariadic { span: variadic_spans });
+        self.dcx().emit_err(errors::BadCVariadic { span: variadic_spans });
     }
 
     fn check_item_named(&self, ident: Ident, kind: &str) {
         if ident.name != kw::Underscore {
             return;
         }
-        self.err_handler().emit_err(errors::ItemUnderscore { span: ident.span, kind });
+        self.dcx().emit_err(errors::ItemUnderscore { span: ident.span, kind });
     }
 
     fn check_nomangle_item_asciionly(&self, ident: Ident, item_span: Span) {
@@ -624,14 +615,14 @@
         let args_len = arg_spans.len();
         let constraint_len = constraint_spans.len();
         // ...and then error:
-        self.err_handler().emit_err(errors::ArgsBeforeConstraint {
+        self.dcx().emit_err(errors::ArgsBeforeConstraint {
             arg_spans: arg_spans.clone(),
             constraints: constraint_spans[0],
             args: *arg_spans.iter().last().unwrap(),
             data: data.span,
             constraint_spans: errors::EmptyLabelManySpans(constraint_spans),
             arg_spans2: errors::EmptyLabelManySpans(arg_spans),
-            suggestion: self.correct_generic_order_suggestion(&data),
+            suggestion: self.correct_generic_order_suggestion(data),
             constraint_len,
             args_len,
         });
@@ -676,7 +667,7 @@
                 }
 
                 if !bounds.iter().any(|b| matches!(b, GenericBound::Trait(..))) {
-                    self.err_handler().emit_err(errors::AtLeastOneTrait { span: ty.span });
+                    self.dcx().emit_err(errors::AtLeastOneTrait { span: ty.span });
                 }
             }
             _ => {}
@@ -706,7 +697,7 @@
 /// Checks that generic parameters are in the correct order,
 /// which is lifetimes, then types and then consts. (`<'a, T, const N: usize>`)
 fn validate_generic_param_order(
-    handler: &rustc_errors::Handler,
+    dcx: &rustc_errors::DiagCtxt,
     generics: &[GenericParam],
     span: Span,
 ) {
@@ -747,7 +738,7 @@
 
             if !bounds.is_empty() {
                 ordered_params += ": ";
-                ordered_params += &pprust::bounds_to_string(&bounds);
+                ordered_params += &pprust::bounds_to_string(bounds);
             }
 
             match kind {
@@ -769,7 +760,7 @@
         ordered_params += ">";
 
         for (param_ord, (max_param, spans)) in &out_of_order {
-            handler.emit_err(errors::OutOfOrderParams {
+            dcx.emit_err(errors::OutOfOrderParams {
                 spans: spans.clone(),
                 sugg_span: span,
                 param_ord,
@@ -832,7 +823,7 @@
                         errors::VisibilityNotPermittedNote::TraitImpl,
                     );
                     if let TyKind::Err = self_ty.kind {
-                        this.err_handler().emit_err(errors::ObsoleteAuto { span: item.span });
+                        this.dcx().emit_err(errors::ObsoleteAuto { span: item.span });
                     }
                     if let (&Unsafe::Yes(span), &ImplPolarity::Negative(sp)) = (unsafety, polarity)
                     {
@@ -845,11 +836,9 @@
 
                     this.visit_vis(&item.vis);
                     this.visit_ident(item.ident);
-                    if let Const::Yes(_) = constness {
-                        this.with_tilde_const_allowed(|this| this.visit_generics(generics));
-                    } else {
-                        this.visit_generics(generics);
-                    }
+                    let disallowed = matches!(constness, Const::No)
+                        .then(|| DisallowTildeConstContext::TraitImpl(item.span));
+                    this.with_tilde_const(disallowed, |this| this.visit_generics(generics));
                     this.visit_trait_ref(t);
                     this.visit_ty(self_ty);
 
@@ -863,10 +852,10 @@
                 polarity,
                 defaultness,
                 constness,
-                generics: _,
+                generics,
                 of_trait: None,
                 self_ty,
-                items: _,
+                items,
             }) => {
                 let error =
                     |annotation_span, annotation, only_trait: bool| errors::InherentImplCannot {
@@ -882,7 +871,7 @@
                     errors::VisibilityNotPermittedNote::IndividualImplItems,
                 );
                 if let &Unsafe::Yes(span) = unsafety {
-                    self.err_handler().emit_err(errors::InherentImplCannotUnsafe {
+                    self.dcx().emit_err(errors::InherentImplCannotUnsafe {
                         span: self_ty.span,
                         annotation_span: span,
                         annotation: "unsafe",
@@ -890,14 +879,24 @@
                     });
                 }
                 if let &ImplPolarity::Negative(span) = polarity {
-                    self.err_handler().emit_err(error(span, "negative", false));
+                    self.dcx().emit_err(error(span, "negative", false));
                 }
                 if let &Defaultness::Default(def_span) = defaultness {
-                    self.err_handler().emit_err(error(def_span, "`default`", true));
+                    self.dcx().emit_err(error(def_span, "`default`", true));
                 }
                 if let &Const::Yes(span) = constness {
-                    self.err_handler().emit_err(error(span, "`const`", true));
+                    self.dcx().emit_err(error(span, "`const`", true));
                 }
+
+                self.visit_vis(&item.vis);
+                self.visit_ident(item.ident);
+                self.with_tilde_const(Some(DisallowTildeConstContext::Impl(item.span)), |this| {
+                    this.visit_generics(generics)
+                });
+                self.visit_ty(self_ty);
+                walk_list!(self, visit_assoc_item, items, AssocCtxt::Impl);
+                walk_list!(self, visit_attribute, &item.attrs);
+                return; // Avoid visiting again.
             }
             ItemKind::Fn(box Fn { defaultness, sig, generics, body }) => {
                 self.check_defaultness(item.span, *defaultness);
@@ -940,7 +939,7 @@
                     errors::VisibilityNotPermittedNote::IndividualForeignItems,
                 );
                 if let &Unsafe::Yes(span) = unsafety {
-                    self.err_handler().emit_err(errors::UnsafeItem { span, kind: "extern block" });
+                    self.dcx().emit_err(errors::UnsafeItem { span, kind: "extern block" });
                 }
                 if abi.is_none() {
                     self.maybe_lint_missing_abi(item.span, item.id);
@@ -978,8 +977,10 @@
                     // context for the supertraits.
                     this.visit_vis(&item.vis);
                     this.visit_ident(item.ident);
-                    this.visit_generics(generics);
-                    this.with_tilde_const_allowed(|this| {
+                    let disallowed =
+                        (!is_const_trait).then(|| DisallowTildeConstContext::Trait(item.span));
+                    this.with_tilde_const(disallowed, |this| {
+                        this.visit_generics(generics);
                         walk_list!(this, visit_param_bound, bounds, BoundKind::SuperTraits)
                     });
                     walk_list!(this, visit_assoc_item, items, AssocCtxt::Trait);
@@ -989,7 +990,7 @@
             }
             ItemKind::Mod(unsafety, mod_kind) => {
                 if let &Unsafe::Yes(span) = unsafety {
-                    self.err_handler().emit_err(errors::UnsafeItem { span, kind: "module" });
+                    self.dcx().emit_err(errors::UnsafeItem { span, kind: "module" });
                 }
                 // Ensure that `path` attributes on modules are recorded as used (cf. issue #35584).
                 if !matches!(mod_kind, ModKind::Loaded(_, Inline::Yes, _))
@@ -999,16 +1000,11 @@
                 }
             }
             ItemKind::Struct(vdata, generics) => match vdata {
-                // Duplicating the `Visitor` logic allows catching all cases
-                // of `Anonymous(Struct, Union)` outside of a field struct or union.
-                //
-                // Inside `visit_ty` the validator catches every `Anonymous(Struct, Union)` it
-                // encounters, and only on `ItemKind::Struct` and `ItemKind::Union`
-                // it uses `visit_ty_common`, which doesn't contain that specific check.
-                VariantData::Struct(fields, ..) => {
+                VariantData::Struct { fields, .. } => {
                     self.visit_vis(&item.vis);
                     self.visit_ident(item.ident);
                     self.visit_generics(generics);
+                    // Permit `Anon{Struct,Union}` as field type.
                     walk_list!(self, visit_struct_field_def, fields);
                     walk_list!(self, visit_attribute, &item.attrs);
                     return;
@@ -1017,13 +1013,14 @@
             },
             ItemKind::Union(vdata, generics) => {
                 if vdata.fields().is_empty() {
-                    self.err_handler().emit_err(errors::FieldlessUnion { span: item.span });
+                    self.dcx().emit_err(errors::FieldlessUnion { span: item.span });
                 }
                 match vdata {
-                    VariantData::Struct(fields, ..) => {
+                    VariantData::Struct { fields, .. } => {
                         self.visit_vis(&item.vis);
                         self.visit_ident(item.ident);
                         self.visit_generics(generics);
+                        // Permit `Anon{Struct,Union}` as field type.
                         walk_list!(self, visit_struct_field_def, fields);
                         walk_list!(self, visit_attribute, &item.attrs);
                         return;
@@ -1031,12 +1028,14 @@
                     _ => {}
                 }
             }
-            ItemKind::Const(box ConstItem { defaultness, expr: None, .. }) => {
+            ItemKind::Const(box ConstItem { defaultness, expr, .. }) => {
                 self.check_defaultness(item.span, *defaultness);
-                self.session.emit_err(errors::ConstWithoutBody {
-                    span: item.span,
-                    replace_span: self.ending_semi_or_hi(item.span),
-                });
+                if expr.is_none() {
+                    self.session.emit_err(errors::ConstWithoutBody {
+                        span: item.span,
+                        replace_span: self.ending_semi_or_hi(item.span),
+                    });
+                }
             }
             ItemKind::Static(box StaticItem { expr: None, .. }) => {
                 self.session.emit_err(errors::StaticWithoutBody {
@@ -1058,10 +1057,10 @@
 
                 if self.features.lazy_type_alias {
                     if let Err(err) = self.check_type_alias_where_clause_location(ty_alias) {
-                        self.err_handler().emit_err(err);
+                        self.dcx().emit_err(err);
                     }
                 } else if where_clauses.1.0 {
-                    self.err_handler().emit_err(errors::WhereClauseAfterTypeAlias {
+                    self.dcx().emit_err(errors::WhereClauseAfterTypeAlias {
                         span: where_clauses.1.1,
                         help: self.session.is_nightly_build().then_some(()),
                     });
@@ -1146,14 +1145,14 @@
                 }
                 GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => {
                     if let Some(span) = prev_param_default {
-                        self.err_handler().emit_err(errors::GenericDefaultTrailing { span });
+                        self.dcx().emit_err(errors::GenericDefaultTrailing { span });
                         break;
                     }
                 }
             }
         }
 
-        validate_generic_param_order(self.err_handler(), &generics.params, generics.span);
+        validate_generic_param_order(self.dcx(), &generics.params, generics.span);
 
         for predicate in &generics.where_clause.predicates {
             if let WherePredicate::EqPredicate(predicate) = predicate {
@@ -1174,7 +1173,7 @@
                             match bound {
                                 GenericBound::Trait(t, _) => {
                                     if !t.bound_generic_params.is_empty() {
-                                        self.err_handler()
+                                        self.dcx()
                                             .emit_err(errors::NestedLifetimes { span: t.span });
                                     }
                                 }
@@ -1200,39 +1199,50 @@
         if let GenericBound::Trait(poly, modify) = bound {
             match (ctxt, modify) {
                 (BoundKind::SuperTraits, TraitBoundModifier::Maybe) => {
-                    self.err_handler().emit_err(errors::OptionalTraitSupertrait {
+                    self.dcx().emit_err(errors::OptionalTraitSupertrait {
                         span: poly.span,
                         path_str: pprust::path_to_string(&poly.trait_ref.path),
                     });
                 }
                 (BoundKind::TraitObject, TraitBoundModifier::Maybe) => {
-                    self.err_handler().emit_err(errors::OptionalTraitObject { span: poly.span });
+                    self.dcx().emit_err(errors::OptionalTraitObject { span: poly.span });
                 }
-                (_, TraitBoundModifier::MaybeConst)
+                (_, &TraitBoundModifier::MaybeConst(span))
                     if let Some(reason) = &self.disallow_tilde_const =>
                 {
                     let reason = match reason {
-                        DisallowTildeConstContext::TraitObject => {
-                            errors::TildeConstReason::TraitObject
-                        }
                         DisallowTildeConstContext::Fn(FnKind::Closure(..)) => {
                             errors::TildeConstReason::Closure
                         }
                         DisallowTildeConstContext::Fn(FnKind::Fn(_, ident, ..)) => {
                             errors::TildeConstReason::Function { ident: ident.span }
                         }
+                        &DisallowTildeConstContext::Trait(span) => {
+                            errors::TildeConstReason::Trait { span }
+                        }
+                        &DisallowTildeConstContext::TraitImpl(span) => {
+                            errors::TildeConstReason::TraitImpl { span }
+                        }
+                        &DisallowTildeConstContext::Impl(span) => {
+                            // FIXME(effects): Consider providing a help message or even a structured
+                            // suggestion for moving such bounds to the assoc const fns if available.
+                            errors::TildeConstReason::Impl { span }
+                        }
+                        DisallowTildeConstContext::TraitObject => {
+                            errors::TildeConstReason::TraitObject
+                        }
+                        DisallowTildeConstContext::Item => errors::TildeConstReason::Item,
                     };
-                    self.err_handler()
-                        .emit_err(errors::TildeConstDisallowed { span: bound.span(), reason });
+                    self.dcx().emit_err(errors::TildeConstDisallowed { span, reason });
                 }
                 (_, TraitBoundModifier::MaybeConstMaybe) => {
-                    self.err_handler().emit_err(errors::OptionalConstExclusive {
+                    self.dcx().emit_err(errors::OptionalConstExclusive {
                         span: bound.span(),
                         modifier: "?",
                     });
                 }
                 (_, TraitBoundModifier::MaybeConstNegative) => {
-                    self.err_handler().emit_err(errors::OptionalConstExclusive {
+                    self.dcx().emit_err(errors::OptionalConstExclusive {
                         span: bound.span(),
                         modifier: "!",
                     });
@@ -1248,7 +1258,7 @@
         {
             for arg in &args.args {
                 if let ast::AngleBracketedArg::Constraint(constraint) = arg {
-                    self.err_handler()
+                    self.dcx()
                         .emit_err(errors::ConstraintOnNegativeBound { span: constraint.span });
                 }
             }
@@ -1267,14 +1277,20 @@
 
         self.check_c_variadic_type(fk);
 
-        // Functions cannot both be `const async`
+        // Functions cannot both be `const async` or `const gen`
         if let Some(&FnHeader {
             constness: Const::Yes(cspan),
-            asyncness: Async::Yes { span: aspan, .. },
+            coroutine_kind: Some(coroutine_kind),
             ..
         }) = fk.header()
         {
-            self.err_handler().emit_err(errors::ConstAndAsync {
+            let aspan = match coroutine_kind {
+                CoroutineKind::Async { span: aspan, .. }
+                | CoroutineKind::Gen { span: aspan, .. }
+                | CoroutineKind::AsyncGen { span: aspan, .. } => aspan,
+            };
+            // FIXME(gen_blocks): Report a different error for `const gen`
+            self.dcx().emit_err(errors::ConstAndAsync {
                 spans: vec![cspan, aspan],
                 cspan,
                 aspan,
@@ -1314,10 +1330,8 @@
                     }
                 } else {
                     match ctxt {
-                        FnCtxt::Foreign => {
-                            self.err_handler().emit_err(errors::PatternInForeign { span })
-                        }
-                        _ => self.err_handler().emit_err(errors::PatternInBodiless { span }),
+                        FnCtxt::Foreign => self.dcx().emit_err(errors::PatternInForeign { span }),
+                        _ => self.dcx().emit_err(errors::PatternInBodiless { span }),
                     };
                 }
             });
@@ -1328,7 +1342,6 @@
                 || matches!(fk.ctxt(), Some(FnCtxt::Assoc(_)) if self.in_const_trait_or_impl);
 
         let disallowed = (!tilde_const_allowed).then(|| DisallowTildeConstContext::Fn(fk));
-
         self.with_tilde_const(disallowed, |this| visit::walk_fn(this, fk));
     }
 
@@ -1397,18 +1410,6 @@
         }
 
         match &item.kind {
-            AssocItemKind::Type(box TyAlias { generics, bounds, ty, .. })
-                if ctxt == AssocCtxt::Trait =>
-            {
-                self.visit_vis(&item.vis);
-                self.visit_ident(item.ident);
-                walk_list!(self, visit_attribute, &item.attrs);
-                self.with_tilde_const_allowed(|this| {
-                    this.visit_generics(generics);
-                    walk_list!(this, visit_param_bound, bounds, BoundKind::Bound);
-                });
-                walk_list!(self, visit_ty, ty);
-            }
             AssocItemKind::Fn(box Fn { sig, generics, body, .. })
                 if self.in_const_trait_or_impl
                     || ctxt == AssocCtxt::Trait
@@ -1461,9 +1462,7 @@
                     id: rustc_ast::node_id::DUMMY_NODE_ID,
                     ident: *ident,
                     gen_args,
-                    kind: AssocConstraintKind::Equality {
-                        term: predicate.rhs_ty.clone().into(),
-                    },
+                    kind: AssocConstraintKind::Equality { term: predicate.rhs_ty.clone().into() },
                     span: ident.span,
                 });
                 // Add `<Bar = RhsTy>` to `Foo`.
@@ -1476,11 +1475,7 @@
                     },
                     empty_args => {
                         *empty_args = Some(
-                            AngleBracketedArgs {
-                                span: ident.span,
-                                args: thin_vec![arg],
-                            }
-                            .into(),
+                            AngleBracketedArgs { span: ident.span, args: thin_vec![arg] }.into(),
                         );
                     }
                 }
@@ -1535,7 +1530,7 @@
             }
         }
     }
-    this.err_handler().emit_err(err);
+    this.dcx().emit_err(err);
 }
 
 pub fn check_crate(
@@ -1552,7 +1547,7 @@
         in_const_trait_or_impl: false,
         has_proc_macro_decls: false,
         outer_impl_trait: None,
-        disallow_tilde_const: None,
+        disallow_tilde_const: Some(DisallowTildeConstContext::Item),
         is_impl_trait_banned: false,
         lint_buffer: lints,
     };
diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs
index d14b62d..4283fc7 100644
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@@ -551,8 +551,6 @@
 
 #[derive(Subdiagnostic)]
 pub enum TildeConstReason {
-    #[note(ast_passes_trait)]
-    TraitObject,
     #[note(ast_passes_closure)]
     Closure,
     #[note(ast_passes_function)]
@@ -560,6 +558,25 @@
         #[primary_span]
         ident: Span,
     },
+    #[note(ast_passes_trait)]
+    Trait {
+        #[primary_span]
+        span: Span,
+    },
+    #[note(ast_passes_trait_impl)]
+    TraitImpl {
+        #[primary_span]
+        span: Span,
+    },
+    #[note(ast_passes_impl)]
+    Impl {
+        #[primary_span]
+        span: Span,
+    },
+    #[note(ast_passes_object)]
+    TraitObject,
+    #[note(ast_passes_item)]
+    Item,
 }
 
 #[derive(Diagnostic)]
@@ -746,3 +763,12 @@
     pub span: Span,
     pub struct_or_union: &'static str,
 }
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_match_arm_with_no_body)]
+pub struct MatchArmWithNoBody {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(code = " => todo!(),", applicability = "has-placeholders")]
+    pub suggestion: Span,
+}
diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs
index e1cf0a2..6900d33 100644
--- a/compiler/rustc_ast_passes/src/feature_gate.rs
+++ b/compiler/rustc_ast_passes/src/feature_gate.rs
@@ -88,7 +88,7 @@
             }
         }
 
-        match abi::is_enabled(&self.features, span, symbol_unescaped.as_str()) {
+        match abi::is_enabled(self.features, span, symbol_unescaped.as_str()) {
             Ok(()) => (),
             Err(abi::AbiDisabled::Unstable { feature, explain }) => {
                 feature_err_issue(
@@ -102,7 +102,7 @@
             }
             Err(abi::AbiDisabled::Unrecognized) => {
                 if self.sess.opts.pretty.map_or(true, |ppm| ppm.needs_hir()) {
-                    self.sess.parse_sess.span_diagnostic.delay_span_bug(
+                    self.sess.dcx().span_delayed_bug(
                         span,
                         format!(
                             "unrecognized ABI not caught in lowering: {}",
@@ -182,7 +182,7 @@
             ..
         }) = attr_info
         {
-            gate_alt!(self, has_feature(&self.features), *name, attr.span, *descr);
+            gate_alt!(self, has_feature(self.features), *name, attr.span, *descr);
         }
         // Check unstable flavors of the `#[doc]` attribute.
         if attr.has_name(sym::doc) {
@@ -300,7 +300,7 @@
             }
 
             ast::ItemKind::TyAlias(box ast::TyAlias { ty: Some(ty), .. }) => {
-                self.check_impl_trait(&ty, false)
+                self.check_impl_trait(ty, false)
             }
 
             _ => {}
@@ -556,6 +556,34 @@
     gate_all!(generic_const_items, "generic const items are experimental");
     gate_all!(unnamed_fields, "unnamed fields are not yet fully implemented");
 
+    if !visitor.features.never_patterns {
+        if let Some(spans) = spans.get(&sym::never_patterns) {
+            for &span in spans {
+                if span.allows_unstable(sym::never_patterns) {
+                    continue;
+                }
+                let sm = sess.source_map();
+                // We gate two types of spans: the span of a `!` pattern, and the span of a
+                // match arm without a body. For the latter we want to give the user a normal
+                // error.
+                if let Ok(snippet) = sm.span_to_snippet(span)
+                    && snippet == "!"
+                {
+                    feature_err(
+                        &sess.parse_sess,
+                        sym::never_patterns,
+                        span,
+                        "`!` patterns are experimental",
+                    )
+                    .emit();
+                } else {
+                    let suggestion = span.shrink_to_hi();
+                    sess.emit_err(errors::MatchArmWithNoBody { span, suggestion });
+                }
+            }
+        }
+    }
+
     if !visitor.features.negative_bounds {
         for &span in spans.get(&sym::negative_bounds).iter().copied().flatten() {
             sess.emit_err(errors::NegativeBoundUnsupported { span });
@@ -627,7 +655,7 @@
             if all_stable {
                 err.sugg = Some(attr.span);
             }
-            sess.parse_sess.span_diagnostic.emit_err(err);
+            sess.dcx().emit_err(err);
         }
     }
 }
diff --git a/compiler/rustc_ast_passes/src/lib.rs b/compiler/rustc_ast_passes/src/lib.rs
index 5147e67..ba09183 100644
--- a/compiler/rustc_ast_passes/src/lib.rs
+++ b/compiler/rustc_ast_passes/src/lib.rs
@@ -4,9 +4,9 @@
 //!
 //! The crate also contains other misc AST visitors, e.g. `node_count` and `show_span`.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![allow(internal_features)]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]
 #![feature(iter_is_partitioned)]
@@ -15,13 +15,10 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 pub mod ast_validation;
 mod errors;
 pub mod feature_gate;
 pub mod node_count;
 pub mod show_span;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_ast_passes/src/show_span.rs b/compiler/rustc_ast_passes/src/show_span.rs
index 280cf32..9882f1d 100644
--- a/compiler/rustc_ast_passes/src/show_span.rs
+++ b/compiler/rustc_ast_passes/src/show_span.rs
@@ -31,37 +31,37 @@
 }
 
 struct ShowSpanVisitor<'a> {
-    span_diagnostic: &'a rustc_errors::Handler,
+    dcx: &'a rustc_errors::DiagCtxt,
     mode: Mode,
 }
 
 impl<'a> Visitor<'a> for ShowSpanVisitor<'a> {
     fn visit_expr(&mut self, e: &'a ast::Expr) {
         if let Mode::Expression = self.mode {
-            self.span_diagnostic.emit_warning(errors::ShowSpan { span: e.span, msg: "expression" });
+            self.dcx.emit_warning(errors::ShowSpan { span: e.span, msg: "expression" });
         }
         visit::walk_expr(self, e);
     }
 
     fn visit_pat(&mut self, p: &'a ast::Pat) {
         if let Mode::Pattern = self.mode {
-            self.span_diagnostic.emit_warning(errors::ShowSpan { span: p.span, msg: "pattern" });
+            self.dcx.emit_warning(errors::ShowSpan { span: p.span, msg: "pattern" });
         }
         visit::walk_pat(self, p);
     }
 
     fn visit_ty(&mut self, t: &'a ast::Ty) {
         if let Mode::Type = self.mode {
-            self.span_diagnostic.emit_warning(errors::ShowSpan { span: t.span, msg: "type" });
+            self.dcx.emit_warning(errors::ShowSpan { span: t.span, msg: "type" });
         }
         visit::walk_ty(self, t);
     }
 }
 
-pub fn run(span_diagnostic: &rustc_errors::Handler, mode: &str, krate: &ast::Crate) {
+pub fn run(dcx: &rustc_errors::DiagCtxt, mode: &str, krate: &ast::Crate) {
     let Ok(mode) = mode.parse() else {
         return;
     };
-    let mut v = ShowSpanVisitor { span_diagnostic, mode };
+    let mut v = ShowSpanVisitor { dcx, mode };
     visit::walk_crate(&mut v, krate);
 }
diff --git a/compiler/rustc_ast_pretty/Cargo.toml b/compiler/rustc_ast_pretty/Cargo.toml
index af1524c..12a08f0 100644
--- a/compiler/rustc_ast_pretty/Cargo.toml
+++ b/compiler/rustc_ast_pretty/Cargo.toml
@@ -5,6 +5,7 @@
 
 [dependencies]
 # tidy-alphabetical-start
+itertools = "0.11"
 rustc_ast = { path = "../rustc_ast" }
 rustc_span = { path = "../rustc_span" }
 thin-vec = "0.2.12"
diff --git a/compiler/rustc_ast_pretty/src/lib.rs b/compiler/rustc_ast_pretty/src/lib.rs
index 475bdb0..670f2a4 100644
--- a/compiler/rustc_ast_pretty/src/lib.rs
+++ b/compiler/rustc_ast_pretty/src/lib.rs
@@ -1,11 +1,9 @@
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
-#![feature(associated_type_bounds)]
 #![feature(box_patterns)]
-#![feature(with_negative_coherence)]
 #![recursion_limit = "256"]
 
 mod helpers;
diff --git a/compiler/rustc_ast_pretty/src/pp.rs b/compiler/rustc_ast_pretty/src/pp.rs
index 7ab8c3e..96f5eff 100644
--- a/compiler/rustc_ast_pretty/src/pp.rs
+++ b/compiler/rustc_ast_pretty/src/pp.rs
@@ -165,20 +165,20 @@
 }
 
 #[derive(Clone, Copy, Default, PartialEq)]
-pub struct BreakToken {
+pub(crate) struct BreakToken {
     offset: isize,
     blank_space: isize,
     pre_break: Option<char>,
 }
 
 #[derive(Clone, Copy, PartialEq)]
-pub struct BeginToken {
+pub(crate) struct BeginToken {
     indent: IndentStyle,
     breaks: Breaks,
 }
 
-#[derive(Clone, PartialEq)]
-pub enum Token {
+#[derive(PartialEq)]
+pub(crate) enum Token {
     // In practice a string token contains either a `&'static str` or a
     // `String`. `Cow` is overkill for this because we never modify the data,
     // but it's more convenient than rolling our own more specialized type.
@@ -229,7 +229,6 @@
     last_printed: Option<Token>,
 }
 
-#[derive(Clone)]
 struct BufEntry {
     token: Token,
     size: isize,
@@ -251,16 +250,16 @@
         }
     }
 
-    pub fn last_token(&self) -> Option<&Token> {
+    pub(crate) fn last_token(&self) -> Option<&Token> {
         self.last_token_still_buffered().or_else(|| self.last_printed.as_ref())
     }
 
-    pub fn last_token_still_buffered(&self) -> Option<&Token> {
+    pub(crate) fn last_token_still_buffered(&self) -> Option<&Token> {
         self.buf.last().map(|last| &last.token)
     }
 
     /// Be very careful with this!
-    pub fn replace_last_token_still_buffered(&mut self, token: Token) {
+    pub(crate) fn replace_last_token_still_buffered(&mut self, token: Token) {
         self.buf.last_mut().unwrap().token = token;
     }
 
@@ -314,7 +313,7 @@
         }
     }
 
-    pub fn offset(&mut self, offset: isize) {
+    pub(crate) fn offset(&mut self, offset: isize) {
         if let Some(BufEntry { token: Token::Break(token), .. }) = &mut self.buf.last_mut() {
             token.offset += offset;
         }
diff --git a/compiler/rustc_ast_pretty/src/pp/convenience.rs b/compiler/rustc_ast_pretty/src/pp/convenience.rs
index 93310dd..c4c4fdc 100644
--- a/compiler/rustc_ast_pretty/src/pp/convenience.rs
+++ b/compiler/rustc_ast_pretty/src/pp/convenience.rs
@@ -66,7 +66,7 @@
         }
     }
 
-    pub fn hardbreak_tok_offset(off: isize) -> Token {
+    pub(crate) fn hardbreak_tok_offset(off: isize) -> Token {
         Token::Break(BreakToken {
             offset: off,
             blank_space: SIZE_INFINITY,
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
index 48421ff..d6c15ec 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -1,13 +1,17 @@
-mod delimited;
+//! AST pretty printing.
+//!
+//! Note that HIR pretty printing is layered on top of this crate.
+
 mod expr;
 mod item;
 
 use crate::pp::Breaks::{Consistent, Inconsistent};
 use crate::pp::{self, Breaks};
+use crate::pprust::state::expr::FixupContext;
 use rustc_ast::attr::AttrIdGenerator;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, BinOpToken, CommentKind, Delimiter, Nonterminal, Token, TokenKind};
-use rustc_ast::tokenstream::{TokenStream, TokenTree};
+use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree};
 use rustc_ast::util::classify;
 use rustc_ast::util::comments::{gather_comments, Comment, CommentStyle};
 use rustc_ast::util::parser;
@@ -23,8 +27,6 @@
 use std::borrow::Cow;
 use thin_vec::ThinVec;
 
-pub use self::delimited::IterDelimited;
-
 pub enum MacHeader<'a> {
     Path(&'a ast::Path),
     Keyword(&'static str),
@@ -46,8 +48,7 @@
     fn post(&self, _state: &mut State<'_>, _node: AnnNode<'_>) {}
 }
 
-#[derive(Copy, Clone)]
-pub struct NoAnn;
+struct NoAnn;
 
 impl PpAnn for NoAnn {}
 
@@ -64,11 +65,11 @@
     }
 
     // FIXME: This shouldn't probably clone lmao
-    pub fn next(&self) -> Option<Comment> {
+    fn next(&self) -> Option<Comment> {
         self.comments.get(self.current).cloned()
     }
 
-    pub fn trailing_comment(
+    fn trailing_comment(
         &self,
         span: rustc_span::Span,
         next_pos: Option<BytePos>,
@@ -95,7 +96,7 @@
     ann: &'a (dyn PpAnn + 'a),
 }
 
-pub(crate) const INDENT_UNIT: isize = 4;
+const INDENT_UNIT: isize = 4;
 
 /// Requires you to pass an input filename and reader so that
 /// it can scan the input text for comments to copy forward.
@@ -151,7 +152,7 @@
 /// Note: some old proc macros parse pretty-printed output, so changes here can
 /// break old code. For example:
 /// - #63896: `#[allow(unused,` must be printed rather than `#[allow(unused ,`
-/// - #73345: `#[allow(unused)] must be printed rather than `# [allow(unused)]
+/// - #73345: `#[allow(unused)]` must be printed rather than `# [allow(unused)]`
 ///
 fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
     use token::*;
@@ -183,10 +184,10 @@
         //
         // FIXME: Incorrect cases:
         // - Let: `let(a, b) = (1, 2)`
-        (Tok(Token { kind: Ident(..), .. }, _), Del(_, Parenthesis, _)) => false,
+        (Tok(Token { kind: Ident(..), .. }, _), Del(_, _, Parenthesis, _)) => false,
 
         // `#` + `[`: `#[attr]`
-        (Tok(Token { kind: Pound, .. }, _), Del(_, Bracket, _)) => false,
+        (Tok(Token { kind: Pound, .. }, _), Del(_, _, Bracket, _)) => false,
 
         _ => true,
     }
@@ -220,7 +221,7 @@
     }
 }
 
-pub fn literal_to_string(lit: token::Lit) -> String {
+fn literal_to_string(lit: token::Lit) -> String {
     let token::Lit { kind, symbol, suffix } = lit;
     let mut out = match kind {
         token::Byte => format!("b'{symbol}'"),
@@ -260,11 +261,17 @@
     }
 }
 
+/// This trait is used for both AST and HIR pretty-printing.
 pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::DerefMut {
     fn comments(&mut self) -> &mut Option<Comments<'a>>;
-    fn print_ident(&mut self, ident: Ident);
+    fn ann_post(&mut self, ident: Ident);
     fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool);
 
+    fn print_ident(&mut self, ident: Ident) {
+        self.word(IdentPrinter::for_ast_ident(ident, ident.is_raw_guess()).to_string());
+        self.ann_post(ident)
+    }
+
     fn strsep<T, F>(
         &mut self,
         sep: &'static str,
@@ -401,15 +408,6 @@
         }
     }
 
-    fn print_meta_item_lit(&mut self, lit: &ast::MetaItemLit) {
-        self.print_token_literal(lit.as_token_lit(), lit.span)
-    }
-
-    fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) {
-        self.maybe_print_comment(span.lo());
-        self.word(token_lit.to_string())
-    }
-
     fn print_string(&mut self, st: &str, style: ast::StrStyle) {
         let st = match style {
             ast::StrStyle::Cooked => format!("\"{}\"", st.escape_debug()),
@@ -420,30 +418,14 @@
         self.word(st)
     }
 
-    fn print_symbol(&mut self, sym: Symbol, style: ast::StrStyle) {
-        self.print_string(sym.as_str(), style);
-    }
-
     fn print_inner_attributes(&mut self, attrs: &[ast::Attribute]) -> bool {
         self.print_either_attributes(attrs, ast::AttrStyle::Inner, false, true)
     }
 
-    fn print_inner_attributes_no_trailing_hardbreak(&mut self, attrs: &[ast::Attribute]) -> bool {
-        self.print_either_attributes(attrs, ast::AttrStyle::Inner, false, false)
-    }
-
     fn print_outer_attributes(&mut self, attrs: &[ast::Attribute]) -> bool {
         self.print_either_attributes(attrs, ast::AttrStyle::Outer, false, true)
     }
 
-    fn print_inner_attributes_inline(&mut self, attrs: &[ast::Attribute]) -> bool {
-        self.print_either_attributes(attrs, ast::AttrStyle::Inner, true, true)
-    }
-
-    fn print_outer_attributes_inline(&mut self, attrs: &[ast::Attribute]) -> bool {
-        self.print_either_attributes(attrs, ast::AttrStyle::Outer, true, true)
-    }
-
     fn print_either_attributes(
         &mut self,
         attrs: &[ast::Attribute],
@@ -467,10 +449,6 @@
         printed
     }
 
-    fn print_attribute(&mut self, attr: &ast::Attribute) {
-        self.print_attribute_inline(attr, false)
-    }
-
     fn print_attribute_inline(&mut self, attr: &ast::Attribute, is_inline: bool) {
         if !is_inline {
             self.hardbreak_if_not_bol();
@@ -525,33 +503,6 @@
         self.end();
     }
 
-    fn print_meta_list_item(&mut self, item: &ast::NestedMetaItem) {
-        match item {
-            ast::NestedMetaItem::MetaItem(mi) => self.print_meta_item(mi),
-            ast::NestedMetaItem::Lit(lit) => self.print_meta_item_lit(lit),
-        }
-    }
-
-    fn print_meta_item(&mut self, item: &ast::MetaItem) {
-        self.ibox(INDENT_UNIT);
-        match &item.kind {
-            ast::MetaItemKind::Word => self.print_path(&item.path, false, 0),
-            ast::MetaItemKind::NameValue(value) => {
-                self.print_path(&item.path, false, 0);
-                self.space();
-                self.word_space("=");
-                self.print_meta_item_lit(value);
-            }
-            ast::MetaItemKind::List(items) => {
-                self.print_path(&item.path, false, 0);
-                self.popen();
-                self.commasep(Consistent, items, |s, i| s.print_meta_list_item(i));
-                self.pclose();
-            }
-        }
-        self.end();
-    }
-
     /// This doesn't deserve to be called "pretty" printing, but it should be
     /// meaning-preserving. A quick hack that might help would be to look at the
     /// spans embedded in the TTs to decide where to put spaces and newlines.
@@ -559,16 +510,17 @@
     /// appropriate macro, transcribe back into the grammar we just parsed from,
     /// and then pretty-print the resulting AST nodes (so, e.g., we print
     /// expression arguments as expressions). It can be done! I think.
-    fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) {
+    fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) -> Spacing {
         match tt {
-            TokenTree::Token(token, _) => {
+            TokenTree::Token(token, spacing) => {
                 let token_str = self.token_to_string_ext(token, convert_dollar_crate);
                 self.word(token_str);
                 if let token::DocComment(..) = token.kind {
                     self.hardbreak()
                 }
+                *spacing
             }
-            TokenTree::Delimited(dspan, delim, tts) => {
+            TokenTree::Delimited(dspan, spacing, delim, tts) => {
                 self.print_mac_common(
                     None,
                     false,
@@ -578,6 +530,7 @@
                     convert_dollar_crate,
                     dspan.entire(),
                 );
+                spacing.close
             }
         }
     }
@@ -585,9 +538,20 @@
     fn print_tts(&mut self, tts: &TokenStream, convert_dollar_crate: bool) {
         let mut iter = tts.trees().peekable();
         while let Some(tt) = iter.next() {
-            self.print_tt(tt, convert_dollar_crate);
+            let spacing = self.print_tt(tt, convert_dollar_crate);
             if let Some(next) = iter.peek() {
-                if space_between(tt, next) {
+                // Should we print a space after `tt`? There are two guiding
+                // factors.
+                // - `spacing` is the more important and accurate one. Most
+                //   tokens have good spacing information, and
+                //   `Joint`/`JointHidden` get used a lot.
+                // - `space_between` is the backup. Code produced by proc
+                //   macros has worse spacing information, with no
+                //   `JointHidden` usage and too much `Alone` usage, which
+                //   would result in over-spaced output such as
+                //   `( x () , y . z )`. `space_between` avoids some of the
+                //   excess whitespace.
+                if spacing == Spacing::Alone && space_between(tt, next) {
                     self.space();
                 }
             }
@@ -825,7 +789,7 @@
             }
             token::Eof => "<eof>".into(),
 
-            token::Interpolated(ref nt) => self.nonterminal_to_string(nt).into(),
+            token::Interpolated(ref nt) => self.nonterminal_to_string(&nt.0).into(),
         }
     }
 
@@ -843,37 +807,18 @@
         Self::to_string(|s| s.print_type(ty))
     }
 
-    fn bounds_to_string(&self, bounds: &[ast::GenericBound]) -> String {
-        Self::to_string(|s| s.print_type_bounds(bounds))
-    }
-
-    fn where_bound_predicate_to_string(
-        &self,
-        where_bound_predicate: &ast::WhereBoundPredicate,
-    ) -> String {
-        Self::to_string(|s| s.print_where_bound_predicate(where_bound_predicate))
-    }
-
     fn pat_to_string(&self, pat: &ast::Pat) -> String {
         Self::to_string(|s| s.print_pat(pat))
     }
 
     fn expr_to_string(&self, e: &ast::Expr) -> String {
-        Self::to_string(|s| s.print_expr(e))
+        Self::to_string(|s| s.print_expr(e, FixupContext::default()))
     }
 
     fn meta_item_lit_to_string(&self, lit: &ast::MetaItemLit) -> String {
         Self::to_string(|s| s.print_meta_item_lit(lit))
     }
 
-    fn tt_to_string(&self, tt: &TokenTree) -> String {
-        Self::to_string(|s| s.print_tt(tt, false))
-    }
-
-    fn tts_to_string(&self, tokens: &TokenStream) -> String {
-        Self::to_string(|s| s.print_tts(tokens, false))
-    }
-
     fn stmt_to_string(&self, stmt: &ast::Stmt) -> String {
         Self::to_string(|s| s.print_stmt(stmt))
     }
@@ -882,26 +827,10 @@
         Self::to_string(|s| s.print_item(i))
     }
 
-    fn assoc_item_to_string(&self, i: &ast::AssocItem) -> String {
-        Self::to_string(|s| s.print_assoc_item(i))
-    }
-
-    fn foreign_item_to_string(&self, i: &ast::ForeignItem) -> String {
-        Self::to_string(|s| s.print_foreign_item(i))
-    }
-
-    fn generic_params_to_string(&self, generic_params: &[ast::GenericParam]) -> String {
-        Self::to_string(|s| s.print_generic_params(generic_params))
-    }
-
     fn path_to_string(&self, p: &ast::Path) -> String {
         Self::to_string(|s| s.print_path(p, false, 0))
     }
 
-    fn path_segment_to_string(&self, p: &ast::PathSegment) -> String {
-        Self::to_string(|s| s.print_path_segment(p, false))
-    }
-
     fn vis_to_string(&self, v: &ast::Visibility) -> String {
         Self::to_string(|s| s.print_visibility(v))
     }
@@ -916,22 +845,10 @@
         })
     }
 
-    fn meta_list_item_to_string(&self, li: &ast::NestedMetaItem) -> String {
-        Self::to_string(|s| s.print_meta_list_item(li))
-    }
-
     fn attr_item_to_string(&self, ai: &ast::AttrItem) -> String {
         Self::to_string(|s| s.print_attr_item(ai, ai.path.span))
     }
 
-    fn attribute_to_string(&self, attr: &ast::Attribute) -> String {
-        Self::to_string(|s| s.print_attribute(attr))
-    }
-
-    fn param_to_string(&self, arg: &ast::Param) -> String {
-        Self::to_string(|s| s.print_param(arg, false))
-    }
-
     fn to_string(f: impl FnOnce(&mut State<'_>)) -> String {
         let mut printer = State::new();
         f(&mut printer);
@@ -944,9 +861,8 @@
         &mut self.comments
     }
 
-    fn print_ident(&mut self, ident: Ident) {
-        self.word(IdentPrinter::for_ast_ident(ident, ident.is_raw_guess()).to_string());
-        self.ann.post(self, AnnNode::Ident(&ident))
+    fn ann_post(&mut self, ident: Ident) {
+        self.ann.post(self, AnnNode::Ident(&ident));
     }
 
     fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool) {
@@ -979,13 +895,8 @@
         State { s: pp::Printer::new(), comments: None, ann: &NoAnn }
     }
 
-    pub(crate) fn commasep_cmnt<T, F, G>(
-        &mut self,
-        b: Breaks,
-        elts: &[T],
-        mut op: F,
-        mut get_span: G,
-    ) where
+    fn commasep_cmnt<T, F, G>(&mut self, b: Breaks, elts: &[T], mut op: F, mut get_span: G)
+    where
         F: FnMut(&mut State<'_>, &T),
         G: FnMut(&T) -> rustc_span::Span,
     {
@@ -1005,8 +916,8 @@
         self.end();
     }
 
-    pub(crate) fn commasep_exprs(&mut self, b: Breaks, exprs: &[P<ast::Expr>]) {
-        self.commasep_cmnt(b, exprs, |s, e| s.print_expr(e), |e| e.span)
+    fn commasep_exprs(&mut self, b: Breaks, exprs: &[P<ast::Expr>]) {
+        self.commasep_cmnt(b, exprs, |s, e| s.print_expr(e, FixupContext::default()), |e| e.span)
     }
 
     pub fn print_opt_lifetime(&mut self, lifetime: &Option<ast::Lifetime>) {
@@ -1043,7 +954,7 @@
         match generic_arg {
             GenericArg::Lifetime(lt) => self.print_lifetime(*lt),
             GenericArg::Type(ty) => self.print_type(ty),
-            GenericArg::Const(ct) => self.print_expr(&ct.value),
+            GenericArg::Const(ct) => self.print_expr(&ct.value, FixupContext::default()),
         }
     }
 
@@ -1078,11 +989,11 @@
             }
             ast::TyKind::AnonStruct(fields) => {
                 self.head("struct");
-                self.print_record_struct_body(&fields, ty.span);
+                self.print_record_struct_body(fields, ty.span);
             }
             ast::TyKind::AnonUnion(fields) => {
                 self.head("union");
-                self.print_record_struct_body(&fields, ty.span);
+                self.print_record_struct_body(fields, ty.span);
             }
             ast::TyKind::Paren(typ) => {
                 self.popen();
@@ -1110,12 +1021,12 @@
                 self.word("[");
                 self.print_type(ty);
                 self.word("; ");
-                self.print_expr(&length.value);
+                self.print_expr(&length.value, FixupContext::default());
                 self.word("]");
             }
             ast::TyKind::Typeof(e) => {
                 self.word("typeof(");
-                self.print_expr(&e.value);
+                self.print_expr(&e.value, FixupContext::default());
                 self.word(")");
             }
             ast::TyKind::Infer => {
@@ -1156,7 +1067,7 @@
         self.print_trait_ref(&t.trait_ref)
     }
 
-    pub(crate) fn print_stmt(&mut self, st: &ast::Stmt) {
+    fn print_stmt(&mut self, st: &ast::Stmt) {
         self.maybe_print_comment(st.span.lo());
         match &st.kind {
             ast::StmtKind::Local(loc) => {
@@ -1171,7 +1082,7 @@
                 if let Some((init, els)) = loc.kind.init_else_opt() {
                     self.nbsp();
                     self.word_space("=");
-                    self.print_expr(init);
+                    self.print_expr(init, FixupContext::default());
                     if let Some(els) = els {
                         self.cbox(INDENT_UNIT);
                         self.ibox(INDENT_UNIT);
@@ -1185,14 +1096,14 @@
             ast::StmtKind::Item(item) => self.print_item(item),
             ast::StmtKind::Expr(expr) => {
                 self.space_if_not_bol();
-                self.print_expr_outer_attr_style(expr, false);
+                self.print_expr_outer_attr_style(expr, false, FixupContext::default());
                 if classify::expr_requires_semi_to_be_stmt(expr) {
                     self.word(";");
                 }
             }
             ast::StmtKind::Semi(expr) => {
                 self.space_if_not_bol();
-                self.print_expr_outer_attr_style(expr, false);
+                self.print_expr_outer_attr_style(expr, false, FixupContext::default());
                 self.word(";");
             }
             ast::StmtKind::Empty => {
@@ -1211,19 +1122,19 @@
         self.maybe_print_trailing_comment(st.span, None)
     }
 
-    pub(crate) fn print_block(&mut self, blk: &ast::Block) {
+    fn print_block(&mut self, blk: &ast::Block) {
         self.print_block_with_attrs(blk, &[])
     }
 
-    pub(crate) fn print_block_unclosed_indent(&mut self, blk: &ast::Block) {
+    fn print_block_unclosed_indent(&mut self, blk: &ast::Block) {
         self.print_block_maybe_unclosed(blk, &[], false)
     }
 
-    pub(crate) fn print_block_with_attrs(&mut self, blk: &ast::Block, attrs: &[ast::Attribute]) {
+    fn print_block_with_attrs(&mut self, blk: &ast::Block, attrs: &[ast::Attribute]) {
         self.print_block_maybe_unclosed(blk, attrs, true)
     }
 
-    pub(crate) fn print_block_maybe_unclosed(
+    fn print_block_maybe_unclosed(
         &mut self,
         blk: &ast::Block,
         attrs: &[ast::Attribute],
@@ -1244,7 +1155,7 @@
                 ast::StmtKind::Expr(expr) if i == blk.stmts.len() - 1 => {
                     self.maybe_print_comment(st.span.lo());
                     self.space_if_not_bol();
-                    self.print_expr_outer_attr_style(expr, false);
+                    self.print_expr_outer_attr_style(expr, false, FixupContext::default());
                     self.maybe_print_trailing_comment(expr.span, Some(blk.span.hi()));
                 }
                 _ => self.print_stmt(st),
@@ -1257,16 +1168,44 @@
     }
 
     /// Print a `let pat = expr` expression.
-    pub(crate) fn print_let(&mut self, pat: &ast::Pat, expr: &ast::Expr) {
+    ///
+    /// Parentheses are inserted surrounding `expr` if a round-trip through the
+    /// parser would otherwise work out the wrong way in a condition position.
+    ///
+    /// For example each of the following would mean the wrong thing without
+    /// parentheses.
+    ///
+    /// ```ignore (illustrative)
+    /// if let _ = (Struct {}) {}
+    ///
+    /// if let _ = (true && false) {}
+    /// ```
+    ///
+    /// In a match guard, the second case still requires parens, but the first
+    /// case no longer does because anything until `=>` is considered part of
+    /// the match guard expression. Parsing of the expression is not terminated
+    /// by `{` in that position.
+    ///
+    /// ```ignore (illustrative)
+    /// match () {
+    ///     () if let _ = Struct {} => {}
+    ///     () if let _ = (true && false) => {}
+    /// }
+    /// ```
+    fn print_let(&mut self, pat: &ast::Pat, expr: &ast::Expr, fixup: FixupContext) {
         self.word("let ");
         self.print_pat(pat);
         self.space();
         self.word_space("=");
-        let npals = || parser::needs_par_as_let_scrutinee(expr.precedence().order());
-        self.print_expr_cond_paren(expr, Self::cond_needs_par(expr) || npals())
+        self.print_expr_cond_paren(
+            expr,
+            fixup.parenthesize_exterior_struct_lit && parser::contains_exterior_struct_lit(expr)
+                || parser::needs_par_as_let_scrutinee(expr.precedence().order()),
+            FixupContext::default(),
+        );
     }
 
-    pub(crate) fn print_mac(&mut self, m: &ast::MacCall) {
+    fn print_mac(&mut self, m: &ast::MacCall) {
         self.print_mac_common(
             Some(MacHeader::Path(&m.path)),
             true,
@@ -1310,7 +1249,7 @@
                         print_reg_or_class(s, reg);
                         s.pclose();
                         s.space();
-                        s.print_expr(expr);
+                        s.print_expr(expr, FixupContext::default());
                     }
                     InlineAsmOperand::Out { reg, late, expr } => {
                         s.word(if *late { "lateout" } else { "out" });
@@ -1319,7 +1258,7 @@
                         s.pclose();
                         s.space();
                         match expr {
-                            Some(expr) => s.print_expr(expr),
+                            Some(expr) => s.print_expr(expr, FixupContext::default()),
                             None => s.word("_"),
                         }
                     }
@@ -1329,7 +1268,7 @@
                         print_reg_or_class(s, reg);
                         s.pclose();
                         s.space();
-                        s.print_expr(expr);
+                        s.print_expr(expr, FixupContext::default());
                     }
                     InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => {
                         s.word(if *late { "inlateout" } else { "inout" });
@@ -1337,18 +1276,18 @@
                         print_reg_or_class(s, reg);
                         s.pclose();
                         s.space();
-                        s.print_expr(in_expr);
+                        s.print_expr(in_expr, FixupContext::default());
                         s.space();
                         s.word_space("=>");
                         match out_expr {
-                            Some(out_expr) => s.print_expr(out_expr),
+                            Some(out_expr) => s.print_expr(out_expr, FixupContext::default()),
                             None => s.word("_"),
                         }
                     }
                     InlineAsmOperand::Const { anon_const } => {
                         s.word("const");
                         s.space();
-                        s.print_expr(&anon_const.value);
+                        s.print_expr(&anon_const.value, FixupContext::default());
                     }
                     InlineAsmOperand::Sym { sym } => {
                         s.word("sym");
@@ -1407,7 +1346,7 @@
         self.pclose();
     }
 
-    pub(crate) fn print_local_decl(&mut self, loc: &ast::Local) {
+    fn print_local_decl(&mut self, loc: &ast::Local) {
         self.print_pat(&loc.pat);
         if let Some(ty) = &loc.ty {
             self.word_space(":");
@@ -1415,7 +1354,7 @@
         }
     }
 
-    pub(crate) fn print_name(&mut self, name: Symbol) {
+    fn print_name(&mut self, name: Symbol) {
         self.word(name.to_string());
         self.ann.post(self, AnnNode::Name(&name))
     }
@@ -1439,13 +1378,14 @@
         }
     }
 
-    pub(crate) fn print_pat(&mut self, pat: &ast::Pat) {
+    fn print_pat(&mut self, pat: &ast::Pat) {
         self.maybe_print_comment(pat.span.lo());
         self.ann.pre(self, AnnNode::Pat(pat));
         /* Pat isn't normalized, but the beauty of it
         is that it doesn't matter */
         match &pat.kind {
             PatKind::Wild => self.word("_"),
+            PatKind::Never => self.word("!"),
             PatKind::Ident(BindingAnnotation(by_ref, mutbl), ident, sub) => {
                 if *by_ref == ByRef::Yes {
                     self.word_nbsp("ref");
@@ -1541,10 +1481,10 @@
                     self.print_pat(inner);
                 }
             }
-            PatKind::Lit(e) => self.print_expr(e),
+            PatKind::Lit(e) => self.print_expr(e, FixupContext::default()),
             PatKind::Range(begin, end, Spanned { node: end_kind, .. }) => {
                 if let Some(e) = begin {
-                    self.print_expr(e);
+                    self.print_expr(e, FixupContext::default());
                 }
                 match end_kind {
                     RangeEnd::Included(RangeSyntax::DotDotDot) => self.word("..."),
@@ -1552,7 +1492,7 @@
                     RangeEnd::Excluded => self.word(".."),
                 }
                 if let Some(e) = end {
-                    self.print_expr(e);
+                    self.print_expr(e, FixupContext::default());
                 }
             }
             PatKind::Slice(elts) => {
@@ -1592,9 +1532,18 @@
         }
     }
 
-    pub(crate) fn print_asyncness(&mut self, asyncness: ast::Async) {
-        if asyncness.is_async() {
-            self.word_nbsp("async");
+    fn print_coroutine_kind(&mut self, coroutine_kind: ast::CoroutineKind) {
+        match coroutine_kind {
+            ast::CoroutineKind::Gen { .. } => {
+                self.word_nbsp("gen");
+            }
+            ast::CoroutineKind::Async { .. } => {
+                self.word_nbsp("async");
+            }
+            ast::CoroutineKind::AsyncGen { .. } => {
+                self.word_nbsp("async");
+                self.word_nbsp("gen");
+            }
         }
     }
 
@@ -1618,7 +1567,7 @@
                         TraitBoundModifier::Maybe => {
                             self.word("?");
                         }
-                        TraitBoundModifier::MaybeConst => {
+                        TraitBoundModifier::MaybeConst(_) => {
                             self.word_space("~const");
                         }
                         TraitBoundModifier::MaybeConstNegative => {
@@ -1637,23 +1586,25 @@
         }
     }
 
-    pub(crate) fn print_lifetime(&mut self, lifetime: ast::Lifetime) {
+    fn print_lifetime(&mut self, lifetime: ast::Lifetime) {
         self.print_name(lifetime.ident.name)
     }
 
-    pub(crate) fn print_lifetime_bounds(&mut self, bounds: &ast::GenericBounds) {
+    fn print_lifetime_bounds(&mut self, bounds: &ast::GenericBounds) {
         for (i, bound) in bounds.iter().enumerate() {
             if i != 0 {
                 self.word(" + ");
             }
             match bound {
                 ast::GenericBound::Outlives(lt) => self.print_lifetime(*lt),
-                _ => panic!(),
+                _ => {
+                    panic!("expected a lifetime bound, found a trait bound")
+                }
             }
         }
     }
 
-    pub(crate) fn print_generic_params(&mut self, generic_params: &[ast::GenericParam]) {
+    fn print_generic_params(&mut self, generic_params: &[ast::GenericParam]) {
         if generic_params.is_empty() {
             return;
         }
@@ -1697,7 +1648,7 @@
                     if let Some(default) = default {
                         s.space();
                         s.word_space("=");
-                        s.print_expr(&default.value);
+                        s.print_expr(&default.value, FixupContext::default());
                     }
                 }
             }
@@ -1717,12 +1668,12 @@
         }
     }
 
-    pub(crate) fn print_mt(&mut self, mt: &ast::MutTy, print_const: bool) {
+    fn print_mt(&mut self, mt: &ast::MutTy, print_const: bool) {
         self.print_mutability(mt.mutbl, print_const);
         self.print_type(&mt.ty)
     }
 
-    pub(crate) fn print_param(&mut self, input: &ast::Param, is_closure: bool) {
+    fn print_param(&mut self, input: &ast::Param, is_closure: bool) {
         self.ibox(INDENT_UNIT);
 
         self.print_outer_attributes_inline(&input.attrs);
@@ -1750,7 +1701,7 @@
         self.end();
     }
 
-    pub(crate) fn print_fn_ret_ty(&mut self, fn_ret_ty: &ast::FnRetTy) {
+    fn print_fn_ret_ty(&mut self, fn_ret_ty: &ast::FnRetTy) {
         if let ast::FnRetTy::Ty(ty) = fn_ret_ty {
             self.space_if_not_bol();
             self.ibox(INDENT_UNIT);
@@ -1761,7 +1712,7 @@
         }
     }
 
-    pub(crate) fn print_ty_fn(
+    fn print_ty_fn(
         &mut self,
         ext: ast::Extern,
         unsafety: ast::Unsafe,
@@ -1785,9 +1736,9 @@
         self.end();
     }
 
-    pub(crate) fn print_fn_header_info(&mut self, header: ast::FnHeader) {
+    fn print_fn_header_info(&mut self, header: ast::FnHeader) {
         self.print_constness(header.constness);
-        self.print_asyncness(header.asyncness);
+        header.coroutine_kind.map(|coroutine_kind| self.print_coroutine_kind(coroutine_kind));
         self.print_unsafety(header.unsafety);
 
         match header.ext {
@@ -1805,24 +1756,109 @@
         self.word("fn")
     }
 
-    pub(crate) fn print_unsafety(&mut self, s: ast::Unsafe) {
+    fn print_unsafety(&mut self, s: ast::Unsafe) {
         match s {
             ast::Unsafe::No => {}
             ast::Unsafe::Yes(_) => self.word_nbsp("unsafe"),
         }
     }
 
-    pub(crate) fn print_constness(&mut self, s: ast::Const) {
+    fn print_constness(&mut self, s: ast::Const) {
         match s {
             ast::Const::No => {}
             ast::Const::Yes(_) => self.word_nbsp("const"),
         }
     }
 
-    pub(crate) fn print_is_auto(&mut self, s: ast::IsAuto) {
+    fn print_is_auto(&mut self, s: ast::IsAuto) {
         match s {
             ast::IsAuto::Yes => self.word_nbsp("auto"),
             ast::IsAuto::No => {}
         }
     }
+
+    fn print_meta_item_lit(&mut self, lit: &ast::MetaItemLit) {
+        self.print_token_literal(lit.as_token_lit(), lit.span)
+    }
+
+    fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) {
+        self.maybe_print_comment(span.lo());
+        self.word(token_lit.to_string())
+    }
+
+    fn print_symbol(&mut self, sym: Symbol, style: ast::StrStyle) {
+        self.print_string(sym.as_str(), style);
+    }
+
+    fn print_inner_attributes_no_trailing_hardbreak(&mut self, attrs: &[ast::Attribute]) -> bool {
+        self.print_either_attributes(attrs, ast::AttrStyle::Inner, false, false)
+    }
+
+    fn print_outer_attributes_inline(&mut self, attrs: &[ast::Attribute]) -> bool {
+        self.print_either_attributes(attrs, ast::AttrStyle::Outer, true, true)
+    }
+
+    fn print_attribute(&mut self, attr: &ast::Attribute) {
+        self.print_attribute_inline(attr, false)
+    }
+
+    fn print_meta_list_item(&mut self, item: &ast::NestedMetaItem) {
+        match item {
+            ast::NestedMetaItem::MetaItem(mi) => self.print_meta_item(mi),
+            ast::NestedMetaItem::Lit(lit) => self.print_meta_item_lit(lit),
+        }
+    }
+
+    fn print_meta_item(&mut self, item: &ast::MetaItem) {
+        self.ibox(INDENT_UNIT);
+        match &item.kind {
+            ast::MetaItemKind::Word => self.print_path(&item.path, false, 0),
+            ast::MetaItemKind::NameValue(value) => {
+                self.print_path(&item.path, false, 0);
+                self.space();
+                self.word_space("=");
+                self.print_meta_item_lit(value);
+            }
+            ast::MetaItemKind::List(items) => {
+                self.print_path(&item.path, false, 0);
+                self.popen();
+                self.commasep(Consistent, items, |s, i| s.print_meta_list_item(i));
+                self.pclose();
+            }
+        }
+        self.end();
+    }
+
+    pub(crate) fn bounds_to_string(&self, bounds: &[ast::GenericBound]) -> String {
+        Self::to_string(|s| s.print_type_bounds(bounds))
+    }
+
+    pub(crate) fn where_bound_predicate_to_string(
+        &self,
+        where_bound_predicate: &ast::WhereBoundPredicate,
+    ) -> String {
+        Self::to_string(|s| s.print_where_bound_predicate(where_bound_predicate))
+    }
+
+    pub(crate) fn tt_to_string(&self, tt: &TokenTree) -> String {
+        Self::to_string(|s| {
+            s.print_tt(tt, false);
+        })
+    }
+
+    pub(crate) fn tts_to_string(&self, tokens: &TokenStream) -> String {
+        Self::to_string(|s| s.print_tts(tokens, false))
+    }
+
+    pub(crate) fn path_segment_to_string(&self, p: &ast::PathSegment) -> String {
+        Self::to_string(|s| s.print_path_segment(p, false))
+    }
+
+    pub(crate) fn meta_list_item_to_string(&self, li: &ast::NestedMetaItem) -> String {
+        Self::to_string(|s| s.print_meta_list_item(li))
+    }
+
+    pub(crate) fn attribute_to_string(&self, attr: &ast::Attribute) -> String {
+        Self::to_string(|s| s.print_attribute(attr))
+    }
 }
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/delimited.rs b/compiler/rustc_ast_pretty/src/pprust/state/delimited.rs
deleted file mode 100644
index fe0640b..0000000
--- a/compiler/rustc_ast_pretty/src/pprust/state/delimited.rs
+++ /dev/null
@@ -1,41 +0,0 @@
-use std::iter::Peekable;
-use std::mem;
-use std::ops::Deref;
-
-pub struct Delimited<I: Iterator> {
-    is_first: bool,
-    iter: Peekable<I>,
-}
-
-pub trait IterDelimited: Iterator + Sized {
-    fn delimited(self) -> Delimited<Self> {
-        Delimited { is_first: true, iter: self.peekable() }
-    }
-}
-
-impl<I: Iterator> IterDelimited for I {}
-
-pub struct IteratorItem<T> {
-    value: T,
-    pub is_first: bool,
-    pub is_last: bool,
-}
-
-impl<I: Iterator> Iterator for Delimited<I> {
-    type Item = IteratorItem<I::Item>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let value = self.iter.next()?;
-        let is_first = mem::replace(&mut self.is_first, false);
-        let is_last = self.iter.peek().is_none();
-        Some(IteratorItem { value, is_first, is_last })
-    }
-}
-
-impl<T> Deref for IteratorItem<T> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        &self.value
-    }
-}
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
index edbc350..f5ffcdd 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
@@ -1,6 +1,6 @@
 use crate::pp::Breaks::Inconsistent;
-use crate::pprust::state::{AnnNode, IterDelimited, PrintState, State, INDENT_UNIT};
-
+use crate::pprust::state::{AnnNode, PrintState, State, INDENT_UNIT};
+use itertools::{Itertools, Position};
 use rustc_ast::ptr::P;
 use rustc_ast::token;
 use rustc_ast::util::literal::escape_byte_str_symbol;
@@ -12,6 +12,19 @@
 };
 use std::fmt::Write;
 
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct FixupContext {
+    pub parenthesize_exterior_struct_lit: bool,
+}
+
+/// The default amount of fixing is minimal fixing. Fixups should be turned on
+/// in a targetted fashion where needed.
+impl Default for FixupContext {
+    fn default() -> Self {
+        FixupContext { parenthesize_exterior_struct_lit: false }
+    }
+}
+
 impl<'a> State<'a> {
     fn print_else(&mut self, els: Option<&ast::Expr>) {
         if let Some(_else) = els {
@@ -55,21 +68,22 @@
         self.pclose()
     }
 
-    fn print_expr_maybe_paren(&mut self, expr: &ast::Expr, prec: i8) {
-        self.print_expr_cond_paren(expr, expr.precedence().order() < prec)
+    fn print_expr_maybe_paren(&mut self, expr: &ast::Expr, prec: i8, fixup: FixupContext) {
+        self.print_expr_cond_paren(expr, expr.precedence().order() < prec, fixup);
     }
 
     /// Prints an expr using syntax that's acceptable in a condition position, such as the `cond` in
     /// `if cond { ... }`.
     fn print_expr_as_cond(&mut self, expr: &ast::Expr) {
-        self.print_expr_cond_paren(expr, Self::cond_needs_par(expr))
+        let fixup = FixupContext { parenthesize_exterior_struct_lit: true };
+        self.print_expr_cond_paren(expr, Self::cond_needs_par(expr), fixup)
     }
 
     /// Does `expr` need parentheses when printed in a condition position?
     ///
     /// These cases need parens due to the parse error observed in #26461: `if return {}`
     /// parses as the erroneous construct `if (return {})`, not `if (return) {}`.
-    pub(super) fn cond_needs_par(expr: &ast::Expr) -> bool {
+    fn cond_needs_par(expr: &ast::Expr) -> bool {
         match expr.kind {
             ast::ExprKind::Break(..)
             | ast::ExprKind::Closure(..)
@@ -80,11 +94,32 @@
     }
 
     /// Prints `expr` or `(expr)` when `needs_par` holds.
-    pub(super) fn print_expr_cond_paren(&mut self, expr: &ast::Expr, needs_par: bool) {
+    pub(super) fn print_expr_cond_paren(
+        &mut self,
+        expr: &ast::Expr,
+        needs_par: bool,
+        fixup: FixupContext,
+    ) {
         if needs_par {
             self.popen();
         }
-        self.print_expr(expr);
+
+        // If we are surrounding the whole cond in parentheses, such as:
+        //
+        //     if (return Struct {}) {}
+        //
+        // then there is no need for parenthesizing the individual struct
+        // expressions within. On the other hand if the whole cond is not
+        // parenthesized, then print_expr must parenthesize exterior struct
+        // literals.
+        //
+        //     if x == (Struct {}) {}
+        //
+        let fixup = FixupContext {
+            parenthesize_exterior_struct_lit: fixup.parenthesize_exterior_struct_lit && !needs_par,
+        };
+        self.print_expr(expr, fixup);
+
         if needs_par {
             self.pclose();
         }
@@ -111,7 +146,7 @@
             self.ibox(0);
             self.print_block_with_attrs(block, attrs);
         } else {
-            self.print_expr(&expr.value);
+            self.print_expr(&expr.value, FixupContext::default());
         }
         self.end();
     }
@@ -119,9 +154,9 @@
     fn print_expr_repeat(&mut self, element: &ast::Expr, count: &ast::AnonConst) {
         self.ibox(INDENT_UNIT);
         self.word("[");
-        self.print_expr(element);
+        self.print_expr(element, FixupContext::default());
         self.word_space(";");
-        self.print_expr(&count.value);
+        self.print_expr(&count.value, FixupContext::default());
         self.word("]");
         self.end();
     }
@@ -149,18 +184,20 @@
             return;
         }
         self.cbox(0);
-        for field in fields.iter().delimited() {
+        for (pos, field) in fields.iter().with_position() {
+            let is_first = matches!(pos, Position::First | Position::Only);
+            let is_last = matches!(pos, Position::Last | Position::Only);
             self.maybe_print_comment(field.span.hi());
             self.print_outer_attributes(&field.attrs);
-            if field.is_first {
+            if is_first {
                 self.space_if_not_bol();
             }
             if !field.is_shorthand {
                 self.print_ident(field.ident);
                 self.word_nbsp(":");
             }
-            self.print_expr(&field.expr);
-            if !field.is_last || has_rest {
+            self.print_expr(&field.expr, FixupContext::default());
+            if !is_last || has_rest {
                 self.word_space(",");
             } else {
                 self.trailing_comma_or_space();
@@ -172,7 +209,7 @@
             }
             self.word("..");
             if let ast::StructRest::Base(expr) = rest {
-                self.print_expr(expr);
+                self.print_expr(expr, FixupContext::default());
             }
             self.space();
         }
@@ -190,13 +227,13 @@
         self.pclose()
     }
 
-    fn print_expr_call(&mut self, func: &ast::Expr, args: &[P<ast::Expr>]) {
+    fn print_expr_call(&mut self, func: &ast::Expr, args: &[P<ast::Expr>], fixup: FixupContext) {
         let prec = match func.kind {
             ast::ExprKind::Field(..) => parser::PREC_FORCE_PAREN,
             _ => parser::PREC_POSTFIX,
         };
 
-        self.print_expr_maybe_paren(func, prec);
+        self.print_expr_maybe_paren(func, prec, fixup);
         self.print_call_post(args)
     }
 
@@ -205,8 +242,9 @@
         segment: &ast::PathSegment,
         receiver: &ast::Expr,
         base_args: &[P<ast::Expr>],
+        fixup: FixupContext,
     ) {
-        self.print_expr_maybe_paren(receiver, parser::PREC_POSTFIX);
+        self.print_expr_maybe_paren(receiver, parser::PREC_POSTFIX, fixup);
         self.word(".");
         self.print_ident(segment.ident);
         if let Some(args) = &segment.args {
@@ -215,7 +253,13 @@
         self.print_call_post(base_args)
     }
 
-    fn print_expr_binary(&mut self, op: ast::BinOp, lhs: &ast::Expr, rhs: &ast::Expr) {
+    fn print_expr_binary(
+        &mut self,
+        op: ast::BinOp,
+        lhs: &ast::Expr,
+        rhs: &ast::Expr,
+        fixup: FixupContext,
+    ) {
         let assoc_op = AssocOp::from_ast_binop(op.node);
         let prec = assoc_op.precedence() as i8;
         let fixity = assoc_op.fixity();
@@ -251,15 +295,15 @@
             _ => left_prec,
         };
 
-        self.print_expr_maybe_paren(lhs, left_prec);
+        self.print_expr_maybe_paren(lhs, left_prec, fixup);
         self.space();
-        self.word_space(op.node.to_string());
-        self.print_expr_maybe_paren(rhs, right_prec)
+        self.word_space(op.node.as_str());
+        self.print_expr_maybe_paren(rhs, right_prec, fixup)
     }
 
-    fn print_expr_unary(&mut self, op: ast::UnOp, expr: &ast::Expr) {
-        self.word(ast::UnOp::to_string(op));
-        self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)
+    fn print_expr_unary(&mut self, op: ast::UnOp, expr: &ast::Expr, fixup: FixupContext) {
+        self.word(op.as_str());
+        self.print_expr_maybe_paren(expr, parser::PREC_PREFIX, fixup)
     }
 
     fn print_expr_addr_of(
@@ -267,6 +311,7 @@
         kind: ast::BorrowKind,
         mutability: ast::Mutability,
         expr: &ast::Expr,
+        fixup: FixupContext,
     ) {
         self.word("&");
         match kind {
@@ -276,14 +321,19 @@
                 self.print_mutability(mutability, true);
             }
         }
-        self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)
+        self.print_expr_maybe_paren(expr, parser::PREC_PREFIX, fixup)
     }
 
-    pub fn print_expr(&mut self, expr: &ast::Expr) {
-        self.print_expr_outer_attr_style(expr, true)
+    pub(super) fn print_expr(&mut self, expr: &ast::Expr, fixup: FixupContext) {
+        self.print_expr_outer_attr_style(expr, true, fixup)
     }
 
-    pub(super) fn print_expr_outer_attr_style(&mut self, expr: &ast::Expr, is_inline: bool) {
+    pub(super) fn print_expr_outer_attr_style(
+        &mut self,
+        expr: &ast::Expr,
+        is_inline: bool,
+        fixup: FixupContext,
+    ) {
         self.maybe_print_comment(expr.span.lo());
 
         let attrs = &expr.attrs;
@@ -312,19 +362,19 @@
                 self.print_expr_tup(exprs);
             }
             ast::ExprKind::Call(func, args) => {
-                self.print_expr_call(func, args);
+                self.print_expr_call(func, args, fixup);
             }
             ast::ExprKind::MethodCall(box ast::MethodCall { seg, receiver, args, .. }) => {
-                self.print_expr_method_call(seg, receiver, args);
+                self.print_expr_method_call(seg, receiver, args, fixup);
             }
             ast::ExprKind::Binary(op, lhs, rhs) => {
-                self.print_expr_binary(*op, lhs, rhs);
+                self.print_expr_binary(*op, lhs, rhs, fixup);
             }
             ast::ExprKind::Unary(op, expr) => {
-                self.print_expr_unary(*op, expr);
+                self.print_expr_unary(*op, expr, fixup);
             }
             ast::ExprKind::AddrOf(k, m, expr) => {
-                self.print_expr_addr_of(*k, *m, expr);
+                self.print_expr_addr_of(*k, *m, expr, fixup);
             }
             ast::ExprKind::Lit(token_lit) => {
                 self.print_token_literal(*token_lit, expr.span);
@@ -335,7 +385,7 @@
             }
             ast::ExprKind::Cast(expr, ty) => {
                 let prec = AssocOp::As.precedence() as i8;
-                self.print_expr_maybe_paren(expr, prec);
+                self.print_expr_maybe_paren(expr, prec, fixup);
                 self.space();
                 self.word_space("as");
                 self.print_type(ty);
@@ -343,7 +393,7 @@
             ast::ExprKind::Type(expr, ty) => {
                 self.word("type_ascribe!(");
                 self.ibox(0);
-                self.print_expr(expr);
+                self.print_expr(expr, FixupContext::default());
 
                 self.word(",");
                 self.space_if_not_bol();
@@ -353,7 +403,7 @@
                 self.word(")");
             }
             ast::ExprKind::Let(pat, scrutinee, _, _) => {
-                self.print_let(pat, scrutinee);
+                self.print_let(pat, scrutinee, fixup);
             }
             ast::ExprKind::If(test, blk, elseopt) => self.print_if(test, blk, elseopt.as_deref()),
             ast::ExprKind::While(test, blk, opt_label) => {
@@ -411,7 +461,7 @@
                 binder,
                 capture_clause,
                 constness,
-                asyncness,
+                coroutine_kind,
                 movability,
                 fn_decl,
                 body,
@@ -421,12 +471,12 @@
                 self.print_closure_binder(binder);
                 self.print_constness(*constness);
                 self.print_movability(*movability);
-                self.print_asyncness(*asyncness);
+                coroutine_kind.map(|coroutine_kind| self.print_coroutine_kind(coroutine_kind));
                 self.print_capture_clause(*capture_clause);
 
                 self.print_fn_params_and_ret(fn_decl, true);
                 self.space();
-                self.print_expr(body);
+                self.print_expr(body, FixupContext::default());
                 self.end(); // need to close a box
 
                 // a box will be closed by print_expr, but we didn't want an overall
@@ -454,33 +504,33 @@
                 self.print_block_with_attrs(blk, attrs);
             }
             ast::ExprKind::Await(expr, _) => {
-                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX);
+                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX, fixup);
                 self.word(".await");
             }
             ast::ExprKind::Assign(lhs, rhs, _) => {
                 let prec = AssocOp::Assign.precedence() as i8;
-                self.print_expr_maybe_paren(lhs, prec + 1);
+                self.print_expr_maybe_paren(lhs, prec + 1, fixup);
                 self.space();
                 self.word_space("=");
-                self.print_expr_maybe_paren(rhs, prec);
+                self.print_expr_maybe_paren(rhs, prec, fixup);
             }
             ast::ExprKind::AssignOp(op, lhs, rhs) => {
                 let prec = AssocOp::Assign.precedence() as i8;
-                self.print_expr_maybe_paren(lhs, prec + 1);
+                self.print_expr_maybe_paren(lhs, prec + 1, fixup);
                 self.space();
-                self.word(op.node.to_string());
+                self.word(op.node.as_str());
                 self.word_space("=");
-                self.print_expr_maybe_paren(rhs, prec);
+                self.print_expr_maybe_paren(rhs, prec, fixup);
             }
             ast::ExprKind::Field(expr, ident) => {
-                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX);
+                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX, fixup);
                 self.word(".");
                 self.print_ident(*ident);
             }
             ast::ExprKind::Index(expr, index, _) => {
-                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX);
+                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX, fixup);
                 self.word("[");
-                self.print_expr(index);
+                self.print_expr(index, FixupContext::default());
                 self.word("]");
             }
             ast::ExprKind::Range(start, end, limits) => {
@@ -490,14 +540,14 @@
                 // a "normal" binop gets parenthesized. (`LOr` is the lowest-precedence binop.)
                 let fake_prec = AssocOp::LOr.precedence() as i8;
                 if let Some(e) = start {
-                    self.print_expr_maybe_paren(e, fake_prec);
+                    self.print_expr_maybe_paren(e, fake_prec, fixup);
                 }
                 match limits {
                     ast::RangeLimits::HalfOpen => self.word(".."),
                     ast::RangeLimits::Closed => self.word("..="),
                 }
                 if let Some(e) = end {
-                    self.print_expr_maybe_paren(e, fake_prec);
+                    self.print_expr_maybe_paren(e, fake_prec, fixup);
                 }
             }
             ast::ExprKind::Underscore => self.word("_"),
@@ -511,7 +561,7 @@
                 }
                 if let Some(expr) = opt_expr {
                     self.space();
-                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP);
+                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP, fixup);
                 }
             }
             ast::ExprKind::Continue(opt_label) => {
@@ -525,7 +575,7 @@
                 self.word("return");
                 if let Some(expr) = result {
                     self.word(" ");
-                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP);
+                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP, fixup);
                 }
             }
             ast::ExprKind::Yeet(result) => {
@@ -534,13 +584,13 @@
                 self.word("yeet");
                 if let Some(expr) = result {
                     self.word(" ");
-                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP);
+                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP, fixup);
                 }
             }
             ast::ExprKind::Become(result) => {
                 self.word("become");
                 self.word(" ");
-                self.print_expr_maybe_paren(result, parser::PREC_JUMP);
+                self.print_expr_maybe_paren(result, parser::PREC_JUMP, fixup);
             }
             ast::ExprKind::InlineAsm(a) => {
                 // FIXME: This should have its own syntax, distinct from a macro invocation.
@@ -555,7 +605,7 @@
                 self.word(reconstruct_format_args_template_string(&fmt.template));
                 for arg in fmt.arguments.all_args() {
                     self.word_space(",");
-                    self.print_expr(&arg.expr);
+                    self.print_expr(&arg.expr, FixupContext::default());
                 }
                 self.end();
                 self.pclose();
@@ -582,7 +632,7 @@
             ast::ExprKind::MacCall(m) => self.print_mac(m),
             ast::ExprKind::Paren(e) => {
                 self.popen();
-                self.print_expr(e);
+                self.print_expr(e, FixupContext::default());
                 self.pclose();
             }
             ast::ExprKind::Yield(e) => {
@@ -590,11 +640,11 @@
 
                 if let Some(expr) = e {
                     self.space();
-                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP);
+                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP, fixup);
                 }
             }
             ast::ExprKind::Try(e) => {
-                self.print_expr_maybe_paren(e, parser::PREC_POSTFIX);
+                self.print_expr_maybe_paren(e, parser::PREC_POSTFIX, fixup);
                 self.word("?")
             }
             ast::ExprKind::TryBlock(blk) => {
@@ -626,31 +676,36 @@
         self.space();
         if let Some(e) = &arm.guard {
             self.word_space("if");
-            self.print_expr(e);
+            self.print_expr(e, FixupContext::default());
             self.space();
         }
-        self.word_space("=>");
 
-        match &arm.body.kind {
-            ast::ExprKind::Block(blk, opt_label) => {
-                if let Some(label) = opt_label {
-                    self.print_ident(label.ident);
-                    self.word_space(":");
+        if let Some(body) = &arm.body {
+            self.word_space("=>");
+
+            match &body.kind {
+                ast::ExprKind::Block(blk, opt_label) => {
+                    if let Some(label) = opt_label {
+                        self.print_ident(label.ident);
+                        self.word_space(":");
+                    }
+
+                    // The block will close the pattern's ibox.
+                    self.print_block_unclosed_indent(blk);
+
+                    // If it is a user-provided unsafe block, print a comma after it.
+                    if let BlockCheckMode::Unsafe(ast::UserProvided) = blk.rules {
+                        self.word(",");
+                    }
                 }
-
-                // The block will close the pattern's ibox.
-                self.print_block_unclosed_indent(blk);
-
-                // If it is a user-provided unsafe block, print a comma after it.
-                if let BlockCheckMode::Unsafe(ast::UserProvided) = blk.rules {
+                _ => {
+                    self.end(); // Close the ibox for the pattern.
+                    self.print_expr(body, FixupContext::default());
                     self.word(",");
                 }
             }
-            _ => {
-                self.end(); // Close the ibox for the pattern.
-                self.print_expr(&arm.body);
-                self.word(",");
-            }
+        } else {
+            self.word(",");
         }
         self.end(); // Close enclosing cbox.
     }
@@ -679,7 +734,7 @@
     }
 }
 
-pub fn reconstruct_format_args_template_string(pieces: &[FormatArgsPiece]) -> String {
+fn reconstruct_format_args_template_string(pieces: &[FormatArgsPiece]) -> String {
     let mut template = "\"".to_string();
     for piece in pieces {
         match piece {
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/item.rs b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
index 3393f03..405ccc7 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/item.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
@@ -1,8 +1,9 @@
 use crate::pp::Breaks::Inconsistent;
-use crate::pprust::state::delimited::IterDelimited;
+use crate::pprust::state::expr::FixupContext;
 use crate::pprust::state::{AnnNode, PrintState, State, INDENT_UNIT};
 
 use ast::StaticItem;
+use itertools::{Itertools, Position};
 use rustc_ast as ast;
 use rustc_ast::GenericBound;
 use rustc_ast::ModKind;
@@ -20,7 +21,7 @@
         }
     }
 
-    pub(crate) fn print_foreign_item(&mut self, item: &ast::ForeignItem) {
+    fn print_foreign_item(&mut self, item: &ast::ForeignItem) {
         let ast::Item { id, span, ident, ref attrs, ref kind, ref vis, tokens: _ } = *item;
         self.ann.pre(self, AnnNode::SubItem(id));
         self.hardbreak_if_not_bol();
@@ -97,7 +98,7 @@
         self.end(); // end the head-ibox
         if let Some(body) = body {
             self.word_space("=");
-            self.print_expr(body);
+            self.print_expr(body, FixupContext::default());
         }
         self.print_where_clause(&generics.where_clause);
         self.word(";");
@@ -368,7 +369,7 @@
                 self.nbsp();
                 if !bounds.is_empty() {
                     self.word_nbsp("=");
-                    self.print_type_bounds(&bounds);
+                    self.print_type_bounds(bounds);
                 }
                 self.print_where_clause(&generics.where_clause);
                 self.word(";");
@@ -499,7 +500,7 @@
                 self.end();
                 self.end(); // Close the outer-box.
             }
-            ast::VariantData::Struct(fields, ..) => {
+            ast::VariantData::Struct { fields, .. } => {
                 self.print_where_clause(&generics.where_clause);
                 self.print_record_struct_body(fields, span);
             }
@@ -514,11 +515,11 @@
         if let Some(d) = &v.disr_expr {
             self.space();
             self.word_space("=");
-            self.print_expr(&d.value)
+            self.print_expr(&d.value, FixupContext::default())
         }
     }
 
-    pub(crate) fn print_assoc_item(&mut self, item: &ast::AssocItem) {
+    fn print_assoc_item(&mut self, item: &ast::AssocItem) {
         let ast::Item { id, span, ident, ref attrs, ref kind, ref vis, tokens: _ } = *item;
         self.ann.pre(self, AnnNode::SubItem(id));
         self.hardbreak_if_not_bol();
@@ -621,7 +622,7 @@
         self.print_where_clause_parts(where_clause.has_where_token, &where_clause.predicates);
     }
 
-    pub(crate) fn print_where_clause_parts(
+    fn print_where_clause_parts(
         &mut self,
         has_where_token: bool,
         predicates: &[ast::WherePredicate],
@@ -668,7 +669,7 @@
         }
     }
 
-    pub fn print_where_bound_predicate(
+    pub(crate) fn print_where_bound_predicate(
         &mut self,
         where_bound_predicate: &ast::WhereBoundPredicate,
     ) {
@@ -712,9 +713,10 @@
                     self.word("{");
                     self.zerobreak();
                     self.ibox(0);
-                    for use_tree in items.iter().delimited() {
+                    for (pos, use_tree) in items.iter().with_position() {
+                        let is_last = matches!(pos, Position::Last | Position::Only);
                         self.print_use_tree(&use_tree.0);
-                        if !use_tree.is_last {
+                        if !is_last {
                             self.word(",");
                             if let ast::UseTreeKind::Nested(_) = use_tree.0.kind {
                                 self.hardbreak();
diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index ad92d58..0959e8d 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -552,7 +552,7 @@
 fn try_gate_cfg(name: Symbol, span: Span, sess: &ParseSess, features: Option<&Features>) {
     let gate = find_gated_cfg(|sym| sym == name);
     if let (Some(feats), Some(gated_cfg)) = (features, gate) {
-        gate_cfg(&gated_cfg, span, sess, feats);
+        gate_cfg(gated_cfg, span, sess, feats);
     }
 }
 
@@ -945,7 +945,7 @@
     assert!(attr.has_name(sym::repr), "expected `#[repr(..)]`, found: {attr:?}");
     use ReprAttr::*;
     let mut acc = Vec::new();
-    let diagnostic = &sess.parse_sess.span_diagnostic;
+    let diagnostic = sess.dcx();
 
     if let Some(items) = attr.meta_item_list() {
         for item in items {
@@ -985,7 +985,7 @@
                         Ok(literal) => acc.push(ReprPacked(literal)),
                         Err(message) => literal_error = Some(message),
                     };
-                } else if matches!(name, sym::C | sym::simd | sym::transparent)
+                } else if matches!(name, sym::Rust | sym::C | sym::simd | sym::transparent)
                     || int_type_of_word(name).is_some()
                 {
                     recognised = true;
@@ -1018,7 +1018,7 @@
                             });
                         } else if matches!(
                             meta_item.name_or_empty(),
-                            sym::C | sym::simd | sym::transparent
+                            sym::Rust | sym::C | sym::simd | sym::transparent
                         ) || int_type_of_word(meta_item.name_or_empty()).is_some()
                         {
                             recognised = true;
@@ -1043,7 +1043,7 @@
                             );
                         } else if matches!(
                             meta_item.name_or_empty(),
-                            sym::C | sym::simd | sym::transparent
+                            sym::Rust | sym::C | sym::simd | sym::transparent
                         ) || int_type_of_word(meta_item.name_or_empty()).is_some()
                         {
                             recognised = true;
@@ -1060,9 +1060,9 @@
                 // Not a word we recognize. This will be caught and reported by
                 // the `check_mod_attrs` pass, but this pass doesn't always run
                 // (e.g. if we only pretty-print the source), so we have to gate
-                // the `delay_span_bug` call as follows:
+                // the `span_delayed_bug` call as follows:
                 if sess.opts.pretty.map_or(true, |pp| pp.needs_analysis()) {
-                    diagnostic.delay_span_bug(item.span(), "unrecognized representation hint");
+                    diagnostic.span_delayed_bug(item.span(), "unrecognized representation hint");
                 }
             }
         }
diff --git a/compiler/rustc_attr/src/lib.rs b/compiler/rustc_attr/src/lib.rs
index 868c041..3c5bcec 100644
--- a/compiler/rustc_attr/src/lib.rs
+++ b/compiler/rustc_attr/src/lib.rs
@@ -4,9 +4,9 @@
 //! The goal is to move the definition of `MetaItem` and things that don't need to be in `syntax`
 //! to this crate.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![feature(let_chains)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -14,9 +14,6 @@
 #[macro_use]
 extern crate rustc_macros;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 mod builtin;
 mod session_diagnostics;
 
@@ -29,4 +26,4 @@
 
 pub(crate) use rustc_session::HashStableContext;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_attr/src/session_diagnostics.rs b/compiler/rustc_attr/src/session_diagnostics.rs
index ca9bbd2..ce8e04d 100644
--- a/compiler/rustc_attr/src/session_diagnostics.rs
+++ b/compiler/rustc_attr/src/session_diagnostics.rs
@@ -2,7 +2,7 @@
 
 use rustc_ast as ast;
 use rustc_errors::{
-    error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
+    error_code, Applicability, DiagCtxt, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic,
 };
 use rustc_macros::Diagnostic;
 use rustc_span::{Span, Symbol};
@@ -51,9 +51,9 @@
 
 // Manual implementation to be able to format `expected` items correctly.
 impl<'a> IntoDiagnostic<'a> for UnknownMetaItem<'_> {
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
         let expected = self.expected.iter().map(|name| format!("`{name}`")).collect::<Vec<_>>();
-        let mut diag = handler.struct_span_err_with_code(
+        let mut diag = dcx.struct_span_err_with_code(
             self.span,
             fluent::attr_unknown_meta_item,
             error_code!(E0541),
@@ -201,8 +201,8 @@
 }
 
 impl<'a> IntoDiagnostic<'a> for UnsupportedLiteral {
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
-        let mut diag = handler.struct_span_err_with_code(
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+        let mut diag = dcx.struct_span_err_with_code(
             self.span,
             match self.reason {
                 UnsupportedLiteralReason::Generic => fluent::attr_unsupported_literal_generic,
diff --git a/compiler/rustc_baked_icu_data/src/lib.rs b/compiler/rustc_baked_icu_data/src/lib.rs
index ae8c062..ffcb290 100644
--- a/compiler/rustc_baked_icu_data/src/lib.rs
+++ b/compiler/rustc_baked_icu_data/src/lib.rs
@@ -20,9 +20,9 @@
 //! --cldr-tag latest --icuexport-tag latest -o src/data
 //! ```
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![allow(elided_lifetimes_in_paths)]
 
 mod data {
diff --git a/compiler/rustc_borrowck/Cargo.toml b/compiler/rustc_borrowck/Cargo.toml
index 636817a..714f462 100644
--- a/compiler/rustc_borrowck/Cargo.toml
+++ b/compiler/rustc_borrowck/Cargo.toml
@@ -6,7 +6,7 @@
 [dependencies]
 # tidy-alphabetical-start
 either = "1.5.0"
-itertools = "0.10.1"
+itertools = "0.11"
 polonius-engine = "0.13.0"
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
@@ -19,7 +19,6 @@
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_mir_dataflow = { path = "../rustc_mir_dataflow" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_borrowck/src/borrow_set.rs b/compiler/rustc_borrowck/src/borrow_set.rs
index 5248a64..948af03 100644
--- a/compiler/rustc_borrowck/src/borrow_set.rs
+++ b/compiler/rustc_borrowck/src/borrow_set.rs
@@ -107,7 +107,7 @@
             LocalsStateAtExit::AllAreInvalidated
         } else {
             let mut has_storage_dead = HasStorageDead(BitSet::new_empty(body.local_decls.len()));
-            has_storage_dead.visit_body(&body);
+            has_storage_dead.visit_body(body);
             let mut has_storage_dead_or_moved = has_storage_dead.0;
             for move_out in &move_data.moves {
                 if let Some(index) = move_data.base_local(move_out.path) {
@@ -128,7 +128,7 @@
     ) -> Self {
         let mut visitor = GatherBorrows {
             tcx,
-            body: &body,
+            body: body,
             location_map: Default::default(),
             activation_map: Default::default(),
             local_map: Default::default(),
@@ -140,7 +140,7 @@
             ),
         };
 
-        for (block, block_data) in traversal::preorder(&body) {
+        for (block, block_data) in traversal::preorder(body) {
             visitor.visit_basic_block_data(block, block_data);
         }
 
diff --git a/compiler/rustc_borrowck/src/constraint_generation.rs b/compiler/rustc_borrowck/src/constraint_generation.rs
deleted file mode 100644
index 1f64209..0000000
--- a/compiler/rustc_borrowck/src/constraint_generation.rs
+++ /dev/null
@@ -1,248 +0,0 @@
-#![deny(rustc::untranslatable_diagnostic)]
-#![deny(rustc::diagnostic_outside_of_impl)]
-use rustc_infer::infer::InferCtxt;
-use rustc_middle::mir::visit::TyContext;
-use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{
-    Body, Local, Location, Place, PlaceRef, ProjectionElem, Rvalue, SourceInfo, Statement,
-    StatementKind, Terminator, TerminatorKind, UserTypeProjection,
-};
-use rustc_middle::ty::visit::TypeVisitable;
-use rustc_middle::ty::GenericArgsRef;
-use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt};
-
-use crate::{
-    borrow_set::BorrowSet, facts::AllFacts, location::LocationTable, places_conflict,
-    region_infer::values::LivenessValues,
-};
-
-pub(super) fn generate_constraints<'tcx>(
-    infcx: &InferCtxt<'tcx>,
-    liveness_constraints: &mut LivenessValues<RegionVid>,
-    all_facts: &mut Option<AllFacts>,
-    location_table: &LocationTable,
-    body: &Body<'tcx>,
-    borrow_set: &BorrowSet<'tcx>,
-) {
-    let mut cg = ConstraintGeneration {
-        borrow_set,
-        infcx,
-        liveness_constraints,
-        location_table,
-        all_facts,
-        body,
-    };
-
-    for (bb, data) in body.basic_blocks.iter_enumerated() {
-        cg.visit_basic_block_data(bb, data);
-    }
-}
-
-/// 'cg = the duration of the constraint generation process itself.
-struct ConstraintGeneration<'cg, 'tcx> {
-    infcx: &'cg InferCtxt<'tcx>,
-    all_facts: &'cg mut Option<AllFacts>,
-    location_table: &'cg LocationTable,
-    liveness_constraints: &'cg mut LivenessValues<RegionVid>,
-    borrow_set: &'cg BorrowSet<'tcx>,
-    body: &'cg Body<'tcx>,
-}
-
-impl<'cg, 'tcx> Visitor<'tcx> for ConstraintGeneration<'cg, 'tcx> {
-    /// We sometimes have `args` within an rvalue, or within a
-    /// call. Make them live at the location where they appear.
-    fn visit_args(&mut self, args: &GenericArgsRef<'tcx>, location: Location) {
-        self.add_regular_live_constraint(*args, location);
-        self.super_args(args);
-    }
-
-    /// We sometimes have `region` within an rvalue, or within a
-    /// call. Make them live at the location where they appear.
-    fn visit_region(&mut self, region: ty::Region<'tcx>, location: Location) {
-        self.add_regular_live_constraint(region, location);
-        self.super_region(region);
-    }
-
-    /// We sometimes have `ty` within an rvalue, or within a
-    /// call. Make them live at the location where they appear.
-    fn visit_ty(&mut self, ty: Ty<'tcx>, ty_context: TyContext) {
-        match ty_context {
-            TyContext::ReturnTy(SourceInfo { span, .. })
-            | TyContext::YieldTy(SourceInfo { span, .. })
-            | TyContext::UserTy(span)
-            | TyContext::LocalDecl { source_info: SourceInfo { span, .. }, .. } => {
-                span_bug!(span, "should not be visiting outside of the CFG: {:?}", ty_context);
-            }
-            TyContext::Location(location) => {
-                self.add_regular_live_constraint(ty, location);
-            }
-        }
-
-        self.super_ty(ty);
-    }
-
-    fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
-        if let Some(all_facts) = self.all_facts {
-            let _prof_timer = self.infcx.tcx.prof.generic_activity("polonius_fact_generation");
-            all_facts.cfg_edge.push((
-                self.location_table.start_index(location),
-                self.location_table.mid_index(location),
-            ));
-
-            all_facts.cfg_edge.push((
-                self.location_table.mid_index(location),
-                self.location_table.start_index(location.successor_within_block()),
-            ));
-
-            // If there are borrows on this now dead local, we need to record them as `killed`.
-            if let StatementKind::StorageDead(local) = statement.kind {
-                record_killed_borrows_for_local(
-                    all_facts,
-                    self.borrow_set,
-                    self.location_table,
-                    local,
-                    location,
-                );
-            }
-        }
-
-        self.super_statement(statement, location);
-    }
-
-    fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) {
-        // When we see `X = ...`, then kill borrows of
-        // `(*X).foo` and so forth.
-        self.record_killed_borrows_for_place(*place, location);
-
-        self.super_assign(place, rvalue, location);
-    }
-
-    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
-        if let Some(all_facts) = self.all_facts {
-            let _prof_timer = self.infcx.tcx.prof.generic_activity("polonius_fact_generation");
-            all_facts.cfg_edge.push((
-                self.location_table.start_index(location),
-                self.location_table.mid_index(location),
-            ));
-
-            let successor_blocks = terminator.successors();
-            all_facts.cfg_edge.reserve(successor_blocks.size_hint().0);
-            for successor_block in successor_blocks {
-                all_facts.cfg_edge.push((
-                    self.location_table.mid_index(location),
-                    self.location_table.start_index(successor_block.start_location()),
-                ));
-            }
-        }
-
-        // A `Call` terminator's return value can be a local which has borrows,
-        // so we need to record those as `killed` as well.
-        if let TerminatorKind::Call { destination, .. } = terminator.kind {
-            self.record_killed_borrows_for_place(destination, location);
-        }
-
-        self.super_terminator(terminator, location);
-    }
-
-    fn visit_ascribe_user_ty(
-        &mut self,
-        _place: &Place<'tcx>,
-        _variance: ty::Variance,
-        _user_ty: &UserTypeProjection,
-        _location: Location,
-    ) {
-    }
-}
-
-impl<'cx, 'tcx> ConstraintGeneration<'cx, 'tcx> {
-    /// Some variable with type `live_ty` is "regular live" at
-    /// `location` -- i.e., it may be used later. This means that all
-    /// regions appearing in the type `live_ty` must be live at
-    /// `location`.
-    fn add_regular_live_constraint<T>(&mut self, live_ty: T, location: Location)
-    where
-        T: TypeVisitable<TyCtxt<'tcx>>,
-    {
-        debug!("add_regular_live_constraint(live_ty={:?}, location={:?})", live_ty, location);
-
-        self.infcx.tcx.for_each_free_region(&live_ty, |live_region| {
-            let vid = live_region.as_var();
-            self.liveness_constraints.add_element(vid, location);
-        });
-    }
-
-    /// When recording facts for Polonius, records the borrows on the specified place
-    /// as `killed`. For example, when assigning to a local, or on a call's return destination.
-    fn record_killed_borrows_for_place(&mut self, place: Place<'tcx>, location: Location) {
-        if let Some(all_facts) = self.all_facts {
-            let _prof_timer = self.infcx.tcx.prof.generic_activity("polonius_fact_generation");
-
-            // Depending on the `Place` we're killing:
-            // - if it's a local, or a single deref of a local,
-            //   we kill all the borrows on the local.
-            // - if it's a deeper projection, we have to filter which
-            //   of the borrows are killed: the ones whose `borrowed_place`
-            //   conflicts with the `place`.
-            match place.as_ref() {
-                PlaceRef { local, projection: &[] }
-                | PlaceRef { local, projection: &[ProjectionElem::Deref] } => {
-                    debug!(
-                        "Recording `killed` facts for borrows of local={:?} at location={:?}",
-                        local, location
-                    );
-
-                    record_killed_borrows_for_local(
-                        all_facts,
-                        self.borrow_set,
-                        self.location_table,
-                        local,
-                        location,
-                    );
-                }
-
-                PlaceRef { local, projection: &[.., _] } => {
-                    // Kill conflicting borrows of the innermost local.
-                    debug!(
-                        "Recording `killed` facts for borrows of \
-                            innermost projected local={:?} at location={:?}",
-                        local, location
-                    );
-
-                    if let Some(borrow_indices) = self.borrow_set.local_map.get(&local) {
-                        for &borrow_index in borrow_indices {
-                            let places_conflict = places_conflict::places_conflict(
-                                self.infcx.tcx,
-                                self.body,
-                                self.borrow_set[borrow_index].borrowed_place,
-                                place,
-                                places_conflict::PlaceConflictBias::NoOverlap,
-                            );
-
-                            if places_conflict {
-                                let location_index = self.location_table.mid_index(location);
-                                all_facts.loan_killed_at.push((borrow_index, location_index));
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-/// When recording facts for Polonius, records the borrows on the specified local as `killed`.
-fn record_killed_borrows_for_local(
-    all_facts: &mut AllFacts,
-    borrow_set: &BorrowSet<'_>,
-    location_table: &LocationTable,
-    local: Local,
-    location: Location,
-) {
-    if let Some(borrow_indices) = borrow_set.local_map.get(&local) {
-        all_facts.loan_killed_at.reserve(borrow_indices.len());
-        for &borrow_index in borrow_indices {
-            let location_index = location_table.mid_index(location);
-            all_facts.loan_killed_at.push((borrow_index, location_index));
-        }
-    }
-}
diff --git a/compiler/rustc_borrowck/src/constraints/mod.rs b/compiler/rustc_borrowck/src/constraints/mod.rs
index 315886b..041ac75 100644
--- a/compiler/rustc_borrowck/src/constraints/mod.rs
+++ b/compiler/rustc_borrowck/src/constraints/mod.rs
@@ -122,6 +122,7 @@
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "ConstraintSccIndex({})"]
     pub struct ConstraintSccIndex {}
 }
diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs
index 8676d2b..1bd891b 100644
--- a/compiler/rustc_borrowck/src/dataflow.rs
+++ b/compiler/rustc_borrowck/src/dataflow.rs
@@ -10,105 +10,93 @@
 use rustc_middle::ty::TyCtxt;
 use rustc_mir_dataflow::impls::{EverInitializedPlaces, MaybeUninitializedPlaces};
 use rustc_mir_dataflow::ResultsVisitable;
-use rustc_mir_dataflow::{self, fmt::DebugWithContext, GenKill};
-use rustc_mir_dataflow::{Analysis, Direction, Results};
+use rustc_mir_dataflow::{fmt::DebugWithContext, GenKill};
+use rustc_mir_dataflow::{Analysis, AnalysisDomain, Results};
 use std::fmt;
 
 use crate::{places_conflict, BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext};
 
-/// A tuple with named fields that can hold either the results or the transient state of the
-/// dataflow analyses used by the borrow checker.
-#[derive(Debug)]
-pub struct BorrowckAnalyses<B, U, E> {
-    pub borrows: B,
-    pub uninits: U,
-    pub ever_inits: E,
-}
-
 /// The results of the dataflow analyses used by the borrow checker.
-pub type BorrowckResults<'mir, 'tcx> = BorrowckAnalyses<
-    Results<'tcx, Borrows<'mir, 'tcx>>,
-    Results<'tcx, MaybeUninitializedPlaces<'mir, 'tcx>>,
-    Results<'tcx, EverInitializedPlaces<'mir, 'tcx>>,
->;
+pub struct BorrowckResults<'mir, 'tcx> {
+    pub(crate) borrows: Results<'tcx, Borrows<'mir, 'tcx>>,
+    pub(crate) uninits: Results<'tcx, MaybeUninitializedPlaces<'mir, 'tcx>>,
+    pub(crate) ever_inits: Results<'tcx, EverInitializedPlaces<'mir, 'tcx>>,
+}
 
 /// The transient state of the dataflow analyses used by the borrow checker.
-pub type BorrowckFlowState<'mir, 'tcx> =
-    <BorrowckResults<'mir, 'tcx> as ResultsVisitable<'tcx>>::FlowState;
-
-macro_rules! impl_visitable {
-    ( $(
-        $T:ident { $( $field:ident : $A:ident ),* $(,)? }
-    )* ) => { $(
-        impl<'tcx, $($A),*, D: Direction> ResultsVisitable<'tcx> for $T<$( Results<'tcx, $A> ),*>
-        where
-            $( $A: Analysis<'tcx, Direction = D>, )*
-        {
-            type Direction = D;
-            type FlowState = $T<$( $A::Domain ),*>;
-
-            fn new_flow_state(&self, body: &mir::Body<'tcx>) -> Self::FlowState {
-                $T {
-                    $( $field: self.$field.analysis.bottom_value(body) ),*
-                }
-            }
-
-            fn reset_to_block_entry(
-                &self,
-                state: &mut Self::FlowState,
-                block: BasicBlock,
-            ) {
-                $( state.$field.clone_from(&self.$field.entry_set_for_block(block)); )*
-            }
-
-            fn reconstruct_before_statement_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                stmt: &mir::Statement<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_before_statement_effect(&mut state.$field, stmt, loc); )*
-            }
-
-            fn reconstruct_statement_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                stmt: &mir::Statement<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_statement_effect(&mut state.$field, stmt, loc); )*
-            }
-
-            fn reconstruct_before_terminator_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                term: &mir::Terminator<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_before_terminator_effect(&mut state.$field, term, loc); )*
-            }
-
-            fn reconstruct_terminator_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                term: &mir::Terminator<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_terminator_effect(&mut state.$field, term, loc); )*
-            }
-        }
-    )* }
+#[derive(Debug)]
+pub struct BorrowckFlowState<'mir, 'tcx> {
+    pub(crate) borrows: <Borrows<'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+    pub(crate) uninits: <MaybeUninitializedPlaces<'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+    pub(crate) ever_inits: <EverInitializedPlaces<'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
 }
 
-impl_visitable! {
-    BorrowckAnalyses { borrows: B, uninits: U, ever_inits: E }
+impl<'mir, 'tcx> ResultsVisitable<'tcx> for BorrowckResults<'mir, 'tcx> {
+    // All three analyses are forward, but we have to use just one here.
+    type Direction = <Borrows<'mir, 'tcx> as AnalysisDomain<'tcx>>::Direction;
+    type FlowState = BorrowckFlowState<'mir, 'tcx>;
+
+    fn new_flow_state(&self, body: &mir::Body<'tcx>) -> Self::FlowState {
+        BorrowckFlowState {
+            borrows: self.borrows.analysis.bottom_value(body),
+            uninits: self.uninits.analysis.bottom_value(body),
+            ever_inits: self.ever_inits.analysis.bottom_value(body),
+        }
+    }
+
+    fn reset_to_block_entry(&self, state: &mut Self::FlowState, block: BasicBlock) {
+        state.borrows.clone_from(&self.borrows.entry_set_for_block(block));
+        state.uninits.clone_from(&self.uninits.entry_set_for_block(block));
+        state.ever_inits.clone_from(&self.ever_inits.entry_set_for_block(block));
+    }
+
+    fn reconstruct_before_statement_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        stmt: &mir::Statement<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_before_statement_effect(&mut state.borrows, stmt, loc);
+        self.uninits.analysis.apply_before_statement_effect(&mut state.uninits, stmt, loc);
+        self.ever_inits.analysis.apply_before_statement_effect(&mut state.ever_inits, stmt, loc);
+    }
+
+    fn reconstruct_statement_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        stmt: &mir::Statement<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_statement_effect(&mut state.borrows, stmt, loc);
+        self.uninits.analysis.apply_statement_effect(&mut state.uninits, stmt, loc);
+        self.ever_inits.analysis.apply_statement_effect(&mut state.ever_inits, stmt, loc);
+    }
+
+    fn reconstruct_before_terminator_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        term: &mir::Terminator<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_before_terminator_effect(&mut state.borrows, term, loc);
+        self.uninits.analysis.apply_before_terminator_effect(&mut state.uninits, term, loc);
+        self.ever_inits.analysis.apply_before_terminator_effect(&mut state.ever_inits, term, loc);
+    }
+
+    fn reconstruct_terminator_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        term: &mir::Terminator<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_terminator_effect(&mut state.borrows, term, loc);
+        self.uninits.analysis.apply_terminator_effect(&mut state.uninits, term, loc);
+        self.ever_inits.analysis.apply_terminator_effect(&mut state.ever_inits, term, loc);
+    }
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "bw{}"]
     pub struct BorrowIndex {}
 }
@@ -120,24 +108,24 @@
 /// `BorrowIndex`, and maps each such index to a `BorrowData`
 /// describing the borrow. These indexes are used for representing the
 /// borrows in compact bitvectors.
-pub struct Borrows<'a, 'tcx> {
+pub struct Borrows<'mir, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    body: &'a Body<'tcx>,
+    body: &'mir Body<'tcx>,
 
-    borrow_set: &'a BorrowSet<'tcx>,
+    borrow_set: &'mir BorrowSet<'tcx>,
     borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
-struct OutOfScopePrecomputer<'a, 'tcx> {
+struct OutOfScopePrecomputer<'mir, 'tcx> {
     visited: BitSet<mir::BasicBlock>,
     visit_stack: Vec<mir::BasicBlock>,
-    body: &'a Body<'tcx>,
-    regioncx: &'a RegionInferenceContext<'tcx>,
+    body: &'mir Body<'tcx>,
+    regioncx: &'mir RegionInferenceContext<'tcx>,
     borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
-impl<'a, 'tcx> OutOfScopePrecomputer<'a, 'tcx> {
-    fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self {
+impl<'mir, 'tcx> OutOfScopePrecomputer<'mir, 'tcx> {
+    fn new(body: &'mir Body<'tcx>, regioncx: &'mir RegionInferenceContext<'tcx>) -> Self {
         OutOfScopePrecomputer {
             visited: BitSet::new_empty(body.basic_blocks.len()),
             visit_stack: vec![],
@@ -240,17 +228,17 @@
     prec.borrows_out_of_scope_at_location
 }
 
-struct PoloniusOutOfScopePrecomputer<'a, 'tcx> {
+struct PoloniusOutOfScopePrecomputer<'mir, 'tcx> {
     visited: BitSet<mir::BasicBlock>,
     visit_stack: Vec<mir::BasicBlock>,
-    body: &'a Body<'tcx>,
-    regioncx: &'a RegionInferenceContext<'tcx>,
+    body: &'mir Body<'tcx>,
+    regioncx: &'mir RegionInferenceContext<'tcx>,
 
     loans_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
-impl<'a, 'tcx> PoloniusOutOfScopePrecomputer<'a, 'tcx> {
-    fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self {
+impl<'mir, 'tcx> PoloniusOutOfScopePrecomputer<'mir, 'tcx> {
+    fn new(body: &'mir Body<'tcx>, regioncx: &'mir RegionInferenceContext<'tcx>) -> Self {
         Self {
             visited: BitSet::new_empty(body.basic_blocks.len()),
             visit_stack: vec![],
@@ -403,12 +391,12 @@
     }
 }
 
-impl<'a, 'tcx> Borrows<'a, 'tcx> {
+impl<'mir, 'tcx> Borrows<'mir, 'tcx> {
     pub fn new(
         tcx: TyCtxt<'tcx>,
-        body: &'a Body<'tcx>,
-        regioncx: &'a RegionInferenceContext<'tcx>,
-        borrow_set: &'a BorrowSet<'tcx>,
+        body: &'mir Body<'tcx>,
+        regioncx: &'mir RegionInferenceContext<'tcx>,
+        borrow_set: &'mir BorrowSet<'tcx>,
     ) -> Self {
         let mut borrows_out_of_scope_at_location =
             calculate_borrows_out_of_scope_at_location(body, regioncx, borrow_set);
@@ -431,7 +419,8 @@
 
             assert_eq!(
                 borrows_out_of_scope_at_location, polonius_prec.loans_out_of_scope_at_location,
-                "the loans out of scope must be the same as the borrows out of scope"
+                "polonius loan scopes differ from NLL borrow scopes, for body {:?}",
+                body.span,
             );
 
             borrows_out_of_scope_at_location = polonius_prec.loans_out_of_scope_at_location;
@@ -596,7 +585,7 @@
 
     fn before_terminator_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _terminator: &mir::Terminator<'tcx>,
         location: Location,
     ) {
@@ -623,7 +612,7 @@
 
     fn call_return_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
diff --git a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
index cfcf31f..924e68f 100644
--- a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
@@ -10,6 +10,8 @@
 use rustc_infer::infer::{InferCtxt, RegionResolutionError, SubregionOrigin, TyCtxtInferExt as _};
 use rustc_infer::traits::ObligationCause;
 use rustc_middle::ty::error::TypeError;
+use rustc_middle::ty::RePlaceholder;
+use rustc_middle::ty::Region;
 use rustc_middle::ty::RegionVid;
 use rustc_middle::ty::UniverseIndex;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable};
@@ -205,6 +207,8 @@
         let span = cause.span;
         let nice_error = self.nice_error(mbcx, cause, placeholder_region, error_region);
 
+        debug!(?nice_error);
+
         if let Some(nice_error) = nice_error {
             mbcx.buffer_error(nice_error);
         } else {
@@ -381,7 +385,7 @@
     error_region: Option<ty::Region<'tcx>>,
 ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
     // We generally shouldn't have errors here because the query was
-    // already run, but there's no point using `delay_span_bug`
+    // already run, but there's no point using `span_delayed_bug`
     // when we're going to emit an error here anyway.
     let _errors = ocx.select_all_or_error();
     let region_constraints = ocx.infcx.with_region_constraints(|r| r.clone());
@@ -404,19 +408,41 @@
     mut region_var_origin: impl FnMut(RegionVid) -> RegionVariableOrigin,
     mut universe_of_region: impl FnMut(RegionVid) -> UniverseIndex,
 ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
-    let (sub_region, cause) =
-        region_constraints.constraints.iter().find_map(|(constraint, cause)| {
-            match *constraint {
-                Constraint::RegSubReg(sub, sup) if sup == placeholder_region && sup != sub => {
-                    Some((sub, cause.clone()))
-                }
-                // FIXME: Should this check the universe of the var?
-                Constraint::VarSubReg(vid, sup) if sup == placeholder_region => {
-                    Some((ty::Region::new_var(infcx.tcx, vid), cause.clone()))
-                }
-                _ => None,
+    let matches =
+        |a_region: Region<'tcx>, b_region: Region<'tcx>| match (a_region.kind(), b_region.kind()) {
+            (RePlaceholder(a_p), RePlaceholder(b_p)) => a_p.bound == b_p.bound,
+            _ => a_region == b_region,
+        };
+    let check = |constraint: &Constraint<'tcx>, cause: &SubregionOrigin<'tcx>, exact| {
+        match *constraint {
+            Constraint::RegSubReg(sub, sup)
+                if ((exact && sup == placeholder_region)
+                    || (!exact && matches(sup, placeholder_region)))
+                    && sup != sub =>
+            {
+                Some((sub, cause.clone()))
             }
-        })?;
+            // FIXME: Should this check the universe of the var?
+            Constraint::VarSubReg(vid, sup)
+                if ((exact && sup == placeholder_region)
+                    || (!exact && matches(sup, placeholder_region))) =>
+            {
+                Some((ty::Region::new_var(infcx.tcx, vid), cause.clone()))
+            }
+            _ => None,
+        }
+    };
+    let mut info = region_constraints
+        .constraints
+        .iter()
+        .find_map(|(constraint, cause)| check(constraint, cause, true));
+    if info.is_none() {
+        info = region_constraints
+            .constraints
+            .iter()
+            .find_map(|(constraint, cause)| check(constraint, cause, false));
+    }
+    let (sub_region, cause) = info?;
 
     debug!(?sub_region, "cause = {:#?}", cause);
     let error = match (error_region, *sub_region) {
diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index 9a8f1c9..db0f455 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -18,7 +18,7 @@
     PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
     VarBindingForm,
 };
-use rustc_middle::ty::{self, suggest_constraining_type_params, PredicateKind, Ty};
+use rustc_middle::ty::{self, suggest_constraining_type_params, PredicateKind, Ty, TyCtxt};
 use rustc_middle::util::CallKind;
 use rustc_mir_dataflow::move_paths::{InitKind, MoveOutIndex, MovePathIndex};
 use rustc_span::def_id::LocalDefId;
@@ -398,7 +398,7 @@
                 }
                 let typeck = self.infcx.tcx.typeck(self.mir_def_id());
                 let hir_id = hir.parent_id(expr.hir_id);
-                if let Some(parent) = hir.find(hir_id) {
+                if let Some(parent) = self.infcx.tcx.opt_hir_node(hir_id) {
                     let (def_id, args, offset) = if let hir::Node::Expr(parent_expr) = parent
                         && let hir::ExprKind::MethodCall(_, _, args, _) = parent_expr.kind
                         && let Some(def_id) = typeck.type_dependent_def_id(parent_expr.hir_id)
@@ -413,7 +413,10 @@
                         (None, &[][..], 0)
                     };
                     if let Some(def_id) = def_id
-                        && let Some(node) = hir.find(hir.local_def_id_to_hir_id(def_id))
+                        && let Some(node) = self
+                            .infcx
+                            .tcx
+                            .opt_hir_node(self.infcx.tcx.local_def_id_to_hir_id(def_id))
                         && let Some(fn_sig) = node.fn_sig()
                         && let Some(ident) = node.ident()
                         && let Some(pos) = args.iter().position(|arg| arg.hir_id == expr.hir_id)
@@ -445,7 +448,6 @@
                         && let hir::ExprKind::Path(hir::QPath::LangItem(
                             LangItem::IntoIterIntoIter,
                             _,
-                            _,
                         )) = call_expr.kind
                     {
                         // Do not suggest `.clone()` in a `for` loop, we already suggest borrowing.
@@ -490,7 +492,7 @@
         let mut spans = vec![];
         for init_idx in inits {
             let init = &self.move_data.inits[*init_idx];
-            let span = init.span(&self.body);
+            let span = init.span(self.body);
             if !span.is_dummy() {
                 spans.push(span);
             }
@@ -518,7 +520,7 @@
         let body = map.body(body_id);
 
         let mut visitor = ConditionVisitor { spans: &spans, name: &name, errors: vec![] };
-        visitor.visit_body(&body);
+        visitor.visit_body(body);
 
         let mut show_assign_sugg = false;
         let isnt_initialized = if let InitializationRequiringAction::PartialAssignment
@@ -614,7 +616,7 @@
             }
 
             let mut visitor = LetVisitor { decl_span, sugg_span: None };
-            visitor.visit_body(&body);
+            visitor.visit_body(body);
             if let Some(span) = visitor.sugg_span {
                 self.suggest_assign_value(&mut err, moved_place, span);
             }
@@ -779,7 +781,7 @@
             return;
         };
         // Try to find predicates on *generic params* that would allow copying `ty`
-        let ocx = ObligationCtxt::new(&self.infcx);
+        let ocx = ObligationCtxt::new(self.infcx);
         let copy_did = tcx.require_lang_item(LangItem::Copy, Some(span));
         let cause = ObligationCause::misc(span, self.mir_def_id());
 
@@ -856,7 +858,7 @@
         self.explain_why_borrow_contains_point(location, borrow, None)
             .add_explanation_to_diagnostic(
                 self.infcx.tcx,
-                &self.body,
+                self.body,
                 &self.local_names,
                 &mut err,
                 "",
@@ -903,7 +905,7 @@
         self.explain_why_borrow_contains_point(location, borrow, None)
             .add_explanation_to_diagnostic(
                 self.infcx.tcx,
-                &self.body,
+                self.body,
                 &self.local_names,
                 &mut err,
                 "",
@@ -1136,7 +1138,7 @@
             });
         } else {
             issued_spans.var_subdiag(
-                Some(&self.infcx.tcx.sess.parse_sess.span_diagnostic),
+                Some(self.infcx.tcx.sess.dcx()),
                 &mut err,
                 Some(issued_borrow.kind),
                 |kind, var_span| {
@@ -1153,7 +1155,7 @@
             );
 
             borrow_spans.var_subdiag(
-                Some(&self.infcx.tcx.sess.parse_sess.span_diagnostic),
+                Some(self.infcx.tcx.sess.dcx()),
                 &mut err,
                 Some(gen_borrow_kind),
                 |kind, var_span| {
@@ -1174,7 +1176,7 @@
 
         explanation.add_explanation_to_diagnostic(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &mut err,
             first_borrow_desc,
@@ -1318,7 +1320,7 @@
         let tcx = self.infcx.tcx;
         let hir = tcx.hir();
 
-        let Some(body_id) = hir.get(self.mir_hir_id()).body_id() else { return };
+        let Some(body_id) = tcx.hir_node(self.mir_hir_id()).body_id() else { return };
         let typeck_results = tcx.typeck(self.mir_def_id());
 
         struct ExprFinder<'hir> {
@@ -1346,11 +1348,8 @@
                 // };
                 // corresponding to the desugaring of a for loop `for <pat> in <head> { <body> }`.
                 if let hir::ExprKind::Call(path, [arg]) = ex.kind
-                    && let hir::ExprKind::Path(hir::QPath::LangItem(
-                        LangItem::IntoIterIntoIter,
-                        _,
-                        _,
-                    )) = path.kind
+                    && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IntoIterIntoIter, _)) =
+                        path.kind
                     && arg.span.contains(self.issue_span)
                 {
                     // Find `IntoIterator::into_iter(<head>)`
@@ -1368,10 +1367,10 @@
                         ..
                     }) = stmt.kind
                     && let hir::ExprKind::Call(path, _args) = call.kind
-                    && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IteratorNext, _, _)) =
+                    && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IteratorNext, _)) =
                         path.kind
                     && let hir::PatKind::Struct(path, [field, ..], _) = bind.pat.kind
-                    && let hir::QPath::LangItem(LangItem::OptionSome, pat_span, _) = path
+                    && let hir::QPath::LangItem(LangItem::OptionSome, pat_span) = path
                     && call.span.contains(self.issue_span)
                 {
                     // Find `<pat>` and the span for the whole `for` loop.
@@ -1513,7 +1512,7 @@
         let local_ty = self.body.local_decls[local].ty;
 
         // Get the body the error happens in
-        let Some(body_id) = hir.get(self.mir_hir_id()).body_id() else { return };
+        let Some(body_id) = tcx.hir_node(self.mir_hir_id()).body_id() else { return };
 
         let body_expr = hir.body(body_id).value;
 
@@ -1562,7 +1561,7 @@
         // Check that the parent of the closure is a method call,
         // with receiver matching with local's type (modulo refs)
         let parent = hir.parent_id(closure_expr.hir_id);
-        if let hir::Node::Expr(parent) = hir.get(parent) {
+        if let hir::Node::Expr(parent) = tcx.hir_node(parent) {
             if let hir::ExprKind::MethodCall(_, recv, ..) = parent.kind {
                 let recv_ty = typeck_results.expr_ty(recv);
 
@@ -1578,8 +1577,9 @@
             return;
         };
         let sig = args.as_closure().sig();
-        let tupled_params =
-            tcx.erase_late_bound_regions(sig.inputs().iter().next().unwrap().map_bound(|&b| b));
+        let tupled_params = tcx.instantiate_bound_regions_with_erased(
+            sig.inputs().iter().next().unwrap().map_bound(|&b| b),
+        );
         let ty::Tuple(params) = tupled_params.kind() else { return };
 
         // Find the first argument with a matching type, get its name
@@ -1638,15 +1638,14 @@
         issued_spans: &UseSpans<'tcx>,
     ) {
         let UseSpans::ClosureUse { capture_kind_span, .. } = issued_spans else { return };
-        let hir = self.infcx.tcx.hir();
 
-        struct ExpressionFinder<'hir> {
+        struct ExpressionFinder<'tcx> {
             capture_span: Span,
             closure_change_spans: Vec<Span>,
             closure_arg_span: Option<Span>,
             in_closure: bool,
             suggest_arg: String,
-            hir: rustc_middle::hir::map::Map<'hir>,
+            tcx: TyCtxt<'tcx>,
             closure_local_id: Option<hir::HirId>,
             closure_call_changes: Vec<(Span, String)>,
         }
@@ -1660,7 +1659,7 @@
                         fn_decl: hir::FnDecl { inputs, .. },
                         ..
                     }) = e.kind
-                        && let Some(hir::Node::Expr(body)) = self.hir.find(body.hir_id)
+                        && let Some(hir::Node::Expr(body)) = self.tcx.opt_hir_node(body.hir_id)
                     {
                         self.suggest_arg = "this: &Self".to_string();
                         if inputs.len() > 0 {
@@ -1725,8 +1724,8 @@
         if let Some(hir::Node::ImplItem(hir::ImplItem {
             kind: hir::ImplItemKind::Fn(_fn_sig, body_id),
             ..
-        })) = hir.find(self.mir_hir_id())
-            && let Some(hir::Node::Expr(expr)) = hir.find(body_id.hir_id)
+        })) = self.infcx.tcx.opt_hir_node(self.mir_hir_id())
+            && let Some(hir::Node::Expr(expr)) = self.infcx.tcx.opt_hir_node(body_id.hir_id)
         {
             let mut finder = ExpressionFinder {
                 capture_span: *capture_kind_span,
@@ -1736,7 +1735,7 @@
                 suggest_arg: String::new(),
                 closure_local_id: None,
                 closure_call_changes: vec![],
-                hir,
+                tcx: self.infcx.tcx,
             };
             finder.visit_expr(expr);
 
@@ -1931,7 +1930,7 @@
         let place_desc = self.describe_place(borrow.borrowed_place.as_ref());
 
         let kind_place = kind.filter(|_| place_desc.is_some()).map(|k| (k, place_span.0));
-        let explanation = self.explain_why_borrow_contains_point(location, &borrow, kind_place);
+        let explanation = self.explain_why_borrow_contains_point(location, borrow, kind_place);
 
         debug!(?place_desc, ?explanation);
 
@@ -2000,14 +1999,14 @@
             (Some(name), explanation) => self.report_local_value_does_not_live_long_enough(
                 location,
                 &name,
-                &borrow,
+                borrow,
                 drop_span,
                 borrow_spans,
                 explanation,
             ),
             (None, explanation) => self.report_temporary_value_does_not_live_long_enough(
                 location,
-                &borrow,
+                borrow,
                 drop_span,
                 borrow_spans,
                 proper_span,
@@ -2075,8 +2074,15 @@
                         .map(|name| format!("function `{name}`"))
                         .unwrap_or_else(|| {
                             match &self.infcx.tcx.def_kind(self.mir_def_id()) {
+                                DefKind::Closure
+                                    if self
+                                        .infcx
+                                        .tcx
+                                        .is_coroutine(self.mir_def_id().to_def_id()) =>
+                                {
+                                    "enclosing coroutine"
+                                }
                                 DefKind::Closure => "enclosing closure",
-                                DefKind::Coroutine => "enclosing coroutine",
                                 kind => bug!("expected closure or coroutine, found {:?}", kind),
                             }
                             .to_string()
@@ -2097,7 +2103,7 @@
             } else {
                 explanation.add_explanation_to_diagnostic(
                     self.infcx.tcx,
-                    &self.body,
+                    self.body,
                     &self.local_names,
                     &mut err,
                     "",
@@ -2118,7 +2124,7 @@
 
             explanation.add_explanation_to_diagnostic(
                 self.infcx.tcx,
-                &self.body,
+                self.body,
                 &self.local_names,
                 &mut err,
                 "",
@@ -2179,7 +2185,7 @@
 
         explanation.add_explanation_to_diagnostic(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &mut err,
             "",
@@ -2290,7 +2296,7 @@
                 let proper_span = proper_span.source_callsite();
                 if let Some(scope) = self.body.source_scopes.get(source_info.scope)
                     && let ClearCrossCrate::Set(scope_data) = &scope.local_data
-                    && let Some(node) = self.infcx.tcx.hir().find(scope_data.lint_root)
+                    && let Some(node) = self.infcx.tcx.opt_hir_node(scope_data.lint_root)
                     && let Some(id) = node.body_id()
                     && let hir::ExprKind::Block(block, _) = self.infcx.tcx.hir().body(id).value.kind
                 {
@@ -2348,7 +2354,7 @@
                                     Applicability::MaybeIncorrect,
                                 );
                             } else {
-                                err.note("the result of `format_args!` can only be assigned directly if no placeholders in it's arguments are used");
+                                err.note("the result of `format_args!` can only be assigned directly if no placeholders in its arguments are used");
                                 err.note("to learn more, visit <https://doc.rust-lang.org/std/macro.format_args.html>");
                             }
                             suggested = true;
@@ -2364,7 +2370,7 @@
         }
         explanation.add_explanation_to_diagnostic(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &mut err,
             "",
@@ -2513,12 +2519,23 @@
                 CoroutineKind::Gen(kind) => match kind {
                     CoroutineSource::Block => "gen block",
                     CoroutineSource::Closure => "gen closure",
-                    _ => bug!("gen block/closure expected, but gen function found."),
+                    CoroutineSource::Fn => {
+                        bug!("gen block/closure expected, but gen function found.")
+                    }
+                },
+                CoroutineKind::AsyncGen(kind) => match kind {
+                    CoroutineSource::Block => "async gen block",
+                    CoroutineSource::Closure => "async gen closure",
+                    CoroutineSource::Fn => {
+                        bug!("gen block/closure expected, but gen function found.")
+                    }
                 },
                 CoroutineKind::Async(async_kind) => match async_kind {
                     CoroutineSource::Block => "async block",
                     CoroutineSource::Closure => "async closure",
-                    _ => bug!("async block/closure expected, but async function found."),
+                    CoroutineSource::Fn => {
+                        bug!("async block/closure expected, but async function found.")
+                    }
                 },
                 CoroutineKind::Coroutine => "coroutine",
             },
@@ -2841,7 +2858,7 @@
 
         self.explain_why_borrow_contains_point(location, loan, None).add_explanation_to_diagnostic(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &mut err,
             "",
@@ -3019,7 +3036,7 @@
             }
         }
         let mut visitor = FakeReadCauseFinder { place, cause: None };
-        visitor.visit_body(&self.body);
+        visitor.visit_body(self.body);
         match visitor.cause {
             Some(FakeReadCause::ForMatchGuard) => Some("match guard"),
             Some(FakeReadCause::ForIndex) => Some("indexing expression"),
@@ -3246,7 +3263,7 @@
     ) -> Option<AnnotatedBorrowFnSignature<'tcx>> {
         debug!("annotate_fn_sig: did={:?} sig={:?}", did, sig);
         let is_closure = self.infcx.tcx.is_closure(did.to_def_id());
-        let fn_hir_id = self.infcx.tcx.hir().local_def_id_to_hir_id(did);
+        let fn_hir_id = self.infcx.tcx.local_def_id_to_hir_id(did);
         let fn_decl = self.infcx.tcx.hir().fn_decl_by_hir_id(fn_hir_id)?;
 
         // We need to work out which arguments to highlight. We do this by looking
diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
index 8a930ca..f1e712d 100644
--- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
@@ -5,12 +5,13 @@
 use rustc_hir::intravisit::Visitor;
 use rustc_index::IndexSlice;
 use rustc_infer::infer::NllRegionVariableOrigin;
+use rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault;
 use rustc_middle::mir::{
     Body, CallSource, CastKind, ConstraintCategory, FakeReadCause, Local, LocalInfo, Location,
     Operand, Place, Rvalue, Statement, StatementKind, TerminatorKind,
 };
 use rustc_middle::ty::adjustment::PointerCoercion;
-use rustc_middle::ty::{self, RegionVid, TyCtxt};
+use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt};
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{sym, DesugaringKind, Span};
 use rustc_trait_selection::traits::error_reporting::FindExprBySpan;
@@ -86,7 +87,7 @@
                     if let hir::ExprKind::Path(hir::QPath::Resolved(None, p)) = expr.kind
                         && let [hir::PathSegment { ident, args: None, .. }] = p.segments
                         && let hir::def::Res::Local(hir_id) = p.res
-                        && let Some(hir::Node::Pat(pat)) = tcx.hir().find(hir_id)
+                        && let Some(hir::Node::Pat(pat)) = tcx.opt_hir_node(hir_id)
                     {
                         err.span_label(pat.span, format!("binding `{ident}` declared here"));
                     }
@@ -290,12 +291,69 @@
                     }
                 }
 
+                if let ConstraintCategory::Cast { unsize_to: Some(unsize_ty) } = category {
+                    self.add_object_lifetime_default_note(tcx, err, unsize_ty);
+                }
                 self.add_lifetime_bound_suggestion_to_diagnostic(err, &category, span, region_name);
             }
             _ => {}
         }
     }
 
+    fn add_object_lifetime_default_note(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        err: &mut Diagnostic,
+        unsize_ty: Ty<'tcx>,
+    ) {
+        if let ty::Adt(def, args) = unsize_ty.kind() {
+            // We try to elaborate the object lifetime defaults and present those to the user. This should
+            // make it clear where the region constraint is coming from.
+            let generics = tcx.generics_of(def.did());
+
+            let mut has_dyn = false;
+            let mut failed = false;
+
+            let elaborated_args = std::iter::zip(*args, &generics.params).map(|(arg, param)| {
+                if let Some(ty::Dynamic(obj, _, ty::DynKind::Dyn)) = arg.as_type().map(Ty::kind) {
+                    let default = tcx.object_lifetime_default(param.def_id);
+
+                    let re_static = tcx.lifetimes.re_static;
+
+                    let implied_region = match default {
+                        // This is not entirely precise.
+                        ObjectLifetimeDefault::Empty => re_static,
+                        ObjectLifetimeDefault::Ambiguous => {
+                            failed = true;
+                            re_static
+                        }
+                        ObjectLifetimeDefault::Param(param_def_id) => {
+                            let index = generics.param_def_id_to_index[&param_def_id] as usize;
+                            args.get(index).and_then(|arg| arg.as_region()).unwrap_or_else(|| {
+                                failed = true;
+                                re_static
+                            })
+                        }
+                        ObjectLifetimeDefault::Static => re_static,
+                    };
+
+                    has_dyn = true;
+
+                    Ty::new_dynamic(tcx, obj, implied_region, ty::DynKind::Dyn).into()
+                } else {
+                    arg
+                }
+            });
+            let elaborated_ty = Ty::new_adt(tcx, *def, tcx.mk_args_from_iter(elaborated_args));
+
+            if has_dyn && !failed {
+                err.note(format!(
+                    "due to object lifetime defaults, `{unsize_ty}` actually means `{elaborated_ty}`"
+                ));
+            }
+        }
+    }
+
     fn add_lifetime_bound_suggestion_to_diagnostic(
         &self,
         err: &mut Diagnostic,
@@ -364,7 +422,7 @@
         kind_place: Option<(WriteKind, Place<'tcx>)>,
     ) -> BorrowExplanation<'tcx> {
         let regioncx = &self.regioncx;
-        let body: &Body<'_> = &self.body;
+        let body: &Body<'_> = self.body;
         let tcx = self.infcx.tcx;
 
         let borrow_region_vid = borrow.region;
diff --git a/compiler/rustc_borrowck/src/diagnostics/mod.rs b/compiler/rustc_borrowck/src/diagnostics/mod.rs
index c4323fe..ee32136 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mod.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mod.rs
@@ -10,7 +10,8 @@
 use rustc_hir::def::{CtorKind, Namespace};
 use rustc_hir::CoroutineKind;
 use rustc_index::IndexSlice;
-use rustc_infer::infer::LateBoundRegionConversionTime;
+use rustc_infer::infer::BoundRegionConversionTime;
+use rustc_infer::traits::{FulfillmentErrorCode, SelectionError};
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::{
     AggregateKind, CallSource, ConstOperand, FakeReadCause, Local, LocalInfo, LocalKind, Location,
@@ -24,10 +25,9 @@
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{symbol::sym, Span, Symbol, DUMMY_SP};
 use rustc_target::abi::{FieldIdx, VariantIdx};
-use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
-use rustc_trait_selection::traits::{
-    type_known_to_meet_bound_modulo_regions, Obligation, ObligationCause,
-};
+use rustc_trait_selection::infer::InferCtxtExt;
+use rustc_trait_selection::traits::error_reporting::suggestions::TypeErrCtxtExt as _;
+use rustc_trait_selection::traits::type_known_to_meet_bound_modulo_regions;
 
 use super::borrow_set::BorrowData;
 use super::MirBorrowckCtxt;
@@ -124,7 +124,7 @@
                         let did = did.expect_local();
                         if let Some((span, hir_place)) = self.infcx.tcx.closure_kind_origin(did) {
                             diag.eager_subdiagnostic(
-                                &self.infcx.tcx.sess.parse_sess.span_diagnostic,
+                                self.infcx.tcx.sess.dcx(),
                                 OnClosureNote::InvokedTwice {
                                     place_name: &ty::place_to_string_for_capture(
                                         self.infcx.tcx,
@@ -146,7 +146,7 @@
                 let did = did.expect_local();
                 if let Some((span, hir_place)) = self.infcx.tcx.closure_kind_origin(did) {
                     diag.eager_subdiagnostic(
-                        &self.infcx.tcx.sess.parse_sess.span_diagnostic,
+                        self.infcx.tcx.sess.dcx(),
                         OnClosureNote::MovedTwice {
                             place_name: &ty::place_to_string_for_capture(self.infcx.tcx, hir_place),
                             span: *span,
@@ -217,9 +217,9 @@
                         projection: place.projection.split_at(index + 1).0,
                     }) {
                         let var_index = field.index();
-                        buf = self.upvars[var_index].place.to_string(self.infcx.tcx);
+                        buf = self.upvars[var_index].to_string(self.infcx.tcx);
                         ok = Ok(());
-                        if !self.upvars[var_index].by_ref {
+                        if !self.upvars[var_index].is_by_ref() {
                             buf.insert(0, '*');
                         }
                     } else {
@@ -250,7 +250,7 @@
                         local,
                         projection: place.projection.split_at(index + 1).0,
                     }) {
-                        buf = self.upvars[field.index()].place.to_string(self.infcx.tcx);
+                        buf = self.upvars[field.index()].to_string(self.infcx.tcx);
                         ok = Ok(());
                     } else {
                         let field_name = self.describe_field(
@@ -354,7 +354,7 @@
                 ty::Adt(def, _) => {
                     let variant = if let Some(idx) = variant_index {
                         assert!(def.is_enum());
-                        &def.variant(idx)
+                        def.variant(idx)
                     } else {
                         def.non_enum_variant()
                     };
@@ -462,7 +462,7 @@
         // lifetimes without names with the value `'0`.
         if let ty::Ref(region, ..) = ty.kind() {
             match **region {
-                ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
+                ty::ReBound(_, ty::BoundRegion { kind: br, .. })
                 | ty::RePlaceholder(ty::PlaceholderRegion {
                     bound: ty::BoundRegion { kind: br, .. },
                     ..
@@ -482,7 +482,7 @@
 
         let region = if let ty::Ref(region, ..) = ty.kind() {
             match **region {
-                ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
+                ty::ReBound(_, ty::BoundRegion { kind: br, .. })
                 | ty::RePlaceholder(ty::PlaceholderRegion {
                     bound: ty::BoundRegion { kind: br, .. },
                     ..
@@ -624,7 +624,7 @@
     /// Add a subdiagnostic to the use of the captured variable, if it exists.
     pub(super) fn var_subdiag(
         self,
-        handler: Option<&rustc_errors::Handler>,
+        dcx: Option<&rustc_errors::DiagCtxt>,
         err: &mut Diagnostic,
         kind: Option<rustc_middle::mir::BorrowKind>,
         f: impl FnOnce(Option<CoroutineKind>, Span) -> CaptureVarCause,
@@ -646,7 +646,7 @@
                 });
             };
             let diag = f(coroutine_kind, path_span);
-            match handler {
+            match dcx {
                 Some(hd) => err.eager_subdiagnostic(hd, diag),
                 None => err.subdiagnostic(diag),
             };
@@ -851,7 +851,7 @@
         {
             let Some((method_did, method_args)) = rustc_middle::util::find_self_call(
                 self.infcx.tcx,
-                &self.body,
+                self.body,
                 target_temp,
                 location.block,
             ) else {
@@ -958,7 +958,7 @@
             "closure_span: def_id={:?} target_place={:?} places={:?}",
             def_id, target_place, places
         );
-        let hir_id = self.infcx.tcx.hir().local_def_id_to_hir_id(def_id);
+        let hir_id = self.infcx.tcx.local_def_id_to_hir_id(def_id);
         let expr = &self.infcx.tcx.hir().expect_expr(hir_id).kind;
         debug!("closure_span: hir_id={:?} expr={:?}", hir_id, expr);
         if let hir::ExprKind::Closure(&hir::Closure { body, fn_decl_span, .. }) = expr {
@@ -1043,12 +1043,43 @@
                 }
                 CallKind::Normal { self_arg, desugaring, method_did, method_args } => {
                     let self_arg = self_arg.unwrap();
+                    let mut has_sugg = false;
                     let tcx = self.infcx.tcx;
+                    // Avoid pointing to the same function in multiple different
+                    // error messages.
+                    if span != DUMMY_SP && self.fn_self_span_reported.insert(self_arg.span) {
+                        self.explain_iterator_advancement_in_for_loop_if_applicable(
+                            err,
+                            span,
+                            &move_spans,
+                        );
+
+                        let func = tcx.def_path_str(method_did);
+                        err.subdiagnostic(CaptureReasonNote::FuncTakeSelf {
+                            func,
+                            place_name: place_name.clone(),
+                            span: self_arg.span,
+                        });
+                    }
+                    let parent_did = tcx.parent(method_did);
+                    let parent_self_ty =
+                        matches!(tcx.def_kind(parent_did), rustc_hir::def::DefKind::Impl { .. })
+                            .then_some(parent_did)
+                            .and_then(|did| match tcx.type_of(did).instantiate_identity().kind() {
+                                ty::Adt(def, ..) => Some(def.did()),
+                                _ => None,
+                            });
+                    let is_option_or_result = parent_self_ty.is_some_and(|def_id| {
+                        matches!(tcx.get_diagnostic_name(def_id), Some(sym::Option | sym::Result))
+                    });
+                    if is_option_or_result && maybe_reinitialized_locations_is_empty {
+                        err.subdiagnostic(CaptureReasonLabel::BorrowContent { var_span });
+                    }
                     if let Some((CallDesugaringKind::ForLoopIntoIter, _)) = desugaring {
                         let ty = moved_place.ty(self.body, tcx).ty;
                         let suggest = match tcx.get_diagnostic_item(sym::IntoIterator) {
                             Some(def_id) => type_known_to_meet_bound_modulo_regions(
-                                &self.infcx,
+                                self.infcx,
                                 self.param_env,
                                 Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, ty),
                                 def_id,
@@ -1108,72 +1139,92 @@
                         // Erase and shadow everything that could be passed to the new infcx.
                         let ty = moved_place.ty(self.body, tcx).ty;
 
-                        if let ty::Adt(def, args) = ty.kind()
+                        if let ty::Adt(def, args) = ty.peel_refs().kind()
                             && Some(def.did()) == tcx.lang_items().pin_type()
                             && let ty::Ref(_, _, hir::Mutability::Mut) = args.type_at(0).kind()
                             && let self_ty = self.infcx.instantiate_binder_with_fresh_vars(
                                 fn_call_span,
-                                LateBoundRegionConversionTime::FnCall,
+                                BoundRegionConversionTime::FnCall,
                                 tcx.fn_sig(method_did).instantiate(tcx, method_args).input(0),
                             )
                             && self.infcx.can_eq(self.param_env, ty, self_ty)
                         {
                             err.eager_subdiagnostic(
-                                &self.infcx.tcx.sess.parse_sess.span_diagnostic,
+                                self.infcx.tcx.sess.dcx(),
                                 CaptureReasonSuggest::FreshReborrow {
                                     span: move_span.shrink_to_hi(),
                                 },
                             );
+                            has_sugg = true;
                         }
-                        if let Some(clone_trait) = tcx.lang_items().clone_trait()
-                            && let trait_ref = ty::TraitRef::new(tcx, clone_trait, [ty])
-                            && let o = Obligation::new(
-                                tcx,
-                                ObligationCause::dummy(),
-                                self.param_env,
-                                ty::Binder::dummy(trait_ref),
-                            )
-                            && self.infcx.predicate_must_hold_modulo_regions(&o)
-                        {
-                            err.span_suggestion_verbose(
-                                move_span.shrink_to_hi(),
-                                "you can `clone` the value and consume it, but this might not be \
-                                 your desired behavior",
-                                ".clone()".to_string(),
-                                Applicability::MaybeIncorrect,
-                            );
+                        if let Some(clone_trait) = tcx.lang_items().clone_trait() {
+                            let sugg = if moved_place
+                                .iter_projections()
+                                .any(|(_, elem)| matches!(elem, ProjectionElem::Deref))
+                            {
+                                vec![
+                                    // We use the fully-qualified path because `.clone()` can
+                                    // sometimes choose `<&T as Clone>` instead of `<T as Clone>`
+                                    // when going through auto-deref, so this ensures that doesn't
+                                    // happen, causing suggestions for `.clone().clone()`.
+                                    (move_span.shrink_to_lo(), format!("<{ty} as Clone>::clone(&")),
+                                    (move_span.shrink_to_hi(), ")".to_string()),
+                                ]
+                            } else {
+                                vec![(move_span.shrink_to_hi(), ".clone()".to_string())]
+                            };
+                            if let Some(errors) =
+                                self.infcx.could_impl_trait(clone_trait, ty, self.param_env)
+                                && !has_sugg
+                            {
+                                let msg = match &errors[..] {
+                                    [] => "you can `clone` the value and consume it, but this \
+                                            might not be your desired behavior"
+                                        .to_string(),
+                                    [error] => {
+                                        format!(
+                                            "you could `clone` the value and consume it, if \
+                                                the `{}` trait bound could be satisfied",
+                                            error.obligation.predicate,
+                                        )
+                                    }
+                                    [errors @ .., last] => {
+                                        format!(
+                                            "you could `clone` the value and consume it, if \
+                                                the following trait bounds could be satisfied: {} \
+                                                and `{}`",
+                                            errors
+                                                .iter()
+                                                .map(|e| format!("`{}`", e.obligation.predicate))
+                                                .collect::<Vec<_>>()
+                                                .join(", "),
+                                            last.obligation.predicate,
+                                        )
+                                    }
+                                };
+                                err.multipart_suggestion_verbose(
+                                    msg,
+                                    sugg,
+                                    Applicability::MaybeIncorrect,
+                                );
+                                for error in errors {
+                                    if let FulfillmentErrorCode::CodeSelectionError(
+                                        SelectionError::Unimplemented,
+                                    ) = error.code
+                                        && let ty::PredicateKind::Clause(ty::ClauseKind::Trait(
+                                            pred,
+                                        )) = error.obligation.predicate.kind().skip_binder()
+                                    {
+                                        self.infcx.err_ctxt().suggest_derive(
+                                            &error.obligation,
+                                            err,
+                                            error.obligation.predicate.kind().rebind(pred),
+                                        );
+                                    }
+                                }
+                            }
                         }
                     }
-                    // Avoid pointing to the same function in multiple different
-                    // error messages.
-                    if span != DUMMY_SP && self.fn_self_span_reported.insert(self_arg.span) {
-                        self.explain_iterator_advancement_in_for_loop_if_applicable(
-                            err,
-                            span,
-                            &move_spans,
-                        );
-
-                        let func = tcx.def_path_str(method_did);
-                        err.subdiagnostic(CaptureReasonNote::FuncTakeSelf {
-                            func,
-                            place_name,
-                            span: self_arg.span,
-                        });
-                    }
-                    let parent_did = tcx.parent(method_did);
-                    let parent_self_ty =
-                        matches!(tcx.def_kind(parent_did), rustc_hir::def::DefKind::Impl { .. })
-                            .then_some(parent_did)
-                            .and_then(|did| match tcx.type_of(did).instantiate_identity().kind() {
-                                ty::Adt(def, ..) => Some(def.did()),
-                                _ => None,
-                            });
-                    let is_option_or_result = parent_self_ty.is_some_and(|def_id| {
-                        matches!(tcx.get_diagnostic_name(def_id), Some(sym::Option | sym::Result))
-                    });
-                    if is_option_or_result && maybe_reinitialized_locations_is_empty {
-                        err.subdiagnostic(CaptureReasonLabel::BorrowContent { var_span });
-                    }
                 }
                 // Other desugarings takes &self, which cannot cause a move
                 _ => {}
diff --git a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
index 41d6b98..43487b8 100644
--- a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
@@ -321,7 +321,7 @@
 
         let deref_base = match deref_target_place.projection.as_ref() {
             [proj_base @ .., ProjectionElem::Deref] => {
-                PlaceRef { local: deref_target_place.local, projection: &proj_base }
+                PlaceRef { local: deref_target_place.local, projection: proj_base }
             }
             _ => bug!("deref_target_place is not a deref projection"),
         };
@@ -363,8 +363,8 @@
                     format!("captured variable in an `{closure_kind}` closure");
 
                 let upvar = &self.upvars[upvar_field.unwrap().index()];
-                let upvar_hir_id = upvar.place.get_root_variable();
-                let upvar_name = upvar.place.to_string(self.infcx.tcx);
+                let upvar_hir_id = upvar.get_root_variable();
+                let upvar_name = upvar.to_string(self.infcx.tcx);
                 let upvar_span = self.infcx.tcx.hir().span(upvar_hir_id);
 
                 let place_name = self.describe_any_place(move_place.as_ref());
@@ -583,7 +583,7 @@
                 err.subdiagnostic(crate::session_diagnostics::TypeNoCopy::Label {
                     is_partial_move: false,
                     ty: bind_to.ty,
-                    place: &place_desc,
+                    place: place_desc,
                     span: binding_span,
                 });
             }
@@ -607,7 +607,8 @@
 
         if let Some(adt) = local_ty.ty_adt_def()
             && adt.repr().packed()
-            && let ExpnKind::Macro(MacroKind::Derive, name) = self.body.span.ctxt().outer_expn_data().kind
+            && let ExpnKind::Macro(MacroKind::Derive, name) =
+                self.body.span.ctxt().outer_expn_data().kind
         {
             err.note(format!("`#[derive({name})]` triggers a move because taking references to the fields of a packed struct is undefined behaviour"));
         }
diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index dde46ee..c3c1f12 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -3,8 +3,9 @@
 use rustc_hir as hir;
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::Node;
+use rustc_infer::traits;
 use rustc_middle::mir::{Mutability, Place, PlaceRef, ProjectionElem};
-use rustc_middle::ty::{self, InstanceDef, Ty, TyCtxt};
+use rustc_middle::ty::{self, InstanceDef, ToPredicate, Ty, TyCtxt};
 use rustc_middle::{
     hir::place::PlaceBase,
     mir::{self, BindingForm, Local, LocalDecl, LocalInfo, LocalKind, Location},
@@ -12,6 +13,8 @@
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{sym, BytePos, DesugaringKind, Span};
 use rustc_target::abi::FieldIdx;
+use rustc_trait_selection::infer::InferCtxtExt;
+use rustc_trait_selection::traits::error_reporting::suggestions::TypeErrCtxtExt;
 
 use crate::diagnostics::BorrowedContentSource;
 use crate::util::FindAssignments;
@@ -67,7 +70,6 @@
 
                 let imm_borrow_derefed = self.upvars[upvar_index.index()]
                     .place
-                    .place
                     .deref_tys()
                     .any(|ty| matches!(ty.kind(), ty::Ref(.., hir::Mutability::Not)));
 
@@ -85,7 +87,7 @@
                     if self.is_upvar_field_projection(access_place.as_ref()).is_some() {
                         reason = ", as it is not declared as mutable".to_string();
                     } else {
-                        let name = self.upvars[upvar_index.index()].place.to_string(self.infcx.tcx);
+                        let name = self.upvars[upvar_index.index()].to_string(self.infcx.tcx);
                         reason = format!(", as `{name}` is not declared as mutable");
                     }
                 }
@@ -388,13 +390,13 @@
                     Place::ty_from(local, proj_base, self.body, self.infcx.tcx).ty
                 ));
 
-                let captured_place = &self.upvars[upvar_index.index()].place;
+                let captured_place = self.upvars[upvar_index.index()];
 
                 err.span_label(span, format!("cannot {act}"));
 
                 let upvar_hir_id = captured_place.get_root_variable();
 
-                if let Some(Node::Pat(pat)) = self.infcx.tcx.hir().find(upvar_hir_id)
+                if let Some(Node::Pat(pat)) = self.infcx.tcx.opt_hir_node(upvar_hir_id)
                     && let hir::PatKind::Binding(hir::BindingAnnotation::NONE, _, upvar_ident, _) =
                         pat.kind
                 {
@@ -659,9 +661,8 @@
         if self.body.local_kind(local) != LocalKind::Arg {
             return (false, None);
         }
-        let hir_map = self.infcx.tcx.hir();
         let my_def = self.body.source.def_id();
-        let my_hir = hir_map.local_def_id_to_hir_id(my_def.as_local().unwrap());
+        let my_hir = self.infcx.tcx.local_def_id_to_hir_id(my_def.as_local().unwrap());
         let Some(td) =
             self.infcx.tcx.impl_of_method(my_def).and_then(|x| self.infcx.tcx.trait_id_of_impl(x))
         else {
@@ -669,7 +670,7 @@
         };
         (
             true,
-            td.as_local().and_then(|tld| match hir_map.find_by_def_id(tld) {
+            td.as_local().and_then(|tld| match self.infcx.tcx.opt_hir_node_by_def_id(tld) {
                 Some(Node::Item(hir::Item {
                     kind: hir::ItemKind::Trait(_, _, _, _, items),
                     ..
@@ -680,25 +681,27 @@
                         if !matches!(k, hir::AssocItemKind::Fn { .. }) {
                             continue;
                         }
-                        if hir_map.name(hi) != hir_map.name(my_hir) {
+                        if self.infcx.tcx.hir().name(hi) != self.infcx.tcx.hir().name(my_hir) {
                             continue;
                         }
                         f_in_trait_opt = Some(hi);
                         break;
                     }
-                    f_in_trait_opt.and_then(|f_in_trait| match hir_map.find(f_in_trait) {
-                        Some(Node::TraitItem(hir::TraitItem {
-                            kind:
-                                hir::TraitItemKind::Fn(
-                                    hir::FnSig { decl: hir::FnDecl { inputs, .. }, .. },
-                                    _,
-                                ),
-                            ..
-                        })) => {
-                            let hir::Ty { span, .. } = inputs[local.index() - 1];
-                            Some(span)
+                    f_in_trait_opt.and_then(|f_in_trait| {
+                        match self.infcx.tcx.opt_hir_node(f_in_trait) {
+                            Some(Node::TraitItem(hir::TraitItem {
+                                kind:
+                                    hir::TraitItemKind::Fn(
+                                        hir::FnSig { decl: hir::FnDecl { inputs, .. }, .. },
+                                        _,
+                                    ),
+                                ..
+                            })) => {
+                                let hir::Ty { span, .. } = inputs[local.index() - 1];
+                                Some(span)
+                            }
+                            _ => None,
                         }
-                        _ => None,
                     })
                 }
                 _ => None,
@@ -739,12 +742,11 @@
             }
         }
 
-        let hir_map = self.infcx.tcx.hir();
         let def_id = self.body.source.def_id();
         let hir_id = if let Some(local_def_id) = def_id.as_local()
-            && let Some(body_id) = hir_map.maybe_body_owned_by(local_def_id)
+            && let Some(body_id) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
         {
-            let body = hir_map.body(body_id);
+            let body = self.infcx.tcx.hir().body(body_id);
             let mut v = BindingFinder { span: pat_span, hir_id: None };
             v.visit_body(body);
             v.hir_id
@@ -760,7 +762,7 @@
             && let Some(hir::Node::Local(hir::Local {
                 pat: hir::Pat { kind: hir::PatKind::Ref(_, _), .. },
                 ..
-            })) = hir_map.find(hir_id)
+            })) = self.infcx.tcx.opt_hir_node(hir_id)
             && let Ok(name) =
                 self.infcx.tcx.sess.source_map().span_to_snippet(local_decl.source_info.span)
         {
@@ -940,7 +942,7 @@
         let closure_id = self.mir_hir_id();
         let closure_span = self.infcx.tcx.def_span(self.mir_def_id());
         let fn_call_id = hir.parent_id(closure_id);
-        let node = hir.get(fn_call_id);
+        let node = self.infcx.tcx.hir_node(fn_call_id);
         let def_id = hir.enclosing_body_owner(fn_call_id);
         let mut look_at_return = true;
         // If we can detect the expression to be an `fn` call where the closure was an argument,
@@ -999,7 +1001,7 @@
         if look_at_return && hir.get_return_block(closure_id).is_some() {
             // ...otherwise we are probably in the tail expression of the function, point at the
             // return type.
-            match hir.get_by_def_id(hir.get_parent_item(fn_call_id).def_id) {
+            match self.infcx.tcx.hir_node_by_def_id(hir.get_parent_item(fn_call_id).def_id) {
                 hir::Node::Item(hir::Item { ident, kind: hir::ItemKind::Fn(sig, ..), .. })
                 | hir::Node::TraitItem(hir::TraitItem {
                     ident,
@@ -1197,12 +1199,11 @@
                         hir::intravisit::walk_stmt(self, s);
                     }
                 }
-                let hir_map = self.infcx.tcx.hir();
                 let def_id = self.body.source.def_id();
                 let hir_id = if let Some(local_def_id) = def_id.as_local()
-                    && let Some(body_id) = hir_map.maybe_body_owned_by(local_def_id)
+                    && let Some(body_id) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
                 {
-                    let body = hir_map.body(body_id);
+                    let body = self.infcx.tcx.hir().body(body_id);
                     let mut v = BindingFinder { span: err_label_span, hir_id: None };
                     v.visit_body(body);
                     v.hir_id
@@ -1211,8 +1212,105 @@
                 };
 
                 if let Some(hir_id) = hir_id
-                    && let Some(hir::Node::Local(local)) = hir_map.find(hir_id)
+                    && let Some(hir::Node::Local(local)) = self.infcx.tcx.opt_hir_node(hir_id)
                 {
+                    let tables = self.infcx.tcx.typeck(def_id.as_local().unwrap());
+                    if let Some(clone_trait) = self.infcx.tcx.lang_items().clone_trait()
+                        && let Some(expr) = local.init
+                        && let ty = tables.node_type_opt(expr.hir_id)
+                        && let Some(ty) = ty
+                        && let ty::Ref(..) = ty.kind()
+                    {
+                        match self
+                            .infcx
+                            .could_impl_trait(clone_trait, ty.peel_refs(), self.param_env)
+                            .as_deref()
+                        {
+                            Some([]) => {
+                                // The type implements Clone.
+                                err.span_help(
+                                    expr.span,
+                                    format!(
+                                        "you can `clone` the `{}` value and consume it, but this \
+                                         might not be your desired behavior",
+                                        ty.peel_refs(),
+                                    ),
+                                );
+                            }
+                            None => {
+                                if let hir::ExprKind::MethodCall(segment, _rcvr, [], span) =
+                                    expr.kind
+                                    && segment.ident.name == sym::clone
+                                {
+                                    err.span_help(
+                                        span,
+                                        format!(
+                                            "`{}` doesn't implement `Clone`, so this call clones \
+                                             the reference `{ty}`",
+                                            ty.peel_refs(),
+                                        ),
+                                    );
+                                }
+                                // The type doesn't implement Clone.
+                                let trait_ref = ty::Binder::dummy(ty::TraitRef::new(
+                                    self.infcx.tcx,
+                                    clone_trait,
+                                    [ty.peel_refs()],
+                                ));
+                                let obligation = traits::Obligation::new(
+                                    self.infcx.tcx,
+                                    traits::ObligationCause::dummy(),
+                                    self.param_env,
+                                    trait_ref,
+                                );
+                                self.infcx.err_ctxt().suggest_derive(
+                                    &obligation,
+                                    err,
+                                    trait_ref.to_predicate(self.infcx.tcx),
+                                );
+                            }
+                            Some(errors) => {
+                                if let hir::ExprKind::MethodCall(segment, _rcvr, [], span) =
+                                    expr.kind
+                                    && segment.ident.name == sym::clone
+                                {
+                                    err.span_help(
+                                        span,
+                                        format!(
+                                            "`{}` doesn't implement `Clone` because its \
+                                             implementations trait bounds could not be met, so \
+                                             this call clones the reference `{ty}`",
+                                            ty.peel_refs(),
+                                        ),
+                                    );
+                                    err.note(format!(
+                                        "the following trait bounds weren't met: {}",
+                                        errors
+                                            .iter()
+                                            .map(|e| e.obligation.predicate.to_string())
+                                            .collect::<Vec<_>>()
+                                            .join("\n"),
+                                    ));
+                                }
+                                // The type doesn't implement Clone because of unmet obligations.
+                                for error in errors {
+                                    if let traits::FulfillmentErrorCode::CodeSelectionError(
+                                        traits::SelectionError::Unimplemented,
+                                    ) = error.code
+                                        && let ty::PredicateKind::Clause(ty::ClauseKind::Trait(
+                                            pred,
+                                        )) = error.obligation.predicate.kind().skip_binder()
+                                    {
+                                        self.infcx.err_ctxt().suggest_derive(
+                                            &error.obligation,
+                                            err,
+                                            error.obligation.predicate.kind().rebind(pred),
+                                        );
+                                    }
+                                }
+                            }
+                        }
+                    }
                     let (changing, span, sugg) = match local.ty {
                         Some(ty) => ("changing", ty.span, message),
                         None => {
@@ -1397,7 +1495,7 @@
         && let ty::Adt(def, _) = ty.kind()
         && let field = def.all_fields().nth(field.index())?
         // Use the HIR types to construct the diagnostic message.
-        && let node = tcx.hir().find_by_def_id(field.did.as_local()?)?
+        && let node = tcx.opt_hir_node_by_def_id(field.did.as_local()?)?
         // Now we're dealing with the actual struct that we're going to suggest a change to,
         // we can expect a field that is an immutable reference to a type.
         && let hir::Node::Field(field) = node
diff --git a/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs b/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
index b6eb9ae..6627588 100644
--- a/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
@@ -50,8 +50,8 @@
     // naming the `'self` lifetime in methods, etc.
     fn region_name_is_suggestable(name: &RegionName) -> bool {
         match name.source {
-            RegionNameSource::NamedEarlyBoundRegion(..)
-            | RegionNameSource::NamedFreeRegion(..)
+            RegionNameSource::NamedEarlyParamRegion(..)
+            | RegionNameSource::NamedLateParamRegion(..)
             | RegionNameSource::Static => true,
 
             // Don't give suggestions for upvars, closure return types, or other unnameable
@@ -206,7 +206,7 @@
         // If there is exactly one suggestable constraints, then just suggest it. Otherwise, emit a
         // list of diagnostics.
         let mut diag = if suggested.len() == 1 {
-            mbcx.infcx.tcx.sess.diagnostic().struct_help(match suggested.last().unwrap() {
+            mbcx.infcx.tcx.sess.dcx().struct_help(match suggested.last().unwrap() {
                 SuggestedConstraint::Outlives(a, bs) => {
                     let bs: SmallVec<[String; 2]> = bs.iter().map(|r| r.to_string()).collect();
                     format!("add bound `{a}: {}`", bs.join(" + "))
@@ -223,7 +223,7 @@
                 .infcx
                 .tcx
                 .sess
-                .diagnostic()
+                .dcx()
                 .struct_help("the following changes may resolve your lifetime errors");
 
             // Add suggestions.
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
index a0a8091..759f5e9 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
@@ -35,7 +35,7 @@
     LifetimeReturnCategoryErr, RequireStaticErr, VarHereDenote,
 };
 
-use super::{OutlivesSuggestionBuilder, RegionName};
+use super::{OutlivesSuggestionBuilder, RegionName, RegionNameSource};
 use crate::region_infer::{BlameConstraint, ExtraConstraintInfo};
 use crate::{
     nll::ConstraintDescription,
@@ -53,7 +53,7 @@
             ConstraintCategory::Yield => "yielding this value ",
             ConstraintCategory::UseAsConst => "using this value as a constant ",
             ConstraintCategory::UseAsStatic => "using this value as a static ",
-            ConstraintCategory::Cast => "cast ",
+            ConstraintCategory::Cast { .. } => "cast ",
             ConstraintCategory::CallArgument(_) => "argument ",
             ConstraintCategory::TypeAnnotation => "type annotation ",
             ConstraintCategory::ClosureBounds => "closure body ",
@@ -84,7 +84,7 @@
     #[track_caller]
     pub fn push(&mut self, val: impl Into<RegionErrorKind<'tcx>>) {
         let val = val.into();
-        self.1.sess.delay_span_bug(DUMMY_SP, format!("{val:?}"));
+        self.1.sess.span_delayed_bug(DUMMY_SP, format!("{val:?}"));
         self.0.push(val);
     }
     pub fn is_empty(&self) -> bool {
@@ -95,6 +95,12 @@
     }
 }
 
+impl std::fmt::Debug for RegionErrors<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("RegionErrors").field(&self.0).finish()
+    }
+}
+
 #[derive(Clone, Debug)]
 pub(crate) enum RegionErrorKind<'tcx> {
     /// A generic bound failure for a type test (`T: 'a`).
@@ -181,8 +187,8 @@
 
     /// Returns `true` if a closure is inferred to be an `FnMut` closure.
     fn is_closure_fn_mut(&self, fr: RegionVid) -> bool {
-        if let Some(ty::ReFree(free_region)) = self.to_error_region(fr).as_deref()
-            && let ty::BoundRegionKind::BrEnv = free_region.bound_region
+        if let Some(ty::ReLateParam(late_param)) = self.to_error_region(fr).as_deref()
+            && let ty::BoundRegionKind::BrEnv = late_param.bound_region
             && let DefiningTy::Closure(_, args) = self.regioncx.universal_regions().defining_ty
         {
             return args.as_closure().kind() == ty::ClosureKind::FnMut;
@@ -209,7 +215,7 @@
             .map(|placeholder| {
                 if let Some(id) = placeholder.bound.kind.get_id()
                     && let Some(placeholder_id) = id.as_local()
-                    && let gat_hir_id = hir.local_def_id_to_hir_id(placeholder_id)
+                    && let gat_hir_id = self.infcx.tcx.local_def_id_to_hir_id(placeholder_id)
                     && let Some(generics_impl) = hir.get_parent(gat_hir_id).generics()
                 {
                     Some((gat_hir_id, generics_impl))
@@ -230,7 +236,7 @@
                 };
                 if bound_generic_params
                     .iter()
-                    .rfind(|bgp| hir.local_def_id_to_hir_id(bgp.def_id) == *gat_hir_id)
+                    .rfind(|bgp| self.infcx.tcx.local_def_id_to_hir_id(bgp.def_id) == *gat_hir_id)
                     .is_some()
                 {
                     for bound in *bounds {
@@ -599,7 +605,7 @@
             };
 
             let captured_place = &self.upvars[upvar_field.index()].place;
-            let defined_hir = match captured_place.place.base {
+            let defined_hir = match captured_place.base {
                 PlaceBase::Local(hirid) => Some(hirid),
                 PlaceBase::Upvar(upvar) => Some(upvar.var_path.hir_id),
                 _ => None,
@@ -644,14 +650,14 @@
 
         let fr_name_and_span = self.regioncx.get_var_name_and_span_for_region(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &self.upvars,
             errci.fr,
         );
         let outlived_fr_name_and_span = self.regioncx.get_var_name_and_span_for_region(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &self.upvars,
             errci.outlived_fr,
@@ -757,7 +763,14 @@
         let err = LifetimeOutliveErr { span: *span };
         let mut diag = self.infcx.tcx.sess.create_err(err);
 
-        let fr_name = self.give_region_a_name(*fr).unwrap();
+        // In certain scenarios, such as the one described in issue #118021,
+        // we might encounter a lifetime that cannot be named.
+        // These situations are bound to result in errors.
+        // To prevent an immediate ICE, we opt to create a dummy name instead.
+        let fr_name = self.give_region_a_name(*fr).unwrap_or(RegionName {
+            name: kw::UnderscoreLifetime,
+            source: RegionNameSource::Static,
+        });
         fr_name.highlight_region_name(&mut diag);
         let outlived_fr_name = self.give_region_a_name(*outlived_fr).unwrap();
         outlived_fr_name.highlight_region_name(&mut diag);
@@ -958,7 +971,7 @@
         for found_did in found_dids {
             let mut traits = vec![];
             let mut hir_v = HirTraitObjectVisitor(&mut traits, *found_did);
-            hir_v.visit_ty(&self_ty);
+            hir_v.visit_ty(self_ty);
             debug!("trait spans found: {:?}", traits);
             for span in &traits {
                 let mut multi_span: MultiSpan = vec![*span].into();
@@ -995,7 +1008,7 @@
             .infcx
             .tcx
             .is_suitable_region(sub)
-            .and_then(|anon_reg| find_anon_type(self.infcx.tcx, sub, &anon_reg.boundregion))
+            .and_then(|anon_reg| find_anon_type(self.infcx.tcx, sub, &anon_reg.bound_region))
         else {
             return;
         };
@@ -1004,7 +1017,7 @@
             .infcx
             .tcx
             .is_suitable_region(sup)
-            .and_then(|anon_reg| find_anon_type(self.infcx.tcx, sup, &anon_reg.boundregion))
+            .and_then(|anon_reg| find_anon_type(self.infcx.tcx, sup, &anon_reg.bound_region))
         else {
             return;
         };
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
index d38cfbc..8441dfa 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@@ -23,14 +23,14 @@
 }
 
 /// Denotes the source of a region that is named by a `RegionName`. For example, a free region that
-/// was named by the user would get `NamedFreeRegion` and `'static` lifetime would get `Static`.
+/// was named by the user would get `NamedLateParamRegion` and `'static` lifetime would get `Static`.
 /// This helps to print the right kinds of diagnostics.
 #[derive(Debug, Clone)]
 pub(crate) enum RegionNameSource {
     /// A bound (not free) region that was instantiated at the def site (not an HRTB).
-    NamedEarlyBoundRegion(Span),
+    NamedEarlyParamRegion(Span),
     /// A free region that the user has a name (`'a`) for.
-    NamedFreeRegion(Span),
+    NamedLateParamRegion(Span),
     /// The `'static` region.
     Static,
     /// The free region corresponding to the environment of a closure.
@@ -69,8 +69,8 @@
 impl RegionName {
     pub(crate) fn was_named(&self) -> bool {
         match self.source {
-            RegionNameSource::NamedEarlyBoundRegion(..)
-            | RegionNameSource::NamedFreeRegion(..)
+            RegionNameSource::NamedEarlyParamRegion(..)
+            | RegionNameSource::NamedLateParamRegion(..)
             | RegionNameSource::Static => true,
             RegionNameSource::SynthesizedFreeEnvRegion(..)
             | RegionNameSource::AnonRegionFromArgument(..)
@@ -85,8 +85,8 @@
     pub(crate) fn span(&self) -> Option<Span> {
         match self.source {
             RegionNameSource::Static => None,
-            RegionNameSource::NamedEarlyBoundRegion(span)
-            | RegionNameSource::NamedFreeRegion(span)
+            RegionNameSource::NamedEarlyParamRegion(span)
+            | RegionNameSource::NamedLateParamRegion(span)
             | RegionNameSource::SynthesizedFreeEnvRegion(span, _)
             | RegionNameSource::AnonRegionFromUpvar(span, _)
             | RegionNameSource::AnonRegionFromYieldTy(span, _)
@@ -104,8 +104,8 @@
 
     pub(crate) fn highlight_region_name(&self, diag: &mut Diagnostic) {
         match &self.source {
-            RegionNameSource::NamedFreeRegion(span)
-            | RegionNameSource::NamedEarlyBoundRegion(span) => {
+            RegionNameSource::NamedLateParamRegion(span)
+            | RegionNameSource::NamedEarlyParamRegion(span) => {
                 diag.span_label(*span, format!("lifetime `{self}` defined here"));
             }
             RegionNameSource::SynthesizedFreeEnvRegion(span, note) => {
@@ -199,7 +199,7 @@
     }
 
     pub(crate) fn mir_hir_id(&self) -> hir::HirId {
-        self.infcx.tcx.hir().local_def_id_to_hir_id(self.mir_def_id())
+        self.infcx.tcx.local_def_id_to_hir_id(self.mir_def_id())
     }
 
     /// Generate a synthetic region named `'N`, where `N` is the next value of the counter. Then,
@@ -280,28 +280,31 @@
 
         debug!("give_region_a_name: error_region = {:?}", error_region);
         match *error_region {
-            ty::ReEarlyBound(ebr) => ebr.has_name().then(|| {
+            ty::ReEarlyParam(ebr) => ebr.has_name().then(|| {
                 let span = tcx.hir().span_if_local(ebr.def_id).unwrap_or(DUMMY_SP);
-                RegionName { name: ebr.name, source: RegionNameSource::NamedEarlyBoundRegion(span) }
+                RegionName { name: ebr.name, source: RegionNameSource::NamedEarlyParamRegion(span) }
             }),
 
             ty::ReStatic => {
                 Some(RegionName { name: kw::StaticLifetime, source: RegionNameSource::Static })
             }
 
-            ty::ReFree(free_region) => match free_region.bound_region {
+            ty::ReLateParam(late_param) => match late_param.bound_region {
                 ty::BoundRegionKind::BrNamed(region_def_id, name) => {
                     // Get the span to point to, even if we don't use the name.
                     let span = tcx.hir().span_if_local(region_def_id).unwrap_or(DUMMY_SP);
                     debug!(
                         "bound region named: {:?}, is_named: {:?}",
                         name,
-                        free_region.bound_region.is_named()
+                        late_param.bound_region.is_named()
                     );
 
-                    if free_region.bound_region.is_named() {
+                    if late_param.bound_region.is_named() {
                         // A named region that is actually named.
-                        Some(RegionName { name, source: RegionNameSource::NamedFreeRegion(span) })
+                        Some(RegionName {
+                            name,
+                            source: RegionNameSource::NamedLateParamRegion(span),
+                        })
                     } else if tcx.asyncness(self.mir_hir_id().owner).is_async() {
                         // If we spuriously thought that the region is named, we should let the
                         // system generate a true name for error messages. Currently this can
@@ -357,7 +360,7 @@
                 ty::BoundRegionKind::BrAnon => None,
             },
 
-            ty::ReLateBound(..)
+            ty::ReBound(..)
             | ty::ReVar(..)
             | ty::RePlaceholder(..)
             | ty::ReErased
@@ -384,7 +387,7 @@
         let arg_ty = self.regioncx.universal_regions().unnormalized_input_tys
             [implicit_inputs + argument_index];
         let (_, span) = self.regioncx.get_argument_name_and_span_for_region(
-            &self.body,
+            self.body,
             &self.local_names,
             argument_index,
         );
@@ -616,8 +619,8 @@
                     _,
                 ) => {
                     // HIR lowering sometimes doesn't catch this in erroneous
-                    // programs, so we need to use delay_span_bug here. See #82126.
-                    self.infcx.tcx.sess.delay_span_bug(
+                    // programs, so we need to use span_delayed_bug here. See #82126.
+                    self.infcx.tcx.sess.span_delayed_bug(
                         hir_arg.span(),
                         format!("unmatched arg and hir arg: found {kind:?} vs {hir_arg:?}"),
                     );
@@ -669,7 +672,7 @@
 
         let mir_hir_id = self.mir_hir_id();
 
-        let (return_span, mir_description, hir_ty) = match hir.get(mir_hir_id) {
+        let (return_span, mir_description, hir_ty) = match tcx.hir_node(mir_hir_id) {
             hir::Node::Expr(hir::Expr {
                 kind: hir::ExprKind::Closure(&hir::Closure { fn_decl, body, fn_decl_span, .. }),
                 ..
@@ -681,12 +684,12 @@
                     hir::FnRetTy::Return(hir_ty) => (fn_decl.output.span(), Some(hir_ty)),
                 };
                 let mir_description = match hir.body(body).coroutine_kind {
-                    Some(hir::CoroutineKind::Async(gen)) => match gen {
+                    Some(hir::CoroutineKind::Async(src)) => match src {
                         hir::CoroutineSource::Block => " of async block",
                         hir::CoroutineSource::Closure => " of async closure",
                         hir::CoroutineSource::Fn => {
                             let parent_item =
-                                hir.get_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                                tcx.hir_node_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
                             let output = &parent_item
                                 .fn_decl()
                                 .expect("coroutine lowered from async fn should be in fn")
@@ -698,12 +701,12 @@
                             " of async function"
                         }
                     },
-                    Some(hir::CoroutineKind::Gen(gen)) => match gen {
+                    Some(hir::CoroutineKind::Gen(src)) => match src {
                         hir::CoroutineSource::Block => " of gen block",
                         hir::CoroutineSource::Closure => " of gen closure",
                         hir::CoroutineSource::Fn => {
                             let parent_item =
-                                hir.get_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                                tcx.hir_node_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
                             let output = &parent_item
                                 .fn_decl()
                                 .expect("coroutine lowered from gen fn should be in fn")
@@ -712,6 +715,21 @@
                             " of gen function"
                         }
                     },
+
+                    Some(hir::CoroutineKind::AsyncGen(src)) => match src {
+                        hir::CoroutineSource::Block => " of async gen block",
+                        hir::CoroutineSource::Closure => " of async gen closure",
+                        hir::CoroutineSource::Fn => {
+                            let parent_item =
+                                tcx.hir_node_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                            let output = &parent_item
+                                .fn_decl()
+                                .expect("coroutine lowered from async gen fn should be in fn")
+                                .output;
+                            span = output.span();
+                            " of async gen function"
+                        }
+                    },
                     Some(hir::CoroutineKind::Coroutine) => " of coroutine",
                     None => " of closure",
                 };
@@ -770,28 +788,18 @@
         };
         let opaque_ty = hir.item(id);
         if let hir::ItemKind::OpaqueTy(hir::OpaqueTy {
-            bounds:
-                [
-                    hir::GenericBound::LangItemTrait(
-                        hir::LangItem::Future,
-                        _,
-                        _,
-                        hir::GenericArgs {
-                            bindings:
-                                [
-                                    hir::TypeBinding {
-                                        ident: Ident { name: sym::Output, .. },
-                                        kind:
-                                            hir::TypeBindingKind::Equality { term: hir::Term::Ty(ty) },
-                                        ..
-                                    },
-                                ],
-                            ..
-                        },
-                    ),
-                ],
+            bounds: [hir::GenericBound::Trait(trait_ref, _)],
             ..
         }) = opaque_ty.kind
+            && let Some(segment) = trait_ref.trait_ref.path.segments.last()
+            && let Some(args) = segment.args
+            && let [
+                hir::TypeBinding {
+                    ident: Ident { name: sym::Output, .. },
+                    kind: hir::TypeBindingKind::Equality { term: hir::Term::Ty(ty) },
+                    ..
+                },
+            ] = args.bindings
         {
             ty
         } else {
@@ -823,7 +831,7 @@
         let type_name =
             self.infcx.extract_inference_diagnostics_data(yield_ty.into(), Some(highlight)).name;
 
-        let yield_span = match tcx.hir().get(self.mir_hir_id()) {
+        let yield_span = match tcx.hir_node(self.mir_hir_id()) {
             hir::Node::Expr(hir::Expr {
                 kind: hir::ExprKind::Closure(&hir::Closure { fn_decl_span, .. }),
                 ..
@@ -847,7 +855,7 @@
         &self,
         fr: RegionVid,
     ) -> Option<RegionName> {
-        let ty::ReEarlyBound(region) = *self.to_error_region(fr)? else {
+        let ty::ReEarlyParam(region) = *self.to_error_region(fr)? else {
             return None;
         };
         if region.has_name() {
@@ -862,7 +870,7 @@
 
         let found = tcx
             .any_free_region_meets(&tcx.type_of(region_parent).instantiate_identity(), |r| {
-                *r == ty::ReEarlyBound(region)
+                *r == ty::ReEarlyParam(region)
             });
 
         Some(RegionName {
@@ -881,7 +889,7 @@
         &self,
         fr: RegionVid,
     ) -> Option<RegionName> {
-        let ty::ReEarlyBound(region) = *self.to_error_region(fr)? else {
+        let ty::ReEarlyParam(region) = *self.to_error_region(fr)? else {
             return None;
         };
         if region.has_name() {
@@ -943,7 +951,7 @@
         &self,
         clauses: &[ty::Clause<'tcx>],
         ty: Ty<'tcx>,
-        region: ty::EarlyBoundRegion,
+        region: ty::EarlyParamRegion,
     ) -> bool {
         let tcx = self.infcx.tcx;
         ty.walk().any(|arg| {
@@ -956,7 +964,7 @@
                         ty::ClauseKind::Projection(data) if data.projection_ty.self_ty() == ty => {}
                         _ => return false,
                     }
-                    tcx.any_free_region_meets(pred, |r| *r == ty::ReEarlyBound(region))
+                    tcx.any_free_region_meets(pred, |r| *r == ty::ReEarlyParam(region))
                 })
             } else {
                 false
diff --git a/compiler/rustc_borrowck/src/diagnostics/var_name.rs b/compiler/rustc_borrowck/src/diagnostics/var_name.rs
index 3a104c5..28e07f2 100644
--- a/compiler/rustc_borrowck/src/diagnostics/var_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/var_name.rs
@@ -2,10 +2,9 @@
 #![deny(rustc::diagnostic_outside_of_impl)]
 
 use crate::region_infer::RegionInferenceContext;
-use crate::Upvar;
 use rustc_index::IndexSlice;
 use rustc_middle::mir::{Body, Local};
-use rustc_middle::ty::{RegionVid, TyCtxt};
+use rustc_middle::ty::{self, RegionVid, TyCtxt};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 
@@ -15,7 +14,7 @@
         tcx: TyCtxt<'tcx>,
         body: &Body<'tcx>,
         local_names: &IndexSlice<Local, Option<Symbol>>,
-        upvars: &[Upvar<'tcx>],
+        upvars: &[&ty::CapturedPlace<'tcx>],
         fr: RegionVid,
     ) -> Option<(Option<Symbol>, Span)> {
         debug!("get_var_name_and_span_for_region(fr={fr:?})");
@@ -66,10 +65,10 @@
     pub(crate) fn get_upvar_name_and_span_for_region(
         &self,
         tcx: TyCtxt<'tcx>,
-        upvars: &[Upvar<'tcx>],
+        upvars: &[&ty::CapturedPlace<'tcx>],
         upvar_index: usize,
     ) -> (Symbol, Span) {
-        let upvar_hir_id = upvars[upvar_index].place.get_root_variable();
+        let upvar_hir_id = upvars[upvar_index].get_root_variable();
         debug!("get_upvar_name_and_span_for_region: upvar_hir_id={upvar_hir_id:?}");
 
         let upvar_name = tcx.hir().name(upvar_hir_id);
diff --git a/compiler/rustc_borrowck/src/invalidation.rs b/compiler/rustc_borrowck/src/invalidation.rs
deleted file mode 100644
index ec6d7b7..0000000
--- a/compiler/rustc_borrowck/src/invalidation.rs
+++ /dev/null
@@ -1,441 +0,0 @@
-#![deny(rustc::untranslatable_diagnostic)]
-#![deny(rustc::diagnostic_outside_of_impl)]
-use rustc_data_structures::graph::dominators::Dominators;
-use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{self, BasicBlock, Body, Location, NonDivergingIntrinsic, Place, Rvalue};
-use rustc_middle::mir::{BorrowKind, Mutability, Operand};
-use rustc_middle::mir::{InlineAsmOperand, Terminator, TerminatorKind};
-use rustc_middle::mir::{Statement, StatementKind};
-use rustc_middle::ty::TyCtxt;
-
-use crate::{
-    borrow_set::BorrowSet, facts::AllFacts, location::LocationTable, path_utils::*, AccessDepth,
-    Activation, ArtificialField, BorrowIndex, Deep, LocalMutationIsAllowed, Read, ReadKind,
-    ReadOrWrite, Reservation, Shallow, Write, WriteKind,
-};
-
-pub(super) fn generate_invalidates<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    all_facts: &mut Option<AllFacts>,
-    location_table: &LocationTable,
-    body: &Body<'tcx>,
-    borrow_set: &BorrowSet<'tcx>,
-) {
-    if all_facts.is_none() {
-        // Nothing to do if we don't have any facts
-        return;
-    }
-
-    if let Some(all_facts) = all_facts {
-        let _prof_timer = tcx.prof.generic_activity("polonius_fact_generation");
-        let dominators = body.basic_blocks.dominators();
-        let mut ig = InvalidationGenerator {
-            all_facts,
-            borrow_set,
-            tcx,
-            location_table,
-            body: &body,
-            dominators,
-        };
-        ig.visit_body(body);
-    }
-}
-
-struct InvalidationGenerator<'cx, 'tcx> {
-    tcx: TyCtxt<'tcx>,
-    all_facts: &'cx mut AllFacts,
-    location_table: &'cx LocationTable,
-    body: &'cx Body<'tcx>,
-    dominators: &'cx Dominators<BasicBlock>,
-    borrow_set: &'cx BorrowSet<'tcx>,
-}
-
-/// Visits the whole MIR and generates `invalidates()` facts.
-/// Most of the code implementing this was stolen from `borrow_check/mod.rs`.
-impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
-    fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
-        self.check_activations(location);
-
-        match &statement.kind {
-            StatementKind::Assign(box (lhs, rhs)) => {
-                self.consume_rvalue(location, rhs);
-
-                self.mutate_place(location, *lhs, Shallow(None));
-            }
-            StatementKind::FakeRead(box (_, _)) => {
-                // Only relevant for initialized/liveness/safety checks.
-            }
-            StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
-                self.consume_operand(location, op);
-            }
-            StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
-                src,
-                dst,
-                count,
-            })) => {
-                self.consume_operand(location, src);
-                self.consume_operand(location, dst);
-                self.consume_operand(location, count);
-            }
-            // Only relevant for mir typeck
-            StatementKind::AscribeUserType(..)
-            // Only relevant for liveness and unsafeck
-            | StatementKind::PlaceMention(..)
-            // Doesn't have any language semantics
-            | StatementKind::Coverage(..)
-            // Does not actually affect borrowck
-            | StatementKind::StorageLive(..) => {}
-            StatementKind::StorageDead(local) => {
-                self.access_place(
-                    location,
-                    Place::from(*local),
-                    (Shallow(None), Write(WriteKind::StorageDeadOrDrop)),
-                    LocalMutationIsAllowed::Yes,
-                );
-            }
-            StatementKind::ConstEvalCounter
-            | StatementKind::Nop
-            | StatementKind::Retag { .. }
-            | StatementKind::Deinit(..)
-            | StatementKind::SetDiscriminant { .. } => {
-                bug!("Statement not allowed in this MIR phase")
-            }
-        }
-
-        self.super_statement(statement, location);
-    }
-
-    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
-        self.check_activations(location);
-
-        match &terminator.kind {
-            TerminatorKind::SwitchInt { discr, targets: _ } => {
-                self.consume_operand(location, discr);
-            }
-            TerminatorKind::Drop { place: drop_place, target: _, unwind: _, replace } => {
-                let write_kind =
-                    if *replace { WriteKind::Replace } else { WriteKind::StorageDeadOrDrop };
-                self.access_place(
-                    location,
-                    *drop_place,
-                    (AccessDepth::Drop, Write(write_kind)),
-                    LocalMutationIsAllowed::Yes,
-                );
-            }
-            TerminatorKind::Call {
-                func,
-                args,
-                destination,
-                target: _,
-                unwind: _,
-                call_source: _,
-                fn_span: _,
-            } => {
-                self.consume_operand(location, func);
-                for arg in args {
-                    self.consume_operand(location, arg);
-                }
-                self.mutate_place(location, *destination, Deep);
-            }
-            TerminatorKind::Assert { cond, expected: _, msg, target: _, unwind: _ } => {
-                self.consume_operand(location, cond);
-                use rustc_middle::mir::AssertKind;
-                if let AssertKind::BoundsCheck { len, index } = &**msg {
-                    self.consume_operand(location, len);
-                    self.consume_operand(location, index);
-                }
-            }
-            TerminatorKind::Yield { value, resume, resume_arg, drop: _ } => {
-                self.consume_operand(location, value);
-
-                // Invalidate all borrows of local places
-                let borrow_set = self.borrow_set;
-                let resume = self.location_table.start_index(resume.start_location());
-                for (i, data) in borrow_set.iter_enumerated() {
-                    if borrow_of_local_data(data.borrowed_place) {
-                        self.all_facts.loan_invalidated_at.push((resume, i));
-                    }
-                }
-
-                self.mutate_place(location, *resume_arg, Deep);
-            }
-            TerminatorKind::UnwindResume
-            | TerminatorKind::Return
-            | TerminatorKind::CoroutineDrop => {
-                // Invalidate all borrows of local places
-                let borrow_set = self.borrow_set;
-                let start = self.location_table.start_index(location);
-                for (i, data) in borrow_set.iter_enumerated() {
-                    if borrow_of_local_data(data.borrowed_place) {
-                        self.all_facts.loan_invalidated_at.push((start, i));
-                    }
-                }
-            }
-            TerminatorKind::InlineAsm {
-                template: _,
-                operands,
-                options: _,
-                line_spans: _,
-                destination: _,
-                unwind: _,
-            } => {
-                for op in operands {
-                    match op {
-                        InlineAsmOperand::In { reg: _, value } => {
-                            self.consume_operand(location, value);
-                        }
-                        InlineAsmOperand::Out { reg: _, late: _, place, .. } => {
-                            if let &Some(place) = place {
-                                self.mutate_place(location, place, Shallow(None));
-                            }
-                        }
-                        InlineAsmOperand::InOut { reg: _, late: _, in_value, out_place } => {
-                            self.consume_operand(location, in_value);
-                            if let &Some(out_place) = out_place {
-                                self.mutate_place(location, out_place, Shallow(None));
-                            }
-                        }
-                        InlineAsmOperand::Const { value: _ }
-                        | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
-                    }
-                }
-            }
-            TerminatorKind::Goto { target: _ }
-            | TerminatorKind::UnwindTerminate(_)
-            | TerminatorKind::Unreachable
-            | TerminatorKind::FalseEdge { real_target: _, imaginary_target: _ }
-            | TerminatorKind::FalseUnwind { real_target: _, unwind: _ } => {
-                // no data used, thus irrelevant to borrowck
-            }
-        }
-
-        self.super_terminator(terminator, location);
-    }
-}
-
-impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
-    /// Simulates mutation of a place.
-    fn mutate_place(&mut self, location: Location, place: Place<'tcx>, kind: AccessDepth) {
-        self.access_place(
-            location,
-            place,
-            (kind, Write(WriteKind::Mutate)),
-            LocalMutationIsAllowed::ExceptUpvars,
-        );
-    }
-
-    /// Simulates consumption of an operand.
-    fn consume_operand(&mut self, location: Location, operand: &Operand<'tcx>) {
-        match *operand {
-            Operand::Copy(place) => {
-                self.access_place(
-                    location,
-                    place,
-                    (Deep, Read(ReadKind::Copy)),
-                    LocalMutationIsAllowed::No,
-                );
-            }
-            Operand::Move(place) => {
-                self.access_place(
-                    location,
-                    place,
-                    (Deep, Write(WriteKind::Move)),
-                    LocalMutationIsAllowed::Yes,
-                );
-            }
-            Operand::Constant(_) => {}
-        }
-    }
-
-    // Simulates consumption of an rvalue
-    fn consume_rvalue(&mut self, location: Location, rvalue: &Rvalue<'tcx>) {
-        match rvalue {
-            &Rvalue::Ref(_ /*rgn*/, bk, place) => {
-                let access_kind = match bk {
-                    BorrowKind::Fake => {
-                        (Shallow(Some(ArtificialField::FakeBorrow)), Read(ReadKind::Borrow(bk)))
-                    }
-                    BorrowKind::Shared => (Deep, Read(ReadKind::Borrow(bk))),
-                    BorrowKind::Mut { .. } => {
-                        let wk = WriteKind::MutableBorrow(bk);
-                        if allow_two_phase_borrow(bk) {
-                            (Deep, Reservation(wk))
-                        } else {
-                            (Deep, Write(wk))
-                        }
-                    }
-                };
-
-                self.access_place(location, place, access_kind, LocalMutationIsAllowed::No);
-            }
-
-            &Rvalue::AddressOf(mutability, place) => {
-                let access_kind = match mutability {
-                    Mutability::Mut => (
-                        Deep,
-                        Write(WriteKind::MutableBorrow(BorrowKind::Mut {
-                            kind: mir::MutBorrowKind::Default,
-                        })),
-                    ),
-                    Mutability::Not => (Deep, Read(ReadKind::Borrow(BorrowKind::Shared))),
-                };
-
-                self.access_place(location, place, access_kind, LocalMutationIsAllowed::No);
-            }
-
-            Rvalue::ThreadLocalRef(_) => {}
-
-            Rvalue::Use(operand)
-            | Rvalue::Repeat(operand, _)
-            | Rvalue::UnaryOp(_ /*un_op*/, operand)
-            | Rvalue::Cast(_ /*cast_kind*/, operand, _ /*ty*/)
-            | Rvalue::ShallowInitBox(operand, _ /*ty*/) => self.consume_operand(location, operand),
-
-            &Rvalue::CopyForDeref(place) => {
-                let op = &Operand::Copy(place);
-                self.consume_operand(location, op);
-            }
-
-            &(Rvalue::Len(place) | Rvalue::Discriminant(place)) => {
-                let af = match rvalue {
-                    Rvalue::Len(..) => Some(ArtificialField::ArrayLength),
-                    Rvalue::Discriminant(..) => None,
-                    _ => unreachable!(),
-                };
-                self.access_place(
-                    location,
-                    place,
-                    (Shallow(af), Read(ReadKind::Copy)),
-                    LocalMutationIsAllowed::No,
-                );
-            }
-
-            Rvalue::BinaryOp(_bin_op, box (operand1, operand2))
-            | Rvalue::CheckedBinaryOp(_bin_op, box (operand1, operand2)) => {
-                self.consume_operand(location, operand1);
-                self.consume_operand(location, operand2);
-            }
-
-            Rvalue::NullaryOp(_op, _ty) => {}
-
-            Rvalue::Aggregate(_, operands) => {
-                for operand in operands {
-                    self.consume_operand(location, operand);
-                }
-            }
-        }
-    }
-
-    /// Simulates an access to a place.
-    fn access_place(
-        &mut self,
-        location: Location,
-        place: Place<'tcx>,
-        kind: (AccessDepth, ReadOrWrite),
-        _is_local_mutation_allowed: LocalMutationIsAllowed,
-    ) {
-        let (sd, rw) = kind;
-        // note: not doing check_access_permissions checks because they don't generate invalidates
-        self.check_access_for_conflict(location, place, sd, rw);
-    }
-
-    fn check_access_for_conflict(
-        &mut self,
-        location: Location,
-        place: Place<'tcx>,
-        sd: AccessDepth,
-        rw: ReadOrWrite,
-    ) {
-        debug!(
-            "invalidation::check_access_for_conflict(location={:?}, place={:?}, sd={:?}, \
-             rw={:?})",
-            location, place, sd, rw,
-        );
-        let tcx = self.tcx;
-        let body = self.body;
-        let borrow_set = self.borrow_set;
-        each_borrow_involving_path(
-            self,
-            tcx,
-            body,
-            location,
-            (sd, place),
-            borrow_set,
-            |_| true,
-            |this, borrow_index, borrow| {
-                match (rw, borrow.kind) {
-                    // Obviously an activation is compatible with its own
-                    // reservation (or even prior activating uses of same
-                    // borrow); so don't check if they interfere.
-                    //
-                    // NOTE: *reservations* do conflict with themselves;
-                    // thus aren't injecting unsoundness w/ this check.)
-                    (Activation(_, activating), _) if activating == borrow_index => {
-                        // Activating a borrow doesn't generate any invalidations, since we
-                        // have already taken the reservation
-                    }
-
-                    (Read(_), BorrowKind::Fake | BorrowKind::Shared)
-                    | (Read(ReadKind::Borrow(BorrowKind::Fake)), BorrowKind::Mut { .. }) => {
-                        // Reads don't invalidate shared or shallow borrows
-                    }
-
-                    (Read(_), BorrowKind::Mut { .. }) => {
-                        // Reading from mere reservations of mutable-borrows is OK.
-                        if !is_active(&this.dominators, borrow, location) {
-                            // If the borrow isn't active yet, reads don't invalidate it
-                            assert!(allow_two_phase_borrow(borrow.kind));
-                            return Control::Continue;
-                        }
-
-                        // Unique and mutable borrows are invalidated by reads from any
-                        // involved path
-                        this.emit_loan_invalidated_at(borrow_index, location);
-                    }
-
-                    (Reservation(_) | Activation(_, _) | Write(_), _) => {
-                        // unique or mutable borrows are invalidated by writes.
-                        // Reservations count as writes since we need to check
-                        // that activating the borrow will be OK
-                        // FIXME(bob_twinkles) is this actually the right thing to do?
-                        this.emit_loan_invalidated_at(borrow_index, location);
-                    }
-                }
-                Control::Continue
-            },
-        );
-    }
-
-    /// Generates a new `loan_invalidated_at(L, B)` fact.
-    fn emit_loan_invalidated_at(&mut self, b: BorrowIndex, l: Location) {
-        let lidx = self.location_table.start_index(l);
-        self.all_facts.loan_invalidated_at.push((lidx, b));
-    }
-
-    fn check_activations(&mut self, location: Location) {
-        // Two-phase borrow support: For each activation that is newly
-        // generated at this statement, check if it interferes with
-        // another borrow.
-        for &borrow_index in self.borrow_set.activations_at_location(location) {
-            let borrow = &self.borrow_set[borrow_index];
-
-            // only mutable borrows should be 2-phase
-            assert!(match borrow.kind {
-                BorrowKind::Shared | BorrowKind::Fake => false,
-                BorrowKind::Mut { .. } => true,
-            });
-
-            self.access_place(
-                location,
-                borrow.borrowed_place,
-                (Deep, Activation(WriteKind::MutableBorrow(borrow.kind), borrow_index)),
-                LocalMutationIsAllowed::No,
-            );
-
-            // We do not need to call `check_if_path_or_subpath_is_moved`
-            // again, as we already called it when we made the
-            // initial reservation.
-        }
-    }
-}
diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index 5787ea1..43f48f5 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -1,8 +1,8 @@
 //! This query borrow-checks the MIR to (further) ensure it is not broken.
 
 #![allow(internal_features)]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(let_chains)]
@@ -22,8 +22,7 @@
 
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
-use rustc_errors::{Diagnostic, DiagnosticBuilder, DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
+use rustc_errors::{Diagnostic, DiagnosticBuilder};
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::bit_set::{BitSet, ChunkedBitSet};
@@ -35,7 +34,7 @@
 use rustc_middle::mir::*;
 use rustc_middle::query::Providers;
 use rustc_middle::traits::DefiningAnchor;
-use rustc_middle::ty::{self, CapturedPlace, ParamEnv, RegionVid, TyCtxt};
+use rustc_middle::ty::{self, ParamEnv, RegionVid, TyCtxt};
 use rustc_session::lint::builtin::UNUSED_MUT;
 use rustc_span::{Span, Symbol};
 use rustc_target::abi::FieldIdx;
@@ -43,6 +42,7 @@
 use smallvec::SmallVec;
 use std::cell::RefCell;
 use std::collections::BTreeMap;
+use std::marker::PhantomData;
 use std::ops::Deref;
 use std::rc::Rc;
 
@@ -65,19 +65,18 @@
 
 pub mod borrow_set;
 mod borrowck_errors;
-mod constraint_generation;
 mod constraints;
 mod dataflow;
 mod def_use;
 mod diagnostics;
 mod facts;
-mod invalidation;
 mod location;
 mod member_constraints;
 mod nll;
 mod path_utils;
 mod place_ext;
 mod places_conflict;
+mod polonius;
 mod prefixes;
 mod region_infer;
 mod renumber;
@@ -98,19 +97,10 @@
 use region_infer::RegionInferenceContext;
 use renumber::RegionCtxt;
 
-fluent_messages! { "../messages.ftl" }
-
-// FIXME(eddyb) perhaps move this somewhere more centrally.
-#[derive(Debug)]
-struct Upvar<'tcx> {
-    place: CapturedPlace<'tcx>,
-
-    /// If true, the capture is behind a reference.
-    by_ref: bool,
-}
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 /// Associate some local constants with the `'tcx` lifetime
-struct TyCtxtConsts<'tcx>(TyCtxt<'tcx>);
+struct TyCtxtConsts<'tcx>(PhantomData<&'tcx ()>);
 impl<'tcx> TyCtxtConsts<'tcx> {
     const DEREF_PROJECTION: &'tcx [PlaceElem<'tcx>; 1] = &[ProjectionElem::Deref];
 }
@@ -135,7 +125,7 @@
         return tcx.arena.alloc(result);
     }
 
-    let hir_owner = tcx.hir().local_def_id_to_hir_id(def).owner;
+    let hir_owner = tcx.local_def_id_to_hir_id(def).owner;
 
     let infcx =
         tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bind(hir_owner.def_id)).build();
@@ -193,18 +183,6 @@
         infcx.set_tainted_by_errors(e);
         errors.set_tainted_by_errors(e);
     }
-    let upvars: Vec<_> = tcx
-        .closure_captures(def)
-        .iter()
-        .map(|&captured_place| {
-            let capture = captured_place.info.capture_kind;
-            let by_ref = match capture {
-                ty::UpvarCapture::ByValue => false,
-                ty::UpvarCapture::ByRef(..) => true,
-            };
-            Upvar { place: captured_place.clone(), by_ref }
-        })
-        .collect();
 
     // Replace all regions with fresh inference variables. This
     // requires first making our own copy of the MIR. This copy will
@@ -216,21 +194,20 @@
         nll::replace_regions_in_mir(&infcx, param_env, &mut body_owned, &mut promoted);
     let body = &body_owned; // no further changes
 
-    let location_table_owned = LocationTable::new(body);
-    let location_table = &location_table_owned;
+    let location_table = LocationTable::new(body);
 
-    let move_data = MoveData::gather_moves(&body, tcx, param_env, |_| true);
+    let move_data = MoveData::gather_moves(body, tcx, param_env, |_| true);
     let promoted_move_data = promoted
         .iter_enumerated()
-        .map(|(idx, body)| (idx, MoveData::gather_moves(&body, tcx, param_env, |_| true)));
+        .map(|(idx, body)| (idx, MoveData::gather_moves(body, tcx, param_env, |_| true)));
 
     let mdpe = MoveDataParamEnv { move_data, param_env };
 
-    let mut flow_inits = MaybeInitializedPlaces::new(tcx, &body, &mdpe)
-        .into_engine(tcx, &body)
+    let mut flow_inits = MaybeInitializedPlaces::new(tcx, body, &mdpe)
+        .into_engine(tcx, body)
         .pass_name("borrowck")
         .iterate_to_fixpoint()
-        .into_results_cursor(&body);
+        .into_results_cursor(body);
 
     let locals_are_invalidated_at_exit = tcx.hir().body_owner_kind(def).is_fn_or_closure();
     let borrow_set =
@@ -249,24 +226,24 @@
         free_regions,
         body,
         &promoted,
-        location_table,
+        &location_table,
         param_env,
         &mut flow_inits,
         &mdpe.move_data,
         &borrow_set,
-        &upvars,
+        tcx.closure_captures(def),
         consumer_options,
     );
 
     // Dump MIR results into a file, if that is enabled. This let us
     // write unit-tests, as well as helping with debugging.
-    nll::dump_mir_results(&infcx, &body, &regioncx, &opt_closure_req);
+    nll::dump_mir_results(&infcx, body, &regioncx, &opt_closure_req);
 
     // We also have a `#[rustc_regions]` annotation that causes us to dump
     // information.
     nll::dump_annotation(
         &infcx,
-        &body,
+        body,
         &regioncx,
         &opt_closure_req,
         &opaque_type_values,
@@ -288,7 +265,7 @@
         .into_engine(tcx, body)
         .pass_name("borrowck")
         .iterate_to_fixpoint();
-    let flow_ever_inits = EverInitializedPlaces::new(tcx, body, &mdpe)
+    let flow_ever_inits = EverInitializedPlaces::new(body, &mdpe)
         .into_engine(tcx, body)
         .pass_name("borrowck")
         .iterate_to_fixpoint();
@@ -313,7 +290,7 @@
             param_env,
             body: promoted_body,
             move_data: &move_data,
-            location_table, // no need to create a real one for the promoted, it is not used
+            location_table: &location_table, // no need to create a real one for the promoted, it is not used
             movable_coroutine,
             fn_self_span_reported: Default::default(),
             locals_are_invalidated_at_exit,
@@ -324,7 +301,7 @@
             used_mut: Default::default(),
             used_mut_upvars: SmallVec::new(),
             borrow_set: Rc::clone(&borrow_set),
-            upvars: Vec::new(),
+            upvars: &[],
             local_names: IndexVec::from_elem(None, &promoted_body.local_decls),
             region_names: RefCell::default(),
             next_region_name: RefCell::new(1),
@@ -354,7 +331,7 @@
         param_env,
         body,
         move_data: &mdpe.move_data,
-        location_table,
+        location_table: &location_table,
         movable_coroutine,
         locals_are_invalidated_at_exit,
         fn_self_span_reported: Default::default(),
@@ -365,7 +342,7 @@
         used_mut: Default::default(),
         used_mut_upvars: SmallVec::new(),
         borrow_set: Rc::clone(&borrow_set),
-        upvars,
+        upvars: tcx.closure_captures(def),
         local_names,
         region_names: RefCell::default(),
         next_region_name: RefCell::new(1),
@@ -456,7 +433,7 @@
             promoted,
             borrow_set,
             region_inference_context: regioncx,
-            location_table: polonius_input.as_ref().map(|_| location_table_owned),
+            location_table: polonius_input.as_ref().map(|_| location_table),
             input_facts: polonius_input,
             output_facts,
         }))
@@ -584,7 +561,7 @@
     borrow_set: Rc<BorrowSet<'tcx>>,
 
     /// Information about upvars not necessarily preserved in types or MIR
-    upvars: Vec<Upvar<'tcx>>,
+    upvars: &'tcx [&'tcx ty::CapturedPlace<'tcx>],
 
     /// Names of local (user) variables (extracted from `var_debug_info`).
     local_names: IndexVec<Local, Option<Symbol>>,
@@ -1041,9 +1018,7 @@
         flow_state: &Flows<'cx, 'tcx>,
     ) -> bool {
         let mut error_reported = false;
-        let tcx = self.infcx.tcx;
-        let body = self.body;
-        let borrow_set = self.borrow_set.clone();
+        let borrow_set = Rc::clone(&self.borrow_set);
 
         // Use polonius output if it has been enabled.
         let mut polonius_output;
@@ -1060,8 +1035,8 @@
 
         each_borrow_involving_path(
             self,
-            tcx,
-            body,
+            self.infcx.tcx,
+            self.body,
             location,
             (sd, place_span.0),
             &borrow_set,
@@ -1538,7 +1513,7 @@
 
         if places_conflict::borrow_conflicts_with_place(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             place,
             borrow.kind,
             root_place,
@@ -2155,11 +2130,11 @@
                     && !self.has_buffered_errors()
                 {
                     // rust-lang/rust#46908: In pure NLL mode this code path should be
-                    // unreachable, but we use `delay_span_bug` because we can hit this when
+                    // unreachable, but we use `span_delayed_bug` because we can hit this when
                     // dereferencing a non-Copy raw pointer *and* have `-Ztreat-err-as-bug`
                     // enabled. We don't want to ICE for that case, as other errors will have
                     // been emitted (#52262).
-                    self.infcx.tcx.sess.delay_span_bug(
+                    self.infcx.tcx.sess.span_delayed_bug(
                         span,
                         format!(
                             "Accessing `{place:?}` with the kind `{kind:?}` shouldn't be possible",
@@ -2193,7 +2168,7 @@
                 // If this is a mutate access to an immutable local variable with no projections
                 // report the error as an illegal reassignment
                 let init = &self.move_data.inits[init_index];
-                let assigned_span = init.span(&self.body);
+                let assigned_span = init.span(self.body);
                 self.report_illegal_reassignment(location, (place, span), assigned_span, place);
             } else {
                 self.report_mutability_error(place, span, the_place_err, error_access, location)
@@ -2294,7 +2269,9 @@
                                     // unique path to the `&mut`
                                     hir::Mutability::Mut => {
                                         let mode = match self.is_upvar_field_projection(place) {
-                                            Some(field) if self.upvars[field.index()].by_ref => {
+                                            Some(field)
+                                                if self.upvars[field.index()].is_by_ref() =>
+                                            {
                                                 is_local_mutation_allowed
                                             }
                                             _ => LocalMutationIsAllowed::Yes,
@@ -2342,7 +2319,7 @@
                                  place={:?}, place_base={:?}",
                                 upvar, is_local_mutation_allowed, place, place_base
                             );
-                            match (upvar.place.mutability, is_local_mutation_allowed) {
+                            match (upvar.mutability, is_local_mutation_allowed) {
                                 (
                                     Mutability::Not,
                                     LocalMutationIsAllowed::No
@@ -2451,7 +2428,7 @@
 
         pub fn buffer_error(&mut self, t: DiagnosticBuilder<'_, ErrorGuaranteed>) {
             if let None = self.tainted_by_errors {
-                self.tainted_by_errors = Some(self.tcx.sess.delay_span_bug(
+                self.tainted_by_errors = Some(self.tcx.sess.span_delayed_bug(
                     t.span.clone_ignoring_labels(),
                     "diagnostic buffered but not emitted",
                 ))
@@ -2525,8 +2502,8 @@
             if !self.errors.buffered.is_empty() {
                 self.errors.buffered.sort_by_key(|diag| diag.sort_span);
 
-                for mut diag in self.errors.buffered.drain(..) {
-                    self.infcx.tcx.sess.diagnostic().emit_diagnostic(&mut diag);
+                for diag in self.errors.buffered.drain(..) {
+                    self.infcx.tcx.sess.dcx().emit_diagnostic(diag);
                 }
             }
 
diff --git a/compiler/rustc_borrowck/src/location.rs b/compiler/rustc_borrowck/src/location.rs
index 0e669ab..6f09393 100644
--- a/compiler/rustc_borrowck/src/location.rs
+++ b/compiler/rustc_borrowck/src/location.rs
@@ -20,6 +20,7 @@
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "LocationIndex({})"]
     pub struct LocationIndex {}
 }
diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index 0ea4401..6781c6a 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -2,16 +2,17 @@
 #![deny(rustc::diagnostic_outside_of_impl)]
 //! The entry point of the NLL borrow checker.
 
+use polonius_engine::{Algorithm, Output};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::IndexSlice;
 use rustc_middle::mir::{create_dump_file, dump_enabled, dump_mir, PassWhere};
-use rustc_middle::mir::{
-    Body, ClosureOutlivesSubject, ClosureRegionRequirements, LocalKind, Location, Promoted,
-    START_BLOCK,
-};
+use rustc_middle::mir::{Body, ClosureOutlivesSubject, ClosureRegionRequirements, Promoted};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, OpaqueHiddenType, TyCtxt};
+use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
+use rustc_mir_dataflow::move_paths::MoveData;
+use rustc_mir_dataflow::ResultsCursor;
 use rustc_span::symbol::sym;
 use std::env;
 use std::io;
@@ -19,25 +20,18 @@
 use std::rc::Rc;
 use std::str::FromStr;
 
-use polonius_engine::{Algorithm, Output};
-
-use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
-use rustc_mir_dataflow::move_paths::{InitKind, InitLocation, MoveData};
-use rustc_mir_dataflow::ResultsCursor;
-
 use crate::{
     borrow_set::BorrowSet,
-    constraint_generation,
     consumers::ConsumerOptions,
     diagnostics::RegionErrors,
     facts::{AllFacts, AllFactsExt, RustcFacts},
-    invalidation,
     location::LocationTable,
+    polonius,
     region_infer::{values::RegionValueElements, RegionInferenceContext},
     renumber,
     type_check::{self, MirTypeckRegionConstraints, MirTypeckResults},
     universal_regions::UniversalRegions,
-    BorrowckInferCtxt, Upvar,
+    BorrowckInferCtxt,
 };
 
 pub type PoloniusOutput = Output<RustcFacts>;
@@ -78,81 +72,6 @@
     universal_regions
 }
 
-// This function populates an AllFacts instance with base facts related to
-// MovePaths and needed for the move analysis.
-fn populate_polonius_move_facts(
-    all_facts: &mut AllFacts,
-    move_data: &MoveData<'_>,
-    location_table: &LocationTable,
-    body: &Body<'_>,
-) {
-    all_facts
-        .path_is_var
-        .extend(move_data.rev_lookup.iter_locals_enumerated().map(|(l, r)| (r, l)));
-
-    for (child, move_path) in move_data.move_paths.iter_enumerated() {
-        if let Some(parent) = move_path.parent {
-            all_facts.child_path.push((child, parent));
-        }
-    }
-
-    let fn_entry_start =
-        location_table.start_index(Location { block: START_BLOCK, statement_index: 0 });
-
-    // initialized_at
-    for init in move_data.inits.iter() {
-        match init.location {
-            InitLocation::Statement(location) => {
-                let block_data = &body[location.block];
-                let is_terminator = location.statement_index == block_data.statements.len();
-
-                if is_terminator && init.kind == InitKind::NonPanicPathOnly {
-                    // We are at the terminator of an init that has a panic path,
-                    // and where the init should not happen on panic
-
-                    for successor in block_data.terminator().successors() {
-                        if body[successor].is_cleanup {
-                            continue;
-                        }
-
-                        // The initialization happened in (or rather, when arriving at)
-                        // the successors, but not in the unwind block.
-                        let first_statement = Location { block: successor, statement_index: 0 };
-                        all_facts
-                            .path_assigned_at_base
-                            .push((init.path, location_table.start_index(first_statement)));
-                    }
-                } else {
-                    // In all other cases, the initialization just happens at the
-                    // midpoint, like any other effect.
-                    all_facts
-                        .path_assigned_at_base
-                        .push((init.path, location_table.mid_index(location)));
-                }
-            }
-            // Arguments are initialized on function entry
-            InitLocation::Argument(local) => {
-                assert!(body.local_kind(local) == LocalKind::Arg);
-                all_facts.path_assigned_at_base.push((init.path, fn_entry_start));
-            }
-        }
-    }
-
-    for (local, path) in move_data.rev_lookup.iter_locals_enumerated() {
-        if body.local_kind(local) != LocalKind::Arg {
-            // Non-arguments start out deinitialised; we simulate this with an
-            // initial move:
-            all_facts.path_moved_at_base.push((path, fn_entry_start));
-        }
-    }
-
-    // moved_out_at
-    // deinitialisation is assumed to always happen!
-    all_facts
-        .path_moved_at_base
-        .extend(move_data.moves.iter().map(|mo| (mo.path, location_table.mid_index(mo.source))));
-}
-
 /// Computes the (non-lexical) regions from the input MIR.
 ///
 /// This may result in errors being reported.
@@ -166,7 +85,7 @@
     flow_inits: &mut ResultsCursor<'cx, 'tcx, MaybeInitializedPlaces<'cx, 'tcx>>,
     move_data: &MoveData<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
-    upvars: &[Upvar<'tcx>],
+    upvars: &[&ty::CapturedPlace<'tcx>],
     consumer_options: Option<ConsumerOptions>,
 ) -> NllOutput<'tcx> {
     let is_polonius_legacy_enabled = infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled();
@@ -179,70 +98,26 @@
 
     let universal_regions = Rc::new(universal_regions);
 
-    let elements = &Rc::new(RegionValueElements::new(&body));
+    let elements = &Rc::new(RegionValueElements::new(body));
 
     // Run the MIR type-checker.
-    let MirTypeckResults {
-        constraints,
-        universal_region_relations,
-        opaque_type_values,
-        live_loans,
-    } = type_check::type_check(
-        infcx,
-        param_env,
-        body,
-        promoted,
-        &universal_regions,
-        location_table,
-        borrow_set,
-        &mut all_facts,
-        flow_inits,
-        move_data,
-        elements,
-        upvars,
-        polonius_input,
-    );
-
-    if let Some(all_facts) = &mut all_facts {
-        let _prof_timer = infcx.tcx.prof.generic_activity("polonius_fact_generation");
-        all_facts.universal_region.extend(universal_regions.universal_regions());
-        populate_polonius_move_facts(all_facts, move_data, location_table, &body);
-
-        // Emit universal regions facts, and their relations, for Polonius.
-        //
-        // 1: universal regions are modeled in Polonius as a pair:
-        // - the universal region vid itself.
-        // - a "placeholder loan" associated to this universal region. Since they don't exist in
-        //   the `borrow_set`, their `BorrowIndex` are synthesized as the universal region index
-        //   added to the existing number of loans, as if they succeeded them in the set.
-        //
-        let borrow_count = borrow_set.len();
-        debug!(
-            "compute_regions: polonius placeholders, num_universals={}, borrow_count={}",
-            universal_regions.len(),
-            borrow_count
+    let MirTypeckResults { constraints, universal_region_relations, opaque_type_values } =
+        type_check::type_check(
+            infcx,
+            param_env,
+            body,
+            promoted,
+            &universal_regions,
+            location_table,
+            borrow_set,
+            &mut all_facts,
+            flow_inits,
+            move_data,
+            elements,
+            upvars,
+            polonius_input,
         );
 
-        for universal_region in universal_regions.universal_regions() {
-            let universal_region_idx = universal_region.index();
-            let placeholder_loan_idx = borrow_count + universal_region_idx;
-            all_facts.placeholder.push((universal_region, placeholder_loan_idx.into()));
-        }
-
-        // 2: the universal region relations `outlives` constraints are emitted as
-        //  `known_placeholder_subset` facts.
-        for (fr1, fr2) in universal_region_relations.known_outlives() {
-            if fr1 != fr2 {
-                debug!(
-                    "compute_regions: emitting polonius `known_placeholder_subset` \
-                     fr1={:?}, fr2={:?}",
-                    fr1, fr2
-                );
-                all_facts.known_placeholder_subset.push((fr1, fr2));
-            }
-        }
-    }
-
     // Create the region inference context, taking ownership of the
     // region inference data that was contained in `infcx`, and the
     // base constraints generated by the type-check.
@@ -250,7 +125,7 @@
     let MirTypeckRegionConstraints {
         placeholder_indices,
         placeholder_index_to_region: _,
-        mut liveness_constraints,
+        liveness_constraints,
         outlives_constraints,
         member_constraints,
         universe_causes,
@@ -258,13 +133,16 @@
     } = constraints;
     let placeholder_indices = Rc::new(placeholder_indices);
 
-    constraint_generation::generate_constraints(
-        infcx,
-        &mut liveness_constraints,
+    // If requested, emit legacy polonius facts.
+    polonius::emit_facts(
         &mut all_facts,
+        infcx.tcx,
         location_table,
-        &body,
+        body,
         borrow_set,
+        move_data,
+        &universal_regions,
+        &universal_region_relations,
     );
 
     let mut regioncx = RegionInferenceContext::new(
@@ -279,17 +157,12 @@
         type_tests,
         liveness_constraints,
         elements,
-        live_loans,
     );
 
-    // Generate various additional constraints.
-    invalidation::generate_invalidates(infcx.tcx, &mut all_facts, location_table, body, borrow_set);
-
-    let def_id = body.source.def_id();
-
-    // Dump facts if requested.
+    // If requested: dump NLL facts, and run legacy polonius analysis.
     let polonius_output = all_facts.as_ref().and_then(|all_facts| {
         if infcx.tcx.sess.opts.unstable_opts.nll_facts {
+            let def_id = body.source.def_id();
             let def_path = infcx.tcx.def_path(def_id);
             let dir_path = PathBuf::from(&infcx.tcx.sess.opts.unstable_opts.nll_facts_dir)
                 .join(def_path.to_filename_friendly_no_crate());
@@ -302,7 +175,7 @@
             let algorithm = Algorithm::from_str(&algorithm).unwrap();
             debug!("compute_regions: using polonius algorithm {:?}", algorithm);
             let _prof_timer = infcx.tcx.prof.generic_activity("polonius_analysis");
-            Some(Rc::new(Output::compute(&all_facts, algorithm, false)))
+            Some(Rc::new(Output::compute(all_facts, algorithm, false)))
         } else {
             None
         }
@@ -310,17 +183,17 @@
 
     // Solve the region constraints.
     let (closure_region_requirements, nll_errors) =
-        regioncx.solve(infcx, param_env, &body, polonius_output.clone());
+        regioncx.solve(infcx, param_env, body, polonius_output.clone());
 
     if !nll_errors.is_empty() {
         // Suppress unhelpful extra errors in `infer_opaque_types`.
-        infcx.set_tainted_by_errors(infcx.tcx.sess.delay_span_bug(
+        infcx.set_tainted_by_errors(infcx.tcx.sess.span_delayed_bug(
             body.span,
             "`compute_regions` tainted `infcx` with errors but did not emit any errors",
         ));
     }
 
-    let remapped_opaque_tys = regioncx.infer_opaque_types(&infcx, opaque_type_values);
+    let remapped_opaque_tys = regioncx.infer_opaque_types(infcx, opaque_type_values);
 
     NllOutput {
         regioncx,
@@ -407,7 +280,7 @@
 
     let def_span = tcx.def_span(body.source.def_id());
     let mut err = if let Some(closure_region_requirements) = closure_region_requirements {
-        let mut err = tcx.sess.diagnostic().span_note_diag(def_span, "external requirements");
+        let mut err = tcx.sess.dcx().struct_span_note(def_span, "external requirements");
 
         regioncx.annotate(tcx, &mut err);
 
@@ -426,7 +299,7 @@
 
         err
     } else {
-        let mut err = tcx.sess.diagnostic().span_note_diag(def_span, "no external requirements");
+        let mut err = tcx.sess.dcx().struct_span_note(def_span, "no external requirements");
         regioncx.annotate(tcx, &mut err);
 
         err
diff --git a/compiler/rustc_borrowck/src/path_utils.rs b/compiler/rustc_borrowck/src/path_utils.rs
index 51e318f..2d997df 100644
--- a/compiler/rustc_borrowck/src/path_utils.rs
+++ b/compiler/rustc_borrowck/src/path_utils.rs
@@ -4,7 +4,6 @@
 use crate::places_conflict;
 use crate::AccessDepth;
 use crate::BorrowIndex;
-use crate::Upvar;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_middle::mir::BorrowKind;
 use rustc_middle::mir::{BasicBlock, Body, Location, Place, PlaceRef, ProjectionElem};
@@ -150,7 +149,7 @@
 /// of a closure type.
 pub(crate) fn is_upvar_field_projection<'tcx>(
     tcx: TyCtxt<'tcx>,
-    upvars: &[Upvar<'tcx>],
+    upvars: &[&rustc_middle::ty::CapturedPlace<'tcx>],
     place_ref: PlaceRef<'tcx>,
     body: &Body<'tcx>,
 ) -> Option<FieldIdx> {
@@ -166,7 +165,7 @@
         Some((place_base, ProjectionElem::Field(field, _ty))) => {
             let base_ty = place_base.ty(body, tcx).ty;
             if (base_ty.is_closure() || base_ty.is_coroutine())
-                && (!by_ref || upvars[field.index()].by_ref)
+                && (!by_ref || upvars[field.index()].is_by_ref())
             {
                 Some(field)
             } else {
diff --git a/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs
new file mode 100644
index 0000000..232bd74
--- /dev/null
+++ b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs
@@ -0,0 +1,424 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
+use rustc_data_structures::graph::dominators::Dominators;
+use rustc_middle::mir::visit::Visitor;
+use rustc_middle::mir::{self, BasicBlock, Body, Location, NonDivergingIntrinsic, Place, Rvalue};
+use rustc_middle::mir::{BorrowKind, Mutability, Operand};
+use rustc_middle::mir::{InlineAsmOperand, Terminator, TerminatorKind};
+use rustc_middle::mir::{Statement, StatementKind};
+use rustc_middle::ty::TyCtxt;
+
+use crate::{
+    borrow_set::BorrowSet, facts::AllFacts, location::LocationTable, path_utils::*, AccessDepth,
+    Activation, ArtificialField, BorrowIndex, Deep, LocalMutationIsAllowed, Read, ReadKind,
+    ReadOrWrite, Reservation, Shallow, Write, WriteKind,
+};
+
+/// Emit `loan_invalidated_at` facts.
+pub(super) fn emit_loan_invalidations<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    all_facts: &mut AllFacts,
+    location_table: &LocationTable,
+    body: &Body<'tcx>,
+    borrow_set: &BorrowSet<'tcx>,
+) {
+    let dominators = body.basic_blocks.dominators();
+    let mut visitor =
+        LoanInvalidationsGenerator { all_facts, borrow_set, tcx, location_table, body, dominators };
+    visitor.visit_body(body);
+}
+
+struct LoanInvalidationsGenerator<'cx, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    all_facts: &'cx mut AllFacts,
+    location_table: &'cx LocationTable,
+    body: &'cx Body<'tcx>,
+    dominators: &'cx Dominators<BasicBlock>,
+    borrow_set: &'cx BorrowSet<'tcx>,
+}
+
+/// Visits the whole MIR and generates `invalidates()` facts.
+/// Most of the code implementing this was stolen from `borrow_check/mod.rs`.
+impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
+    fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
+        self.check_activations(location);
+
+        match &statement.kind {
+            StatementKind::Assign(box (lhs, rhs)) => {
+                self.consume_rvalue(location, rhs);
+
+                self.mutate_place(location, *lhs, Shallow(None));
+            }
+            StatementKind::FakeRead(box (_, _)) => {
+                // Only relevant for initialized/liveness/safety checks.
+            }
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
+                self.consume_operand(location, op);
+            }
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
+                src,
+                dst,
+                count,
+            })) => {
+                self.consume_operand(location, src);
+                self.consume_operand(location, dst);
+                self.consume_operand(location, count);
+            }
+            // Only relevant for mir typeck
+            StatementKind::AscribeUserType(..)
+            // Only relevant for liveness and unsafeck
+            | StatementKind::PlaceMention(..)
+            // Doesn't have any language semantics
+            | StatementKind::Coverage(..)
+            // Does not actually affect borrowck
+            | StatementKind::StorageLive(..) => {}
+            StatementKind::StorageDead(local) => {
+                self.access_place(
+                    location,
+                    Place::from(*local),
+                    (Shallow(None), Write(WriteKind::StorageDeadOrDrop)),
+                    LocalMutationIsAllowed::Yes,
+                );
+            }
+            StatementKind::ConstEvalCounter
+            | StatementKind::Nop
+            | StatementKind::Retag { .. }
+            | StatementKind::Deinit(..)
+            | StatementKind::SetDiscriminant { .. } => {
+                bug!("Statement not allowed in this MIR phase")
+            }
+        }
+
+        self.super_statement(statement, location);
+    }
+
+    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
+        self.check_activations(location);
+
+        match &terminator.kind {
+            TerminatorKind::SwitchInt { discr, targets: _ } => {
+                self.consume_operand(location, discr);
+            }
+            TerminatorKind::Drop { place: drop_place, target: _, unwind: _, replace } => {
+                let write_kind =
+                    if *replace { WriteKind::Replace } else { WriteKind::StorageDeadOrDrop };
+                self.access_place(
+                    location,
+                    *drop_place,
+                    (AccessDepth::Drop, Write(write_kind)),
+                    LocalMutationIsAllowed::Yes,
+                );
+            }
+            TerminatorKind::Call {
+                func,
+                args,
+                destination,
+                target: _,
+                unwind: _,
+                call_source: _,
+                fn_span: _,
+            } => {
+                self.consume_operand(location, func);
+                for arg in args {
+                    self.consume_operand(location, arg);
+                }
+                self.mutate_place(location, *destination, Deep);
+            }
+            TerminatorKind::Assert { cond, expected: _, msg, target: _, unwind: _ } => {
+                self.consume_operand(location, cond);
+                use rustc_middle::mir::AssertKind;
+                if let AssertKind::BoundsCheck { len, index } = &**msg {
+                    self.consume_operand(location, len);
+                    self.consume_operand(location, index);
+                }
+            }
+            TerminatorKind::Yield { value, resume, resume_arg, drop: _ } => {
+                self.consume_operand(location, value);
+
+                // Invalidate all borrows of local places
+                let borrow_set = self.borrow_set;
+                let resume = self.location_table.start_index(resume.start_location());
+                for (i, data) in borrow_set.iter_enumerated() {
+                    if borrow_of_local_data(data.borrowed_place) {
+                        self.all_facts.loan_invalidated_at.push((resume, i));
+                    }
+                }
+
+                self.mutate_place(location, *resume_arg, Deep);
+            }
+            TerminatorKind::UnwindResume
+            | TerminatorKind::Return
+            | TerminatorKind::CoroutineDrop => {
+                // Invalidate all borrows of local places
+                let borrow_set = self.borrow_set;
+                let start = self.location_table.start_index(location);
+                for (i, data) in borrow_set.iter_enumerated() {
+                    if borrow_of_local_data(data.borrowed_place) {
+                        self.all_facts.loan_invalidated_at.push((start, i));
+                    }
+                }
+            }
+            TerminatorKind::InlineAsm {
+                template: _,
+                operands,
+                options: _,
+                line_spans: _,
+                destination: _,
+                unwind: _,
+            } => {
+                for op in operands {
+                    match op {
+                        InlineAsmOperand::In { reg: _, value } => {
+                            self.consume_operand(location, value);
+                        }
+                        InlineAsmOperand::Out { reg: _, late: _, place, .. } => {
+                            if let &Some(place) = place {
+                                self.mutate_place(location, place, Shallow(None));
+                            }
+                        }
+                        InlineAsmOperand::InOut { reg: _, late: _, in_value, out_place } => {
+                            self.consume_operand(location, in_value);
+                            if let &Some(out_place) = out_place {
+                                self.mutate_place(location, out_place, Shallow(None));
+                            }
+                        }
+                        InlineAsmOperand::Const { value: _ }
+                        | InlineAsmOperand::SymFn { value: _ }
+                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                    }
+                }
+            }
+            TerminatorKind::Goto { target: _ }
+            | TerminatorKind::UnwindTerminate(_)
+            | TerminatorKind::Unreachable
+            | TerminatorKind::FalseEdge { real_target: _, imaginary_target: _ }
+            | TerminatorKind::FalseUnwind { real_target: _, unwind: _ } => {
+                // no data used, thus irrelevant to borrowck
+            }
+        }
+
+        self.super_terminator(terminator, location);
+    }
+}
+
+impl<'cx, 'tcx> LoanInvalidationsGenerator<'cx, 'tcx> {
+    /// Simulates mutation of a place.
+    fn mutate_place(&mut self, location: Location, place: Place<'tcx>, kind: AccessDepth) {
+        self.access_place(
+            location,
+            place,
+            (kind, Write(WriteKind::Mutate)),
+            LocalMutationIsAllowed::ExceptUpvars,
+        );
+    }
+
+    /// Simulates consumption of an operand.
+    fn consume_operand(&mut self, location: Location, operand: &Operand<'tcx>) {
+        match *operand {
+            Operand::Copy(place) => {
+                self.access_place(
+                    location,
+                    place,
+                    (Deep, Read(ReadKind::Copy)),
+                    LocalMutationIsAllowed::No,
+                );
+            }
+            Operand::Move(place) => {
+                self.access_place(
+                    location,
+                    place,
+                    (Deep, Write(WriteKind::Move)),
+                    LocalMutationIsAllowed::Yes,
+                );
+            }
+            Operand::Constant(_) => {}
+        }
+    }
+
+    // Simulates consumption of an rvalue
+    fn consume_rvalue(&mut self, location: Location, rvalue: &Rvalue<'tcx>) {
+        match rvalue {
+            &Rvalue::Ref(_ /*rgn*/, bk, place) => {
+                let access_kind = match bk {
+                    BorrowKind::Fake => {
+                        (Shallow(Some(ArtificialField::FakeBorrow)), Read(ReadKind::Borrow(bk)))
+                    }
+                    BorrowKind::Shared => (Deep, Read(ReadKind::Borrow(bk))),
+                    BorrowKind::Mut { .. } => {
+                        let wk = WriteKind::MutableBorrow(bk);
+                        if allow_two_phase_borrow(bk) {
+                            (Deep, Reservation(wk))
+                        } else {
+                            (Deep, Write(wk))
+                        }
+                    }
+                };
+
+                self.access_place(location, place, access_kind, LocalMutationIsAllowed::No);
+            }
+
+            &Rvalue::AddressOf(mutability, place) => {
+                let access_kind = match mutability {
+                    Mutability::Mut => (
+                        Deep,
+                        Write(WriteKind::MutableBorrow(BorrowKind::Mut {
+                            kind: mir::MutBorrowKind::Default,
+                        })),
+                    ),
+                    Mutability::Not => (Deep, Read(ReadKind::Borrow(BorrowKind::Shared))),
+                };
+
+                self.access_place(location, place, access_kind, LocalMutationIsAllowed::No);
+            }
+
+            Rvalue::ThreadLocalRef(_) => {}
+
+            Rvalue::Use(operand)
+            | Rvalue::Repeat(operand, _)
+            | Rvalue::UnaryOp(_ /*un_op*/, operand)
+            | Rvalue::Cast(_ /*cast_kind*/, operand, _ /*ty*/)
+            | Rvalue::ShallowInitBox(operand, _ /*ty*/) => self.consume_operand(location, operand),
+
+            &Rvalue::CopyForDeref(place) => {
+                let op = &Operand::Copy(place);
+                self.consume_operand(location, op);
+            }
+
+            &(Rvalue::Len(place) | Rvalue::Discriminant(place)) => {
+                let af = match rvalue {
+                    Rvalue::Len(..) => Some(ArtificialField::ArrayLength),
+                    Rvalue::Discriminant(..) => None,
+                    _ => unreachable!(),
+                };
+                self.access_place(
+                    location,
+                    place,
+                    (Shallow(af), Read(ReadKind::Copy)),
+                    LocalMutationIsAllowed::No,
+                );
+            }
+
+            Rvalue::BinaryOp(_bin_op, box (operand1, operand2))
+            | Rvalue::CheckedBinaryOp(_bin_op, box (operand1, operand2)) => {
+                self.consume_operand(location, operand1);
+                self.consume_operand(location, operand2);
+            }
+
+            Rvalue::NullaryOp(_op, _ty) => {}
+
+            Rvalue::Aggregate(_, operands) => {
+                for operand in operands {
+                    self.consume_operand(location, operand);
+                }
+            }
+        }
+    }
+
+    /// Simulates an access to a place.
+    fn access_place(
+        &mut self,
+        location: Location,
+        place: Place<'tcx>,
+        kind: (AccessDepth, ReadOrWrite),
+        _is_local_mutation_allowed: LocalMutationIsAllowed,
+    ) {
+        let (sd, rw) = kind;
+        // note: not doing check_access_permissions checks because they don't generate invalidates
+        self.check_access_for_conflict(location, place, sd, rw);
+    }
+
+    fn check_access_for_conflict(
+        &mut self,
+        location: Location,
+        place: Place<'tcx>,
+        sd: AccessDepth,
+        rw: ReadOrWrite,
+    ) {
+        debug!(
+            "check_access_for_conflict(location={:?}, place={:?}, sd={:?}, rw={:?})",
+            location, place, sd, rw,
+        );
+        each_borrow_involving_path(
+            self,
+            self.tcx,
+            self.body,
+            location,
+            (sd, place),
+            self.borrow_set,
+            |_| true,
+            |this, borrow_index, borrow| {
+                match (rw, borrow.kind) {
+                    // Obviously an activation is compatible with its own
+                    // reservation (or even prior activating uses of same
+                    // borrow); so don't check if they interfere.
+                    //
+                    // NOTE: *reservations* do conflict with themselves;
+                    // thus aren't injecting unsoundness w/ this check.)
+                    (Activation(_, activating), _) if activating == borrow_index => {
+                        // Activating a borrow doesn't generate any invalidations, since we
+                        // have already taken the reservation
+                    }
+
+                    (Read(_), BorrowKind::Fake | BorrowKind::Shared)
+                    | (Read(ReadKind::Borrow(BorrowKind::Fake)), BorrowKind::Mut { .. }) => {
+                        // Reads don't invalidate shared or shallow borrows
+                    }
+
+                    (Read(_), BorrowKind::Mut { .. }) => {
+                        // Reading from mere reservations of mutable-borrows is OK.
+                        if !is_active(this.dominators, borrow, location) {
+                            // If the borrow isn't active yet, reads don't invalidate it
+                            assert!(allow_two_phase_borrow(borrow.kind));
+                            return Control::Continue;
+                        }
+
+                        // Unique and mutable borrows are invalidated by reads from any
+                        // involved path
+                        this.emit_loan_invalidated_at(borrow_index, location);
+                    }
+
+                    (Reservation(_) | Activation(_, _) | Write(_), _) => {
+                        // unique or mutable borrows are invalidated by writes.
+                        // Reservations count as writes since we need to check
+                        // that activating the borrow will be OK
+                        // FIXME(bob_twinkles) is this actually the right thing to do?
+                        this.emit_loan_invalidated_at(borrow_index, location);
+                    }
+                }
+                Control::Continue
+            },
+        );
+    }
+
+    /// Generates a new `loan_invalidated_at(L, B)` fact.
+    fn emit_loan_invalidated_at(&mut self, b: BorrowIndex, l: Location) {
+        let lidx = self.location_table.start_index(l);
+        self.all_facts.loan_invalidated_at.push((lidx, b));
+    }
+
+    fn check_activations(&mut self, location: Location) {
+        // Two-phase borrow support: For each activation that is newly
+        // generated at this statement, check if it interferes with
+        // another borrow.
+        for &borrow_index in self.borrow_set.activations_at_location(location) {
+            let borrow = &self.borrow_set[borrow_index];
+
+            // only mutable borrows should be 2-phase
+            assert!(match borrow.kind {
+                BorrowKind::Shared | BorrowKind::Fake => false,
+                BorrowKind::Mut { .. } => true,
+            });
+
+            self.access_place(
+                location,
+                borrow.borrowed_place,
+                (Deep, Activation(WriteKind::MutableBorrow(borrow.kind), borrow_index)),
+                LocalMutationIsAllowed::No,
+            );
+
+            // We do not need to call `check_if_path_or_subpath_is_moved`
+            // again, as we already called it when we made the
+            // initial reservation.
+        }
+    }
+}
diff --git a/compiler/rustc_borrowck/src/polonius/loan_kills.rs b/compiler/rustc_borrowck/src/polonius/loan_kills.rs
new file mode 100644
index 0000000..5df9438
--- /dev/null
+++ b/compiler/rustc_borrowck/src/polonius/loan_kills.rs
@@ -0,0 +1,147 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
+use rustc_middle::mir::visit::Visitor;
+use rustc_middle::mir::{
+    Body, Local, Location, Place, PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind,
+    Terminator, TerminatorKind,
+};
+use rustc_middle::ty::TyCtxt;
+
+use crate::{borrow_set::BorrowSet, facts::AllFacts, location::LocationTable, places_conflict};
+
+/// Emit `loan_killed_at` and `cfg_edge` facts at the same time.
+pub(super) fn emit_loan_kills<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    all_facts: &mut AllFacts,
+    location_table: &LocationTable,
+    body: &Body<'tcx>,
+    borrow_set: &BorrowSet<'tcx>,
+) {
+    let mut visitor = LoanKillsGenerator { borrow_set, tcx, location_table, all_facts, body };
+    for (bb, data) in body.basic_blocks.iter_enumerated() {
+        visitor.visit_basic_block_data(bb, data);
+    }
+}
+
+struct LoanKillsGenerator<'cx, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    all_facts: &'cx mut AllFacts,
+    location_table: &'cx LocationTable,
+    borrow_set: &'cx BorrowSet<'tcx>,
+    body: &'cx Body<'tcx>,
+}
+
+impl<'cx, 'tcx> Visitor<'tcx> for LoanKillsGenerator<'cx, 'tcx> {
+    fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
+        // Also record CFG facts here.
+        self.all_facts.cfg_edge.push((
+            self.location_table.start_index(location),
+            self.location_table.mid_index(location),
+        ));
+
+        self.all_facts.cfg_edge.push((
+            self.location_table.mid_index(location),
+            self.location_table.start_index(location.successor_within_block()),
+        ));
+
+        // If there are borrows on this now dead local, we need to record them as `killed`.
+        if let StatementKind::StorageDead(local) = statement.kind {
+            self.record_killed_borrows_for_local(local, location);
+        }
+
+        self.super_statement(statement, location);
+    }
+
+    fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) {
+        // When we see `X = ...`, then kill borrows of
+        // `(*X).foo` and so forth.
+        self.record_killed_borrows_for_place(*place, location);
+        self.super_assign(place, rvalue, location);
+    }
+
+    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
+        // Also record CFG facts here.
+        self.all_facts.cfg_edge.push((
+            self.location_table.start_index(location),
+            self.location_table.mid_index(location),
+        ));
+
+        let successor_blocks = terminator.successors();
+        self.all_facts.cfg_edge.reserve(successor_blocks.size_hint().0);
+        for successor_block in successor_blocks {
+            self.all_facts.cfg_edge.push((
+                self.location_table.mid_index(location),
+                self.location_table.start_index(successor_block.start_location()),
+            ));
+        }
+
+        // A `Call` terminator's return value can be a local which has borrows,
+        // so we need to record those as `killed` as well.
+        if let TerminatorKind::Call { destination, .. } = terminator.kind {
+            self.record_killed_borrows_for_place(destination, location);
+        }
+
+        self.super_terminator(terminator, location);
+    }
+}
+
+impl<'tcx> LoanKillsGenerator<'_, 'tcx> {
+    /// Records the borrows on the specified place as `killed`. For example, when assigning to a
+    /// local, or on a call's return destination.
+    fn record_killed_borrows_for_place(&mut self, place: Place<'tcx>, location: Location) {
+        // Depending on the `Place` we're killing:
+        // - if it's a local, or a single deref of a local,
+        //   we kill all the borrows on the local.
+        // - if it's a deeper projection, we have to filter which
+        //   of the borrows are killed: the ones whose `borrowed_place`
+        //   conflicts with the `place`.
+        match place.as_ref() {
+            PlaceRef { local, projection: &[] }
+            | PlaceRef { local, projection: &[ProjectionElem::Deref] } => {
+                debug!(
+                    "Recording `killed` facts for borrows of local={:?} at location={:?}",
+                    local, location
+                );
+
+                self.record_killed_borrows_for_local(local, location);
+            }
+
+            PlaceRef { local, projection: &[.., _] } => {
+                // Kill conflicting borrows of the innermost local.
+                debug!(
+                    "Recording `killed` facts for borrows of \
+                            innermost projected local={:?} at location={:?}",
+                    local, location
+                );
+
+                if let Some(borrow_indices) = self.borrow_set.local_map.get(&local) {
+                    for &borrow_index in borrow_indices {
+                        let places_conflict = places_conflict::places_conflict(
+                            self.tcx,
+                            self.body,
+                            self.borrow_set[borrow_index].borrowed_place,
+                            place,
+                            places_conflict::PlaceConflictBias::NoOverlap,
+                        );
+
+                        if places_conflict {
+                            let location_index = self.location_table.mid_index(location);
+                            self.all_facts.loan_killed_at.push((borrow_index, location_index));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /// Records the borrows on the specified local as `killed`.
+    fn record_killed_borrows_for_local(&mut self, local: Local, location: Location) {
+        if let Some(borrow_indices) = self.borrow_set.local_map.get(&local) {
+            let location_index = self.location_table.mid_index(location);
+            self.all_facts.loan_killed_at.reserve(borrow_indices.len());
+            for &borrow_index in borrow_indices {
+                self.all_facts.loan_killed_at.push((borrow_index, location_index));
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs
new file mode 100644
index 0000000..40126d5
--- /dev/null
+++ b/compiler/rustc_borrowck/src/polonius/mod.rs
@@ -0,0 +1,188 @@
+//! Functions dedicated to fact generation for the `-Zpolonius=legacy` datalog implementation.
+//!
+//! Will be removed in the future, once the in-tree `-Zpolonius=next` implementation reaches feature
+//! parity.
+
+use rustc_middle::mir::{Body, LocalKind, Location, START_BLOCK};
+use rustc_middle::ty::TyCtxt;
+use rustc_mir_dataflow::move_paths::{InitKind, InitLocation, MoveData};
+
+use crate::borrow_set::BorrowSet;
+use crate::facts::AllFacts;
+use crate::location::LocationTable;
+use crate::type_check::free_region_relations::UniversalRegionRelations;
+use crate::universal_regions::UniversalRegions;
+
+mod loan_invalidations;
+mod loan_kills;
+
+/// When requested, emit most of the facts needed by polonius:
+/// - moves and assignments
+/// - universal regions and their relations
+/// - CFG points and edges
+/// - loan kills
+/// - loan invalidations
+///
+/// The rest of the facts are emitted during typeck and liveness.
+pub(crate) fn emit_facts<'tcx>(
+    all_facts: &mut Option<AllFacts>,
+    tcx: TyCtxt<'tcx>,
+    location_table: &LocationTable,
+    body: &Body<'tcx>,
+    borrow_set: &BorrowSet<'tcx>,
+    move_data: &MoveData<'_>,
+    universal_regions: &UniversalRegions<'_>,
+    universal_region_relations: &UniversalRegionRelations<'_>,
+) {
+    let Some(all_facts) = all_facts else {
+        // We don't do anything if there are no facts to fill.
+        return;
+    };
+    let _prof_timer = tcx.prof.generic_activity("polonius_fact_generation");
+    emit_move_facts(all_facts, move_data, location_table, body);
+    emit_universal_region_facts(
+        all_facts,
+        borrow_set,
+        &universal_regions,
+        &universal_region_relations,
+    );
+    emit_cfg_and_loan_kills_facts(all_facts, tcx, location_table, body, borrow_set);
+    emit_loan_invalidations_facts(all_facts, tcx, location_table, body, borrow_set);
+}
+
+/// Emit facts needed for move/init analysis: moves and assignments.
+fn emit_move_facts(
+    all_facts: &mut AllFacts,
+    move_data: &MoveData<'_>,
+    location_table: &LocationTable,
+    body: &Body<'_>,
+) {
+    all_facts
+        .path_is_var
+        .extend(move_data.rev_lookup.iter_locals_enumerated().map(|(l, r)| (r, l)));
+
+    for (child, move_path) in move_data.move_paths.iter_enumerated() {
+        if let Some(parent) = move_path.parent {
+            all_facts.child_path.push((child, parent));
+        }
+    }
+
+    let fn_entry_start =
+        location_table.start_index(Location { block: START_BLOCK, statement_index: 0 });
+
+    // initialized_at
+    for init in move_data.inits.iter() {
+        match init.location {
+            InitLocation::Statement(location) => {
+                let block_data = &body[location.block];
+                let is_terminator = location.statement_index == block_data.statements.len();
+
+                if is_terminator && init.kind == InitKind::NonPanicPathOnly {
+                    // We are at the terminator of an init that has a panic path,
+                    // and where the init should not happen on panic
+
+                    for successor in block_data.terminator().successors() {
+                        if body[successor].is_cleanup {
+                            continue;
+                        }
+
+                        // The initialization happened in (or rather, when arriving at)
+                        // the successors, but not in the unwind block.
+                        let first_statement = Location { block: successor, statement_index: 0 };
+                        all_facts
+                            .path_assigned_at_base
+                            .push((init.path, location_table.start_index(first_statement)));
+                    }
+                } else {
+                    // In all other cases, the initialization just happens at the
+                    // midpoint, like any other effect.
+                    all_facts
+                        .path_assigned_at_base
+                        .push((init.path, location_table.mid_index(location)));
+                }
+            }
+            // Arguments are initialized on function entry
+            InitLocation::Argument(local) => {
+                assert!(body.local_kind(local) == LocalKind::Arg);
+                all_facts.path_assigned_at_base.push((init.path, fn_entry_start));
+            }
+        }
+    }
+
+    for (local, path) in move_data.rev_lookup.iter_locals_enumerated() {
+        if body.local_kind(local) != LocalKind::Arg {
+            // Non-arguments start out deinitialised; we simulate this with an
+            // initial move:
+            all_facts.path_moved_at_base.push((path, fn_entry_start));
+        }
+    }
+
+    // moved_out_at
+    // deinitialisation is assumed to always happen!
+    all_facts
+        .path_moved_at_base
+        .extend(move_data.moves.iter().map(|mo| (mo.path, location_table.mid_index(mo.source))));
+}
+
+/// Emit universal regions facts, and their relations.
+fn emit_universal_region_facts(
+    all_facts: &mut AllFacts,
+    borrow_set: &BorrowSet<'_>,
+    universal_regions: &UniversalRegions<'_>,
+    universal_region_relations: &UniversalRegionRelations<'_>,
+) {
+    // 1: universal regions are modeled in Polonius as a pair:
+    // - the universal region vid itself.
+    // - a "placeholder loan" associated to this universal region. Since they don't exist in
+    //   the `borrow_set`, their `BorrowIndex` are synthesized as the universal region index
+    //   added to the existing number of loans, as if they succeeded them in the set.
+    //
+    all_facts.universal_region.extend(universal_regions.universal_regions());
+    let borrow_count = borrow_set.len();
+    debug!(
+        "emit_universal_region_facts: polonius placeholders, num_universals={}, borrow_count={}",
+        universal_regions.len(),
+        borrow_count
+    );
+
+    for universal_region in universal_regions.universal_regions() {
+        let universal_region_idx = universal_region.index();
+        let placeholder_loan_idx = borrow_count + universal_region_idx;
+        all_facts.placeholder.push((universal_region, placeholder_loan_idx.into()));
+    }
+
+    // 2: the universal region relations `outlives` constraints are emitted as
+    //  `known_placeholder_subset` facts.
+    for (fr1, fr2) in universal_region_relations.known_outlives() {
+        if fr1 != fr2 {
+            debug!(
+                "emit_universal_region_facts: emitting polonius `known_placeholder_subset` \
+                     fr1={:?}, fr2={:?}",
+                fr1, fr2
+            );
+            all_facts.known_placeholder_subset.push((fr1, fr2));
+        }
+    }
+}
+
+/// Emit facts about loan invalidations.
+fn emit_loan_invalidations_facts<'tcx>(
+    all_facts: &mut AllFacts,
+    tcx: TyCtxt<'tcx>,
+    location_table: &LocationTable,
+    body: &Body<'tcx>,
+    borrow_set: &BorrowSet<'tcx>,
+) {
+    loan_invalidations::emit_loan_invalidations(tcx, all_facts, location_table, body, borrow_set);
+}
+
+/// Emit facts about CFG points and edges, as well as locations where loans are killed.
+fn emit_cfg_and_loan_kills_facts<'tcx>(
+    all_facts: &mut AllFacts,
+    tcx: TyCtxt<'tcx>,
+    location_table: &LocationTable,
+    body: &Body<'tcx>,
+    borrow_set: &BorrowSet<'tcx>,
+) {
+    loan_kills::emit_loan_kills(tcx, all_facts, location_table, body, borrow_set);
+}
diff --git a/compiler/rustc_borrowck/src/region_infer/dump_mir.rs b/compiler/rustc_borrowck/src/region_infer/dump_mir.rs
index 4d620ac..cfbb276 100644
--- a/compiler/rustc_borrowck/src/region_infer/dump_mir.rs
+++ b/compiler/rustc_borrowck/src/region_infer/dump_mir.rs
@@ -67,7 +67,7 @@
         with_msg: &mut dyn FnMut(&str) -> io::Result<()>,
     ) -> io::Result<()> {
         for region in self.definitions.indices() {
-            let value = self.liveness_constraints.region_value_str(region);
+            let value = self.liveness_constraints.pretty_print_live_points(region);
             if value != "{}" {
                 with_msg(&format!("{region:?} live at {value}"))?;
             }
diff --git a/compiler/rustc_borrowck/src/region_infer/graphviz.rs b/compiler/rustc_borrowck/src/region_infer/graphviz.rs
index a0cf22e..408c839 100644
--- a/compiler/rustc_borrowck/src/region_infer/graphviz.rs
+++ b/compiler/rustc_borrowck/src/region_infer/graphviz.rs
@@ -8,7 +8,6 @@
 use std::io::{self, Write};
 
 use super::*;
-use crate::constraints::OutlivesConstraint;
 use rustc_graphviz as dot;
 
 impl<'tcx> RegionInferenceContext<'tcx> {
diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs
index b1f91a0..948221e 100644
--- a/compiler/rustc_borrowck/src/region_infer/mod.rs
+++ b/compiler/rustc_borrowck/src/region_infer/mod.rs
@@ -7,7 +7,6 @@
 use rustc_data_structures::graph::scc::Sccs;
 use rustc_errors::Diagnostic;
 use rustc_hir::def_id::CRATE_DEF_ID;
-use rustc_index::bit_set::SparseBitMatrix;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_infer::infer::outlives::test_type_match;
 use rustc_infer::infer::region_constraints::{GenericKind, VarInfos, VerifyBound, VerifyIfEq};
@@ -31,8 +30,8 @@
     nll::PoloniusOutput,
     region_infer::reverse_sccs::ReverseSccGraph,
     region_infer::values::{
-        LivenessValues, PlaceholderIndices, PointIndex, RegionElement, RegionValueElements,
-        RegionValues, ToElementIndex,
+        LivenessValues, PlaceholderIndices, RegionElement, RegionValueElements, RegionValues,
+        ToElementIndex,
     },
     type_check::{free_region_relations::UniversalRegionRelations, Locations},
     universal_regions::UniversalRegions,
@@ -59,7 +58,7 @@
     /// regions, these start out empty and steadily grow, though for
     /// each universally quantified region R they start out containing
     /// the entire CFG and `end(R)`.
-    liveness_constraints: LivenessValues<RegionVid>,
+    liveness_constraints: LivenessValues,
 
     /// The outlives constraints computed by the type-check.
     constraints: Frozen<OutlivesConstraintSet<'tcx>>,
@@ -120,9 +119,6 @@
     /// Information about how the universally quantified regions in
     /// scope on this function relate to one another.
     universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
-
-    /// The set of loans that are live at a given point in the CFG, when using `-Zpolonius=next`.
-    live_loans: SparseBitMatrix<PointIndex, BorrowIndex>,
 }
 
 /// Each time that `apply_member_constraint` is successful, it appends
@@ -151,6 +147,7 @@
     pub(crate) member_constraint_index: NllMemberConstraintIndex,
 }
 
+#[derive(Debug)]
 pub(crate) struct RegionDefinition<'tcx> {
     /// What kind of variable is this -- a free region? existential
     /// variable? etc. (See the `NllRegionVariableOrigin` for more
@@ -332,9 +329,8 @@
         member_constraints_in: MemberConstraintSet<'tcx, RegionVid>,
         universe_causes: FxIndexMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
         type_tests: Vec<TypeTest<'tcx>>,
-        liveness_constraints: LivenessValues<RegionVid>,
+        liveness_constraints: LivenessValues,
         elements: &Rc<RegionValueElements>,
-        live_loans: SparseBitMatrix<PointIndex, BorrowIndex>,
     ) -> Self {
         debug!("universal_regions: {:#?}", universal_regions);
         debug!("outlives constraints: {:#?}", outlives_constraints);
@@ -359,7 +355,7 @@
         let mut scc_values =
             RegionValues::new(elements, universal_regions.len(), &placeholder_indices);
 
-        for region in liveness_constraints.rows() {
+        for region in liveness_constraints.regions() {
             let scc = constraint_sccs.scc(region);
             scc_values.merge_liveness(scc, region, &liveness_constraints);
         }
@@ -388,7 +384,6 @@
             type_tests,
             universal_regions,
             universal_region_relations,
-            live_loans,
         };
 
         result.init_free_and_bound_regions();
@@ -679,13 +674,10 @@
         // eagerly.
         let mut outlives_requirements = infcx.tcx.is_typeck_child(mir_def_id).then(Vec::new);
 
-        self.check_type_tests(
-            infcx,
-            param_env,
-            body,
-            outlives_requirements.as_mut(),
-            &mut errors_buffer,
-        );
+        self.check_type_tests(infcx, body, outlives_requirements.as_mut(), &mut errors_buffer);
+
+        debug!(?errors_buffer);
+        debug!(?outlives_requirements);
 
         // In Polonius mode, the errors about missing universal region relations are in the output
         // and need to be emitted or propagated. Otherwise, we need to check whether the
@@ -700,10 +692,14 @@
             self.check_universal_regions(outlives_requirements.as_mut(), &mut errors_buffer);
         }
 
+        debug!(?errors_buffer);
+
         if errors_buffer.is_empty() {
             self.check_member_constraints(infcx, &mut errors_buffer);
         }
 
+        debug!(?errors_buffer);
+
         let outlives_requirements = outlives_requirements.unwrap_or_default();
 
         if outlives_requirements.is_empty() {
@@ -936,7 +932,6 @@
     fn check_type_tests(
         &self,
         infcx: &InferCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
         body: &Body<'tcx>,
         mut propagated_outlives_requirements: Option<&mut Vec<ClosureOutlivesRequirement<'tcx>>>,
         errors_buffer: &mut RegionErrors<'tcx>,
@@ -954,7 +949,6 @@
             let generic_ty = type_test.generic_kind.to_ty(tcx);
             if self.eval_verify_bound(
                 infcx,
-                param_env,
                 generic_ty,
                 type_test.lower_bound,
                 &type_test.verify_bound,
@@ -965,7 +959,6 @@
             if let Some(propagated_outlives_requirements) = &mut propagated_outlives_requirements {
                 if self.try_promote_type_test(
                     infcx,
-                    param_env,
                     body,
                     type_test,
                     propagated_outlives_requirements,
@@ -1023,7 +1016,6 @@
     fn try_promote_type_test(
         &self,
         infcx: &InferCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
         body: &Body<'tcx>,
         type_test: &TypeTest<'tcx>,
         propagated_outlives_requirements: &mut Vec<ClosureOutlivesRequirement<'tcx>>,
@@ -1085,7 +1077,7 @@
             // where `ur` is a local bound -- we are sometimes in a
             // position to prove things that our caller cannot. See
             // #53570 for an example.
-            if self.eval_verify_bound(infcx, param_env, generic_ty, ur, &type_test.verify_bound) {
+            if self.eval_verify_bound(infcx, generic_ty, ur, &type_test.verify_bound) {
                 continue;
             }
 
@@ -1268,7 +1260,6 @@
     fn eval_verify_bound(
         &self,
         infcx: &InferCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
         generic_ty: Ty<'tcx>,
         lower_bound: RegionVid,
         verify_bound: &VerifyBound<'tcx>,
@@ -1277,7 +1268,7 @@
 
         match verify_bound {
             VerifyBound::IfEq(verify_if_eq_b) => {
-                self.eval_if_eq(infcx, param_env, generic_ty, lower_bound, *verify_if_eq_b)
+                self.eval_if_eq(infcx, generic_ty, lower_bound, *verify_if_eq_b)
             }
 
             VerifyBound::IsEmpty => {
@@ -1291,11 +1282,11 @@
             }
 
             VerifyBound::AnyBound(verify_bounds) => verify_bounds.iter().any(|verify_bound| {
-                self.eval_verify_bound(infcx, param_env, generic_ty, lower_bound, verify_bound)
+                self.eval_verify_bound(infcx, generic_ty, lower_bound, verify_bound)
             }),
 
             VerifyBound::AllBounds(verify_bounds) => verify_bounds.iter().all(|verify_bound| {
-                self.eval_verify_bound(infcx, param_env, generic_ty, lower_bound, verify_bound)
+                self.eval_verify_bound(infcx, generic_ty, lower_bound, verify_bound)
             }),
         }
     }
@@ -1303,19 +1294,13 @@
     fn eval_if_eq(
         &self,
         infcx: &InferCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
         generic_ty: Ty<'tcx>,
         lower_bound: RegionVid,
         verify_if_eq_b: ty::Binder<'tcx, VerifyIfEq<'tcx>>,
     ) -> bool {
         let generic_ty = self.normalize_to_scc_representatives(infcx.tcx, generic_ty);
         let verify_if_eq_b = self.normalize_to_scc_representatives(infcx.tcx, verify_if_eq_b);
-        match test_type_match::extract_verify_if_eq(
-            infcx.tcx,
-            param_env,
-            &verify_if_eq_b,
-            generic_ty,
-        ) {
+        match test_type_match::extract_verify_if_eq(infcx.tcx, &verify_if_eq_b, generic_ty) {
             Some(r) => {
                 let r_vid = self.to_region_vid(r);
                 self.eval_outlives(r_vid, lower_bound)
@@ -1457,6 +1442,7 @@
         errors_buffer: &mut RegionErrors<'tcx>,
     ) {
         for (fr, fr_definition) in self.definitions.iter_enumerated() {
+            debug!(?fr, ?fr_definition);
             match fr_definition.origin {
                 NllRegionVariableOrigin::FreeRegion => {
                     // Go through each of the universal regions `fr` and check that
@@ -1963,15 +1949,15 @@
         None
     }
 
-    /// Finds some region R such that `fr1: R` and `R` is live at `elem`.
+    /// Finds some region R such that `fr1: R` and `R` is live at `location`.
     #[instrument(skip(self), level = "trace", ret)]
-    pub(crate) fn find_sub_region_live_at(&self, fr1: RegionVid, elem: Location) -> RegionVid {
+    pub(crate) fn find_sub_region_live_at(&self, fr1: RegionVid, location: Location) -> RegionVid {
         trace!(scc = ?self.constraint_sccs.scc(fr1));
         trace!(universe = ?self.scc_universes[self.constraint_sccs.scc(fr1)]);
         self.find_constraint_paths_between_regions(fr1, |r| {
-            // First look for some `r` such that `fr1: r` and `r` is live at `elem`
-            trace!(?r, liveness_constraints=?self.liveness_constraints.region_value_str(r));
-            self.liveness_constraints.contains(r, elem)
+            // First look for some `r` such that `fr1: r` and `r` is live at `location`
+            trace!(?r, liveness_constraints=?self.liveness_constraints.pretty_print_live_points(r));
+            self.liveness_constraints.is_live_at(r, location)
         })
         .or_else(|| {
             // If we fail to find that, we may find some `r` such that
@@ -2316,7 +2302,7 @@
     /// Note: for now, the sets of live loans is only available when using `-Zpolonius=next`.
     pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, location: Location) -> bool {
         let point = self.liveness_constraints.point_from_location(location);
-        self.live_loans.contains(point, loan_idx)
+        self.liveness_constraints.is_loan_live_at(loan_idx, point)
     }
 }
 
diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
index fb0e581..3764e4c 100644
--- a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
+++ b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
@@ -36,7 +36,7 @@
     /// call `infer_opaque_definition_from_instantiation` to get the inferred
     /// type of `_Return<'_a>`. `infer_opaque_definition_from_instantiation`
     /// compares lifetimes directly, so we need to map the inference variables
-    /// back to concrete lifetimes: `'static`, `ReEarlyBound` or `ReFree`.
+    /// back to concrete lifetimes: `'static`, `ReEarlyParam` or `ReLateParam`.
     ///
     /// First we map all the lifetimes in the concrete type to an equal
     /// universal region that occurs in the concrete type's args, in this case
@@ -315,7 +315,7 @@
         parent_def_id = tcx.local_parent(parent_def_id);
     }
 
-    // FIXME(-Ztrait-solver=next): We probably should use `DefiningAnchor::Error`
+    // FIXME(-Znext-solver): We probably should use `DefiningAnchor::Error`
     // and prepopulate this `InferCtxt` with known opaque values, rather than
     // using the `Bind` anchor here. For now it's fine.
     let infcx = tcx
@@ -386,7 +386,7 @@
         let arg_is_param = match arg.unpack() {
             GenericArgKind::Type(ty) => matches!(ty.kind(), ty::Param(_)),
             GenericArgKind::Lifetime(lt) if is_ty_alias => {
-                matches!(*lt, ty::ReEarlyBound(_) | ty::ReFree(_))
+                matches!(*lt, ty::ReEarlyParam(_) | ty::ReLateParam(_))
             }
             // FIXME(#113916): we can't currently check for unique lifetime params,
             // see that issue for more. We will also have to ignore unused lifetime
diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs
index 38452df..dc3ee84 100644
--- a/compiler/rustc_borrowck/src/region_infer/values.rs
+++ b/compiler/rustc_borrowck/src/region_infer/values.rs
@@ -11,6 +11,8 @@
 use std::fmt::Debug;
 use std::rc::Rc;
 
+use crate::dataflow::BorrowIndex;
+
 /// Maps between a `Location` and a `PointIndex` (and vice versa).
 pub(crate) struct RegionValueElements {
     /// For each basic block, how many points are contained within?
@@ -90,6 +92,7 @@
 rustc_index::newtype_index! {
     /// A single integer representing a `Location` in the MIR control-flow
     /// graph. Constructed efficiently from `RegionValueElements`.
+    #[orderable]
     #[debug_format = "PointIndex({})"]
     pub struct PointIndex {}
 }
@@ -116,71 +119,132 @@
     PlaceholderRegion(ty::PlaceholderRegion),
 }
 
-/// When we initially compute liveness, we use an interval matrix storing
-/// liveness ranges for each region-vid.
-pub(crate) struct LivenessValues<N: Idx> {
+/// Records the CFG locations where each region is live. When we initially compute liveness, we use
+/// an interval matrix storing liveness ranges for each region-vid.
+pub(crate) struct LivenessValues {
+    /// The map from locations to points.
     elements: Rc<RegionValueElements>,
-    points: SparseIntervalMatrix<N, PointIndex>,
+
+    /// For each region: the points where it is live.
+    points: SparseIntervalMatrix<RegionVid, PointIndex>,
+
+    /// When using `-Zpolonius=next`, for each point: the loans flowing into the live regions at
+    /// that point.
+    pub(crate) loans: Option<LiveLoans>,
 }
 
-impl<N: Idx> LivenessValues<N> {
-    /// Creates a new set of "region values" that tracks causal information.
-    /// Each of the regions in num_region_variables will be initialized with an
-    /// empty set of points and no causal information.
+/// Data used to compute the loans that are live at a given point in the CFG, when using
+/// `-Zpolonius=next`.
+pub(crate) struct LiveLoans {
+    /// The set of loans that flow into a given region. When individual regions are marked as live
+    /// in the CFG, these inflowing loans are recorded as live.
+    pub(crate) inflowing_loans: SparseBitMatrix<RegionVid, BorrowIndex>,
+
+    /// The set of loans that are live at a given point in the CFG.
+    pub(crate) live_loans: SparseBitMatrix<PointIndex, BorrowIndex>,
+}
+
+impl LiveLoans {
+    pub(crate) fn new(num_loans: usize) -> Self {
+        LiveLoans {
+            live_loans: SparseBitMatrix::new(num_loans),
+            inflowing_loans: SparseBitMatrix::new(num_loans),
+        }
+    }
+}
+
+impl LivenessValues {
+    /// Create an empty map of regions to locations where they're live.
     pub(crate) fn new(elements: Rc<RegionValueElements>) -> Self {
-        Self { points: SparseIntervalMatrix::new(elements.num_points), elements }
+        LivenessValues {
+            points: SparseIntervalMatrix::new(elements.num_points),
+            elements,
+            loans: None,
+        }
     }
 
     /// Iterate through each region that has a value in this set.
-    pub(crate) fn rows(&self) -> impl Iterator<Item = N> {
+    pub(crate) fn regions(&self) -> impl Iterator<Item = RegionVid> {
         self.points.rows()
     }
 
-    /// Adds the given element to the value for the given region. Returns whether
-    /// the element is newly added (i.e., was not already present).
-    pub(crate) fn add_element(&mut self, row: N, location: Location) -> bool {
-        debug!("LivenessValues::add(r={:?}, location={:?})", row, location);
-        let index = self.elements.point_from_location(location);
-        self.points.insert(row, index)
+    /// Records `region` as being live at the given `location`.
+    pub(crate) fn add_location(&mut self, region: RegionVid, location: Location) {
+        debug!("LivenessValues::add_location(region={:?}, location={:?})", region, location);
+        let point = self.elements.point_from_location(location);
+        self.points.insert(region, point);
+
+        // When available, record the loans flowing into this region as live at the given point.
+        if let Some(loans) = self.loans.as_mut() {
+            if let Some(inflowing) = loans.inflowing_loans.row(region) {
+                loans.live_loans.union_row(point, inflowing);
+            }
+        }
     }
 
-    /// Adds all the elements in the given bit array into the given
-    /// region. Returns whether any of them are newly added.
-    pub(crate) fn add_elements(&mut self, row: N, locations: &IntervalSet<PointIndex>) -> bool {
-        debug!("LivenessValues::add_elements(row={:?}, locations={:?})", row, locations);
-        self.points.union_row(row, locations)
+    /// Records `region` as being live at all the given `points`.
+    pub(crate) fn add_points(&mut self, region: RegionVid, points: &IntervalSet<PointIndex>) {
+        debug!("LivenessValues::add_points(region={:?}, points={:?})", region, points);
+        self.points.union_row(region, points);
+
+        // When available, record the loans flowing into this region as live at the given points.
+        if let Some(loans) = self.loans.as_mut() {
+            if let Some(inflowing) = loans.inflowing_loans.row(region) {
+                if !inflowing.is_empty() {
+                    for point in points.iter() {
+                        loans.live_loans.union_row(point, inflowing);
+                    }
+                }
+            }
+        }
     }
 
-    /// Adds all the control-flow points to the values for `r`.
-    pub(crate) fn add_all_points(&mut self, row: N) {
-        self.points.insert_all_into_row(row);
+    /// Records `region` as being live at all the control-flow points.
+    pub(crate) fn add_all_points(&mut self, region: RegionVid) {
+        self.points.insert_all_into_row(region);
     }
 
-    /// Returns `true` if the region `r` contains the given element.
-    pub(crate) fn contains(&self, row: N, location: Location) -> bool {
-        let index = self.elements.point_from_location(location);
-        self.points.row(row).is_some_and(|r| r.contains(index))
+    /// Returns whether `region` is marked live at the given `location`.
+    pub(crate) fn is_live_at(&self, region: RegionVid, location: Location) -> bool {
+        let point = self.elements.point_from_location(location);
+        self.points.row(region).is_some_and(|r| r.contains(point))
     }
 
-    /// Returns an iterator of all the elements contained by the region `r`
-    pub(crate) fn get_elements(&self, row: N) -> impl Iterator<Item = Location> + '_ {
+    /// Returns whether `region` is marked live at any location.
+    pub(crate) fn is_live_anywhere(&self, region: RegionVid) -> bool {
+        self.live_points(region).next().is_some()
+    }
+
+    /// Returns an iterator of all the points where `region` is live.
+    fn live_points(&self, region: RegionVid) -> impl Iterator<Item = PointIndex> + '_ {
         self.points
-            .row(row)
+            .row(region)
             .into_iter()
             .flat_map(|set| set.iter())
-            .take_while(move |&p| self.elements.point_in_range(p))
-            .map(move |p| self.elements.to_location(p))
+            .take_while(|&p| self.elements.point_in_range(p))
     }
 
-    /// Returns a "pretty" string value of the region. Meant for debugging.
-    pub(crate) fn region_value_str(&self, r: N) -> String {
-        region_value_str(self.get_elements(r).map(RegionElement::Location))
+    /// For debugging purposes, returns a pretty-printed string of the points where the `region` is
+    /// live.
+    pub(crate) fn pretty_print_live_points(&self, region: RegionVid) -> String {
+        pretty_print_region_elements(
+            self.live_points(region).map(|p| RegionElement::Location(self.elements.to_location(p))),
+        )
     }
 
     #[inline]
     pub(crate) fn point_from_location(&self, location: Location) -> PointIndex {
         self.elements.point_from_location(location)
     }
+
+    /// When using `-Zpolonius=next`, returns whether the `loan_idx` is live at the given `point`.
+    pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, point: PointIndex) -> bool {
+        self.loans
+            .as_ref()
+            .expect("Accessing live loans requires `-Zpolonius=next`")
+            .live_loans
+            .contains(point, loan_idx)
+    }
 }
 
 /// Maps from `ty::PlaceholderRegion` values that are used in the rest of
@@ -307,7 +371,7 @@
     /// `self[to] |= values[from]`, essentially: that is, take all the
     /// elements for the region `from` from `values` and add them to
     /// the region `to` in `self`.
-    pub(crate) fn merge_liveness<M: Idx>(&mut self, to: N, from: M, values: &LivenessValues<M>) {
+    pub(crate) fn merge_liveness(&mut self, to: N, from: RegionVid, values: &LivenessValues) {
         if let Some(set) = values.points.row(from) {
             self.points.union_row(to, set);
         }
@@ -376,7 +440,7 @@
 
     /// Returns a "pretty" string value of the region. Meant for debugging.
     pub(crate) fn region_value_str(&self, r: N) -> String {
-        region_value_str(self.elements_contained_in(r))
+        pretty_print_region_elements(self.elements_contained_in(r))
     }
 }
 
@@ -420,11 +484,12 @@
     }
 }
 
-pub(crate) fn location_set_str(
+/// For debugging purposes, returns a pretty-printed string of the given points.
+pub(crate) fn pretty_print_points(
     elements: &RegionValueElements,
     points: impl IntoIterator<Item = PointIndex>,
 ) -> String {
-    region_value_str(
+    pretty_print_region_elements(
         points
             .into_iter()
             .take_while(|&p| elements.point_in_range(p))
@@ -433,7 +498,8 @@
     )
 }
 
-fn region_value_str(elements: impl IntoIterator<Item = RegionElement>) -> String {
+/// For debugging purposes, returns a pretty-printed string of the given region elements.
+fn pretty_print_region_elements(elements: impl IntoIterator<Item = RegionElement>) -> String {
     let mut result = String::new();
     result.push('{');
 
diff --git a/compiler/rustc_borrowck/src/renumber.rs b/compiler/rustc_borrowck/src/renumber.rs
index ec0131c..a31d39e 100644
--- a/compiler/rustc_borrowck/src/renumber.rs
+++ b/compiler/rustc_borrowck/src/renumber.rs
@@ -28,6 +28,9 @@
     renumberer.visit_body(body);
 }
 
+// FIXME(@lcnr): A lot of these variants overlap and it seems like
+// this type is only used to decide which region should be used
+// as representative. This should be cleaned up.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
 pub(crate) enum RegionCtxt {
     Location(Location),
diff --git a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
index c84256f..4a76d87 100644
--- a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
+++ b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
@@ -7,7 +7,7 @@
 use rustc_infer::infer::InferCtxt;
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::traits::query::OutlivesBound;
-use rustc_middle::ty::{self, RegionVid, Ty};
+use rustc_middle::ty::{self, RegionVid, Ty, TypeVisitableExt};
 use rustc_span::{ErrorGuaranteed, Span, DUMMY_SP};
 use rustc_trait_selection::traits::query::type_op::{self, TypeOp};
 use std::rc::Rc;
@@ -303,7 +303,7 @@
             Locations::All(span),
             span,
             ConstraintCategory::Internal,
-            &mut self.constraints,
+            self.constraints,
         )
         .convert_all(data);
     }
@@ -321,6 +321,9 @@
             .map_err(|_: ErrorGuaranteed| debug!("failed to compute implied bounds {:?}", ty))
             .ok()?;
         debug!(?bounds, ?constraints);
+        // Because of #109628, we may have unexpected placeholders. Ignore them!
+        // FIXME(#109628): panic in this case once the issue is fixed.
+        let bounds = bounds.into_iter().filter(|bound| !bound.has_placeholders());
         self.add_outlives_bounds(bounds);
         constraints
     }
diff --git a/compiler/rustc_borrowck/src/type_check/input_output.rs b/compiler/rustc_borrowck/src/type_check/input_output.rs
index d053d0a..8e141bb 100644
--- a/compiler/rustc_borrowck/src/type_check/input_output.rs
+++ b/compiler/rustc_borrowck/src/type_check/input_output.rs
@@ -7,7 +7,7 @@
 //! `RETURN_PLACE` the MIR arguments) are always fully normalized (and
 //! contain revealed `impl Trait` values).
 
-use rustc_infer::infer::LateBoundRegionConversionTime;
+use rustc_infer::infer::BoundRegionConversionTime;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty};
 use rustc_span::Span;
@@ -35,7 +35,7 @@
             .instantiate_canonical_with_fresh_inference_vars(body.span, &user_provided_poly_sig);
         let user_provided_sig = self.infcx.instantiate_binder_with_fresh_vars(
             body.span,
-            LateBoundRegionConversionTime::FnCall,
+            BoundRegionConversionTime::FnCall,
             user_provided_sig,
         );
 
@@ -77,7 +77,7 @@
             if argument_index + 1 >= body.local_decls.len() {
                 self.tcx()
                     .sess
-                    .delay_span_bug(body.span, "found more normalized_input_ty than local_decls");
+                    .span_delayed_bug(body.span, "found more normalized_input_ty than local_decls");
                 break;
             }
 
@@ -101,10 +101,10 @@
         );
 
         // We will not have a universal_regions.yield_ty if we yield (by accident)
-        // outside of a coroutine and return an `impl Trait`, so emit a delay_span_bug
+        // outside of a coroutine and return an `impl Trait`, so emit a span_delayed_bug
         // because we don't want to panic in an assert here if we've already got errors.
         if body.yield_ty().is_some() != universal_regions.yield_ty.is_some() {
-            self.tcx().sess.delay_span_bug(
+            self.tcx().sess.span_delayed_bug(
                 body.span,
                 format!(
                     "Expected body to have yield_ty ({:?}) iff we have a UR yield_ty ({:?})",
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs b/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
index a9ca945..7433c94 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
@@ -80,7 +80,7 @@
         live_locals.iter().for_each(|&local| locals_with_use_data[local] = true);
 
         LocalUseMapBuild { local_use_map: &mut local_use_map, elements, locals_with_use_data }
-            .visit_body(&body);
+            .visit_body(body);
 
         local_use_map
     }
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
index f1ad0ca..dc4695f 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
@@ -1,7 +1,9 @@
 use itertools::{Either, Itertools};
 use rustc_data_structures::fx::FxHashSet;
-use rustc_middle::mir::{Body, Local};
-use rustc_middle::ty::{RegionVid, TyCtxt};
+use rustc_middle::mir::visit::{TyContext, Visitor};
+use rustc_middle::mir::{Body, Local, Location, SourceInfo};
+use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::{GenericArgsRef, Region, RegionVid, Ty, TyCtxt};
 use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::MoveData;
 use rustc_mir_dataflow::ResultsCursor;
@@ -11,7 +13,7 @@
     constraints::OutlivesConstraintSet,
     facts::{AllFacts, AllFactsExt},
     location::LocationTable,
-    region_infer::values::RegionValueElements,
+    region_infer::values::{LivenessValues, RegionValueElements},
     universal_regions::UniversalRegions,
 };
 
@@ -42,11 +44,11 @@
 
     let free_regions = regions_that_outlive_free_regions(
         typeck.infcx.num_region_vars(),
-        &typeck.borrowck_context.universal_regions,
+        typeck.borrowck_context.universal_regions,
         &typeck.borrowck_context.constraints.outlives_constraints,
     );
     let (relevant_live_locals, boring_locals) =
-        compute_relevant_live_locals(typeck.tcx(), &free_regions, &body);
+        compute_relevant_live_locals(typeck.tcx(), &free_regions, body);
     let facts_enabled = use_polonius || AllFacts::enabled(typeck.tcx());
 
     let polonius_drop_used = facts_enabled.then(|| {
@@ -65,6 +67,14 @@
         boring_locals,
         polonius_drop_used,
     );
+
+    // Mark regions that should be live where they appear within rvalues or within a call: like
+    // args, regions, and types.
+    record_regular_live_regions(
+        typeck.tcx(),
+        &mut typeck.borrowck_context.constraints.liveness_constraints,
+        body,
+    );
 }
 
 // The purpose of `compute_relevant_live_locals` is to define the subset of `Local`
@@ -132,3 +142,71 @@
     // Return the final set of things we visited.
     outlives_free_region
 }
+
+/// Some variables are "regular live" at `location` -- i.e., they may be used later. This means that
+/// all regions appearing in their type must be live at `location`.
+fn record_regular_live_regions<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    liveness_constraints: &mut LivenessValues,
+    body: &Body<'tcx>,
+) {
+    let mut visitor = LiveVariablesVisitor { tcx, liveness_constraints };
+    for (bb, data) in body.basic_blocks.iter_enumerated() {
+        visitor.visit_basic_block_data(bb, data);
+    }
+}
+
+/// Visitor looking for regions that should be live within rvalues or calls.
+struct LiveVariablesVisitor<'cx, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    liveness_constraints: &'cx mut LivenessValues,
+}
+
+impl<'cx, 'tcx> Visitor<'tcx> for LiveVariablesVisitor<'cx, 'tcx> {
+    /// We sometimes have `args` within an rvalue, or within a
+    /// call. Make them live at the location where they appear.
+    fn visit_args(&mut self, args: &GenericArgsRef<'tcx>, location: Location) {
+        self.record_regions_live_at(*args, location);
+        self.super_args(args);
+    }
+
+    /// We sometimes have `region`s within an rvalue, or within a
+    /// call. Make them live at the location where they appear.
+    fn visit_region(&mut self, region: Region<'tcx>, location: Location) {
+        self.record_regions_live_at(region, location);
+        self.super_region(region);
+    }
+
+    /// We sometimes have `ty`s within an rvalue, or within a
+    /// call. Make them live at the location where they appear.
+    fn visit_ty(&mut self, ty: Ty<'tcx>, ty_context: TyContext) {
+        match ty_context {
+            TyContext::ReturnTy(SourceInfo { span, .. })
+            | TyContext::YieldTy(SourceInfo { span, .. })
+            | TyContext::UserTy(span)
+            | TyContext::LocalDecl { source_info: SourceInfo { span, .. }, .. } => {
+                span_bug!(span, "should not be visiting outside of the CFG: {:?}", ty_context);
+            }
+            TyContext::Location(location) => {
+                self.record_regions_live_at(ty, location);
+            }
+        }
+
+        self.super_ty(ty);
+    }
+}
+
+impl<'cx, 'tcx> LiveVariablesVisitor<'cx, 'tcx> {
+    /// Some variable is "regular live" at `location` -- i.e., it may be used later. This means that
+    /// all regions appearing in the type of `value` must be live at `location`.
+    fn record_regions_live_at<T>(&mut self, value: T, location: Location)
+    where
+        T: TypeVisitable<TyCtxt<'tcx>>,
+    {
+        debug!("record_regions_live_at(value={:?}, location={:?})", value, location);
+        self.tcx.for_each_free_region(&value, |live_region| {
+            let live_region_vid = live_region.as_var();
+            self.liveness_constraints.add_location(live_region_vid, location);
+        });
+    }
+}
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/polonius.rs b/compiler/rustc_borrowck/src/type_check/liveness/polonius.rs
index c621df3..45f7b07 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/polonius.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/polonius.rs
@@ -100,7 +100,7 @@
             location_table,
             move_data,
         };
-        extractor.visit_body(&body);
+        extractor.visit_body(body);
 
         facts.var_dropped_at.extend(
             dropped_at.iter().map(|&(local, location)| (local, location_table.mid_index(location))),
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index e616449..c718d57 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -1,12 +1,12 @@
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::WithSuccessors;
-use rustc_index::bit_set::{HybridBitSet, SparseBitMatrix};
+use rustc_index::bit_set::HybridBitSet;
 use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_infer::infer::outlives::for_liveness;
 use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
 use rustc_middle::traits::query::DropckOutlivesResult;
-use rustc_middle::ty::{RegionVid, Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
+use rustc_middle::ty::{Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
 use rustc_span::DUMMY_SP;
 use rustc_trait_selection::traits::query::type_op::outlives::DropckOutlives;
 use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput};
@@ -16,9 +16,8 @@
 use rustc_mir_dataflow::move_paths::{HasMoveData, MoveData, MovePathIndex};
 use rustc_mir_dataflow::ResultsCursor;
 
-use crate::dataflow::BorrowIndex;
 use crate::{
-    region_infer::values::{self, PointIndex, RegionValueElements},
+    region_infer::values::{self, LiveLoans, PointIndex, RegionValueElements},
     type_check::liveness::local_use_map::LocalUseMap,
     type_check::liveness::polonius,
     type_check::NormalizeLocation,
@@ -49,22 +48,17 @@
     boring_locals: Vec<Local>,
     polonius_drop_used: Option<Vec<(Local, Location)>>,
 ) {
-    debug!("trace()");
-
     let local_use_map = &LocalUseMap::build(&relevant_live_locals, elements, body);
 
     // When using `-Zpolonius=next`, compute the set of loans that can reach a given region.
-    let num_loans = typeck.borrowck_context.borrow_set.len();
-    let mut inflowing_loans = SparseBitMatrix::new(num_loans);
     if typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() {
-        let borrowck_context = &typeck.borrowck_context;
+        let borrowck_context = &mut typeck.borrowck_context;
         let borrow_set = &borrowck_context.borrow_set;
-        let constraint_set = &borrowck_context.constraints.outlives_constraints;
-
-        let num_region_vars = typeck.infcx.num_region_vars();
-        let graph = constraint_set.graph(num_region_vars);
+        let mut live_loans = LiveLoans::new(borrow_set.len());
+        let outlives_constraints = &borrowck_context.constraints.outlives_constraints;
+        let graph = outlives_constraints.graph(typeck.infcx.num_region_vars());
         let region_graph =
-            graph.region_graph(&constraint_set, borrowck_context.universal_regions.fr_static);
+            graph.region_graph(outlives_constraints, borrowck_context.universal_regions.fr_static);
 
         // Traverse each issuing region's constraints, and record the loan as flowing into the
         // outlived region.
@@ -75,9 +69,13 @@
                     continue;
                 }
 
-                inflowing_loans.insert(succ, loan);
+                live_loans.inflowing_loans.insert(succ, loan);
             }
         }
+
+        // Store the inflowing loans in the liveness constraints: they will be used to compute live
+        // loans when liveness data is recorded there.
+        borrowck_context.constraints.liveness_constraints.loans = Some(live_loans);
     };
 
     let cx = LivenessContext {
@@ -88,7 +86,6 @@
         local_use_map,
         move_data,
         drop_data: FxIndexMap::default(),
-        inflowing_loans,
     };
 
     let mut results = LivenessResults::new(cx);
@@ -126,9 +123,6 @@
     /// Index indicating where each variable is assigned, used, or
     /// dropped.
     local_use_map: &'me LocalUseMap,
-
-    /// Set of loans that flow into a given region, when using `-Zpolonius=next`.
-    inflowing_loans: SparseBitMatrix<RegionVid, BorrowIndex>,
 }
 
 struct DropData<'tcx> {
@@ -489,7 +483,7 @@
         }
 
         let move_paths = &self.flow_inits.analysis().move_data().move_paths;
-        move_paths[mpi].find_descendant(&move_paths, |mpi| state.contains(mpi)).is_some()
+        move_paths[mpi].find_descendant(move_paths, |mpi| state.contains(mpi)).is_some()
     }
 
     /// Returns `true` if the local variable (or some part of it) is initialized in
@@ -519,14 +513,7 @@
         live_at: &IntervalSet<PointIndex>,
     ) {
         debug!("add_use_live_facts_for(value={:?})", value);
-
-        Self::make_all_regions_live(
-            self.elements,
-            &mut self.typeck,
-            value,
-            live_at,
-            &self.inflowing_loans,
-        );
+        Self::make_all_regions_live(self.elements, self.typeck, value, live_at);
     }
 
     /// Some variable with type `live_ty` is "drop live" at `location`
@@ -550,7 +537,7 @@
             dropped_local,
             dropped_ty,
             drop_locations,
-            values::location_set_str(self.elements, live_at.iter()),
+            values::pretty_print_points(self.elements, live_at.iter()),
         );
 
         let drop_data = self.drop_data.entry(dropped_ty).or_insert_with({
@@ -577,15 +564,8 @@
         // All things in the `outlives` array may be touched by
         // the destructor and must be live at this point.
         for &kind in &drop_data.dropck_result.kinds {
-            Self::make_all_regions_live(
-                self.elements,
-                &mut self.typeck,
-                kind,
-                live_at,
-                &self.inflowing_loans,
-            );
-
-            polonius::add_drop_of_var_derefs_origin(&mut self.typeck, dropped_local, &kind);
+            Self::make_all_regions_live(self.elements, self.typeck, kind, live_at);
+            polonius::add_drop_of_var_derefs_origin(self.typeck, dropped_local, &kind);
         }
     }
 
@@ -594,20 +574,13 @@
         typeck: &mut TypeChecker<'_, 'tcx>,
         value: impl TypeVisitable<TyCtxt<'tcx>>,
         live_at: &IntervalSet<PointIndex>,
-        inflowing_loans: &SparseBitMatrix<RegionVid, BorrowIndex>,
     ) {
         debug!("make_all_regions_live(value={:?})", value);
         debug!(
             "make_all_regions_live: live_at={}",
-            values::location_set_str(elements, live_at.iter()),
+            values::pretty_print_points(elements, live_at.iter()),
         );
 
-        // When using `-Zpolonius=next`, we want to record the loans that flow into this value's
-        // regions as being live at the given `live_at` points: this will be used to compute the
-        // location where a loan goes out of scope.
-        let num_loans = typeck.borrowck_context.borrow_set.len();
-        let value_loans = &mut HybridBitSet::new_empty(num_loans);
-
         value.visit_with(&mut for_liveness::FreeRegionsVisitor {
             tcx: typeck.tcx(),
             param_env: typeck.param_env,
@@ -618,22 +591,9 @@
                     .borrowck_context
                     .constraints
                     .liveness_constraints
-                    .add_elements(live_region_vid, live_at);
-
-                // There can only be inflowing loans for this region when we are using
-                // `-Zpolonius=next`.
-                if let Some(inflowing) = inflowing_loans.row(live_region_vid) {
-                    value_loans.union(inflowing);
-                }
+                    .add_points(live_region_vid, live_at);
             },
         });
-
-        // Record the loans reaching the value.
-        if !value_loans.is_empty() {
-            for point in live_at.iter() {
-                typeck.borrowck_context.live_loans.union_row(point, value_loans);
-            }
-        }
     }
 
     fn compute_drop_data(
diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index 608d010..5247d5f 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -14,18 +14,16 @@
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::lang_items::LangItem;
-use rustc_index::bit_set::SparseBitMatrix;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_infer::infer::outlives::env::RegionBoundPairs;
 use rustc_infer::infer::region_constraints::RegionConstraintData;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::infer::{
-    InferCtxt, LateBoundRegion, LateBoundRegionConversionTime, NllRegionVariableOrigin,
+    BoundRegion, BoundRegionConversionTime, InferCtxt, NllRegionVariableOrigin,
 };
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
-use rustc_middle::mir::AssertKind;
 use rustc_middle::mir::*;
 use rustc_middle::traits::query::NoSolution;
 use rustc_middle::traits::ObligationCause;
@@ -51,8 +49,6 @@
 use rustc_mir_dataflow::move_paths::MoveData;
 use rustc_mir_dataflow::ResultsCursor;
 
-use crate::dataflow::BorrowIndex;
-use crate::region_infer::values::PointIndex;
 use crate::session_diagnostics::{MoveUnsized, SimdShuffleLastConst};
 use crate::{
     borrow_set::BorrowSet,
@@ -68,7 +64,7 @@
     region_infer::TypeTest,
     type_check::free_region_relations::{CreateResult, UniversalRegionRelations},
     universal_regions::{DefiningTy, UniversalRegions},
-    BorrowckInferCtxt, Upvar,
+    BorrowckInferCtxt,
 };
 
 macro_rules! span_mirbug {
@@ -138,7 +134,7 @@
     flow_inits: &mut ResultsCursor<'mir, 'tcx, MaybeInitializedPlaces<'mir, 'tcx>>,
     move_data: &MoveData<'tcx>,
     elements: &Rc<RegionValueElements>,
-    upvars: &[Upvar<'tcx>],
+    upvars: &[&ty::CapturedPlace<'tcx>],
     use_polonius: bool,
 ) -> MirTypeckResults<'tcx> {
     let implicit_region_bound = ty::Region::new_var(infcx.tcx, universal_regions.fr_fn_body);
@@ -166,9 +162,6 @@
 
     debug!(?normalized_inputs_and_output);
 
-    // When using `-Zpolonius=next`, liveness will record the set of live loans per point.
-    let mut live_loans = SparseBitMatrix::new(borrow_set.len());
-
     let mut borrowck_context = BorrowCheckContext {
         universal_regions,
         location_table,
@@ -176,7 +169,6 @@
         all_facts,
         constraints: &mut constraints,
         upvars,
-        live_loans: &mut live_loans,
     };
 
     let mut checker = TypeChecker::new(
@@ -191,11 +183,11 @@
     checker.check_user_type_annotations();
 
     let mut verifier = TypeVerifier::new(&mut checker, promoted);
-    verifier.visit_body(&body);
+    verifier.visit_body(body);
 
     checker.typeck_mir(body);
-    checker.equate_inputs_and_outputs(&body, universal_regions, &normalized_inputs_and_output);
-    checker.check_signature_annotation(&body);
+    checker.equate_inputs_and_outputs(body, universal_regions, &normalized_inputs_and_output);
+    checker.check_signature_annotation(body);
 
     liveness::generate(
         &mut checker,
@@ -232,7 +224,7 @@
             let mut hidden_type = infcx.resolve_vars_if_possible(decl.hidden_type);
             trace!("finalized opaque type {:?} to {:#?}", opaque_type_key, hidden_type.ty.kind());
             if hidden_type.has_non_region_infer() {
-                let reported = infcx.tcx.sess.delay_span_bug(
+                let reported = infcx.tcx.sess.span_delayed_bug(
                     decl.hidden_type.span,
                     format!("could not resolve {:#?}", hidden_type.ty.kind()),
                 );
@@ -243,7 +235,7 @@
         })
         .collect();
 
-    MirTypeckResults { constraints, universal_region_relations, opaque_type_values, live_loans }
+    MirTypeckResults { constraints, universal_region_relations, opaque_type_values }
 }
 
 fn translate_outlives_facts(typeck: &mut TypeChecker<'_, '_>) {
@@ -274,9 +266,9 @@
 #[track_caller]
 fn mirbug(tcx: TyCtxt<'_>, span: Span, msg: String) {
     // We sometimes see MIR failures (notably predicate failures) due to
-    // the fact that we check rvalue sized predicates here. So use `delay_span_bug`
+    // the fact that we check rvalue sized predicates here. So use `span_delayed_bug`
     // to avoid reporting bugs in those cases.
-    tcx.sess.diagnostic().delay_span_bug(span, msg);
+    tcx.sess.dcx().span_delayed_bug(span, msg);
 }
 
 enum FieldAccessError {
@@ -318,7 +310,7 @@
                 .borrowck_context
                 .constraints
                 .liveness_constraints
-                .add_element(live_region_vid, location);
+                .add_location(live_region_vid, location);
         });
 
         // HACK(compiler-errors): Constants that are gathered into Body.required_consts
@@ -389,7 +381,7 @@
                     self.cx.ascribe_user_type(
                         constant.const_.ty(),
                         UserType::TypeOf(uv.def, UserArgs { args: uv.args, user_self_ty: None }),
-                        locations.span(&self.cx.body),
+                        locations.span(self.cx.body),
                     );
                 }
             } else if let Some(static_def_id) = constant.check_static_ptr(tcx) {
@@ -553,7 +545,7 @@
         let all_facts = &mut None;
         let mut constraints = Default::default();
         let mut liveness_constraints =
-            LivenessValues::new(Rc::new(RegionValueElements::new(&promoted_body)));
+            LivenessValues::new(Rc::new(RegionValueElements::new(promoted_body)));
         // Don't try to add borrow_region facts for the promoted MIR
 
         let mut swap_constraints = |this: &mut Self| {
@@ -570,7 +562,7 @@
 
         swap_constraints(self);
 
-        self.visit_body(&promoted_body);
+        self.visit_body(promoted_body);
 
         self.cx.typeck_mir(promoted_body);
 
@@ -592,16 +584,16 @@
             }
             self.cx.borrowck_context.constraints.outlives_constraints.push(constraint)
         }
-        for region in liveness_constraints.rows() {
+        for region in liveness_constraints.regions() {
             // If the region is live at at least one location in the promoted MIR,
             // then add a liveness constraint to the main MIR for this region
             // at the location provided as an argument to this method
-            if liveness_constraints.get_elements(region).next().is_some() {
+            if liveness_constraints.is_live_anywhere(region) {
                 self.cx
                     .borrowck_context
                     .constraints
                     .liveness_constraints
-                    .add_element(region, location);
+                    .add_location(region, location);
             }
         }
     }
@@ -857,11 +849,7 @@
     all_facts: &'a mut Option<AllFacts>,
     borrow_set: &'a BorrowSet<'tcx>,
     pub(crate) constraints: &'a mut MirTypeckRegionConstraints<'tcx>,
-    upvars: &'a [Upvar<'tcx>],
-
-    /// The set of loans that are live at a given point in the CFG, filled in by `liveness::trace`,
-    /// when using `-Zpolonius=next`.
-    pub(crate) live_loans: &'a mut SparseBitMatrix<PointIndex, BorrowIndex>,
+    upvars: &'a [&'a ty::CapturedPlace<'tcx>],
 }
 
 /// Holder struct for passing results from MIR typeck to the rest of the non-lexical regions
@@ -870,9 +858,6 @@
     pub(crate) constraints: MirTypeckRegionConstraints<'tcx>,
     pub(crate) universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
     pub(crate) opaque_type_values: FxIndexMap<OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>>,
-
-    /// The set of loans that are live at a given point in the CFG, when using `-Zpolonius=next`.
-    pub(crate) live_loans: SparseBitMatrix<PointIndex, BorrowIndex>,
 }
 
 /// A collection of region constraints that must be satisfied for the
@@ -899,7 +884,7 @@
     /// not otherwise appear in the MIR -- in particular, the
     /// late-bound regions that it instantiates at call-sites -- and
     /// hence it must report on their liveness constraints.
-    pub(crate) liveness_constraints: LivenessValues<RegionVid>,
+    pub(crate) liveness_constraints: LivenessValues,
 
     pub(crate) outlives_constraints: OutlivesConstraintSet<'tcx>,
 
@@ -1018,16 +1003,16 @@
             reported_errors: Default::default(),
         };
 
-        // FIXME(-Ztrait-solver=next): A bit dubious that we're only registering
+        // FIXME(-Znext-solver): A bit dubious that we're only registering
         // predefined opaques in the typeck root.
         if infcx.next_trait_solver() && !infcx.tcx.is_typeck_child(body.source.def_id()) {
-            checker.register_predefined_opaques_in_new_solver();
+            checker.register_predefined_opaques_for_next_solver();
         }
 
         checker
     }
 
-    pub(super) fn register_predefined_opaques_in_new_solver(&mut self) {
+    pub(super) fn register_predefined_opaques_for_next_solver(&mut self) {
         // OK to use the identity arguments for each opaque type key, since
         // we remap opaques from HIR typeck back to their definition params.
         let opaques: Vec<_> = self
@@ -1082,7 +1067,7 @@
         );
 
         if result.is_err() {
-            self.infcx.tcx.sess.delay_span_bug(
+            self.infcx.tcx.sess.span_delayed_bug(
                 self.body.span,
                 "failed re-defining predefined opaques in mir typeck",
             );
@@ -1127,7 +1112,7 @@
             locations,
             locations.span(self.body),
             category,
-            &mut self.borrowck_context.constraints,
+            self.borrowck_context.constraints,
         )
         .convert_all(data);
     }
@@ -1202,7 +1187,7 @@
         self.infcx.tcx
     }
 
-    #[instrument(skip(self, body, location), level = "debug")]
+    #[instrument(skip(self, body), level = "debug")]
     fn check_stmt(&mut self, body: &Body<'tcx>, stmt: &Statement<'tcx>, location: Location) {
         let tcx = self.tcx();
         debug!("stmt kind: {:?}", stmt.kind);
@@ -1387,7 +1372,7 @@
                         return;
                     }
                 };
-                let (sig, map) = tcx.replace_late_bound_regions(sig, |br| {
+                let (sig, map) = tcx.instantiate_bound_regions(sig, |br| {
                     use crate::renumber::RegionCtxt;
 
                     let region_ctxt_fn = || {
@@ -1401,10 +1386,10 @@
                     };
 
                     self.infcx.next_region_var(
-                        LateBoundRegion(
+                        BoundRegion(
                             term.source_info.span,
                             br.kind,
-                            LateBoundRegionConversionTime::FnCall,
+                            BoundRegionConversionTime::FnCall,
                         ),
                         region_ctxt_fn,
                     )
@@ -1443,7 +1428,7 @@
                     self.borrowck_context
                         .constraints