[CodeGen] Align MIR parser and printer ownership

1 files changed, 0 insertions, 1694 deletions

diff --git a/backend/lib/MIR/MIPrinter.cpp b/backend/lib/MIR/MIPrinter.cpp
deleted file mode 100644
index 0265564..0000000
--- a/backend/lib/MIR/MIPrinter.cpp
+++ /dev/null
@@ -1,1694 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "rcc/MIR/MIPrinter.h"
-
-#include "rcc/CodeGen/InlineAsm.h"
-#include "rcc/CodeGen/InstructionDescriptor.h"
-#include "rcc/CodeGen/MachineBasicBlock.h"
-#include "rcc/CodeGen/MachineCFIInfo.h"
-#include "rcc/CodeGen/MachineFunction.h"
-#include "rcc/CodeGen/MachineInstr.h"
-#include "rcc/CodeGen/MachineMemOperand.h"
-#include "rcc/CodeGen/MachineOperand.h"
-#include "rcc/CodeGen/MachineRegisterInfo.h"
-#include "rcc/CodeGen/MachineVerifier.h"
-#include "rcc/CodeGen/TargetDescription.h"
-#include "rcc/CodeGen/TargetMIRTextInfo.h"
-#include "rcc/CodeGen/TargetRegisterInfo.h"
-#include "rcc/MIR/LLVMConstantSyntax.h"
-#include "rcc/MIR/MIDebugSyntax.h"
-
-#include <algorithm>
-#include <array>
-#include <bit>
-#include <cstddef>
-#include <cstdint>
-#include <limits>
-#include <optional>
-#include <ranges>
-#include <span>
-#include <string>
-#include <string_view>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-namespace rcc::mir {
-namespace {
-
-[[nodiscard]] bool isBareNameCharacter(char Character) noexcept {
-  return (Character >= 'a' && Character <= 'z') || (Character >= 'A' && Character <= 'Z') ||
-         (Character >= '0' && Character <= '9') || Character == '_' || Character == '-' ||
-         Character == '.' || Character == '$';
-}
-
-[[nodiscard]] std::string quoteName(std::string_view Name) {
-  if (!Name.empty() && std::ranges::all_of(Name, isBareNameCharacter))
-    return std::string(Name);
-
-  constexpr std::string_view Hex = "0123456789ABCDEF";
-  std::string Result;
-  Result.reserve(Name.size() + 2U);
-  Result.push_back('"');
-  for (const char RawCharacter : Name) {
-    const auto Character = static_cast<unsigned char>(RawCharacter);
-    if (Character >= 0x20U && Character <= 0x7EU && Character != '"' && Character != '\\') {
-      Result.push_back(static_cast<char>(Character));
-      continue;
-    }
-    Result.push_back('\\');
-    Result.push_back(Hex[Character >> 4U]);
-    Result.push_back(Hex[Character & 0xFU]);
-  }
-  Result.push_back('"');
-  return Result;
-}
-
-[[nodiscard]] std::string quoteString(std::string_view Value) {
-  std::string Result = quoteName(Value);
-  if (!Result.empty() && Result.front() == '"')
-    return Result;
-  Result.insert(Result.begin(), '"');
-  Result.push_back('"');
-  return Result;
-}
-
-[[nodiscard]] std::string hexadecimalWords(std::span<const std::uint64_t> Words,
-                                           std::uint32_t BitWidth) {
-  constexpr std::string_view Hex = "0123456789ABCDEF";
-  const std::size_t DigitCount = (static_cast<std::size_t>(BitWidth) + 3U) / 4U;
-  std::string Result(DigitCount, '0');
-  for (std::size_t Digit = 0; Digit < DigitCount; ++Digit) {
-    const std::size_t BitOffset = Digit * 4U;
-    const std::uint64_t Nibble = (Words[BitOffset / 64U] >> (BitOffset % 64U)) & 0xFU;
-    Result[DigitCount - Digit - 1U] = Hex[Nibble];
-  }
-  return Result;
-}
-
-[[nodiscard]] std::string_view integerPredicateName(IntegerPredicate Predicate) noexcept {
-  switch (Predicate) {
-  case IntegerPredicate::Equal:
-    return "eq";
-  case IntegerPredicate::NotEqual:
-    return "ne";
-  case IntegerPredicate::SignedGreaterThan:
-    return "sgt";
-  case IntegerPredicate::SignedGreaterEqual:
-    return "sge";
-  case IntegerPredicate::SignedLessThan:
-    return "slt";
-  case IntegerPredicate::SignedLessEqual:
-    return "sle";
-  case IntegerPredicate::UnsignedGreaterThan:
-    return "ugt";
-  case IntegerPredicate::UnsignedGreaterEqual:
-    return "uge";
-  case IntegerPredicate::UnsignedLessThan:
-    return "ult";
-  case IntegerPredicate::UnsignedLessEqual:
-    return "ule";
-  case IntegerPredicate::Count:
-    return {};
-  }
-  return {};
-}
-
-[[nodiscard]] std::string_view floatingPredicateName(FloatingPredicate Predicate) noexcept {
-  switch (Predicate) {
-  case FloatingPredicate::AlwaysFalse:
-    return "false";
-  case FloatingPredicate::OrderedEqual:
-    return "oeq";
-  case FloatingPredicate::OrderedGreaterThan:
-    return "ogt";
-  case FloatingPredicate::OrderedGreaterEqual:
-    return "oge";
-  case FloatingPredicate::OrderedLessThan:
-    return "olt";
-  case FloatingPredicate::OrderedLessEqual:
-    return "ole";
-  case FloatingPredicate::OrderedNotEqual:
-    return "one";
-  case FloatingPredicate::Ordered:
-    return "ord";
-  case FloatingPredicate::Unordered:
-    return "uno";
-  case FloatingPredicate::UnorderedEqual:
-    return "ueq";
-  case FloatingPredicate::UnorderedGreaterThan:
-    return "ugt";
-  case FloatingPredicate::UnorderedGreaterEqual:
-    return "uge";
-  case FloatingPredicate::UnorderedLessThan:
-    return "ult";
-  case FloatingPredicate::UnorderedLessEqual:
-    return "ule";
-  case FloatingPredicate::UnorderedNotEqual:
-    return "une";
-  case FloatingPredicate::AlwaysTrue:
-    return "true";
-  case FloatingPredicate::Count:
-    return {};
-  }
-  return {};
-}
-
-[[nodiscard]] std::string_view atomicOrderingName(AtomicOrdering Ordering) noexcept {
-  switch (Ordering) {
-  case AtomicOrdering::Unordered:
-    return "unordered";
-  case AtomicOrdering::Monotonic:
-    return "monotonic";
-  case AtomicOrdering::Acquire:
-    return "acquire";
-  case AtomicOrdering::Release:
-    return "release";
-  case AtomicOrdering::AcquireRelease:
-    return "acq_rel";
-  case AtomicOrdering::SequentiallyConsistent:
-    return "seq_cst";
-  case AtomicOrdering::NotAtomic:
-    return {};
-  }
-  return {};
-}
-
-[[nodiscard]] Expected<std::string> llvmMIRScalarLowLevelType(LLT Type) {
-  if (Type.isToken())
-    return "s0";
-  if (Type.isAnyScalar())
-    return "s" + std::to_string(Type.scalarBitWidth());
-  if (Type.isInteger())
-    return "i" + std::to_string(Type.scalarBitWidth());
-  if (Type.isPointer()) {
-    const auto AddressSpace = Type.addressSpace();
-    if (!AddressSpace) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "pointer low-level type has no address space");
-    }
-    return "p" + std::to_string(*AddressSpace);
-  }
-  if (!Type.isFloat()) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "invalid scalar low-level type cannot be printed as LLVM MIR");
-  }
-
-  const auto Semantics = Type.floatSemantics();
-  if (!Semantics) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "floating low-level type has no floating-point semantics");
-  }
-  switch (*Semantics) {
-  case FloatSemantics::IEEEHalf:
-    return "f16";
-  case FloatSemantics::BFloat:
-    return "bf16";
-  case FloatSemantics::IEEESingle:
-    return "f32";
-  case FloatSemantics::IEEEDouble:
-    return "f64";
-  case FloatSemantics::IEEEQuad:
-    return "f128";
-  case FloatSemantics::PPCDoubleDouble:
-  case FloatSemantics::Float8E5M2:
-  case FloatSemantics::Float8E5M2FNUZ:
-  case FloatSemantics::Float8E4M3:
-  case FloatSemantics::Float8E4M3FN:
-  case FloatSemantics::Float8E4M3FNUZ:
-  case FloatSemantics::Float8E4M3B11FNUZ:
-  case FloatSemantics::Float8E3M4:
-  case FloatSemantics::FloatTF32:
-  case FloatSemantics::Float8E8M0FNU:
-  case FloatSemantics::Float6E3M2FN:
-  case FloatSemantics::Float6E2M3FN:
-  case FloatSemantics::Float4E2M1FN:
-  case FloatSemantics::X87DoubleExtended:
-  case FloatSemantics::Count:
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "fixed LLVM MIR cannot spell this floating low-level type");
-  }
-  return makeUnexpected(ErrorCode::InternalError, "floating low-level type is unhandled");
-}
-
-[[nodiscard]] Expected<std::string> llvmMIRLowLevelType(LLT Type) {
-  if (!Type.isValid()) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "invalid low-level type cannot be printed as LLVM MIR");
-  }
-  if (!Type.isVector())
-    return llvmMIRScalarLowLevelType(Type);
-
-  const auto ElementType = Type.scalarElementType();
-  if (!ElementType) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "vector low-level type has no scalar element type");
-  }
-  auto Scalar = llvmMIRScalarLowLevelType(*ElementType);
-  if (!Scalar)
-    return std::unexpected(Scalar.error());
-
-  std::string Result = "<";
-  if (Type.isScalableVector())
-    Result.append("vscale x ");
-  Result.append(std::to_string(Type.minimumElementCount()));
-  Result.append(" x ");
-  Result.append(*Scalar);
-  Result.push_back('>');
-  return Result;
-}
-
-class BodyPrinter final {
-public:
-  BodyPrinter(const MachineFunction &Function, const TargetDescription &Target,
-              const MIParserContext &Context, MIPrinterOptions Options)
-      : Function(Function), Target(Target), Context(Context), Options(Options) {}
-
-  [[nodiscard]] Expected<std::string> run();
-
-private:
-  [[nodiscard]] Expected<void> initialize();
-  [[nodiscard]] Expected<void> printBlock(const MachineBasicBlock &Block);
-  [[nodiscard]] Expected<void> printInstruction(const MachineInstr &Instruction);
-  [[nodiscard]] Expected<void> printOperand(const MachineOperand &Operand, bool IsExplicitDef,
-                                            bool PrintVirtualType = true);
-  [[nodiscard]] Expected<void> printInlineAsmExtraInfo(const MachineOperand &Operand);
-  [[nodiscard]] Expected<std::size_t> printInlineAsmOperandFlag(const MachineOperand &Operand);
-  [[nodiscard]] Expected<void> printRegisterOperand(const MachineOperand &Operand,
-                                                    bool IsExplicitDef, bool PrintVirtualType);
-  [[nodiscard]] Expected<void> printTargetFlags(std::uint32_t Flags);
-  [[nodiscard]] Expected<void> printMetadataReference(MachineMetadataReference Reference);
-  [[nodiscard]] Expected<void> printDILocation(const MachineDILocation &Location,
-                                               std::size_t FrameIndex);
-  [[nodiscard]] Expected<void> printCFIInstruction(CFIInstructionIndex Index);
-  [[nodiscard]] Expected<void> printMemoryOperand(const MachineMemOperand &Operand);
-  [[nodiscard]] Expected<void> printMemoryBase(const MemoryBase &Base, std::int64_t Offset);
-
-  [[nodiscard]] Expected<std::uint32_t> blockID(const MachineBasicBlock &Block) const;
-  [[nodiscard]] Expected<std::uint32_t> virtualRegisterID(Register Reg) const;
-  [[nodiscard]] Expected<std::uint32_t> frameID(FrameIndex Index) const;
-  [[nodiscard]] Expected<std::uint32_t> constantPoolID(ConstantPoolIndex Index) const;
-  [[nodiscard]] Expected<std::uint32_t> jumpTableID(JumpTableIndex Index) const;
-  [[nodiscard]] Expected<std::uint32_t> metadataID(MachineMetadataReference Reference) const;
-  [[nodiscard]] Expected<void> appendPhysicalRegister(PhysicalRegister Register);
-  void appendBlockReference(const MachineBasicBlock &Block);
-  void appendOffset(std::int64_t Offset);
-
-  const MachineFunction &Function;
-  const TargetDescription &Target;
-  const MIParserContext &Context;
-  MIPrinterOptions Options;
-  std::string Output;
-  std::unordered_map<const MachineBasicBlock *, std::uint32_t> BlockIDs;
-};
-
-Expected<void> BodyPrinter::initialize() {
-  std::uint64_t NextID = 0;
-  for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
-       Block = Block->next()) {
-    if (NextID > std::numeric_limits<std::uint32_t>::max()) {
-      return makeUnexpected(ErrorCode::InvalidArgument,
-                            "machine function has too many blocks to print");
-    }
-    BlockIDs.emplace(Block, static_cast<std::uint32_t>(NextID));
-    ++NextID;
-  }
-  return {};
-}
-
-Expected<std::uint32_t> BodyPrinter::blockID(const MachineBasicBlock &Block) const {
-  const auto Found = BlockIDs.find(&Block);
-  if (Found == BlockIDs.end())
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "machine operand references an unlinked basic block");
-  return Found->second;
-}
-
-Expected<std::uint32_t> BodyPrinter::virtualRegisterID(Register Reg) const {
-  if (!Context.VirtualRegisters.empty()) {
-    const auto Found =
-        std::ranges::find_if(Context.VirtualRegisters,
-                             [Reg](const MIVirtualRegisterSlot &Slot) { return Slot.Reg == Reg; });
-    if (Found == Context.VirtualRegisters.end()) {
-      return makeUnexpected(ErrorCode::VerificationError, "virtual register has no MIR text slot");
-    }
-    return Found->TextID;
-  }
-  const auto VirtualIndex = Reg.virtualIndex();
-  if (!VirtualIndex)
-    return makeUnexpected(ErrorCode::VerificationError, "virtual register has no numeric index");
-  return *VirtualIndex;
-}
-
-Expected<std::uint32_t> BodyPrinter::frameID(FrameIndex Index) const {
-  const auto InStack = std::ranges::find_if(
-      Context.StackObjects, [Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; });
-  if (InStack != Context.StackObjects.end())
-    return InStack->TextID;
-  const auto InFixed =
-      std::ranges::find_if(Context.FixedStackObjects,
-                           [Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; });
-  if (InFixed != Context.FixedStackObjects.end())
-    return InFixed->TextID;
-  return makeUnexpected(ErrorCode::VerificationError, "frame index has no MIR text slot");
-}
-
-Expected<std::uint32_t> BodyPrinter::constantPoolID(ConstantPoolIndex Index) const {
-  const auto Found =
-      std::ranges::find_if(Context.ConstantPoolItems,
-                           [Index](const MIConstantPoolSlot &Slot) { return Slot.Index == Index; });
-  if (Found == Context.ConstantPoolItems.end()) {
-    return makeUnexpected(ErrorCode::VerificationError, "constant-pool index has no MIR text slot");
-  }
-  return Found->TextID;
-}
-
-Expected<std::uint32_t> BodyPrinter::jumpTableID(JumpTableIndex Index) const {
-  const auto Found = std::ranges::find_if(
-      Context.JumpTables, [Index](const MIJumpTableSlot &Slot) { return Slot.Index == Index; });
-  if (Found == Context.JumpTables.end())
-    return makeUnexpected(ErrorCode::VerificationError, "jump-table index has no MIR text slot");
-  return Found->TextID;
-}
-
-Expected<std::uint32_t> BodyPrinter::metadataID(MachineMetadataReference Reference) const {
-  const auto Found =
-      std::ranges::find_if(Context.Metadata, [Reference](const MIMetadataSlot &Slot) {
-        return Slot.Reference == Reference;
-      });
-  if (Found != Context.Metadata.end())
-    return Found->TextID;
-  if (Reference.scope() == MachineMetadataScope::MachineFunction)
-    return Reference.slot();
-  return makeUnexpected(ErrorCode::VerificationError,
-                        "embedded metadata reference has no MIR text slot");
-}
-
-Expected<void> BodyPrinter::appendPhysicalRegister(PhysicalRegister Register) {
-  const auto Name = Target.registers().physicalRegisterMIRName(Register);
-  if (!Name)
-    return makeUnexpected(ErrorCode::VerificationError, "physical register has no MIR name");
-  Output.push_back('$');
-  Output.append(*Name);
-  return {};
-}
-
-void BodyPrinter::appendBlockReference(const MachineBasicBlock &Block) {
-  Output.append("%bb.");
-  Output.append(std::to_string(BlockIDs.at(&Block)));
-}
-
-void BodyPrinter::appendOffset(std::int64_t Offset) {
-  if (Offset == 0)
-    return;
-  if (Offset > 0) {
-    Output.append(" + ");
-    Output.append(std::to_string(Offset));
-    return;
-  }
-  Output.append(" - ");
-  const std::uint64_t Magnitude = Offset == std::numeric_limits<std::int64_t>::min()
-                                      ? std::uint64_t{1} << 63U
-                                      : static_cast<std::uint64_t>(-Offset);
-  Output.append(std::to_string(Magnitude));
-}
-
-Expected<void> BodyPrinter::printTargetFlags(std::uint32_t Flags) {
-  if (Flags == 0)
-    return {};
-  auto Names = Target.mirText().operandFlagNames(Flags);
-  if (!Names)
-    return std::unexpected(Names.error());
-  Output.append("target-flags(");
-  for (std::size_t Index = 0; Index < Names->size(); ++Index) {
-    if (Index != 0)
-      Output.append(", ");
-    Output.append((*Names)[Index]);
-  }
-  Output.append(") ");
-  return {};
-}
-
-Expected<void> BodyPrinter::printMetadataReference(MachineMetadataReference Reference) {
-  auto ID = metadataID(Reference);
-  if (!ID)
-    return std::unexpected(ID.error());
-  Output.push_back('!');
-  Output.append(std::to_string(*ID));
-  return {};
-}
-
-Expected<void> BodyPrinter::printDILocation(const MachineDILocation &Location,
-                                            std::size_t FrameIndex) {
-  const auto Frames = Location.inlineChain();
-  if (FrameIndex >= Frames.size()) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "inline DILocation has an invalid frame index");
-  }
-  const MachineDILocationFrame &Frame = Frames[FrameIndex];
-  Output.append("!DILocation(line: ");
-  Output.append(std::to_string(Frame.Line));
-  if (Options.Mode == MIPrintMode::Complete || Frame.Column != 0) {
-    Output.append(", column: ");
-    Output.append(std::to_string(Frame.Column));
-  }
-  Output.append(", scope: ");
-  if (auto Printed = printMetadataReference(Frame.Scope); !Printed)
-    return Printed;
-  if (FrameIndex + 1U < Frames.size()) {
-    Output.append(", inlinedAt: ");
-    if (auto Printed = printDILocation(Location, FrameIndex + 1U); !Printed)
-      return Printed;
-  } else if (Location.terminalInlinedAt()) {
-    Output.append(", inlinedAt: ");
-    if (auto Printed = printMetadataReference(*Location.terminalInlinedAt()); !Printed) {
-      return Printed;
-    }
-  }
-  if (Options.Mode == MIPrintMode::Complete || Frame.IsImplicitCode) {
-    Output.append(", isImplicitCode: ");
-    Output.append(Frame.IsImplicitCode ? "true" : "false");
-  }
-  if (Options.Mode == MIPrintMode::Complete || Frame.AtomGroup != 0) {
-    Output.append(", atomGroup: ");
-    Output.append(std::to_string(Frame.AtomGroup));
-  }
-  if (Options.Mode == MIPrintMode::Complete || Frame.AtomRank != 0) {
-    Output.append(", atomRank: ");
-    Output.append(std::to_string(Frame.AtomRank));
-  }
-  Output.push_back(')');
-  return {};
-}
-
-Expected<void> BodyPrinter::printRegisterOperand(const MachineOperand &Operand, bool IsExplicitDef,
-                                                 bool PrintVirtualType) {
-  const RegisterOperandFlags Flags = Operand.registerFlags();
-  if (Flags.has(RegisterOperandFlag::Implicit) && Flags.has(RegisterOperandFlag::Define)) {
-    Output.append("implicit-def ");
-  } else {
-    if (!IsExplicitDef && Flags.has(RegisterOperandFlag::Define))
-      Output.append("def ");
-    if (Flags.has(RegisterOperandFlag::Implicit))
-      Output.append("implicit ");
-  }
-  if (Flags.has(RegisterOperandFlag::Dead))
-    Output.append("dead ");
-  if (Flags.has(RegisterOperandFlag::Kill))
-    Output.append("killed ");
-  if (Flags.has(RegisterOperandFlag::Undef))
-    Output.append("undef ");
-  if (Flags.has(RegisterOperandFlag::InternalRead))
-    Output.append("internal ");
-  if (Flags.has(RegisterOperandFlag::EarlyClobber))
-    Output.append("early-clobber ");
-  if (Flags.has(RegisterOperandFlag::Debug))
-    Output.append("debug-use ");
-  if (Flags.has(RegisterOperandFlag::Renamable))
-    Output.append("renamable ");
-
-  const auto RegisterValue = Operand.registerValue();
-  if (!RegisterValue) {
-    return makeUnexpected(ErrorCode::VerificationError, "register operand has no register value");
-  }
-  const Register Reg = *RegisterValue;
-  if (Reg.isNoRegister()) {
-    Output.append("$noreg");
-  } else if (Reg.isPhysical()) {
-    const auto Physical = Reg.physicalRegister();
-    if (!Physical) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "physical register has no physical-register value");
-    }
-    auto Printed = appendPhysicalRegister(*Physical);
-    if (!Printed)
-      return Printed;
-  } else {
-    auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
-    if (!Info)
-      return std::unexpected(Info.error());
-    const VirtualRegisterInfo &RegisterInfo = Info->get();
-    Output.push_back('%');
-    if (RegisterInfo.Name) {
-      Output.append(*RegisterInfo.Name);
-    } else {
-      auto VirtualIndex = virtualRegisterID(Reg);
-      if (!VirtualIndex)
-        return std::unexpected(VirtualIndex.error());
-      Output.append(std::to_string(*VirtualIndex));
-    }
-    if (Operand.subReg().value() != 0) {
-      const SubRegIndexDesc *SubReg = Target.registers().subRegIndex(Operand.subReg());
-      if (SubReg == nullptr)
-        return makeUnexpected(ErrorCode::VerificationError,
-                              "virtual register has an unknown subregister");
-      Output.push_back('.');
-      Output.append(SubReg->Name);
-    }
-
-    const bool PrintRegisterConstraint =
-        IsExplicitDef || Function.registerInfo().getUniqueVRegDef(Reg) == nullptr;
-    if (PrintRegisterConstraint && RegisterInfo.RegClass) {
-      const auto Name = Target.registers().registerClassMIRName(*RegisterInfo.RegClass);
-      if (!Name)
-        return makeUnexpected(ErrorCode::VerificationError,
-                              "virtual register class has no MIR name");
-      Output.push_back(':');
-      Output.append(*Name);
-    } else if (PrintRegisterConstraint && RegisterInfo.Bank) {
-      const auto Name = Target.registers().registerBankMIRName(*RegisterInfo.Bank);
-      if (!Name)
-        return makeUnexpected(ErrorCode::VerificationError,
-                              "virtual register bank has no MIR name");
-      Output.push_back(':');
-      Output.append(*Name);
-    } else if (PrintRegisterConstraint && RegisterInfo.Type) {
-      Output.append(":_");
-    }
-    if (PrintVirtualType && RegisterInfo.Type) {
-      auto Type = llvmMIRLowLevelType(*RegisterInfo.Type);
-      if (!Type)
-        return std::unexpected(Type.error());
-      Output.push_back('(');
-      Output.append(*Type);
-      Output.push_back(')');
-    }
-  }
-
-  if (Operand.tiedOperandIndex() && Operand.isUse()) {
-    Output.append("(tied-def ");
-    Output.append(std::to_string(*Operand.tiedOperandIndex()));
-    Output.push_back(')');
-  }
-  return {};
-}
-
-Expected<void> BodyPrinter::printCFIInstruction(CFIInstructionIndex Index) {
-  const MachineCFIInstruction *Instruction = Function.cfiInfo().instruction(Index);
-  if (Instruction == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "CFI operand references an unknown instruction");
-  const auto PrintRegister = [this](DwarfRegisterNumber Dwarf) -> Expected<void> {
-    const auto Register = Target.mirText().physicalRegister(Dwarf.value());
-    if (!Register) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "CFI instruction has no canonical physical register");
-    }
-    return appendPhysicalRegister(*Register);
-  };
-  const auto PrintRegisterOffset =
-      [this, &PrintRegister](std::string_view Name,
-                             const MachineCFIRegisterOffsetRule &Rule) -> Expected<void> {
-    Output.append(Name);
-    Output.push_back(' ');
-    if (auto Printed = PrintRegister(Rule.Register); !Printed)
-      return Printed;
-    Output.append(", ");
-    Output.append(std::to_string(Rule.Offset));
-    return {};
-  };
-
-  switch (Instruction->operation()) {
-  case MachineCFIOperation::SameValue: {
-    Output.append("same_value ");
-    return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register);
-  }
-  case MachineCFIOperation::RememberState:
-    Output.append("remember_state");
-    return {};
-  case MachineCFIOperation::RestoreState:
-    Output.append("restore_state");
-    return {};
-  case MachineCFIOperation::Offset:
-    return PrintRegisterOffset("offset", *Instruction->getIf<MachineCFIRegisterOffsetRule>());
-  case MachineCFIOperation::LLVMDefAddressSpaceCFA: {
-    const auto &Rule = *Instruction->getIf<MachineCFIAddressSpaceCFARule>();
-    Output.append("llvm_def_aspace_cfa ");
-    if (auto Printed = PrintRegister(Rule.Register); !Printed)
-      return Printed;
-    Output.append(", ");
-    Output.append(std::to_string(Rule.Offset));
-    Output.append(", ");
-    Output.append(std::to_string(Rule.AddressSpace));
-    return {};
-  }
-  case MachineCFIOperation::DefCFARegister:
-    Output.append("def_cfa_register ");
-    return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register);
-  case MachineCFIOperation::DefCFAOffset:
-    Output.append("def_cfa_offset ");
-    Output.append(std::to_string(Instruction->getIf<MachineCFIOffsetRule>()->Offset));
-    return {};
-  case MachineCFIOperation::DefCFA:
-    return PrintRegisterOffset("def_cfa", *Instruction->getIf<MachineCFIRegisterOffsetRule>());
-  case MachineCFIOperation::RelativeOffset:
-    return PrintRegisterOffset("rel_offset", *Instruction->getIf<MachineCFIRegisterOffsetRule>());
-  case MachineCFIOperation::AdjustCFAOffset:
-    Output.append("adjust_cfa_offset ");
-    Output.append(std::to_string(Instruction->getIf<MachineCFIOffsetRule>()->Offset));
-    return {};
-  case MachineCFIOperation::Escape: {
-    Output.append("escape ");
-    constexpr std::string_view Hex = "0123456789ABCDEF";
-    const auto &Bytes = Instruction->getIf<MachineCFIEscapeRule>()->Bytes;
-    for (std::size_t ByteIndex = 0; ByteIndex < Bytes.size(); ++ByteIndex) {
-      if (ByteIndex != 0)
-        Output.append(", ");
-      Output.append("0x");
-      Output.push_back(Hex[Bytes[ByteIndex] >> 4U]);
-      Output.push_back(Hex[Bytes[ByteIndex] & 0xFU]);
-    }
-    return {};
-  }
-  case MachineCFIOperation::Restore:
-    Output.append("restore ");
-    return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register);
-  case MachineCFIOperation::Undefined:
-    Output.append("undefined ");
-    return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register);
-  case MachineCFIOperation::Register: {
-    const auto &Rule = *Instruction->getIf<MachineCFIRegisterRegisterRule>();
-    Output.append("register ");
-    if (auto Printed = PrintRegister(Rule.Register); !Printed)
-      return Printed;
-    Output.append(", ");
-    return PrintRegister(Rule.SourceRegister);
-  }
-  case MachineCFIOperation::WindowSave:
-    Output.append("window_save");
-    return {};
-  case MachineCFIOperation::NegateReturnAddressState:
-    Output.append("negate_ra_sign_state");
-    return {};
-  case MachineCFIOperation::NegateReturnAddressStateWithPC:
-    Output.append("negate_ra_sign_state_with_pc");
-    return {};
-  case MachineCFIOperation::GNUArgsSize:
-  case MachineCFIOperation::Label:
-  case MachineCFIOperation::ValueOffset:
-    return makeUnexpected(
-        ErrorCode::Unsupported,
-        "CFI operation has no textual representation in the fixed LLVM MIR baseline");
-  case MachineCFIOperation::LLVMRegisterPair: {
-    const auto &Rule = *Instruction->getIf<MachineCFIRegisterPairRule>();
-    Output.append("llvm_register_pair ");
-    if (auto Printed = PrintRegister(Rule.Register); !Printed)
-      return Printed;
-    Output.append(", ");
-    if (auto Printed = PrintRegister(Rule.FirstRegister); !Printed)
-      return Printed;
-    Output.append(", ");
-    Output.append(std::to_string(Rule.FirstSizeInBits));
-    Output.append(", ");
-    if (auto Printed = PrintRegister(Rule.SecondRegister); !Printed)
-      return Printed;
-    Output.append(", ");
-    Output.append(std::to_string(Rule.SecondSizeInBits));
-    return {};
-  }
-  case MachineCFIOperation::LLVMVectorRegisters: {
-    const auto &Rule = *Instruction->getIf<MachineCFIVectorRegistersRule>();
-    Output.append("llvm_vector_registers ");
-    if (auto Printed = PrintRegister(Rule.Register); !Printed)
-      return Printed;
-    for (const MachineCFIVectorRegisterLane &Lane : Rule.VectorRegisters) {
-      Output.append(", ");
-      if (auto Printed = PrintRegister(Lane.Register); !Printed)
-        return Printed;
-      Output.append(", ");
-      Output.append(std::to_string(Lane.Lane));
-      Output.append(", ");
-      Output.append(std::to_string(Lane.SizeInBits));
-    }
-    return {};
-  }
-  case MachineCFIOperation::LLVMVectorOffset: {
-    const auto &Rule = *Instruction->getIf<MachineCFIVectorOffsetRule>();
-    Output.append("llvm_vector_offset ");
-    if (auto Printed = PrintRegister(Rule.Register); !Printed)
-      return Printed;
-    Output.append(", ");
-    Output.append(std::to_string(Rule.RegisterSizeInBits));
-    Output.append(", ");
-    if (auto Printed = PrintRegister(Rule.MaskRegister); !Printed)
-      return Printed;
-    Output.append(", ");
-    Output.append(std::to_string(Rule.MaskRegisterSizeInBits));
-    Output.append(", ");
-    Output.append(std::to_string(Rule.Offset));
-    return {};
-  }
-  case MachineCFIOperation::LLVMVectorRegisterMask: {
-    const auto &Rule = *Instruction->getIf<MachineCFIVectorRegisterMaskRule>();
-    Output.append("llvm_vector_register_mask ");
-    if (auto Printed = PrintRegister(Rule.Register); !Printed)
-      return Printed;
-    Output.append(", ");
-    if (auto Printed = PrintRegister(Rule.SpillRegister); !Printed)
-      return Printed;
-    Output.append(", ");
-    Output.append(std::to_string(Rule.SpillRegisterLaneSizeInBits));
-    Output.append(", ");
-    if (auto Printed = PrintRegister(Rule.MaskRegister); !Printed)
-      return Printed;
-    Output.append(", ");
-    Output.append(std::to_string(Rule.MaskRegisterSizeInBits));
-    return {};
-  }
-  case MachineCFIOperation::Count:
-    return makeUnexpected(ErrorCode::VerificationError, "CFI instruction has an invalid operation");
-  }
-  return makeUnexpected(ErrorCode::InternalError, "CFI printer did not handle an operation");
-}
-
-Expected<void> BodyPrinter::printOperand(const MachineOperand &Operand, bool IsExplicitDef,
-                                         bool PrintVirtualType) {
-  if (Operand.isRegister())
-    return printRegisterOperand(Operand, IsExplicitDef, PrintVirtualType);
-  if (auto Flags = printTargetFlags(Operand.targetFlags()); !Flags)
-    return Flags;
-
-  switch (Operand.kind()) {
-  case MachineOperand::Kind::Register:
-    return makeUnexpected(ErrorCode::InternalError,
-                          "register operand reached the non-register printer");
-  case MachineOperand::Kind::Immediate:
-    Output.append(std::to_string(*Operand.getIf<std::int64_t>()));
-    return {};
-  case MachineOperand::Kind::IntegerImmediate: {
-    const IntegerImmediate &Value = Operand.getIf<IntegerImmediateOperand>()->Value;
-    Output.push_back('i');
-    Output.append(std::to_string(Value.bitWidth()));
-    Output.push_back(' ');
-    if (Value.bitWidth() == 1U)
-      Output.append(Value.isZero() ? "false" : "true");
-    else
-      Output.append(Value.toSignedDecimal());
-    return {};
-  }
-  case MachineOperand::Kind::FloatingImmediate: {
-    const FloatingPointBits &Bits = Operand.getIf<FloatingImmediateOperand>()->Bits;
-    auto Constant = GenericConstantData::floating(Bits);
-    if (!Constant)
-      return std::unexpected(Constant.error());
-    auto Text = printLLVMConstant(*Constant);
-    if (!Text)
-      return std::unexpected(Text.error());
-    Output.append(*Text);
-    return {};
-  }
-  case MachineOperand::Kind::BasicBlock: {
-    const MachineBasicBlock *Block = Operand.getIf<BasicBlockOperand>()->Block;
-    if (Block == nullptr || !BlockIDs.contains(Block)) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "basic-block operand references an unlinked block");
-    }
-    appendBlockReference(*Block);
-    return {};
-  }
-  case MachineOperand::Kind::FrameIndex: {
-    const auto &Frame = *Operand.getIf<FrameIndexOperand>();
-    auto ID = frameID(Frame.Index);
-    if (!ID)
-      return std::unexpected(ID.error());
-    const auto Stack = std::ranges::find_if(
-        Context.StackObjects,
-        [Index = Frame.Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; });
-    if (Stack != Context.StackObjects.end()) {
-      Output.append("%stack.");
-      Output.append(std::to_string(*ID));
-      if (Stack->Name) {
-        Output.push_back('.');
-        Output.append(quoteName(*Stack->Name));
-      }
-    } else {
-      Output.append("%fixed-stack.");
-      Output.append(std::to_string(*ID));
-    }
-    appendOffset(Frame.Offset);
-    return {};
-  }
-  case MachineOperand::Kind::ConstantPoolIndex: {
-    const auto &Constant = *Operand.getIf<ConstantPoolOperand>();
-    auto ID = constantPoolID(Constant.Index);
-    if (!ID)
-      return std::unexpected(ID.error());
-    Output.append("%const.");
-    Output.append(std::to_string(*ID));
-    appendOffset(Constant.Offset);
-    return {};
-  }
-  case MachineOperand::Kind::JumpTableIndex: {
-    auto ID = jumpTableID(Operand.getIf<JumpTableOperand>()->Index);
-    if (!ID)
-      return std::unexpected(ID.error());
-    Output.append("%jump-table.");
-    Output.append(std::to_string(*ID));
-    return {};
-  }
-  case MachineOperand::Kind::GlobalSymbol: {
-    const auto &Global = *Operand.getIf<GlobalSymbolOperand>();
-    Output.push_back('@');
-    Output.append(quoteName(Global.Name));
-    appendOffset(Global.Offset);
-    return {};
-  }
-  case MachineOperand::Kind::ExternalSymbol: {
-    const auto &External = *Operand.getIf<ExternalSymbolOperand>();
-    Output.push_back('&');
-    Output.append(quoteName(External.Name));
-    appendOffset(External.Offset);
-    return {};
-  }
-  case MachineOperand::Kind::BlockAddress: {
-    const auto &Address = *Operand.getIf<BlockAddressOperand>();
-    Output.append("blockaddress(@");
-    Output.append(quoteName(Address.FunctionName));
-    Output.append(", %ir-block.");
-    Output.append(quoteName(Address.BlockName));
-    Output.push_back(')');
-    appendOffset(Address.Offset);
-    return {};
-  }
-  case MachineOperand::Kind::RegisterMask: {
-    const RegisterMask &Mask = *Operand.getIf<RegisterMaskOperand>()->Mask;
-    if (const auto Name = Target.mirText().registerMaskName(Mask)) {
-      Output.append(*Name);
-      return {};
-    }
-    Output.append("CustomRegMask(");
-    bool First = true;
-    for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) {
-      if (!Mask.preserves(Register.Reg))
-        continue;
-      if (!First)
-        Output.append(", ");
-      First = false;
-      if (auto Printed = appendPhysicalRegister(Register.Reg); !Printed)
-        return Printed;
-    }
-    Output.push_back(')');
-    return {};
-  }
-  case MachineOperand::Kind::LiveOutRegisterMask: {
-    const RegisterMask &Mask = *Operand.getIf<LiveOutRegisterMaskOperand>()->Mask;
-    Output.append("liveout(");
-    bool First = true;
-    for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) {
-      if (!Mask.preserves(Register.Reg))
-        continue;
-      if (!First)
-        Output.append(", ");
-      First = false;
-      if (auto Printed = appendPhysicalRegister(Register.Reg); !Printed)
-        return Printed;
-    }
-    Output.push_back(')');
-    return {};
-  }
-  case MachineOperand::Kind::Metadata: {
-    const MachineMetadataValue &Value = Operand.getIf<MetadataOperand>()->Value;
-    if (const auto *Reference = Value.reference())
-      return printMetadataReference(*Reference);
-    const MachineDIExpression *Expression = Value.expression();
-    if (Expression == nullptr) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "metadata operand has no printable value");
-    }
-    Output.append(formatDIExpression(Expression->elements()));
-    return {};
-  }
-  case MachineOperand::Kind::TargetIndex: {
-    const auto &TargetIndexValue = *Operand.getIf<TargetIndexOperand>();
-    const auto Name = Target.mirText().targetIndexName(TargetIndexValue.Index);
-    if (!Name)
-      return makeUnexpected(ErrorCode::VerificationError, "target index has no MIR name");
-    Output.append("target-index(");
-    Output.append(*Name);
-    Output.push_back(')');
-    appendOffset(TargetIndexValue.Offset);
-    return {};
-  }
-  case MachineOperand::Kind::MCSymbol: {
-    const auto &Symbol = *Operand.getIf<MCSymbolOperand>();
-    Output.append("<mcsymbol ");
-    Output.append(quoteName(Symbol.Name));
-    Output.push_back('>');
-    appendOffset(Symbol.Offset);
-    return {};
-  }
-  case MachineOperand::Kind::CFIIndex:
-    return printCFIInstruction(Operand.getIf<CFIIndexOperand>()->Index);
-  case MachineOperand::Kind::Intrinsic: {
-    const auto Name = Target.mirText().intrinsicName(Operand.getIf<IntrinsicOperand>()->ID);
-    if (!Name)
-      return makeUnexpected(ErrorCode::VerificationError, "intrinsic has no MIR name");
-    Output.append("intrinsic(@");
-    Output.append(*Name);
-    Output.push_back(')');
-    return {};
-  }
-  case MachineOperand::Kind::Predicate: {
-    const PredicateOperand &Predicate = *Operand.getIf<PredicateOperand>();
-    if (const auto *Integer = Predicate.integerPredicate()) {
-      Output.append("intpred(");
-      Output.append(integerPredicateName(*Integer));
-    } else {
-      Output.append("floatpred(");
-      Output.append(floatingPredicateName(*Predicate.floatingPredicate()));
-    }
-    Output.push_back(')');
-    return {};
-  }
-  case MachineOperand::Kind::ShuffleMask: {
-    Output.append("shufflemask(");
-    const auto &Elements = Operand.getIf<ShuffleMaskOperand>()->Elements;
-    for (std::size_t Index = 0; Index < Elements.size(); ++Index) {
-      if (Index != 0)
-        Output.append(", ");
-      if (Elements[Index] == ShuffleMaskOperand::Undef)
-        Output.append("undef");
-      else
-        Output.append(std::to_string(Elements[Index]));
-    }
-    Output.push_back(')');
-    return {};
-  }
-  case MachineOperand::Kind::DebugInstrRef: {
-    const auto &Reference = *Operand.getIf<DebugInstrRefOperand>();
-    Output.append("dbg-instr-ref(");
-    Output.append(std::to_string(Reference.InstructionNumber));
-    Output.append(", ");
-    Output.append(std::to_string(Reference.OperandNumber));
-    Output.push_back(')');
-    return {};
-  }
-  case MachineOperand::Kind::LaneMask:
-    Output.append("lanemask(0x");
-    Output.append(
-        hexadecimalWords(std::array{Operand.getIf<LaneMaskOperand>()->Mask.rawValue()}, 64));
-    Output.push_back(')');
-    return {};
-  case MachineOperand::Kind::Count:
-    return makeUnexpected(ErrorCode::VerificationError, "machine operand has an invalid kind");
-  }
-  return makeUnexpected(ErrorCode::InternalError, "machine operand printer did not handle a kind");
-}
-
-Expected<void> BodyPrinter::printInlineAsmExtraInfo(const MachineOperand &Operand) {
-  const auto *Value = Operand.getIf<std::int64_t>();
-  if (Value == nullptr || Operand.targetFlags() != 0) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "inline assembly extra info must be an unflagged immediate");
-  }
-  const InlineAsmExtraInfo ExtraInfo(static_cast<std::uint32_t>(*Value));
-  const std::vector<std::string_view> Names = ExtraInfo.names();
-  for (std::size_t Index = 0; Index < Names.size(); ++Index) {
-    if (Index != 0)
-      Output.push_back(' ');
-    Output.append(Names[Index]);
-  }
-  return {};
-}
-
-Expected<std::size_t> BodyPrinter::printInlineAsmOperandFlag(const MachineOperand &Operand) {
-  const auto *Value = Operand.getIf<std::int64_t>();
-  if (Value == nullptr || *Value < 0 ||
-      static_cast<std::uint64_t>(*Value) > std::numeric_limits<std::uint32_t>::max() ||
-      Operand.targetFlags() != 0) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "inline assembly operand flag must be an unflagged 32-bit immediate");
-  }
-
-  const InlineAsmOperandFlag Flag(static_cast<std::uint32_t>(*Value));
-  const std::string_view KindName = InlineAsmOperandFlag::kindName(Flag.kind());
-  if (KindName.empty() || Flag.operandCount() == 0) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "inline assembly operand flag has an invalid kind or operand count");
-  }
-  Output.append(KindName);
-
-  if (const auto MatchingOperand = Flag.matchingOperand()) {
-    Output.append(" tiedto:$");
-    Output.append(std::to_string(*MatchingOperand));
-    return Flag.operandCount();
-  }
-  if (Flag.kind() == InlineAsmOperandKind::Memory ||
-      Flag.kind() == InlineAsmOperandKind::Function) {
-    const auto Constraint = InlineAsmOperandFlag::memoryConstraintName(Flag.memoryConstraint());
-    if (Constraint) {
-      Output.push_back(':');
-      Output.append(*Constraint);
-    }
-    return Flag.operandCount();
-  }
-  if (Flag.isRegisterKind()) {
-    if (const auto RegisterClass = Flag.registerClass()) {
-      const auto Name =
-          Target.registers().registerClassMIRName(RegisterClassID::fromValue(*RegisterClass));
-      if (!Name) {
-        return makeUnexpected(ErrorCode::VerificationError,
-                              "inline assembly operand flag references an unknown register class");
-      }
-      Output.push_back(':');
-      Output.append(*Name);
-    }
-  }
-  return Flag.operandCount();
-}
-
-Expected<void> BodyPrinter::printMemoryBase(const MemoryBase &Base, std::int64_t Offset) {
-  switch (Base.kind()) {
-  case MemoryBase::Kind::Unknown:
-    Output.append("unknown-address");
-    break;
-  case MemoryBase::Kind::FrameIndex: {
-    const auto Frame = Base.frameIndex();
-    if (!Frame) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "frame-index memory base has no frame index");
-    }
-    const FrameIndex Index = *Frame;
-    auto ID = frameID(Index);
-    if (!ID)
-      return std::unexpected(ID.error());
-    const auto Stack =
-        std::ranges::find_if(Context.StackObjects, [Index](const MIFrameObjectSlot &Slot) {
-          return Slot.Index == Index;
-        });
-    if (Stack != Context.StackObjects.end()) {
-      Output.append("%stack.");
-      Output.append(std::to_string(*ID));
-      if (Stack->Name) {
-        Output.push_back('.');
-        Output.append(quoteName(*Stack->Name));
-      }
-    } else {
-      Output.append("%fixed-stack.");
-      Output.append(std::to_string(*ID));
-    }
-    break;
-  }
-  case MemoryBase::Kind::ConstantPool: {
-    const auto Constant = Base.constantPoolIndex();
-    if (!Constant) {
-      return makeUnexpected(ErrorCode::VerificationError, "constant-pool memory base has no index");
-    }
-    auto ID = constantPoolID(*Constant);
-    if (!ID)
-      return std::unexpected(ID.error());
-    Output.append("%const.");
-    Output.append(std::to_string(*ID));
-    break;
-  }
-  case MemoryBase::Kind::JumpTable: {
-    const auto JumpTable = Base.jumpTableIndex();
-    if (!JumpTable) {
-      return makeUnexpected(ErrorCode::VerificationError, "jump-table memory base has no index");
-    }
-    auto ID = jumpTableID(*JumpTable);
-    if (!ID)
-      return std::unexpected(ID.error());
-    Output.append("%jump-table.");
-    Output.append(std::to_string(*ID));
-    break;
-  }
-  case MemoryBase::Kind::GlobalSymbol: {
-    const auto &Name = Base.symbolName();
-    if (!Name) {
-      return makeUnexpected(ErrorCode::VerificationError, "global memory base has no symbol name");
-    }
-    Output.push_back('@');
-    Output.append(quoteName(*Name));
-    break;
-  }
-  case MemoryBase::Kind::ExternalSymbol: {
-    const auto &Name = Base.symbolName();
-    if (!Name) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "external memory base has no symbol name");
-    }
-    Output.push_back('&');
-    Output.append(quoteName(*Name));
-    break;
-  }
-  case MemoryBase::Kind::IRValue: {
-    const auto &Name = Base.symbolName();
-    if (!Name) {
-      return makeUnexpected(ErrorCode::VerificationError, "IR-value memory base has no value name");
-    }
-    Output.append("%ir.");
-    Output.append(quoteName(*Name));
-    break;
-  }
-  case MemoryBase::Kind::IRConstant: {
-    const GenericConstantData *Constant = Base.constantValue();
-    if (Constant == nullptr) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "IR-constant memory base has no constant value");
-    }
-    auto Printed = printLLVMConstant(*Constant);
-    if (!Printed)
-      return std::unexpected(Printed.error());
-    Output.push_back('`');
-    Output.append(*Printed);
-    Output.push_back('`');
-    break;
-  }
-  case MemoryBase::Kind::Stack:
-    Output.append("stack");
-    break;
-  case MemoryBase::Kind::GOT:
-    Output.append("got");
-    break;
-  case MemoryBase::Kind::ConstantPoolPseudo:
-    Output.append("constant-pool");
-    break;
-  case MemoryBase::Kind::JumpTablePseudo:
-    Output.append("jump-table");
-    break;
-  case MemoryBase::Kind::CallEntryGlobal: {
-    const auto &Name = Base.symbolName();
-    if (!Name) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "global call-entry memory base has no symbol name");
-    }
-    Output.append("call-entry @");
-    Output.append(quoteName(*Name));
-    break;
-  }
-  case MemoryBase::Kind::CallEntryExternal: {
-    const auto &Name = Base.symbolName();
-    if (!Name) {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "external call-entry memory base has no symbol name");
-    }
-    Output.append("call-entry &");
-    Output.append(quoteName(*Name));
-    break;
-  }
-  case MemoryBase::Kind::Custom: {
-    const auto &Name = Base.symbolName();
-    if (!Name) {
-      return makeUnexpected(ErrorCode::VerificationError, "custom memory base has no name");
-    }
-    Output.append("custom ");
-    Output.append(quoteName(*Name));
-    break;
-  }
-  }
-  appendOffset(Offset);
-  return {};
-}
-
-Expected<void> BodyPrinter::printMemoryOperand(const MachineMemOperand &Operand) {
-  const MachineMemOperandDesc &Description = Operand.description();
-  Output.push_back('(');
-  if (Description.Accesses.has(MemoryAccessFlag::Volatile))
-    Output.append("volatile ");
-  if (Description.Accesses.has(MemoryAccessFlag::NonTemporal))
-    Output.append("non-temporal ");
-  if (Description.Accesses.has(MemoryAccessFlag::Dereferenceable))
-    Output.append("dereferenceable ");
-  if (Description.Accesses.has(MemoryAccessFlag::Invariant))
-    Output.append("invariant ");
-  if (Description.TargetFlags != 0) {
-    auto Names = Target.mirText().machineMemOperandFlagNames(Description.TargetFlags);
-    if (!Names)
-      return std::unexpected(Names.error());
-    for (std::string_view Name : *Names) {
-      Output.append(quoteName(Name));
-      Output.push_back(' ');
-    }
-  }
-
-  if (Operand.isLoad())
-    Output.append("load");
-  else
-    Output.append("store");
-  if (Operand.isLoad() && Operand.isStore())
-    Output.append(" store");
-
-  if (Description.SyncScope.value() != 0) {
-    const auto Scope =
-        std::ranges::find_if(Context.SyncScopes, [&Description](const MISyncScopeSlot &Slot) {
-          return Slot.ID == Description.SyncScope;
-        });
-    if (Scope == Context.SyncScopes.end()) {
-      return makeUnexpected(ErrorCode::VerificationError, "synchronization scope has no MIR name");
-    }
-    Output.append(" syncscope(");
-    Output.append(quoteString(Scope->Name));
-    Output.push_back(')');
-  }
-  if (Description.SuccessOrdering != AtomicOrdering::NotAtomic) {
-    Output.push_back(' ');
-    Output.append(atomicOrderingName(Description.SuccessOrdering));
-  }
-  if (Description.FailureOrdering != AtomicOrdering::NotAtomic) {
-    Output.push_back(' ');
-    Output.append(atomicOrderingName(Description.FailureOrdering));
-  }
-
-  Output.push_back(' ');
-  if (Description.MemoryType) {
-    auto Type = llvmMIRLowLevelType(*Description.MemoryType);
-    if (!Type)
-      return std::unexpected(Type.error());
-    Output.push_back('(');
-    Output.append(*Type);
-    Output.push_back(')');
-  } else {
-    switch (Description.Size.kind()) {
-    case MemorySize::Kind::Unknown:
-      Output.append("unknown-size");
-      break;
-    case MemorySize::Kind::Fixed:
-      Output.append(std::to_string(Description.Size.minimumBytes()));
-      break;
-    case MemorySize::Kind::Scalable:
-      Output.append("(<vscale x ");
-      Output.append(std::to_string(Description.Size.minimumBytes()));
-      Output.append(" x s8>)");
-      break;
-    }
-  }
-
-  const std::string_view Preposition =
-      Operand.isLoad() && Operand.isStore() ? " on " : (Operand.isLoad() ? " from " : " into ");
-  if (Description.Base.kind() != MemoryBase::Kind::Unknown || Description.Offset != 0) {
-    Output.append(Preposition);
-    if (auto Printed = printMemoryBase(Description.Base, Description.Offset); !Printed) {
-      return Printed;
-    }
-  }
-
-  if (!Description.Size.isKnown() ||
-      (Description.Size.minimumBytes() != 0 &&
-       Description.Align.bytes() != Description.Size.minimumBytes())) {
-    Output.append(", align ");
-    Output.append(std::to_string(Description.Align.bytes()));
-  }
-  if (Description.BaseAlign && *Description.BaseAlign != Description.Align) {
-    Output.append(", basealign ");
-    Output.append(std::to_string(Description.BaseAlign->bytes()));
-  }
-  if (Description.AddressSpace != 0) {
-    Output.append(", addrspace ");
-    Output.append(std::to_string(Description.AddressSpace));
-  }
-
-  const auto PrintMetadata =
-      [this](std::string_view Name,
-             const std::optional<MachineMetadataReference> &Reference) -> Expected<void> {
-    if (!Reference)
-      return {};
-    Output.append(", ");
-    Output.append(Name);
-    Output.push_back(' ');
-    return printMetadataReference(*Reference);
-  };
-  if (auto Printed = PrintMetadata("!tbaa", Description.TBAA); !Printed)
-    return Printed;
-  if (auto Printed = PrintMetadata("!alias.scope", Description.AliasScope); !Printed) {
-    return Printed;
-  }
-  if (auto Printed = PrintMetadata("!noalias", Description.NoAlias); !Printed)
-    return Printed;
-  if (auto Printed = PrintMetadata("!noalias.addrspace", Description.NoAliasAddressSpace);
-      !Printed) {
-    return Printed;
-  }
-  if (auto Printed = PrintMetadata("!range", Description.RangeMetadata); !Printed) {
-    return Printed;
-  }
-  Output.push_back(')');
-  return {};
-}
-
-Expected<void> BodyPrinter::printInstruction(const MachineInstr &Instruction) {
-  const InstructionDescriptor *Descriptor = Target.instructions().descriptor(Instruction.opcode());
-  if (Descriptor == nullptr) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "machine instruction has no target descriptor");
-  }
-  std::size_t ExplicitDefs = 0;
-  while (ExplicitDefs < Instruction.operandCount()) {
-    const MachineOperand *Operand = Instruction.operandAt(ExplicitDefs);
-    if (Operand == nullptr || !Operand->isRegister() || Operand->isImplicit() ||
-        !Operand->isDef() || Operand->isUse())
-      break;
-    ++ExplicitDefs;
-  }
-
-  Output.append("  ");
-  for (std::size_t Index = 0; Index < ExplicitDefs; ++Index) {
-    if (Index != 0)
-      Output.append(", ");
-    if (auto Printed = printOperand(*Instruction.operandAt(Index), true); !Printed) {
-      return Printed;
-    }
-  }
-  if (ExplicitDefs != 0)
-    Output.append(" = ");
-
-  const MachineInstrFlags Flags = Instruction.flags();
-  constexpr std::array FlagSpellings{
-      std::pair{MachineInstrFlag::FrameSetup, std::string_view{"frame-setup"}},
-      std::pair{MachineInstrFlag::FrameDestroy, std::string_view{"frame-destroy"}},
-      std::pair{MachineInstrFlag::FastMathNoNaNs, std::string_view{"nnan"}},
-      std::pair{MachineInstrFlag::FastMathNoInfs, std::string_view{"ninf"}},
-      std::pair{MachineInstrFlag::FastMathNoSignedZeros, std::string_view{"nsz"}},
-      std::pair{MachineInstrFlag::FastMathAllowReciprocal, std::string_view{"arcp"}},
-      std::pair{MachineInstrFlag::FastMathAllowContract, std::string_view{"contract"}},
-      std::pair{MachineInstrFlag::FastMathApproxFunc, std::string_view{"afn"}},
-      std::pair{MachineInstrFlag::FastMathAllowReassoc, std::string_view{"reassoc"}},
-      std::pair{MachineInstrFlag::NoUnsignedWrap, std::string_view{"nuw"}},
-      std::pair{MachineInstrFlag::NoSignedWrap, std::string_view{"nsw"}},
-      std::pair{MachineInstrFlag::NoUSWrap, std::string_view{"nusw"}},
-      std::pair{MachineInstrFlag::Exact, std::string_view{"exact"}},
-      std::pair{MachineInstrFlag::NoFPExcept, std::string_view{"nofpexcept"}},
-      std::pair{MachineInstrFlag::Unpredictable, std::string_view{"unpredictable"}},
-      std::pair{MachineInstrFlag::NoConvergent, std::string_view{"noconvergent"}},
-      std::pair{MachineInstrFlag::NonNeg, std::string_view{"nneg"}},
-      std::pair{MachineInstrFlag::Disjoint, std::string_view{"disjoint"}},
-      std::pair{MachineInstrFlag::SameSign, std::string_view{"samesign"}},
-      std::pair{MachineInstrFlag::InBounds, std::string_view{"inbounds"}},
-  };
-  for (const auto &[Flag, Spelling] : FlagSpellings) {
-    if (Flags.has(Flag)) {
-      Output.append(Spelling);
-      Output.push_back(' ');
-    }
-  }
-  Output.append(Descriptor->Name);
-
-  bool HasPrintedTail = false;
-  const bool IsInlineAsm = Descriptor->Name == "INLINEASM" || Descriptor->Name == "INLINEASM_BR";
-  std::size_t InlineAsmPayloadRemaining = 0;
-  for (std::size_t Index = ExplicitDefs; Index < Instruction.operandCount(); ++Index) {
-    Output.append(HasPrintedTail ? ", " : " ");
-    HasPrintedTail = true;
-    const MachineOperand &Operand = *Instruction.operandAt(Index);
-    if (IsInlineAsm && Index == InlineAsmExtraInfoOperandIndex) {
-      auto Printed = printInlineAsmExtraInfo(Operand);
-      if (!Printed)
-        return Printed;
-      continue;
-    }
-    if (IsInlineAsm && Index >= InlineAsmFirstConstraintOperandIndex &&
-        InlineAsmPayloadRemaining == 0) {
-      auto OperandCount = printInlineAsmOperandFlag(Operand);
-      if (!OperandCount)
-        return std::unexpected(OperandCount.error());
-      InlineAsmPayloadRemaining = *OperandCount;
-      continue;
-    }
-    bool PrintVirtualType = true;
-    if (Descriptor->Flags.has(InstructionFlag::Generic) &&
-        !Descriptor->Flags.has(InstructionFlag::Phi) && Index < Descriptor->Operands.size()) {
-      const std::optional<OperandTypeConstraint> &UseConstraint =
-          Descriptor->Operands[Index].TypeConstraint;
-      if (UseConstraint) {
-        for (std::size_t DefIndex = 0; DefIndex < ExplicitDefs; ++DefIndex) {
-          if (DefIndex >= Descriptor->Operands.size())
-            break;
-          const std::optional<OperandTypeConstraint> &DefConstraint =
-              Descriptor->Operands[DefIndex].TypeConstraint;
-          if (DefConstraint && DefConstraint->EqualityGroup == UseConstraint->EqualityGroup) {
-            PrintVirtualType = false;
-            break;
-          }
-        }
-      } else if (const Register *UseRegister = Operand.getIf<Register>();
-                 UseRegister != nullptr && UseRegister->isVirtual()) {
-        const auto UseInfo = Function.registerInfo().virtualRegisterInfo(*UseRegister);
-        for (std::size_t DefIndex = 0; DefIndex < ExplicitDefs && UseInfo && UseInfo->get().Type;
-             ++DefIndex) {
-          if (DefIndex >= Descriptor->Operands.size() ||
-              Descriptor->Operands[DefIndex].TypeConstraint)
-            continue;
-          const MachineOperand *DefOperand = Instruction.operandAt(DefIndex);
-          const Register *DefRegister =
-              DefOperand == nullptr ? nullptr : DefOperand->getIf<Register>();
-          if (DefRegister == nullptr || !DefRegister->isVirtual())
-            continue;
-          const auto DefInfo = Function.registerInfo().virtualRegisterInfo(*DefRegister);
-          if (DefInfo && DefInfo->get().Type == UseInfo->get().Type) {
-            PrintVirtualType = false;
-            break;
-          }
-        }
-      }
-    }
-    auto Printed = printOperand(Operand, false, PrintVirtualType);
-    if (!Printed)
-      return Printed;
-    if (IsInlineAsm && InlineAsmPayloadRemaining != 0)
-      --InlineAsmPayloadRemaining;
-  }
-  if (InlineAsmPayloadRemaining != 0)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "inline assembly operand flag has missing payload operands");
-
-  const auto BeginAttachment = [this, &HasPrintedTail](std::string_view Name) {
-    Output.append(HasPrintedTail ? ", " : " ");
-    HasPrintedTail = true;
-    Output.append(Name);
-    Output.push_back(' ');
-  };
-  if (Instruction.preInstrSymbol()) {
-    BeginAttachment("pre-instr-symbol");
-    Output.append("<mcsymbol ");
-    Output.append(quoteName(*Instruction.preInstrSymbol()));
-    Output.push_back('>');
-  }
-  if (Instruction.postInstrSymbol()) {
-    BeginAttachment("post-instr-symbol");
-    Output.append("<mcsymbol ");
-    Output.append(quoteName(*Instruction.postInstrSymbol()));
-    Output.push_back('>');
-  }
-  if (Instruction.heapAllocMarker()) {
-    BeginAttachment("heap-alloc-marker");
-    if (auto Printed = printMetadataReference(*Instruction.heapAllocMarker()); !Printed) {
-      return Printed;
-    }
-  }
-  if (Instruction.pcSections()) {
-    BeginAttachment("pcsections");
-    if (auto Printed = printMetadataReference(*Instruction.pcSections()); !Printed) {
-      return Printed;
-    }
-  }
-  if (Instruction.mmra()) {
-    BeginAttachment("mmra");
-    if (auto Printed = printMetadataReference(*Instruction.mmra()); !Printed)
-      return Printed;
-  }
-  if (Instruction.cfiType()) {
-    BeginAttachment("cfi-type");
-    Output.append(std::to_string(*Instruction.cfiType()));
-  }
-  if (Instruction.deactivationSymbol()) {
-    BeginAttachment("deactivation-symbol");
-    Output.push_back('@');
-    Output.append(quoteName(*Instruction.deactivationSymbol()));
-  }
-  if (Instruction.debugInstrNumber() != 0) {
-    BeginAttachment("debug-instr-number");
-    Output.append(std::to_string(Instruction.debugInstrNumber()));
-  }
-  if (Instruction.debugLocation()) {
-    BeginAttachment("debug-location");
-    if (const MachineMetadataReference *Reference = Instruction.debugLocation()->reference()) {
-      if (auto Printed = printMetadataReference(*Reference); !Printed)
-        return Printed;
-    } else if (const MachineDILocation *Location = Instruction.debugLocation()->inlineLocation()) {
-      if (auto Printed = printDILocation(*Location, 0); !Printed)
-        return Printed;
-    } else {
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "machine debug location has no printable value");
-    }
-  }
-
-  if (!Instruction.memOperands().empty()) {
-    Output.append(" :: ");
-    for (std::size_t Index = 0; Index < Instruction.memOperands().size(); ++Index) {
-      if (Index != 0)
-        Output.append(", ");
-      if (auto Printed = printMemoryOperand(*Instruction.memOperands()[Index]); !Printed) {
-        return Printed;
-      }
-    }
-  }
-  return {};
-}
-
-Expected<void> BodyPrinter::printBlock(const MachineBasicBlock &Block) {
-  auto ID = blockID(Block);
-  if (!ID)
-    return std::unexpected(ID.error());
-  Output.append("bb.");
-  Output.append(std::to_string(*ID));
-  const auto &Name = Block.attributes().name();
-  if (Name) {
-    Output.push_back('.');
-    Output.append(quoteName(*Name));
-  }
-
-  std::vector<std::string> Attributes;
-  const auto AddFlag = [&Block, &Attributes](MachineBasicBlockAttribute Attribute,
-                                             std::string_view Spelling) {
-    if (Block.attributes().has(Attribute))
-      Attributes.emplace_back(Spelling);
-  };
-  AddFlag(MachineBasicBlockAttribute::MachineBlockAddressTaken, "machine-block-address-taken");
-  if (Block.attributes().has(MachineBasicBlockAttribute::IRBlockAddressTaken)) {
-    std::string Attribute = "ir-block-address-taken";
-    const auto &IRBlockName = Block.attributes().irBlockName();
-    if (IRBlockName) {
-      Attribute.append(" %ir-block.");
-      Attribute.append(quoteName(*IRBlockName));
-    }
-    Attributes.push_back(std::move(Attribute));
-  } else {
-    const auto &IRBlockName = Block.attributes().irBlockName();
-    if (IRBlockName)
-      Attributes.push_back("%ir-block." + quoteName(*IRBlockName));
-  }
-  AddFlag(MachineBasicBlockAttribute::LandingPad, "landing-pad");
-  AddFlag(MachineBasicBlockAttribute::InlineAsmBrIndirectTarget, "inlineasm-br-indirect-target");
-  AddFlag(MachineBasicBlockAttribute::EHFuncletEntry, "ehfunclet-entry");
-  AddFlag(MachineBasicBlockAttribute::EHScopeEntry, "ehscope-entry");
-  if (Block.attributes().alignment().bytes() != 1) {
-    Attributes.push_back("align " + std::to_string(Block.attributes().alignment().bytes()));
-  }
-  const MachineBasicBlockSectionID Section = Block.attributes().sectionID();
-  if (!Section.isDefault()) {
-    std::string Value;
-    if (Section.kind() == MachineBasicBlockSectionKind::Exception)
-      Value = "Exception";
-    else if (Section.kind() == MachineBasicBlockSectionKind::Cold)
-      Value = "Cold";
-    else
-      Value = std::to_string(Section.number());
-    Attributes.push_back("bbsections " + Value);
-  }
-  const auto &ProfileID = Block.attributes().profileID();
-  if (ProfileID) {
-    const MachineBasicBlockProfileID Profile = *ProfileID;
-    Attributes.push_back("bb_id " + std::to_string(Profile.BaseID) + "." +
-                         std::to_string(Profile.CloneID));
-  }
-  if (Block.attributes().callFrameSize() != 0) {
-    Attributes.push_back("call-frame-size " + std::to_string(Block.attributes().callFrameSize()));
-  }
-  if (!Attributes.empty()) {
-    Output.append(" (");
-    for (std::size_t Index = 0; Index < Attributes.size(); ++Index) {
-      if (Index != 0)
-        Output.append(", ");
-      Output.append(Attributes[Index]);
-    }
-    Output.push_back(')');
-  }
-  Output.append(":\n");
-
-  bool HasLineAttributes = false;
-  if (!Block.successors().empty()) {
-    Output.append("  successors: ");
-    for (std::size_t Index = 0; Index < Block.successors().size(); ++Index) {
-      if (Index != 0)
-        Output.append(", ");
-      const MachineSuccessor &Successor = Block.successors()[Index];
-      if (Successor.Block == nullptr || !BlockIDs.contains(Successor.Block)) {
-        return makeUnexpected(ErrorCode::VerificationError,
-                              "successor references an unlinked block");
-      }
-      appendBlockReference(*Successor.Block);
-      Output.append("(0x");
-      Output.append(hexadecimalWords(
-          std::array{static_cast<std::uint64_t>(Successor.Probability.rawValue())}, 32));
-      Output.push_back(')');
-    }
-    Output.push_back('\n');
-    HasLineAttributes = true;
-  }
-  if (!Block.liveIns().empty()) {
-    Output.append("  liveins: ");
-    for (std::size_t Index = 0; Index < Block.liveIns().size(); ++Index) {
-      if (Index != 0)
-        Output.append(", ");
-      const MachineBasicBlockLiveIn &LiveIn = Block.liveIns()[Index];
-      if (auto Printed = appendPhysicalRegister(LiveIn.PhysReg); !Printed)
-        return Printed;
-      if (LiveIn.Lanes != LaneMask::all()) {
-        Output.append(": 0x");
-        Output.append(hexadecimalWords(std::array{LiveIn.Lanes.rawValue()}, 64));
-      }
-    }
-    Output.push_back('\n');
-    HasLineAttributes = true;
-  }
-
-  if (HasLineAttributes && Block.firstInstruction() != nullptr)
-    Output.push_back('\n');
-
-  bool InBundle = false;
-  for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr;
-       Instruction = Instruction->next()) {
-    if (auto Printed = printInstruction(*Instruction); !Printed)
-      return Printed;
-    const bool HasPredecessor = Instruction->flags().has(MachineInstrFlag::BundledWithPred);
-    const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc);
-    if (!InBundle && HasSuccessor && !HasPredecessor) {
-      Output.append(" {");
-      InBundle = true;
-    }
-    Output.push_back('\n');
-    if (InBundle && !HasSuccessor) {
-      Output.append("  }\n");
-      InBundle = false;
-    }
-  }
-  if (InBundle) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "unterminated instruction bundle cannot be printed");
-  }
-  return {};
-}
-
-Expected<std::string> BodyPrinter::run() {
-  auto Verification = MachineVerifier::verify(
-      Function, Target,
-      MachineVerifierOptions{
-          .Mode = MachineVerifierMode::Generic, .Stage = std::nullopt, .VerifyProperties = false});
-  if (!Verification)
-    return std::unexpected(Verification.error());
-  if (auto Initialized = initialize(); !Initialized)
-    return std::unexpected(Initialized.error());
-
-  bool First = true;
-  for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
-       Block = Block->next()) {
-    if (!First)
-      Output.push_back('\n');
-    First = false;
-    if (auto Printed = printBlock(*Block); !Printed)
-      return std::unexpected(Printed.error());
-  }
-  return std::move(Output);
-}
-
-} // namespace
-
-Expected<std::string> printMachineFunctionBody(const MachineFunction &Function,
-                                               const TargetDescription &Target,
-                                               const MIParserContext &Context,
-                                               MIPrinterOptions Options) {
-  return BodyPrinter(Function, Target, Context, Options).run();
-}
-
-} // namespace rcc::mir