8 files changed, 600 insertions, 1 deletions

diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt
index 08ca00e..d0f725d 100644
--- a/backend/lib/CodeGen/CMakeLists.txt
+++ b/backend/lib/CodeGen/CMakeLists.txt
@@ -6,6 +6,7 @@ set(
   InstructionDescriptor.cpp
   LowLevelType.cpp
   MachineBasicBlockAttributes.cpp
+  MachineCFIInfo.cpp
   MachineCompilationStatus.cpp
   MachineBasicBlock.cpp
   MachineCFGUpdater.cpp
@@ -39,6 +40,7 @@ set(
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/InstructionDescriptor.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LowLevelType.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlockAttributes.h"
+  "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFIInfo.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCompilationStatus.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlock.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFGUpdater.h"
@@ -101,6 +103,7 @@ rcc_backend_register_format_files(
   "${CMAKE_CURRENT_SOURCE_DIR}/InstructionDescriptor.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/LowLevelType.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlockAttributes.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFIInfo.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp"
@@ -134,6 +137,7 @@ rcc_backend_register_tidy_files(
   "${CMAKE_CURRENT_SOURCE_DIR}/InstructionDescriptor.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/LowLevelType.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlockAttributes.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFIInfo.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp"
diff --git a/backend/lib/CodeGen/MachineCFIInfo.cpp b/backend/lib/CodeGen/MachineCFIInfo.cpp
new file mode 100644
index 0000000..db9f319
--- /dev/null
+++ b/backend/lib/CodeGen/MachineCFIInfo.cpp
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/MachineCFIInfo.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <string>
+#include <utility>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] Expected<void> verifyVectorRegisters(const MachineCFIVectorRegistersRule &Rule,
+                                                   ErrorCode Code) {
+  if (Rule.VectorRegisters.empty()) {
+    return makeUnexpected(Code, "CFI vector-register rule must contain at least one lane");
+  }
+  return {};
+}
+
+} // namespace
+
+MachineCFIInstruction MachineCFIInstruction::sameValue(DwarfRegisterNumber Register) {
+  return MachineCFIInstruction(MachineCFIOperation::SameValue, MachineCFIRegisterRule{Register});
+}
+
+MachineCFIInstruction MachineCFIInstruction::rememberState() {
+  return MachineCFIInstruction(MachineCFIOperation::RememberState, MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::restoreState() {
+  return MachineCFIInstruction(MachineCFIOperation::RestoreState, MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::offset(DwarfRegisterNumber Register,
+                                                    std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::Offset,
+                               MachineCFIRegisterOffsetRule{Register, Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::llvmDefAddressSpaceCFA(DwarfRegisterNumber Register,
+                                                                    std::int64_t Offset,
+                                                                    std::uint32_t AddressSpace) {
+  return MachineCFIInstruction(MachineCFIOperation::LLVMDefAddressSpaceCFA,
+                               MachineCFIAddressSpaceCFARule{Register, Offset, AddressSpace});
+}
+
+MachineCFIInstruction MachineCFIInstruction::defCFARegister(DwarfRegisterNumber Register) {
+  return MachineCFIInstruction(MachineCFIOperation::DefCFARegister,
+                               MachineCFIRegisterRule{Register});
+}
+
+MachineCFIInstruction MachineCFIInstruction::defCFAOffset(std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::DefCFAOffset, MachineCFIOffsetRule{Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::defCFA(DwarfRegisterNumber Register,
+                                                    std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::DefCFA,
+                               MachineCFIRegisterOffsetRule{Register, Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::relativeOffset(DwarfRegisterNumber Register,
+                                                            std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::RelativeOffset,
+                               MachineCFIRegisterOffsetRule{Register, Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::adjustCFAOffset(std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::AdjustCFAOffset, MachineCFIOffsetRule{Offset});
+}
+
+Expected<MachineCFIInstruction> MachineCFIInstruction::escape(std::vector<std::uint8_t> Bytes) {
+  if (Bytes.empty())
+    return makeUnexpected(ErrorCode::InvalidArgument, "CFI escape byte sequence cannot be empty");
+  return MachineCFIInstruction(MachineCFIOperation::Escape, MachineCFIEscapeRule{std::move(Bytes)});
+}
+
+MachineCFIInstruction MachineCFIInstruction::restore(DwarfRegisterNumber Register) {
+  return MachineCFIInstruction(MachineCFIOperation::Restore, MachineCFIRegisterRule{Register});
+}
+
+MachineCFIInstruction MachineCFIInstruction::undefined(DwarfRegisterNumber Register) {
+  return MachineCFIInstruction(MachineCFIOperation::Undefined, MachineCFIRegisterRule{Register});
+}
+
+MachineCFIInstruction MachineCFIInstruction::registerRule(DwarfRegisterNumber Register,
+                                                          DwarfRegisterNumber SourceRegister) {
+  return MachineCFIInstruction(MachineCFIOperation::Register,
+                               MachineCFIRegisterRegisterRule{Register, SourceRegister});
+}
+
+MachineCFIInstruction MachineCFIInstruction::windowSave() {
+  return MachineCFIInstruction(MachineCFIOperation::WindowSave, MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::negateReturnAddressState() {
+  return MachineCFIInstruction(MachineCFIOperation::NegateReturnAddressState,
+                               MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::negateReturnAddressStateWithPC() {
+  return MachineCFIInstruction(MachineCFIOperation::NegateReturnAddressStateWithPC,
+                               MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::gnuArgsSize(std::int64_t Size) {
+  return MachineCFIInstruction(MachineCFIOperation::GNUArgsSize, MachineCFIOffsetRule{Size});
+}
+
+Expected<MachineCFIInstruction> MachineCFIInstruction::label(std::string Symbol) {
+  if (Symbol.empty())
+    return makeUnexpected(ErrorCode::InvalidArgument, "CFI label symbol cannot be empty");
+  return MachineCFIInstruction(MachineCFIOperation::Label, MachineCFILabelRule{std::move(Symbol)});
+}
+
+MachineCFIInstruction MachineCFIInstruction::valueOffset(DwarfRegisterNumber Register,
+                                                         std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::ValueOffset,
+                               MachineCFIRegisterOffsetRule{Register, Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::llvmRegisterPair(MachineCFIRegisterPairRule Rule) {
+  return MachineCFIInstruction(MachineCFIOperation::LLVMRegisterPair, Rule);
+}
+
+Expected<MachineCFIInstruction>
+MachineCFIInstruction::llvmVectorRegisters(MachineCFIVectorRegistersRule Rule) {
+  auto Verification = verifyVectorRegisters(Rule, ErrorCode::InvalidArgument);
+  if (!Verification)
+    return std::unexpected(Verification.error());
+  return MachineCFIInstruction(MachineCFIOperation::LLVMVectorRegisters, std::move(Rule));
+}
+
+MachineCFIInstruction MachineCFIInstruction::llvmVectorOffset(MachineCFIVectorOffsetRule Rule) {
+  return MachineCFIInstruction(MachineCFIOperation::LLVMVectorOffset, Rule);
+}
+
+MachineCFIInstruction
+MachineCFIInstruction::llvmVectorRegisterMask(MachineCFIVectorRegisterMaskRule Rule) {
+  return MachineCFIInstruction(MachineCFIOperation::LLVMVectorRegisterMask, Rule);
+}
+
+Expected<void> MachineCFIInstruction::verify() const {
+  switch (Operation) {
+  case MachineCFIOperation::SameValue:
+  case MachineCFIOperation::DefCFARegister:
+  case MachineCFIOperation::Restore:
+  case MachineCFIOperation::Undefined:
+    if (getIf<MachineCFIRegisterRule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::RememberState:
+  case MachineCFIOperation::RestoreState:
+  case MachineCFIOperation::WindowSave:
+  case MachineCFIOperation::NegateReturnAddressState:
+  case MachineCFIOperation::NegateReturnAddressStateWithPC:
+    if (getIf<MachineCFINoOperands>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::Offset:
+  case MachineCFIOperation::DefCFA:
+  case MachineCFIOperation::RelativeOffset:
+  case MachineCFIOperation::ValueOffset:
+    if (getIf<MachineCFIRegisterOffsetRule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::LLVMDefAddressSpaceCFA:
+    if (getIf<MachineCFIAddressSpaceCFARule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::DefCFAOffset:
+  case MachineCFIOperation::AdjustCFAOffset:
+  case MachineCFIOperation::GNUArgsSize:
+    if (getIf<MachineCFIOffsetRule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::Escape: {
+    const auto *Rule = getIf<MachineCFIEscapeRule>();
+    if (Rule == nullptr)
+      break;
+    if (Rule->Bytes.empty()) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "CFI escape byte sequence cannot be empty");
+    }
+    return {};
+  }
+  case MachineCFIOperation::Register:
+    if (getIf<MachineCFIRegisterRegisterRule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::Label: {
+    const auto *Rule = getIf<MachineCFILabelRule>();
+    if (Rule == nullptr)
+      break;
+    if (Rule->Symbol.empty())
+      return makeUnexpected(ErrorCode::VerificationError, "CFI label symbol cannot be empty");
+    return {};
+  }
+  case MachineCFIOperation::LLVMRegisterPair: {
+    const auto *Rule = getIf<MachineCFIRegisterPairRule>();
+    if (Rule == nullptr)
+      break;
+    return {};
+  }
+  case MachineCFIOperation::LLVMVectorRegisters: {
+    const auto *Rule = getIf<MachineCFIVectorRegistersRule>();
+    if (Rule == nullptr)
+      break;
+    return verifyVectorRegisters(*Rule, ErrorCode::VerificationError);
+  }
+  case MachineCFIOperation::LLVMVectorOffset: {
+    const auto *Rule = getIf<MachineCFIVectorOffsetRule>();
+    if (Rule == nullptr)
+      break;
+    return {};
+  }
+  case MachineCFIOperation::LLVMVectorRegisterMask: {
+    const auto *Rule = getIf<MachineCFIVectorRegisterMaskRule>();
+    if (Rule == nullptr)
+      break;
+    return {};
+  }
+  case MachineCFIOperation::Count:
+    break;
+  }
+  return makeUnexpected(ErrorCode::VerificationError,
+                        "CFI instruction operation and payload do not match");
+}
+
+Expected<CFIInstructionIndex> MachineCFIInfo::addInstruction(MachineCFIInstruction Instruction) {
+  auto Verification = Instruction.verify();
+  if (!Verification)
+    return std::unexpected(Verification.error());
+  if (Instructions.size() > std::numeric_limits<std::uint32_t>::max()) {
+    return makeUnexpected(ErrorCode::Unsupported, "CFI instruction index space is exhausted");
+  }
+
+  const auto Index =
+      CFIInstructionIndex::fromValue(static_cast<std::uint32_t>(Instructions.size()));
+  Instructions.push_back(std::move(Instruction));
+  return Index;
+}
+
+const MachineCFIInstruction *MachineCFIInfo::instruction(CFIInstructionIndex Index) const noexcept {
+  if (Index.value() >= Instructions.size())
+    return nullptr;
+  return &Instructions[Index.value()];
+}
+
+Expected<void> MachineCFIInfo::verify() const {
+  for (const MachineCFIInstruction &Instruction : Instructions) {
+    auto Verification = Instruction.verify();
+    if (!Verification)
+      return std::unexpected(Verification.error());
+  }
+  return {};
+}
+
+Expected<MachineCFIInfo> MachineCFIInfo::clone() const {
+  auto Verification = verify();
+  if (!Verification)
+    return std::unexpected(Verification.error());
+
+  MachineCFIInfo Result;
+  for (const MachineCFIInstruction &Instruction : Instructions) {
+    auto Index = Result.addInstruction(Instruction);
+    if (!Index)
+      return std::unexpected(Index.error());
+  }
+  return Result;
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/MachineFunctionCloner.cpp b/backend/lib/CodeGen/MachineFunctionCloner.cpp
index dcf9712..b769f7d 100644
--- a/backend/lib/CodeGen/MachineFunctionCloner.cpp
+++ b/backend/lib/CodeGen/MachineFunctionCloner.cpp
@@ -126,13 +126,21 @@ struct CloneState final {
     return MachineOperand::jumpTable(Position->second, SourceOperand.targetFlags());
   }
   case MachineOperand::Kind::Immediate:
+  case MachineOperand::Kind::IntegerImmediate:
   case MachineOperand::Kind::FloatingImmediate:
   case MachineOperand::Kind::GlobalSymbol:
   case MachineOperand::Kind::ExternalSymbol:
   case MachineOperand::Kind::RegisterMask:
+  case MachineOperand::Kind::LiveOutRegisterMask:
   case MachineOperand::Kind::Metadata:
   case MachineOperand::Kind::TargetIndex:
   case MachineOperand::Kind::MCSymbol:
+  case MachineOperand::Kind::CFIIndex:
+  case MachineOperand::Kind::Intrinsic:
+  case MachineOperand::Kind::Predicate:
+  case MachineOperand::Kind::ShuffleMask:
+  case MachineOperand::Kind::DebugInstrRef:
+  case MachineOperand::Kind::LaneMask:
     return SourceOperand.cloneDetached();
   case MachineOperand::Kind::Count:
     return makeUnexpected(ErrorCode::VerificationError, "source operand has an invalid kind");
@@ -636,6 +644,10 @@ Expected<MachineFunctionCloneResult> cloneMachineFunction(const MachineFunction
   if (!Metadata)
     return std::unexpected(Metadata.error());
   Result.ClonedFunction->metadata() = std::move(*Metadata);
+  auto CFIInfo = Source.cfiInfo().clone();
+  if (!CFIInfo)
+    return std::unexpected(CFIInfo.error());
+  Result.ClonedFunction->cfiInfo() = std::move(*CFIInfo);
   Result.ClonedFunction->properties() = Source.properties();
   Result.ClonedFunction->attributes() = Source.attributes();
   auto DestinationVerification = MachineVerifier::verify(*Result.ClonedFunction, Target);
diff --git a/backend/lib/CodeGen/MachineIRBuilder.cpp b/backend/lib/CodeGen/MachineIRBuilder.cpp
index 5afdd72..e1e8006 100644
--- a/backend/lib/CodeGen/MachineIRBuilder.cpp
+++ b/backend/lib/CodeGen/MachineIRBuilder.cpp
@@ -162,6 +162,11 @@ Expected<void> MachineInstrBuilder::addImmediate(std::int64_t Value, std::uint32
   return addOperand(MachineOperand::immediate(Value, TargetFlags));
 }
 
+Expected<void> MachineInstrBuilder::addIntegerImmediate(IntegerImmediate Value,
+                                                        std::uint32_t TargetFlags) {
+  return addOperand(MachineOperand::integerImmediate(std::move(Value), TargetFlags));
+}
+
 Expected<void> MachineInstrBuilder::addFloatingImmediate(FloatingPointBits Bits,
                                                          std::uint32_t TargetFlags) {
   return addOperand(MachineOperand::floatingImmediate(Bits, TargetFlags));
@@ -220,6 +225,14 @@ Expected<void> MachineInstrBuilder::addRegisterMask(std::shared_ptr<const Regist
   return addOperand(std::move(*Operand));
 }
 
+Expected<void> MachineInstrBuilder::addLiveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask,
+                                                           std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::liveOutRegisterMask(std::move(Mask), TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
 Expected<void> MachineInstrBuilder::addMetadata(MachineMetadataValue Value,
                                                 std::uint32_t TargetFlags) {
   auto Operand = MachineOperand::metadata(std::move(Value), TargetFlags);
@@ -241,6 +254,55 @@ Expected<void> MachineInstrBuilder::addMCSymbol(std::string Name, std::int64_t O
   return addOperand(std::move(*Operand));
 }
 
+Expected<void> MachineInstrBuilder::addCFIInstruction(CFIInstructionIndex Index,
+                                                      std::uint32_t TargetFlags) {
+  return addOperand(MachineOperand::cfiIndex(Index, TargetFlags));
+}
+
+Expected<void> MachineInstrBuilder::addIntrinsic(IntrinsicID ID, std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::intrinsic(ID, TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addIntegerPredicate(IntegerPredicate Predicate,
+                                                        std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::integerPredicate(Predicate, TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addFloatingPredicate(FloatingPredicate Predicate,
+                                                         std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::floatingPredicate(Predicate, TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addShuffleMask(std::vector<std::int32_t> Elements,
+                                                   std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::shuffleMask(std::move(Elements), TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addDebugInstrRef(std::uint64_t InstructionNumber,
+                                                     std::uint64_t OperandNumber,
+                                                     std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::debugInstrRef(InstructionNumber, OperandNumber, TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addLaneMask(LaneMask Mask, std::uint32_t TargetFlags) {
+  return addOperand(MachineOperand::laneMask(Mask, TargetFlags));
+}
+
 Expected<void> MachineInstrBuilder::tieOperands(std::size_t DefOperand, std::size_t UseOperand) {
   auto Pending = ensurePending();
   if (!Pending)
@@ -377,6 +439,14 @@ Expected<void> MachineInstrBuilder::validateJumpTableReference(JumpTableIndex In
   return {};
 }
 
+Expected<void> MachineInstrBuilder::validateCFIReference(CFIInstructionIndex Index) const {
+  if (Function->cfiInfo().instruction(Index) == nullptr) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "builder references an unknown CFI instruction");
+  }
+  return {};
+}
+
 Expected<void> MachineInstrBuilder::validateOperandOwnership(const MachineOperand &Operand) const {
   if (Operand.parent() != nullptr) {
     return makeUnexpected(ErrorCode::InvalidArgument,
@@ -409,6 +479,11 @@ Expected<void> MachineInstrBuilder::validateOperandOwnership(const MachineOperan
     if (!JumpTableResult)
       return JumpTableResult;
   }
+  if (const auto *CFI = Operand.getIf<CFIIndexOperand>(); CFI != nullptr) {
+    auto CFIResult = validateCFIReference(CFI->Index);
+    if (!CFIResult)
+      return CFIResult;
+  }
   return {};
 }
 
diff --git a/backend/lib/CodeGen/MachineOperand.cpp b/backend/lib/CodeGen/MachineOperand.cpp
index cd2d217..6cce188 100644
--- a/backend/lib/CodeGen/MachineOperand.cpp
+++ b/backend/lib/CodeGen/MachineOperand.cpp
@@ -9,11 +9,103 @@
 #include <algorithm>
 #include <array>
 #include <cstdint>
+#include <limits>
 #include <memory>
 #include <span>
 #include <utility>
+#include <vector>
 
 namespace rcc {
+namespace {
+
+[[nodiscard]] constexpr std::size_t integerWordCount(std::uint32_t BitWidth) noexcept {
+  return (static_cast<std::size_t>(BitWidth) + 63U) / 64U;
+}
+
+[[nodiscard]] constexpr std::uint64_t integerHighWordMask(std::uint32_t BitWidth) noexcept {
+  const std::uint32_t UsedBits = BitWidth % 64U;
+  if (UsedBits == 0)
+    return std::numeric_limits<std::uint64_t>::max();
+  return (std::uint64_t{1} << UsedBits) - 1U;
+}
+
+[[nodiscard]] Expected<void> verifyIntegerImmediate(std::uint32_t BitWidth,
+                                                    std::span<const std::uint64_t> Words,
+                                                    ErrorCode Code) {
+  if (BitWidth == 0 || BitWidth > IntegerImmediate::MaxBitWidth) {
+    return makeUnexpected(Code, "integer immediate bit width is outside the supported range");
+  }
+  if (Words.size() != integerWordCount(BitWidth)) {
+    return makeUnexpected(Code, "integer immediate bit pattern has the wrong word count");
+  }
+  if ((Words.back() & ~integerHighWordMask(BitWidth)) != 0) {
+    return makeUnexpected(Code, "integer immediate has nonzero bits above its declared width");
+  }
+  return {};
+}
+
+} // namespace
+
+Expected<IntegerImmediate> IntegerImmediate::create(std::uint32_t BitWidth,
+                                                    std::span<const std::uint64_t> Words) {
+  auto Verification = verifyIntegerImmediate(BitWidth, Words, ErrorCode::InvalidArgument);
+  if (!Verification)
+    return std::unexpected(Verification.error());
+  return IntegerImmediate(BitWidth, std::vector<std::uint64_t>(Words.begin(), Words.end()));
+}
+
+Expected<IntegerImmediate> IntegerImmediate::fromUnsigned(std::uint32_t BitWidth,
+                                                          std::uint64_t Value) {
+  if (BitWidth == 0 || BitWidth > MaxBitWidth) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "integer immediate bit width is outside the supported range");
+  }
+  if (BitWidth < 64U && (Value >> BitWidth) != 0) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "unsigned value does not fit the integer immediate bit width");
+  }
+
+  std::vector<std::uint64_t> Words(integerWordCount(BitWidth), 0);
+  Words.front() = Value;
+  return create(BitWidth, Words);
+}
+
+Expected<IntegerImmediate> IntegerImmediate::fromSigned(std::uint32_t BitWidth,
+                                                        std::int64_t Value) {
+  if (BitWidth == 0 || BitWidth > MaxBitWidth) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "integer immediate bit width is outside the supported range");
+  }
+  if (BitWidth < 64U) {
+    const std::uint32_t MagnitudeBits = BitWidth - 1U;
+    const std::int64_t Minimum = -(std::int64_t{1} << MagnitudeBits);
+    const std::int64_t Maximum = (std::int64_t{1} << MagnitudeBits) - 1;
+    if (Value < Minimum || Value > Maximum) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "signed value does not fit the integer immediate bit width");
+    }
+  }
+
+  const std::uint64_t Fill =
+      Value < 0 ? std::numeric_limits<std::uint64_t>::max() : std::uint64_t{0};
+  std::vector<std::uint64_t> Words(integerWordCount(BitWidth), Fill);
+  Words.front() = static_cast<std::uint64_t>(Value);
+  Words.back() &= integerHighWordMask(BitWidth);
+  return create(BitWidth, Words);
+}
+
+bool IntegerImmediate::isZero() const noexcept {
+  return std::ranges::all_of(Words, [](std::uint64_t Word) { return Word == 0; });
+}
+
+bool IntegerImmediate::isNegative() const noexcept {
+  const std::uint32_t SignBit = BitWidth - 1U;
+  return (Words[SignBit / 64U] & (std::uint64_t{1} << (SignBit % 64U))) != 0;
+}
+
+Expected<void> IntegerImmediate::verify() const {
+  return verifyIntegerImmediate(BitWidth, Words, ErrorCode::VerificationError);
+}
 
 Expected<FloatingPointBits> FloatingPointBits::create(FloatSemantics Semantics,
                                                       std::span<const std::uint64_t> Words) {
@@ -45,6 +137,20 @@ RegisterOperandFlags::RegisterOperandFlags(
     set(Flag);
 }
 
+Expected<void> PredicateOperand::verify() const {
+  if (const auto *Integer = integerPredicate();
+      Integer != nullptr &&
+      static_cast<std::uint8_t>(*Integer) < static_cast<std::uint8_t>(IntegerPredicate::Count)) {
+    return {};
+  }
+  if (const auto *Floating = floatingPredicate();
+      Floating != nullptr &&
+      static_cast<std::uint8_t>(*Floating) < static_cast<std::uint8_t>(FloatingPredicate::Count)) {
+    return {};
+  }
+  return makeUnexpected(ErrorCode::VerificationError, "predicate operand contains an invalid kind");
+}
+
 Expected<void> MachineOperand::verifyRegisterFlags(RegisterOperandFlags Flags) {
   const bool IsDef = Flags.has(RegisterOperandFlag::Define);
   const bool IsUse = Flags.has(RegisterOperandFlag::Use);
@@ -96,6 +202,11 @@ MachineOperand MachineOperand::immediate(std::int64_t Value, std::uint32_t Targe
   return MachineOperand(Kind::Immediate, Value, TargetFlags);
 }
 
+MachineOperand MachineOperand::integerImmediate(IntegerImmediate Value, std::uint32_t TargetFlags) {
+  return MachineOperand(Kind::IntegerImmediate, IntegerImmediateOperand{std::move(Value)},
+                        TargetFlags);
+}
+
 MachineOperand MachineOperand::floatingImmediate(FloatingPointBits Bits,
                                                  std::uint32_t TargetFlags) {
   return MachineOperand(Kind::FloatingImmediate, FloatingImmediateOperand{Bits}, TargetFlags);
@@ -158,6 +269,17 @@ Expected<MachineOperand> MachineOperand::registerMask(std::shared_ptr<const Regi
   return MachineOperand(Kind::RegisterMask, RegisterMaskOperand{std::move(Mask)}, TargetFlags);
 }
 
+Expected<MachineOperand>
+MachineOperand::liveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask,
+                                    std::uint32_t TargetFlags) {
+  if (!Mask) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "live-out register-mask operand cannot be null");
+  }
+  return MachineOperand(Kind::LiveOutRegisterMask, LiveOutRegisterMaskOperand{std::move(Mask)},
+                        TargetFlags);
+}
+
 Expected<MachineOperand> MachineOperand::metadata(MachineMetadataValue Value,
                                                   std::uint32_t TargetFlags) {
   if (!Value.isValid()) {
@@ -179,6 +301,65 @@ Expected<MachineOperand> MachineOperand::mcSymbol(std::string Name, std::int64_t
   return MachineOperand(Kind::MCSymbol, MCSymbolOperand{std::move(Name), Offset}, TargetFlags);
 }
 
+MachineOperand MachineOperand::cfiIndex(CFIInstructionIndex Index, std::uint32_t TargetFlags) {
+  return MachineOperand(Kind::CFIIndex, CFIIndexOperand{Index}, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::intrinsic(IntrinsicID ID, std::uint32_t TargetFlags) {
+  if (ID.value() == 0) {
+    return makeUnexpected(ErrorCode::InvalidArgument, "intrinsic operand ID must be nonzero");
+  }
+  return MachineOperand(Kind::Intrinsic, IntrinsicOperand{ID}, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::integerPredicate(IntegerPredicate Predicate,
+                                                          std::uint32_t TargetFlags) {
+  PredicateOperand Value(Predicate);
+  auto Verification = Value.verify();
+  if (!Verification)
+    return makeUnexpected(ErrorCode::InvalidArgument, "integer predicate operand is invalid");
+  return MachineOperand(Kind::Predicate, Value, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::floatingPredicate(FloatingPredicate Predicate,
+                                                           std::uint32_t TargetFlags) {
+  PredicateOperand Value(Predicate);
+  auto Verification = Value.verify();
+  if (!Verification)
+    return makeUnexpected(ErrorCode::InvalidArgument, "floating predicate operand is invalid");
+  return MachineOperand(Kind::Predicate, Value, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::shuffleMask(std::vector<std::int32_t> Elements,
+                                                     std::uint32_t TargetFlags) {
+  if (Elements.size() < 2) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "shuffle-mask operand requires at least two elements");
+  }
+  return MachineOperand(Kind::ShuffleMask, ShuffleMaskOperand{std::move(Elements)}, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::debugInstrRef(std::uint64_t InstructionNumber,
+                                                       std::uint64_t OperandNumber,
+                                                       std::uint32_t TargetFlags) {
+  if (InstructionNumber > std::numeric_limits<std::uint32_t>::max()) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "debug instruction reference instruction number must fit in uint32");
+  }
+  if (OperandNumber > std::numeric_limits<std::uint32_t>::max()) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "debug instruction reference operand number must fit in uint32");
+  }
+  return MachineOperand(Kind::DebugInstrRef,
+                        DebugInstrRefOperand{static_cast<std::uint32_t>(InstructionNumber),
+                                             static_cast<std::uint32_t>(OperandNumber)},
+                        TargetFlags);
+}
+
+MachineOperand MachineOperand::laneMask(LaneMask Mask, std::uint32_t TargetFlags) {
+  return MachineOperand(Kind::LaneMask, LaneMaskOperand{Mask}, TargetFlags);
+}
+
 MachineOperand::MachineOperand(MachineOperand &&Other) noexcept
     : OperandKind(Other.OperandKind), Storage(std::move(Other.Storage)), RegFlags(Other.RegFlags),
       RegSubReg(Other.RegSubReg), TiedOperandIndex(Other.TiedOperandIndex),
diff --git a/backend/lib/CodeGen/MachineVerifier.cpp b/backend/lib/CodeGen/MachineVerifier.cpp
index 7e050ec..98fb9f7 100644
--- a/backend/lib/CodeGen/MachineVerifier.cpp
+++ b/backend/lib/CodeGen/MachineVerifier.cpp
@@ -40,6 +40,9 @@ public:
     auto MetadataResult = Function.metadata().verify();
     if (!MetadataResult)
       return failWith("machine metadata", MetadataResult.error());
+    auto CFIResult = Function.cfiInfo().verify();
+    if (!CFIResult)
+      return failWith("CFI information", CFIResult.error());
     if (Function.targetState().hasValue()) {
       auto TargetStateResult = Function.targetState().verify();
       if (!TargetStateResult)
@@ -103,6 +106,8 @@ private:
       return Operand.getIf<Register>() != nullptr;
     case MachineOperand::Kind::Immediate:
       return Operand.getIf<std::int64_t>() != nullptr;
+    case MachineOperand::Kind::IntegerImmediate:
+      return Operand.getIf<IntegerImmediateOperand>() != nullptr;
     case MachineOperand::Kind::FloatingImmediate:
       return Operand.getIf<FloatingImmediateOperand>() != nullptr;
     case MachineOperand::Kind::BasicBlock:
@@ -121,12 +126,26 @@ private:
       return Operand.getIf<BlockAddressOperand>() != nullptr;
     case MachineOperand::Kind::RegisterMask:
       return Operand.getIf<RegisterMaskOperand>() != nullptr;
+    case MachineOperand::Kind::LiveOutRegisterMask:
+      return Operand.getIf<LiveOutRegisterMaskOperand>() != nullptr;
     case MachineOperand::Kind::Metadata:
       return Operand.getIf<MetadataOperand>() != nullptr;
     case MachineOperand::Kind::TargetIndex:
       return Operand.getIf<TargetIndexOperand>() != nullptr;
     case MachineOperand::Kind::MCSymbol:
       return Operand.getIf<MCSymbolOperand>() != nullptr;
+    case MachineOperand::Kind::CFIIndex:
+      return Operand.getIf<CFIIndexOperand>() != nullptr;
+    case MachineOperand::Kind::Intrinsic:
+      return Operand.getIf<IntrinsicOperand>() != nullptr;
+    case MachineOperand::Kind::Predicate:
+      return Operand.getIf<PredicateOperand>() != nullptr;
+    case MachineOperand::Kind::ShuffleMask:
+      return Operand.getIf<ShuffleMaskOperand>() != nullptr;
+    case MachineOperand::Kind::DebugInstrRef:
+      return Operand.getIf<DebugInstrRefOperand>() != nullptr;
+    case MachineOperand::Kind::LaneMask:
+      return Operand.getIf<LaneMaskOperand>() != nullptr;
     case MachineOperand::Kind::Count:
       return false;
     }
@@ -286,6 +305,15 @@ private:
       if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); Mask != nullptr && !Mask->Mask) {
         return fail("register-mask operand contains a null mask");
       }
+      if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>();
+          Mask != nullptr && !Mask->Mask) {
+        return fail("live-out register-mask operand contains a null mask");
+      }
+      if (const auto *Integer = Operand.getIf<IntegerImmediateOperand>(); Integer != nullptr) {
+        auto IntegerResult = Integer->Value.verify();
+        if (!IntegerResult)
+          return failWith("integer immediate operand", IntegerResult.error());
+      }
       if (const auto *Global = Operand.getIf<GlobalSymbolOperand>();
           Global != nullptr && Global->Name.empty()) {
         return fail("global symbol operand has an empty name");
@@ -307,6 +335,19 @@ private:
           Symbol != nullptr && Symbol->Name.empty()) {
         return fail("MC symbol operand has an empty name");
       }
+      if (const auto *Intrinsic = Operand.getIf<IntrinsicOperand>();
+          Intrinsic != nullptr && Intrinsic->ID.value() == 0) {
+        return fail("intrinsic operand has an invalid ID");
+      }
+      if (const auto *Predicate = Operand.getIf<PredicateOperand>(); Predicate != nullptr) {
+        auto PredicateResult = Predicate->verify();
+        if (!PredicateResult)
+          return failWith("predicate operand", PredicateResult.error());
+      }
+      if (const auto *Shuffle = Operand.getIf<ShuffleMaskOperand>();
+          Shuffle != nullptr && Shuffle->Elements.size() < 2) {
+        return fail("shuffle-mask operand has fewer than two elements");
+      }
     }
 
     for (const MachineMemOperand *MemOperand : Instruction.memOperands()) {
@@ -497,6 +538,10 @@ private:
         if (!JumpTableResult)
           return JumpTableResult;
       }
+      if (const auto *CFI = Operand.getIf<CFIIndexOperand>();
+          CFI != nullptr && Function.cfiInfo().instruction(CFI->Index) == nullptr) {
+        return fail("live instruction references an unknown CFI instruction");
+      }
 
       if (const auto *Block = Operand.getIf<BasicBlockOperand>();
           Block != nullptr &&
diff --git a/backend/lib/CodeGen/TargetDescription.cpp b/backend/lib/CodeGen/TargetDescription.cpp
index 3a6ea31..a32534d 100644
--- a/backend/lib/CodeGen/TargetDescription.cpp
+++ b/backend/lib/CodeGen/TargetDescription.cpp
@@ -203,6 +203,13 @@ Expected<void> TargetDescription::validateTargetOperand(const OperandDescriptor
     }
     return Registers.validateRegisterMask(*Mask->Mask);
   }
+  if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); Mask != nullptr) {
+    if (!Mask->Mask) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "live-out register-mask operand contains a null mask");
+    }
+    return Registers.validateRegisterMask(*Mask->Mask);
+  }
 
   const Register *Reg = Operand.getIf<Register>();
   if (Reg == nullptr)
diff --git a/backend/lib/CodeGen/TargetRegisterInfo.cpp b/backend/lib/CodeGen/TargetRegisterInfo.cpp
index 7bd4834..7c8af82 100644
--- a/backend/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/backend/lib/CodeGen/TargetRegisterInfo.cpp
@@ -592,7 +592,7 @@ Expected<void> TargetRegisterInfo::validateRegisterMask(const RegisterMask &Mask
       const auto Reg = PhysicalRegister::create(RegisterNumber);
       if (!Reg || !isKnownPhysicalRegister(*Reg)) {
         return makeUnexpected(ErrorCode::VerificationError,
-                              "register mask preserves an unknown physical register");
+                              "register mask contains an unknown physical register");
       }
       Word &= Word - 1;
     }