[CodeGen] Model complete MIR operands

15 files changed, 1572 insertions, 5 deletions

diff --git a/backend/include/rcc/CodeGen/MachineCFIInfo.h b/backend/include/rcc/CodeGen/MachineCFIInfo.h
new file mode 100644
index 0000000..1819daf
--- /dev/null
+++ b/backend/include/rcc/CodeGen/MachineCFIInfo.h
@@ -0,0 +1,269 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef RCC_CODEGEN_MACHINECFIINFO_H
+#define RCC_CODEGEN_MACHINECFIINFO_H
+
+#include "rcc/CodeGen/MachineIRTypes.h"
+#include "rcc/Support/Error.h"
+
+#include <cstdint>
+#include <span>
+#include <string>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace rcc {
+
+class DwarfRegisterNumber final {
+public:
+  constexpr DwarfRegisterNumber() noexcept = default;
+
+  [[nodiscard]] static constexpr DwarfRegisterNumber fromValue(std::uint32_t Value) noexcept {
+    return DwarfRegisterNumber(Value);
+  }
+
+  [[nodiscard]] constexpr std::uint32_t value() const noexcept {
+    return Value;
+  }
+
+  friend constexpr bool operator==(DwarfRegisterNumber Left,
+                                   DwarfRegisterNumber Right) noexcept = default;
+
+private:
+  explicit constexpr DwarfRegisterNumber(std::uint32_t Value) noexcept : Value(Value) {}
+
+  std::uint32_t Value = 0;
+};
+
+enum class MachineCFIOperation : std::uint8_t {
+  SameValue,
+  RememberState,
+  RestoreState,
+  Offset,
+  LLVMDefAddressSpaceCFA,
+  DefCFARegister,
+  DefCFAOffset,
+  DefCFA,
+  RelativeOffset,
+  AdjustCFAOffset,
+  Escape,
+  Restore,
+  Undefined,
+  Register,
+  WindowSave,
+  NegateReturnAddressState,
+  NegateReturnAddressStateWithPC,
+  GNUArgsSize,
+  Label,
+  ValueOffset,
+  LLVMRegisterPair,
+  LLVMVectorRegisters,
+  LLVMVectorOffset,
+  LLVMVectorRegisterMask,
+  Count,
+};
+
+struct MachineCFINoOperands final {
+  friend constexpr bool operator==(MachineCFINoOperands Left,
+                                   MachineCFINoOperands Right) noexcept = default;
+};
+
+struct MachineCFIRegisterRule final {
+  DwarfRegisterNumber Register;
+
+  friend constexpr bool operator==(MachineCFIRegisterRule Left,
+                                   MachineCFIRegisterRule Right) noexcept = default;
+};
+
+struct MachineCFIOffsetRule final {
+  std::int64_t Offset = 0;
+
+  friend constexpr bool operator==(MachineCFIOffsetRule Left,
+                                   MachineCFIOffsetRule Right) noexcept = default;
+};
+
+struct MachineCFIRegisterOffsetRule final {
+  DwarfRegisterNumber Register;
+  std::int64_t Offset = 0;
+
+  friend constexpr bool operator==(MachineCFIRegisterOffsetRule Left,
+                                   MachineCFIRegisterOffsetRule Right) noexcept = default;
+};
+
+struct MachineCFIRegisterRegisterRule final {
+  DwarfRegisterNumber Register;
+  DwarfRegisterNumber SourceRegister;
+
+  friend constexpr bool operator==(MachineCFIRegisterRegisterRule Left,
+                                   MachineCFIRegisterRegisterRule Right) noexcept = default;
+};
+
+struct MachineCFIAddressSpaceCFARule final {
+  DwarfRegisterNumber Register;
+  std::int64_t Offset = 0;
+  std::uint32_t AddressSpace = 0;
+
+  friend constexpr bool operator==(MachineCFIAddressSpaceCFARule Left,
+                                   MachineCFIAddressSpaceCFARule Right) noexcept = default;
+};
+
+struct MachineCFIEscapeRule final {
+  std::vector<std::uint8_t> Bytes;
+
+  friend bool operator==(const MachineCFIEscapeRule &Left,
+                         const MachineCFIEscapeRule &Right) = default;
+};
+
+struct MachineCFILabelRule final {
+  std::string Symbol;
+
+  friend bool operator==(const MachineCFILabelRule &Left,
+                         const MachineCFILabelRule &Right) = default;
+};
+
+struct MachineCFIRegisterPairRule final {
+  DwarfRegisterNumber Register;
+  DwarfRegisterNumber FirstRegister;
+  std::uint32_t FirstSizeInBits = 0;
+  DwarfRegisterNumber SecondRegister;
+  std::uint32_t SecondSizeInBits = 0;
+
+  friend constexpr bool operator==(MachineCFIRegisterPairRule Left,
+                                   MachineCFIRegisterPairRule Right) noexcept = default;
+};
+
+struct MachineCFIVectorRegisterLane final {
+  DwarfRegisterNumber Register;
+  std::uint32_t Lane = 0;
+  std::uint32_t SizeInBits = 0;
+
+  friend constexpr bool operator==(MachineCFIVectorRegisterLane Left,
+                                   MachineCFIVectorRegisterLane Right) noexcept = default;
+};
+
+struct MachineCFIVectorRegistersRule final {
+  DwarfRegisterNumber Register;
+  std::vector<MachineCFIVectorRegisterLane> VectorRegisters;
+
+  friend bool operator==(const MachineCFIVectorRegistersRule &Left,
+                         const MachineCFIVectorRegistersRule &Right) = default;
+};
+
+struct MachineCFIVectorOffsetRule final {
+  DwarfRegisterNumber Register;
+  std::uint32_t RegisterSizeInBits = 0;
+  DwarfRegisterNumber MaskRegister;
+  std::uint32_t MaskRegisterSizeInBits = 0;
+  std::int64_t Offset = 0;
+
+  friend constexpr bool operator==(MachineCFIVectorOffsetRule Left,
+                                   MachineCFIVectorOffsetRule Right) noexcept = default;
+};
+
+struct MachineCFIVectorRegisterMaskRule final {
+  DwarfRegisterNumber Register;
+  DwarfRegisterNumber SpillRegister;
+  std::uint32_t SpillRegisterLaneSizeInBits = 0;
+  DwarfRegisterNumber MaskRegister;
+  std::uint32_t MaskRegisterSizeInBits = 0;
+
+  friend constexpr bool operator==(MachineCFIVectorRegisterMaskRule Left,
+                                   MachineCFIVectorRegisterMaskRule Right) noexcept = default;
+};
+
+class MachineCFIInstruction final {
+public:
+  using Payload =
+      std::variant<MachineCFINoOperands, MachineCFIRegisterRule, MachineCFIOffsetRule,
+                   MachineCFIRegisterOffsetRule, MachineCFIRegisterRegisterRule,
+                   MachineCFIAddressSpaceCFARule, MachineCFIEscapeRule, MachineCFILabelRule,
+                   MachineCFIRegisterPairRule, MachineCFIVectorRegistersRule,
+                   MachineCFIVectorOffsetRule, MachineCFIVectorRegisterMaskRule>;
+
+  [[nodiscard]] static MachineCFIInstruction sameValue(DwarfRegisterNumber Register);
+  [[nodiscard]] static MachineCFIInstruction rememberState();
+  [[nodiscard]] static MachineCFIInstruction restoreState();
+  [[nodiscard]] static MachineCFIInstruction offset(DwarfRegisterNumber Register,
+                                                    std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction llvmDefAddressSpaceCFA(DwarfRegisterNumber Register,
+                                                                    std::int64_t Offset,
+                                                                    std::uint32_t AddressSpace);
+  [[nodiscard]] static MachineCFIInstruction defCFARegister(DwarfRegisterNumber Register);
+  [[nodiscard]] static MachineCFIInstruction defCFAOffset(std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction defCFA(DwarfRegisterNumber Register,
+                                                    std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction relativeOffset(DwarfRegisterNumber Register,
+                                                            std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction adjustCFAOffset(std::int64_t Offset);
+  [[nodiscard]] static Expected<MachineCFIInstruction> escape(std::vector<std::uint8_t> Bytes);
+  [[nodiscard]] static MachineCFIInstruction restore(DwarfRegisterNumber Register);
+  [[nodiscard]] static MachineCFIInstruction undefined(DwarfRegisterNumber Register);
+  [[nodiscard]] static MachineCFIInstruction registerRule(DwarfRegisterNumber Register,
+                                                          DwarfRegisterNumber SourceRegister);
+  [[nodiscard]] static MachineCFIInstruction windowSave();
+  [[nodiscard]] static MachineCFIInstruction negateReturnAddressState();
+  [[nodiscard]] static MachineCFIInstruction negateReturnAddressStateWithPC();
+  [[nodiscard]] static MachineCFIInstruction gnuArgsSize(std::int64_t Size);
+  [[nodiscard]] static Expected<MachineCFIInstruction> label(std::string Symbol);
+  [[nodiscard]] static MachineCFIInstruction valueOffset(DwarfRegisterNumber Register,
+                                                         std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction llvmRegisterPair(MachineCFIRegisterPairRule Rule);
+  [[nodiscard]] static Expected<MachineCFIInstruction>
+  llvmVectorRegisters(MachineCFIVectorRegistersRule Rule);
+  [[nodiscard]] static MachineCFIInstruction llvmVectorOffset(MachineCFIVectorOffsetRule Rule);
+  [[nodiscard]] static MachineCFIInstruction
+  llvmVectorRegisterMask(MachineCFIVectorRegisterMaskRule Rule);
+
+  [[nodiscard]] MachineCFIOperation operation() const noexcept {
+    return Operation;
+  }
+
+  template <typename PayloadT> [[nodiscard]] const PayloadT *getIf() const noexcept {
+    return std::get_if<PayloadT>(&Data);
+  }
+
+  [[nodiscard]] Expected<void> verify() const;
+
+  friend bool operator==(const MachineCFIInstruction &Left,
+                         const MachineCFIInstruction &Right) = default;
+
+private:
+  MachineCFIInstruction(MachineCFIOperation Operation, Payload Data)
+      : Operation(Operation), Data(std::move(Data)) {}
+
+  MachineCFIOperation Operation;
+  Payload Data;
+};
+
+class MachineCFIInfo final {
+public:
+  MachineCFIInfo() = default;
+  MachineCFIInfo(const MachineCFIInfo &) = delete;
+  MachineCFIInfo &operator=(const MachineCFIInfo &) = delete;
+  MachineCFIInfo(MachineCFIInfo &&) noexcept = default;
+  MachineCFIInfo &operator=(MachineCFIInfo &&) noexcept = default;
+  ~MachineCFIInfo() = default;
+
+  [[nodiscard]] Expected<CFIInstructionIndex> addInstruction(MachineCFIInstruction Instruction);
+
+  [[nodiscard]] const MachineCFIInstruction *instruction(CFIInstructionIndex Index) const noexcept;
+
+  [[nodiscard]] std::span<const MachineCFIInstruction> instructions() const noexcept {
+    return Instructions;
+  }
+
+  [[nodiscard]] Expected<void> verify() const;
+  [[nodiscard]] Expected<MachineCFIInfo> clone() const;
+
+  friend bool operator==(const MachineCFIInfo &Left, const MachineCFIInfo &Right) {
+    return Left.Instructions == Right.Instructions;
+  }
+
+private:
+  std::vector<MachineCFIInstruction> Instructions;
+};
+
+} // namespace rcc
+
+#endif // RCC_CODEGEN_MACHINECFIINFO_H
diff --git a/backend/include/rcc/CodeGen/MachineFunction.h b/backend/include/rcc/CodeGen/MachineFunction.h
index 7b325d4..c188d4a 100644
--- a/backend/include/rcc/CodeGen/MachineFunction.h
+++ b/backend/include/rcc/CodeGen/MachineFunction.h
@@ -3,6 +3,7 @@
 #ifndef RCC_CODEGEN_MACHINEFUNCTION_H
 #define RCC_CODEGEN_MACHINEFUNCTION_H
 
+#include "rcc/CodeGen/MachineCFIInfo.h"
 #include "rcc/CodeGen/MachineCompilationStatus.h"
 #include "rcc/CodeGen/MachineConstantPool.h"
 #include "rcc/CodeGen/MachineFrameInfo.h"
@@ -117,6 +118,14 @@ public:
     return Metadata;
   }
 
+  [[nodiscard]] MachineCFIInfo &cfiInfo() noexcept {
+    return CFIInfo;
+  }
+
+  [[nodiscard]] const MachineCFIInfo &cfiInfo() const noexcept {
+    return CFIInfo;
+  }
+
   [[nodiscard]] const TargetFunctionState &targetState() const noexcept {
     return TargetState;
   }
@@ -199,6 +208,7 @@ private:
   MachineFunctionAttributes Attributes;
   MachineFunctionRecords Records;
   MachineMetadataInfo Metadata;
+  MachineCFIInfo CFIInfo;
   TargetFunctionState TargetState;
   MachineCompilationStatus CompilationStatus;
   MachineBasicBlock *FirstBlock = nullptr;
diff --git a/backend/include/rcc/CodeGen/MachineIRBuilder.h b/backend/include/rcc/CodeGen/MachineIRBuilder.h
index 918f389..2a5979b 100644
--- a/backend/include/rcc/CodeGen/MachineIRBuilder.h
+++ b/backend/include/rcc/CodeGen/MachineIRBuilder.h
@@ -52,6 +52,8 @@ public:
                                               std::uint32_t TargetFlags = 0);
 
   [[nodiscard]] Expected<void> addImmediate(std::int64_t Value, std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addIntegerImmediate(IntegerImmediate Value,
+                                                   std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addFloatingImmediate(FloatingPointBits Bits,
                                                     std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addBasicBlock(MachineBasicBlock &Block,
@@ -70,12 +72,27 @@ public:
                                                std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addRegisterMask(std::shared_ptr<const RegisterMask> Mask,
                                                std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addLiveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask,
+                                                      std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addMetadata(MachineMetadataValue Value,
                                            std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addTargetIndex(TargetIndex Index, std::int64_t Offset = 0,
                                               std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addMCSymbol(std::string Name, std::int64_t Offset = 0,
                                            std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addCFIInstruction(CFIInstructionIndex Index,
+                                                 std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addIntrinsic(IntrinsicID ID, std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addIntegerPredicate(IntegerPredicate Predicate,
+                                                   std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addFloatingPredicate(FloatingPredicate Predicate,
+                                                    std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addShuffleMask(std::vector<std::int32_t> Elements,
+                                              std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addDebugInstrRef(std::uint64_t InstructionNumber,
+                                                std::uint64_t OperandNumber,
+                                                std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addLaneMask(LaneMask Mask, std::uint32_t TargetFlags = 0);
 
   [[nodiscard]] Expected<void> tieOperands(std::size_t DefOperand, std::size_t UseOperand);
   [[nodiscard]] Expected<void> addMemOperand(const MachineMemOperand &MemOperand);
@@ -102,6 +119,7 @@ private:
   [[nodiscard]] Expected<void> validateFrameReference(FrameIndex Index) const;
   [[nodiscard]] Expected<void> validateConstantPoolReference(ConstantPoolIndex Index) const;
   [[nodiscard]] Expected<void> validateJumpTableReference(JumpTableIndex Index) const;
+  [[nodiscard]] Expected<void> validateCFIReference(CFIInstructionIndex Index) const;
   [[nodiscard]] Expected<void> validateOperandOwnership(const MachineOperand &Operand) const;
   [[nodiscard]] std::vector<MachineOperandTie> collectOperandTies() const;
   [[nodiscard]] Expected<void> appendRegister(Register Reg, RegisterOperandFlags Flags,
diff --git a/backend/include/rcc/CodeGen/MachineIRTypes.h b/backend/include/rcc/CodeGen/MachineIRTypes.h
index 90aa2b7..f7095cc 100644
--- a/backend/include/rcc/CodeGen/MachineIRTypes.h
+++ b/backend/include/rcc/CodeGen/MachineIRTypes.h
@@ -43,6 +43,8 @@ struct FrameIndexTag;
 struct ConstantPoolIndexTag;
 struct JumpTableIndexTag;
 struct TargetIndexTag;
+struct CFIInstructionIndexTag;
+struct IntrinsicIDTag;
 struct SubRegIndexTag;
 struct RegisterUnitIDTag;
 struct RegisterBankIDTag;
@@ -66,6 +68,8 @@ using FrameIndex = MachineIndex<detail::FrameIndexTag>;
 using ConstantPoolIndex = MachineIndex<detail::ConstantPoolIndexTag>;
 using JumpTableIndex = MachineIndex<detail::JumpTableIndexTag>;
 using TargetIndex = MachineIndex<detail::TargetIndexTag>;
+using CFIInstructionIndex = MachineIndex<detail::CFIInstructionIndexTag>;
+using IntrinsicID = MachineIndex<detail::IntrinsicIDTag>;
 using SubRegIndex = MachineIndex<detail::SubRegIndexTag>;
 using RegisterUnitID = MachineIndex<detail::RegisterUnitIDTag>;
 using RegisterBankID = MachineIndex<detail::RegisterBankIDTag>;
diff --git a/backend/include/rcc/CodeGen/MachineOperand.h b/backend/include/rcc/CodeGen/MachineOperand.h
index ce5eb60..7344932 100644
--- a/backend/include/rcc/CodeGen/MachineOperand.h
+++ b/backend/include/rcc/CodeGen/MachineOperand.h
@@ -21,6 +21,7 @@
 #include <string_view>
 #include <utility>
 #include <variant>
+#include <vector>
 
 namespace rcc {
 
@@ -58,6 +59,39 @@ private:
   std::uint8_t WordCount;
 };
 
+class IntegerImmediate final {
+public:
+  static constexpr std::uint32_t MaxBitWidth = std::uint32_t{1} << 23U;
+
+  [[nodiscard]] static Expected<IntegerImmediate> create(std::uint32_t BitWidth,
+                                                         std::span<const std::uint64_t> Words);
+  [[nodiscard]] static Expected<IntegerImmediate> fromUnsigned(std::uint32_t BitWidth,
+                                                               std::uint64_t Value);
+  [[nodiscard]] static Expected<IntegerImmediate> fromSigned(std::uint32_t BitWidth,
+                                                             std::int64_t Value);
+
+  [[nodiscard]] std::uint32_t bitWidth() const noexcept {
+    return BitWidth;
+  }
+
+  [[nodiscard]] std::span<const std::uint64_t> words() const noexcept {
+    return Words;
+  }
+
+  [[nodiscard]] bool isZero() const noexcept;
+  [[nodiscard]] bool isNegative() const noexcept;
+  [[nodiscard]] Expected<void> verify() const;
+
+  friend bool operator==(const IntegerImmediate &Left, const IntegerImmediate &Right) = default;
+
+private:
+  IntegerImmediate(std::uint32_t BitWidth, std::vector<std::uint64_t> Words)
+      : BitWidth(BitWidth), Words(std::move(Words)) {}
+
+  std::uint32_t BitWidth;
+  std::vector<std::uint64_t> Words;
+};
+
 enum class RegisterOperandFlag : std::uint8_t {
   Define,
   Use,
@@ -115,6 +149,13 @@ struct FloatingImmediateOperand final {
                          const FloatingImmediateOperand &Right) = default;
 };
 
+struct IntegerImmediateOperand final {
+  IntegerImmediate Value;
+
+  friend bool operator==(const IntegerImmediateOperand &Left,
+                         const IntegerImmediateOperand &Right) = default;
+};
+
 struct BasicBlockOperand final {
   MachineBasicBlock *Block = nullptr;
 
@@ -176,6 +217,13 @@ struct RegisterMaskOperand final {
                          const RegisterMaskOperand &Right) = default;
 };
 
+struct LiveOutRegisterMaskOperand final {
+  std::shared_ptr<const RegisterMask> Mask;
+
+  friend bool operator==(const LiveOutRegisterMaskOperand &Left,
+                         const LiveOutRegisterMaskOperand &Right) = default;
+};
+
 struct MetadataOperand final {
   MachineMetadataValue Value;
 
@@ -197,11 +245,106 @@ struct MCSymbolOperand final {
   friend bool operator==(const MCSymbolOperand &Left, const MCSymbolOperand &Right) = default;
 };
 
+struct CFIIndexOperand final {
+  CFIInstructionIndex Index;
+
+  friend constexpr bool operator==(CFIIndexOperand Left, CFIIndexOperand Right) noexcept = default;
+};
+
+struct IntrinsicOperand final {
+  IntrinsicID ID;
+
+  friend constexpr bool operator==(IntrinsicOperand Left,
+                                   IntrinsicOperand Right) noexcept = default;
+};
+
+enum class IntegerPredicate : std::uint8_t {
+  Equal,
+  NotEqual,
+  SignedGreaterThan,
+  SignedGreaterEqual,
+  SignedLessThan,
+  SignedLessEqual,
+  UnsignedGreaterThan,
+  UnsignedGreaterEqual,
+  UnsignedLessThan,
+  UnsignedLessEqual,
+  Count,
+};
+
+enum class FloatingPredicate : std::uint8_t {
+  AlwaysFalse,
+  OrderedEqual,
+  OrderedGreaterThan,
+  OrderedGreaterEqual,
+  OrderedLessThan,
+  OrderedLessEqual,
+  OrderedNotEqual,
+  Ordered,
+  Unordered,
+  UnorderedEqual,
+  UnorderedGreaterThan,
+  UnorderedGreaterEqual,
+  UnorderedLessThan,
+  UnorderedLessEqual,
+  UnorderedNotEqual,
+  AlwaysTrue,
+  Count,
+};
+
+class PredicateOperand final {
+public:
+  explicit PredicateOperand(IntegerPredicate Predicate) : Value(Predicate) {}
+  explicit PredicateOperand(FloatingPredicate Predicate) : Value(Predicate) {}
+
+  [[nodiscard]] bool isInteger() const noexcept {
+    return std::holds_alternative<IntegerPredicate>(Value);
+  }
+
+  [[nodiscard]] const IntegerPredicate *integerPredicate() const noexcept {
+    return std::get_if<IntegerPredicate>(&Value);
+  }
+
+  [[nodiscard]] const FloatingPredicate *floatingPredicate() const noexcept {
+    return std::get_if<FloatingPredicate>(&Value);
+  }
+
+  [[nodiscard]] Expected<void> verify() const;
+
+  friend bool operator==(const PredicateOperand &Left, const PredicateOperand &Right) = default;
+
+private:
+  std::variant<IntegerPredicate, FloatingPredicate> Value;
+};
+
+struct ShuffleMaskOperand final {
+  static constexpr std::int32_t Undef = -1;
+
+  std::vector<std::int32_t> Elements;
+
+  friend bool operator==(const ShuffleMaskOperand &Left, const ShuffleMaskOperand &Right) = default;
+};
+
+struct DebugInstrRefOperand final {
+  std::uint32_t InstructionNumber = 0;
+  std::uint32_t OperandNumber = 0;
+
+  friend constexpr bool operator==(DebugInstrRefOperand Left,
+                                   DebugInstrRefOperand Right) noexcept = default;
+};
+
+struct LaneMaskOperand final {
+  LaneMask Mask;
+
+  friend constexpr bool operator==(LaneMaskOperand Left, LaneMaskOperand Right) noexcept = default;
+};
+
 class MachineOperand final {
 public:
   enum class Kind : std::uint8_t {
     Register,
     Immediate,
+    IntegerImmediate,
     FloatingImmediate,
     BasicBlock,
     FrameIndex,
@@ -211,9 +354,16 @@ public:
     ExternalSymbol,
     BlockAddress,
     RegisterMask,
+    LiveOutRegisterMask,
     Metadata,
     TargetIndex,
     MCSymbol,
+    CFIIndex,
+    Intrinsic,
+    Predicate,
+    ShuffleMask,
+    DebugInstrRef,
+    LaneMask,
     Count,
   };
 
@@ -228,6 +378,8 @@ public:
                                                                 SubRegIndex SubReg = {},
                                                                 std::uint32_t TargetFlags = 0);
   [[nodiscard]] static MachineOperand immediate(std::int64_t Value, std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static MachineOperand integerImmediate(IntegerImmediate Value,
+                                                       std::uint32_t TargetFlags = 0);
   [[nodiscard]] static MachineOperand floatingImmediate(FloatingPointBits Bits,
                                                         std::uint32_t TargetFlags = 0);
   [[nodiscard]] static MachineOperand basicBlock(MachineBasicBlock &Block,
@@ -248,12 +400,28 @@ public:
                                                              std::uint32_t TargetFlags = 0);
   [[nodiscard]] static Expected<MachineOperand>
   registerMask(std::shared_ptr<const RegisterMask> Mask, std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand>
+  liveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask, std::uint32_t TargetFlags = 0);
   [[nodiscard]] static Expected<MachineOperand> metadata(MachineMetadataValue Value,
                                                          std::uint32_t TargetFlags = 0);
   [[nodiscard]] static MachineOperand targetIndex(TargetIndex Index, std::int64_t Offset = 0,
                                                   std::uint32_t TargetFlags = 0);
   [[nodiscard]] static Expected<MachineOperand> mcSymbol(std::string Name, std::int64_t Offset = 0,
                                                          std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static MachineOperand cfiIndex(CFIInstructionIndex Index,
+                                               std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> intrinsic(IntrinsicID ID,
+                                                          std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> integerPredicate(IntegerPredicate Predicate,
+                                                                 std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> floatingPredicate(FloatingPredicate Predicate,
+                                                                  std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> shuffleMask(std::vector<std::int32_t> Elements,
+                                                            std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> debugInstrRef(std::uint64_t InstructionNumber,
+                                                              std::uint64_t OperandNumber,
+                                                              std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static MachineOperand laneMask(LaneMask Mask, std::uint32_t TargetFlags = 0);
 
   [[nodiscard]] Kind kind() const noexcept {
     return OperandKind;
@@ -307,10 +475,12 @@ public:
 
 private:
   using OperandStorage =
-      std::variant<Register, std::int64_t, FloatingImmediateOperand, BasicBlockOperand,
-                   FrameIndexOperand, ConstantPoolOperand, JumpTableOperand, GlobalSymbolOperand,
-                   ExternalSymbolOperand, BlockAddressOperand, RegisterMaskOperand, MetadataOperand,
-                   TargetIndexOperand, MCSymbolOperand>;
+      std::variant<Register, std::int64_t, IntegerImmediateOperand, FloatingImmediateOperand,
+                   BasicBlockOperand, FrameIndexOperand, ConstantPoolOperand, JumpTableOperand,
+                   GlobalSymbolOperand, ExternalSymbolOperand, BlockAddressOperand,
+                   RegisterMaskOperand, LiveOutRegisterMaskOperand, MetadataOperand,
+                   TargetIndexOperand, MCSymbolOperand, CFIIndexOperand, IntrinsicOperand,
+                   PredicateOperand, ShuffleMaskOperand, DebugInstrRefOperand, LaneMaskOperand>;
 
   MachineOperand(Kind OperandKind, OperandStorage Storage, std::uint32_t TargetFlags = 0)
       : OperandKind(OperandKind), Storage(std::move(Storage)), TargetFlags(TargetFlags) {}
diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt
index 08ca00e..d0f725d 100644
--- a/backend/lib/CodeGen/CMakeLists.txt
+++ b/backend/lib/CodeGen/CMakeLists.txt
@@ -6,6 +6,7 @@ set(
   InstructionDescriptor.cpp
   LowLevelType.cpp
   MachineBasicBlockAttributes.cpp
+  MachineCFIInfo.cpp
   MachineCompilationStatus.cpp
   MachineBasicBlock.cpp
   MachineCFGUpdater.cpp
@@ -39,6 +40,7 @@ set(
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/InstructionDescriptor.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LowLevelType.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlockAttributes.h"
+  "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFIInfo.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCompilationStatus.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlock.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFGUpdater.h"
@@ -101,6 +103,7 @@ rcc_backend_register_format_files(
   "${CMAKE_CURRENT_SOURCE_DIR}/InstructionDescriptor.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/LowLevelType.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlockAttributes.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFIInfo.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp"
@@ -134,6 +137,7 @@ rcc_backend_register_tidy_files(
   "${CMAKE_CURRENT_SOURCE_DIR}/InstructionDescriptor.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/LowLevelType.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlockAttributes.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFIInfo.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineCompilationStatus.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineBasicBlock.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineCFGUpdater.cpp"
diff --git a/backend/lib/CodeGen/MachineCFIInfo.cpp b/backend/lib/CodeGen/MachineCFIInfo.cpp
new file mode 100644
index 0000000..db9f319
--- /dev/null
+++ b/backend/lib/CodeGen/MachineCFIInfo.cpp
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/MachineCFIInfo.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <string>
+#include <utility>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] Expected<void> verifyVectorRegisters(const MachineCFIVectorRegistersRule &Rule,
+                                                   ErrorCode Code) {
+  if (Rule.VectorRegisters.empty()) {
+    return makeUnexpected(Code, "CFI vector-register rule must contain at least one lane");
+  }
+  return {};
+}
+
+} // namespace
+
+MachineCFIInstruction MachineCFIInstruction::sameValue(DwarfRegisterNumber Register) {
+  return MachineCFIInstruction(MachineCFIOperation::SameValue, MachineCFIRegisterRule{Register});
+}
+
+MachineCFIInstruction MachineCFIInstruction::rememberState() {
+  return MachineCFIInstruction(MachineCFIOperation::RememberState, MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::restoreState() {
+  return MachineCFIInstruction(MachineCFIOperation::RestoreState, MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::offset(DwarfRegisterNumber Register,
+                                                    std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::Offset,
+                               MachineCFIRegisterOffsetRule{Register, Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::llvmDefAddressSpaceCFA(DwarfRegisterNumber Register,
+                                                                    std::int64_t Offset,
+                                                                    std::uint32_t AddressSpace) {
+  return MachineCFIInstruction(MachineCFIOperation::LLVMDefAddressSpaceCFA,
+                               MachineCFIAddressSpaceCFARule{Register, Offset, AddressSpace});
+}
+
+MachineCFIInstruction MachineCFIInstruction::defCFARegister(DwarfRegisterNumber Register) {
+  return MachineCFIInstruction(MachineCFIOperation::DefCFARegister,
+                               MachineCFIRegisterRule{Register});
+}
+
+MachineCFIInstruction MachineCFIInstruction::defCFAOffset(std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::DefCFAOffset, MachineCFIOffsetRule{Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::defCFA(DwarfRegisterNumber Register,
+                                                    std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::DefCFA,
+                               MachineCFIRegisterOffsetRule{Register, Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::relativeOffset(DwarfRegisterNumber Register,
+                                                            std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::RelativeOffset,
+                               MachineCFIRegisterOffsetRule{Register, Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::adjustCFAOffset(std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::AdjustCFAOffset, MachineCFIOffsetRule{Offset});
+}
+
+Expected<MachineCFIInstruction> MachineCFIInstruction::escape(std::vector<std::uint8_t> Bytes) {
+  if (Bytes.empty())
+    return makeUnexpected(ErrorCode::InvalidArgument, "CFI escape byte sequence cannot be empty");
+  return MachineCFIInstruction(MachineCFIOperation::Escape, MachineCFIEscapeRule{std::move(Bytes)});
+}
+
+MachineCFIInstruction MachineCFIInstruction::restore(DwarfRegisterNumber Register) {
+  return MachineCFIInstruction(MachineCFIOperation::Restore, MachineCFIRegisterRule{Register});
+}
+
+MachineCFIInstruction MachineCFIInstruction::undefined(DwarfRegisterNumber Register) {
+  return MachineCFIInstruction(MachineCFIOperation::Undefined, MachineCFIRegisterRule{Register});
+}
+
+MachineCFIInstruction MachineCFIInstruction::registerRule(DwarfRegisterNumber Register,
+                                                          DwarfRegisterNumber SourceRegister) {
+  return MachineCFIInstruction(MachineCFIOperation::Register,
+                               MachineCFIRegisterRegisterRule{Register, SourceRegister});
+}
+
+MachineCFIInstruction MachineCFIInstruction::windowSave() {
+  return MachineCFIInstruction(MachineCFIOperation::WindowSave, MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::negateReturnAddressState() {
+  return MachineCFIInstruction(MachineCFIOperation::NegateReturnAddressState,
+                               MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::negateReturnAddressStateWithPC() {
+  return MachineCFIInstruction(MachineCFIOperation::NegateReturnAddressStateWithPC,
+                               MachineCFINoOperands{});
+}
+
+MachineCFIInstruction MachineCFIInstruction::gnuArgsSize(std::int64_t Size) {
+  return MachineCFIInstruction(MachineCFIOperation::GNUArgsSize, MachineCFIOffsetRule{Size});
+}
+
+Expected<MachineCFIInstruction> MachineCFIInstruction::label(std::string Symbol) {
+  if (Symbol.empty())
+    return makeUnexpected(ErrorCode::InvalidArgument, "CFI label symbol cannot be empty");
+  return MachineCFIInstruction(MachineCFIOperation::Label, MachineCFILabelRule{std::move(Symbol)});
+}
+
+MachineCFIInstruction MachineCFIInstruction::valueOffset(DwarfRegisterNumber Register,
+                                                         std::int64_t Offset) {
+  return MachineCFIInstruction(MachineCFIOperation::ValueOffset,
+                               MachineCFIRegisterOffsetRule{Register, Offset});
+}
+
+MachineCFIInstruction MachineCFIInstruction::llvmRegisterPair(MachineCFIRegisterPairRule Rule) {
+  return MachineCFIInstruction(MachineCFIOperation::LLVMRegisterPair, Rule);
+}
+
+Expected<MachineCFIInstruction>
+MachineCFIInstruction::llvmVectorRegisters(MachineCFIVectorRegistersRule Rule) {
+  auto Verification = verifyVectorRegisters(Rule, ErrorCode::InvalidArgument);
+  if (!Verification)
+    return std::unexpected(Verification.error());
+  return MachineCFIInstruction(MachineCFIOperation::LLVMVectorRegisters, std::move(Rule));
+}
+
+MachineCFIInstruction MachineCFIInstruction::llvmVectorOffset(MachineCFIVectorOffsetRule Rule) {
+  return MachineCFIInstruction(MachineCFIOperation::LLVMVectorOffset, Rule);
+}
+
+MachineCFIInstruction
+MachineCFIInstruction::llvmVectorRegisterMask(MachineCFIVectorRegisterMaskRule Rule) {
+  return MachineCFIInstruction(MachineCFIOperation::LLVMVectorRegisterMask, Rule);
+}
+
+Expected<void> MachineCFIInstruction::verify() const {
+  switch (Operation) {
+  case MachineCFIOperation::SameValue:
+  case MachineCFIOperation::DefCFARegister:
+  case MachineCFIOperation::Restore:
+  case MachineCFIOperation::Undefined:
+    if (getIf<MachineCFIRegisterRule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::RememberState:
+  case MachineCFIOperation::RestoreState:
+  case MachineCFIOperation::WindowSave:
+  case MachineCFIOperation::NegateReturnAddressState:
+  case MachineCFIOperation::NegateReturnAddressStateWithPC:
+    if (getIf<MachineCFINoOperands>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::Offset:
+  case MachineCFIOperation::DefCFA:
+  case MachineCFIOperation::RelativeOffset:
+  case MachineCFIOperation::ValueOffset:
+    if (getIf<MachineCFIRegisterOffsetRule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::LLVMDefAddressSpaceCFA:
+    if (getIf<MachineCFIAddressSpaceCFARule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::DefCFAOffset:
+  case MachineCFIOperation::AdjustCFAOffset:
+  case MachineCFIOperation::GNUArgsSize:
+    if (getIf<MachineCFIOffsetRule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::Escape: {
+    const auto *Rule = getIf<MachineCFIEscapeRule>();
+    if (Rule == nullptr)
+      break;
+    if (Rule->Bytes.empty()) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "CFI escape byte sequence cannot be empty");
+    }
+    return {};
+  }
+  case MachineCFIOperation::Register:
+    if (getIf<MachineCFIRegisterRegisterRule>() == nullptr)
+      break;
+    return {};
+  case MachineCFIOperation::Label: {
+    const auto *Rule = getIf<MachineCFILabelRule>();
+    if (Rule == nullptr)
+      break;
+    if (Rule->Symbol.empty())
+      return makeUnexpected(ErrorCode::VerificationError, "CFI label symbol cannot be empty");
+    return {};
+  }
+  case MachineCFIOperation::LLVMRegisterPair: {
+    const auto *Rule = getIf<MachineCFIRegisterPairRule>();
+    if (Rule == nullptr)
+      break;
+    return {};
+  }
+  case MachineCFIOperation::LLVMVectorRegisters: {
+    const auto *Rule = getIf<MachineCFIVectorRegistersRule>();
+    if (Rule == nullptr)
+      break;
+    return verifyVectorRegisters(*Rule, ErrorCode::VerificationError);
+  }
+  case MachineCFIOperation::LLVMVectorOffset: {
+    const auto *Rule = getIf<MachineCFIVectorOffsetRule>();
+    if (Rule == nullptr)
+      break;
+    return {};
+  }
+  case MachineCFIOperation::LLVMVectorRegisterMask: {
+    const auto *Rule = getIf<MachineCFIVectorRegisterMaskRule>();
+    if (Rule == nullptr)
+      break;
+    return {};
+  }
+  case MachineCFIOperation::Count:
+    break;
+  }
+  return makeUnexpected(ErrorCode::VerificationError,
+                        "CFI instruction operation and payload do not match");
+}
+
+Expected<CFIInstructionIndex> MachineCFIInfo::addInstruction(MachineCFIInstruction Instruction) {
+  auto Verification = Instruction.verify();
+  if (!Verification)
+    return std::unexpected(Verification.error());
+  if (Instructions.size() > std::numeric_limits<std::uint32_t>::max()) {
+    return makeUnexpected(ErrorCode::Unsupported, "CFI instruction index space is exhausted");
+  }
+
+  const auto Index =
+      CFIInstructionIndex::fromValue(static_cast<std::uint32_t>(Instructions.size()));
+  Instructions.push_back(std::move(Instruction));
+  return Index;
+}
+
+const MachineCFIInstruction *MachineCFIInfo::instruction(CFIInstructionIndex Index) const noexcept {
+  if (Index.value() >= Instructions.size())
+    return nullptr;
+  return &Instructions[Index.value()];
+}
+
+Expected<void> MachineCFIInfo::verify() const {
+  for (const MachineCFIInstruction &Instruction : Instructions) {
+    auto Verification = Instruction.verify();
+    if (!Verification)
+      return std::unexpected(Verification.error());
+  }
+  return {};
+}
+
+Expected<MachineCFIInfo> MachineCFIInfo::clone() const {
+  auto Verification = verify();
+  if (!Verification)
+    return std::unexpected(Verification.error());
+
+  MachineCFIInfo Result;
+  for (const MachineCFIInstruction &Instruction : Instructions) {
+    auto Index = Result.addInstruction(Instruction);
+    if (!Index)
+      return std::unexpected(Index.error());
+  }
+  return Result;
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/MachineFunctionCloner.cpp b/backend/lib/CodeGen/MachineFunctionCloner.cpp
index dcf9712..b769f7d 100644
--- a/backend/lib/CodeGen/MachineFunctionCloner.cpp
+++ b/backend/lib/CodeGen/MachineFunctionCloner.cpp
@@ -126,13 +126,21 @@ struct CloneState final {
     return MachineOperand::jumpTable(Position->second, SourceOperand.targetFlags());
   }
   case MachineOperand::Kind::Immediate:
+  case MachineOperand::Kind::IntegerImmediate:
   case MachineOperand::Kind::FloatingImmediate:
   case MachineOperand::Kind::GlobalSymbol:
   case MachineOperand::Kind::ExternalSymbol:
   case MachineOperand::Kind::RegisterMask:
+  case MachineOperand::Kind::LiveOutRegisterMask:
   case MachineOperand::Kind::Metadata:
   case MachineOperand::Kind::TargetIndex:
   case MachineOperand::Kind::MCSymbol:
+  case MachineOperand::Kind::CFIIndex:
+  case MachineOperand::Kind::Intrinsic:
+  case MachineOperand::Kind::Predicate:
+  case MachineOperand::Kind::ShuffleMask:
+  case MachineOperand::Kind::DebugInstrRef:
+  case MachineOperand::Kind::LaneMask:
     return SourceOperand.cloneDetached();
   case MachineOperand::Kind::Count:
     return makeUnexpected(ErrorCode::VerificationError, "source operand has an invalid kind");
@@ -636,6 +644,10 @@ Expected<MachineFunctionCloneResult> cloneMachineFunction(const MachineFunction
   if (!Metadata)
     return std::unexpected(Metadata.error());
   Result.ClonedFunction->metadata() = std::move(*Metadata);
+  auto CFIInfo = Source.cfiInfo().clone();
+  if (!CFIInfo)
+    return std::unexpected(CFIInfo.error());
+  Result.ClonedFunction->cfiInfo() = std::move(*CFIInfo);
   Result.ClonedFunction->properties() = Source.properties();
   Result.ClonedFunction->attributes() = Source.attributes();
   auto DestinationVerification = MachineVerifier::verify(*Result.ClonedFunction, Target);
diff --git a/backend/lib/CodeGen/MachineIRBuilder.cpp b/backend/lib/CodeGen/MachineIRBuilder.cpp
index 5afdd72..e1e8006 100644
--- a/backend/lib/CodeGen/MachineIRBuilder.cpp
+++ b/backend/lib/CodeGen/MachineIRBuilder.cpp
@@ -162,6 +162,11 @@ Expected<void> MachineInstrBuilder::addImmediate(std::int64_t Value, std::uint32
   return addOperand(MachineOperand::immediate(Value, TargetFlags));
 }
 
+Expected<void> MachineInstrBuilder::addIntegerImmediate(IntegerImmediate Value,
+                                                        std::uint32_t TargetFlags) {
+  return addOperand(MachineOperand::integerImmediate(std::move(Value), TargetFlags));
+}
+
 Expected<void> MachineInstrBuilder::addFloatingImmediate(FloatingPointBits Bits,
                                                          std::uint32_t TargetFlags) {
   return addOperand(MachineOperand::floatingImmediate(Bits, TargetFlags));
@@ -220,6 +225,14 @@ Expected<void> MachineInstrBuilder::addRegisterMask(std::shared_ptr<const Regist
   return addOperand(std::move(*Operand));
 }
 
+Expected<void> MachineInstrBuilder::addLiveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask,
+                                                           std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::liveOutRegisterMask(std::move(Mask), TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
 Expected<void> MachineInstrBuilder::addMetadata(MachineMetadataValue Value,
                                                 std::uint32_t TargetFlags) {
   auto Operand = MachineOperand::metadata(std::move(Value), TargetFlags);
@@ -241,6 +254,55 @@ Expected<void> MachineInstrBuilder::addMCSymbol(std::string Name, std::int64_t O
   return addOperand(std::move(*Operand));
 }
 
+Expected<void> MachineInstrBuilder::addCFIInstruction(CFIInstructionIndex Index,
+                                                      std::uint32_t TargetFlags) {
+  return addOperand(MachineOperand::cfiIndex(Index, TargetFlags));
+}
+
+Expected<void> MachineInstrBuilder::addIntrinsic(IntrinsicID ID, std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::intrinsic(ID, TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addIntegerPredicate(IntegerPredicate Predicate,
+                                                        std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::integerPredicate(Predicate, TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addFloatingPredicate(FloatingPredicate Predicate,
+                                                         std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::floatingPredicate(Predicate, TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addShuffleMask(std::vector<std::int32_t> Elements,
+                                                   std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::shuffleMask(std::move(Elements), TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addDebugInstrRef(std::uint64_t InstructionNumber,
+                                                     std::uint64_t OperandNumber,
+                                                     std::uint32_t TargetFlags) {
+  auto Operand = MachineOperand::debugInstrRef(InstructionNumber, OperandNumber, TargetFlags);
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return addOperand(std::move(*Operand));
+}
+
+Expected<void> MachineInstrBuilder::addLaneMask(LaneMask Mask, std::uint32_t TargetFlags) {
+  return addOperand(MachineOperand::laneMask(Mask, TargetFlags));
+}
+
 Expected<void> MachineInstrBuilder::tieOperands(std::size_t DefOperand, std::size_t UseOperand) {
   auto Pending = ensurePending();
   if (!Pending)
@@ -377,6 +439,14 @@ Expected<void> MachineInstrBuilder::validateJumpTableReference(JumpTableIndex In
   return {};
 }
 
+Expected<void> MachineInstrBuilder::validateCFIReference(CFIInstructionIndex Index) const {
+  if (Function->cfiInfo().instruction(Index) == nullptr) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "builder references an unknown CFI instruction");
+  }
+  return {};
+}
+
 Expected<void> MachineInstrBuilder::validateOperandOwnership(const MachineOperand &Operand) const {
   if (Operand.parent() != nullptr) {
     return makeUnexpected(ErrorCode::InvalidArgument,
@@ -409,6 +479,11 @@ Expected<void> MachineInstrBuilder::validateOperandOwnership(const MachineOperan
     if (!JumpTableResult)
       return JumpTableResult;
   }
+  if (const auto *CFI = Operand.getIf<CFIIndexOperand>(); CFI != nullptr) {
+    auto CFIResult = validateCFIReference(CFI->Index);
+    if (!CFIResult)
+      return CFIResult;
+  }
   return {};
 }
 
diff --git a/backend/lib/CodeGen/MachineOperand.cpp b/backend/lib/CodeGen/MachineOperand.cpp
index cd2d217..6cce188 100644
--- a/backend/lib/CodeGen/MachineOperand.cpp
+++ b/backend/lib/CodeGen/MachineOperand.cpp
@@ -9,11 +9,103 @@
 #include <algorithm>
 #include <array>
 #include <cstdint>
+#include <limits>
 #include <memory>
 #include <span>
 #include <utility>
+#include <vector>
 
 namespace rcc {
+namespace {
+
+[[nodiscard]] constexpr std::size_t integerWordCount(std::uint32_t BitWidth) noexcept {
+  return (static_cast<std::size_t>(BitWidth) + 63U) / 64U;
+}
+
+[[nodiscard]] constexpr std::uint64_t integerHighWordMask(std::uint32_t BitWidth) noexcept {
+  const std::uint32_t UsedBits = BitWidth % 64U;
+  if (UsedBits == 0)
+    return std::numeric_limits<std::uint64_t>::max();
+  return (std::uint64_t{1} << UsedBits) - 1U;
+}
+
+[[nodiscard]] Expected<void> verifyIntegerImmediate(std::uint32_t BitWidth,
+                                                    std::span<const std::uint64_t> Words,
+                                                    ErrorCode Code) {
+  if (BitWidth == 0 || BitWidth > IntegerImmediate::MaxBitWidth) {
+    return makeUnexpected(Code, "integer immediate bit width is outside the supported range");
+  }
+  if (Words.size() != integerWordCount(BitWidth)) {
+    return makeUnexpected(Code, "integer immediate bit pattern has the wrong word count");
+  }
+  if ((Words.back() & ~integerHighWordMask(BitWidth)) != 0) {
+    return makeUnexpected(Code, "integer immediate has nonzero bits above its declared width");
+  }
+  return {};
+}
+
+} // namespace
+
+Expected<IntegerImmediate> IntegerImmediate::create(std::uint32_t BitWidth,
+                                                    std::span<const std::uint64_t> Words) {
+  auto Verification = verifyIntegerImmediate(BitWidth, Words, ErrorCode::InvalidArgument);
+  if (!Verification)
+    return std::unexpected(Verification.error());
+  return IntegerImmediate(BitWidth, std::vector<std::uint64_t>(Words.begin(), Words.end()));
+}
+
+Expected<IntegerImmediate> IntegerImmediate::fromUnsigned(std::uint32_t BitWidth,
+                                                          std::uint64_t Value) {
+  if (BitWidth == 0 || BitWidth > MaxBitWidth) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "integer immediate bit width is outside the supported range");
+  }
+  if (BitWidth < 64U && (Value >> BitWidth) != 0) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "unsigned value does not fit the integer immediate bit width");
+  }
+
+  std::vector<std::uint64_t> Words(integerWordCount(BitWidth), 0);
+  Words.front() = Value;
+  return create(BitWidth, Words);
+}
+
+Expected<IntegerImmediate> IntegerImmediate::fromSigned(std::uint32_t BitWidth,
+                                                        std::int64_t Value) {
+  if (BitWidth == 0 || BitWidth > MaxBitWidth) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "integer immediate bit width is outside the supported range");
+  }
+  if (BitWidth < 64U) {
+    const std::uint32_t MagnitudeBits = BitWidth - 1U;
+    const std::int64_t Minimum = -(std::int64_t{1} << MagnitudeBits);
+    const std::int64_t Maximum = (std::int64_t{1} << MagnitudeBits) - 1;
+    if (Value < Minimum || Value > Maximum) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "signed value does not fit the integer immediate bit width");
+    }
+  }
+
+  const std::uint64_t Fill =
+      Value < 0 ? std::numeric_limits<std::uint64_t>::max() : std::uint64_t{0};
+  std::vector<std::uint64_t> Words(integerWordCount(BitWidth), Fill);
+  Words.front() = static_cast<std::uint64_t>(Value);
+  Words.back() &= integerHighWordMask(BitWidth);
+  return create(BitWidth, Words);
+}
+
+bool IntegerImmediate::isZero() const noexcept {
+  return std::ranges::all_of(Words, [](std::uint64_t Word) { return Word == 0; });
+}
+
+bool IntegerImmediate::isNegative() const noexcept {
+  const std::uint32_t SignBit = BitWidth - 1U;
+  return (Words[SignBit / 64U] & (std::uint64_t{1} << (SignBit % 64U))) != 0;
+}
+
+Expected<void> IntegerImmediate::verify() const {
+  return verifyIntegerImmediate(BitWidth, Words, ErrorCode::VerificationError);
+}
 
 Expected<FloatingPointBits> FloatingPointBits::create(FloatSemantics Semantics,
                                                       std::span<const std::uint64_t> Words) {
@@ -45,6 +137,20 @@ RegisterOperandFlags::RegisterOperandFlags(
     set(Flag);
 }
 
+Expected<void> PredicateOperand::verify() const {
+  if (const auto *Integer = integerPredicate();
+      Integer != nullptr &&
+      static_cast<std::uint8_t>(*Integer) < static_cast<std::uint8_t>(IntegerPredicate::Count)) {
+    return {};
+  }
+  if (const auto *Floating = floatingPredicate();
+      Floating != nullptr &&
+      static_cast<std::uint8_t>(*Floating) < static_cast<std::uint8_t>(FloatingPredicate::Count)) {
+    return {};
+  }
+  return makeUnexpected(ErrorCode::VerificationError, "predicate operand contains an invalid kind");
+}
+
 Expected<void> MachineOperand::verifyRegisterFlags(RegisterOperandFlags Flags) {
   const bool IsDef = Flags.has(RegisterOperandFlag::Define);
   const bool IsUse = Flags.has(RegisterOperandFlag::Use);
@@ -96,6 +202,11 @@ MachineOperand MachineOperand::immediate(std::int64_t Value, std::uint32_t Targe
   return MachineOperand(Kind::Immediate, Value, TargetFlags);
 }
 
+MachineOperand MachineOperand::integerImmediate(IntegerImmediate Value, std::uint32_t TargetFlags) {
+  return MachineOperand(Kind::IntegerImmediate, IntegerImmediateOperand{std::move(Value)},
+                        TargetFlags);
+}
+
 MachineOperand MachineOperand::floatingImmediate(FloatingPointBits Bits,
                                                  std::uint32_t TargetFlags) {
   return MachineOperand(Kind::FloatingImmediate, FloatingImmediateOperand{Bits}, TargetFlags);
@@ -158,6 +269,17 @@ Expected<MachineOperand> MachineOperand::registerMask(std::shared_ptr<const Regi
   return MachineOperand(Kind::RegisterMask, RegisterMaskOperand{std::move(Mask)}, TargetFlags);
 }
 
+Expected<MachineOperand>
+MachineOperand::liveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask,
+                                    std::uint32_t TargetFlags) {
+  if (!Mask) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "live-out register-mask operand cannot be null");
+  }
+  return MachineOperand(Kind::LiveOutRegisterMask, LiveOutRegisterMaskOperand{std::move(Mask)},
+                        TargetFlags);
+}
+
 Expected<MachineOperand> MachineOperand::metadata(MachineMetadataValue Value,
                                                   std::uint32_t TargetFlags) {
   if (!Value.isValid()) {
@@ -179,6 +301,65 @@ Expected<MachineOperand> MachineOperand::mcSymbol(std::string Name, std::int64_t
   return MachineOperand(Kind::MCSymbol, MCSymbolOperand{std::move(Name), Offset}, TargetFlags);
 }
 
+MachineOperand MachineOperand::cfiIndex(CFIInstructionIndex Index, std::uint32_t TargetFlags) {
+  return MachineOperand(Kind::CFIIndex, CFIIndexOperand{Index}, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::intrinsic(IntrinsicID ID, std::uint32_t TargetFlags) {
+  if (ID.value() == 0) {
+    return makeUnexpected(ErrorCode::InvalidArgument, "intrinsic operand ID must be nonzero");
+  }
+  return MachineOperand(Kind::Intrinsic, IntrinsicOperand{ID}, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::integerPredicate(IntegerPredicate Predicate,
+                                                          std::uint32_t TargetFlags) {
+  PredicateOperand Value(Predicate);
+  auto Verification = Value.verify();
+  if (!Verification)
+    return makeUnexpected(ErrorCode::InvalidArgument, "integer predicate operand is invalid");
+  return MachineOperand(Kind::Predicate, Value, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::floatingPredicate(FloatingPredicate Predicate,
+                                                           std::uint32_t TargetFlags) {
+  PredicateOperand Value(Predicate);
+  auto Verification = Value.verify();
+  if (!Verification)
+    return makeUnexpected(ErrorCode::InvalidArgument, "floating predicate operand is invalid");
+  return MachineOperand(Kind::Predicate, Value, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::shuffleMask(std::vector<std::int32_t> Elements,
+                                                     std::uint32_t TargetFlags) {
+  if (Elements.size() < 2) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "shuffle-mask operand requires at least two elements");
+  }
+  return MachineOperand(Kind::ShuffleMask, ShuffleMaskOperand{std::move(Elements)}, TargetFlags);
+}
+
+Expected<MachineOperand> MachineOperand::debugInstrRef(std::uint64_t InstructionNumber,
+                                                       std::uint64_t OperandNumber,
+                                                       std::uint32_t TargetFlags) {
+  if (InstructionNumber > std::numeric_limits<std::uint32_t>::max()) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "debug instruction reference instruction number must fit in uint32");
+  }
+  if (OperandNumber > std::numeric_limits<std::uint32_t>::max()) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "debug instruction reference operand number must fit in uint32");
+  }
+  return MachineOperand(Kind::DebugInstrRef,
+                        DebugInstrRefOperand{static_cast<std::uint32_t>(InstructionNumber),
+                                             static_cast<std::uint32_t>(OperandNumber)},
+                        TargetFlags);
+}
+
+MachineOperand MachineOperand::laneMask(LaneMask Mask, std::uint32_t TargetFlags) {
+  return MachineOperand(Kind::LaneMask, LaneMaskOperand{Mask}, TargetFlags);
+}
+
 MachineOperand::MachineOperand(MachineOperand &&Other) noexcept
     : OperandKind(Other.OperandKind), Storage(std::move(Other.Storage)), RegFlags(Other.RegFlags),
       RegSubReg(Other.RegSubReg), TiedOperandIndex(Other.TiedOperandIndex),
diff --git a/backend/lib/CodeGen/MachineVerifier.cpp b/backend/lib/CodeGen/MachineVerifier.cpp
index 7e050ec..98fb9f7 100644
--- a/backend/lib/CodeGen/MachineVerifier.cpp
+++ b/backend/lib/CodeGen/MachineVerifier.cpp
@@ -40,6 +40,9 @@ public:
     auto MetadataResult = Function.metadata().verify();
     if (!MetadataResult)
       return failWith("machine metadata", MetadataResult.error());
+    auto CFIResult = Function.cfiInfo().verify();
+    if (!CFIResult)
+      return failWith("CFI information", CFIResult.error());
     if (Function.targetState().hasValue()) {
       auto TargetStateResult = Function.targetState().verify();
       if (!TargetStateResult)
@@ -103,6 +106,8 @@ private:
       return Operand.getIf<Register>() != nullptr;
     case MachineOperand::Kind::Immediate:
       return Operand.getIf<std::int64_t>() != nullptr;
+    case MachineOperand::Kind::IntegerImmediate:
+      return Operand.getIf<IntegerImmediateOperand>() != nullptr;
     case MachineOperand::Kind::FloatingImmediate:
       return Operand.getIf<FloatingImmediateOperand>() != nullptr;
     case MachineOperand::Kind::BasicBlock:
@@ -121,12 +126,26 @@ private:
       return Operand.getIf<BlockAddressOperand>() != nullptr;
     case MachineOperand::Kind::RegisterMask:
       return Operand.getIf<RegisterMaskOperand>() != nullptr;
+    case MachineOperand::Kind::LiveOutRegisterMask:
+      return Operand.getIf<LiveOutRegisterMaskOperand>() != nullptr;
     case MachineOperand::Kind::Metadata:
       return Operand.getIf<MetadataOperand>() != nullptr;
     case MachineOperand::Kind::TargetIndex:
       return Operand.getIf<TargetIndexOperand>() != nullptr;
     case MachineOperand::Kind::MCSymbol:
       return Operand.getIf<MCSymbolOperand>() != nullptr;
+    case MachineOperand::Kind::CFIIndex:
+      return Operand.getIf<CFIIndexOperand>() != nullptr;
+    case MachineOperand::Kind::Intrinsic:
+      return Operand.getIf<IntrinsicOperand>() != nullptr;
+    case MachineOperand::Kind::Predicate:
+      return Operand.getIf<PredicateOperand>() != nullptr;
+    case MachineOperand::Kind::ShuffleMask:
+      return Operand.getIf<ShuffleMaskOperand>() != nullptr;
+    case MachineOperand::Kind::DebugInstrRef:
+      return Operand.getIf<DebugInstrRefOperand>() != nullptr;
+    case MachineOperand::Kind::LaneMask:
+      return Operand.getIf<LaneMaskOperand>() != nullptr;
     case MachineOperand::Kind::Count:
       return false;
     }
@@ -286,6 +305,15 @@ private:
       if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); Mask != nullptr && !Mask->Mask) {
         return fail("register-mask operand contains a null mask");
       }
+      if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>();
+          Mask != nullptr && !Mask->Mask) {
+        return fail("live-out register-mask operand contains a null mask");
+      }
+      if (const auto *Integer = Operand.getIf<IntegerImmediateOperand>(); Integer != nullptr) {
+        auto IntegerResult = Integer->Value.verify();
+        if (!IntegerResult)
+          return failWith("integer immediate operand", IntegerResult.error());
+      }
       if (const auto *Global = Operand.getIf<GlobalSymbolOperand>();
           Global != nullptr && Global->Name.empty()) {
         return fail("global symbol operand has an empty name");
@@ -307,6 +335,19 @@ private:
           Symbol != nullptr && Symbol->Name.empty()) {
         return fail("MC symbol operand has an empty name");
       }
+      if (const auto *Intrinsic = Operand.getIf<IntrinsicOperand>();
+          Intrinsic != nullptr && Intrinsic->ID.value() == 0) {
+        return fail("intrinsic operand has an invalid ID");
+      }
+      if (const auto *Predicate = Operand.getIf<PredicateOperand>(); Predicate != nullptr) {
+        auto PredicateResult = Predicate->verify();
+        if (!PredicateResult)
+          return failWith("predicate operand", PredicateResult.error());
+      }
+      if (const auto *Shuffle = Operand.getIf<ShuffleMaskOperand>();
+          Shuffle != nullptr && Shuffle->Elements.size() < 2) {
+        return fail("shuffle-mask operand has fewer than two elements");
+      }
     }
 
     for (const MachineMemOperand *MemOperand : Instruction.memOperands()) {
@@ -497,6 +538,10 @@ private:
         if (!JumpTableResult)
           return JumpTableResult;
       }
+      if (const auto *CFI = Operand.getIf<CFIIndexOperand>();
+          CFI != nullptr && Function.cfiInfo().instruction(CFI->Index) == nullptr) {
+        return fail("live instruction references an unknown CFI instruction");
+      }
 
       if (const auto *Block = Operand.getIf<BasicBlockOperand>();
           Block != nullptr &&
diff --git a/backend/lib/CodeGen/TargetDescription.cpp b/backend/lib/CodeGen/TargetDescription.cpp
index 3a6ea31..a32534d 100644
--- a/backend/lib/CodeGen/TargetDescription.cpp
+++ b/backend/lib/CodeGen/TargetDescription.cpp
@@ -203,6 +203,13 @@ Expected<void> TargetDescription::validateTargetOperand(const OperandDescriptor
     }
     return Registers.validateRegisterMask(*Mask->Mask);
   }
+  if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); Mask != nullptr) {
+    if (!Mask->Mask) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "live-out register-mask operand contains a null mask");
+    }
+    return Registers.validateRegisterMask(*Mask->Mask);
+  }
 
   const Register *Reg = Operand.getIf<Register>();
   if (Reg == nullptr)
diff --git a/backend/lib/CodeGen/TargetRegisterInfo.cpp b/backend/lib/CodeGen/TargetRegisterInfo.cpp
index 7bd4834..7c8af82 100644
--- a/backend/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/backend/lib/CodeGen/TargetRegisterInfo.cpp
@@ -592,7 +592,7 @@ Expected<void> TargetRegisterInfo::validateRegisterMask(const RegisterMask &Mask
       const auto Reg = PhysicalRegister::create(RegisterNumber);
       if (!Reg || !isKnownPhysicalRegister(*Reg)) {
         return makeUnexpected(ErrorCode::VerificationError,
-                              "register mask preserves an unknown physical register");
+                              "register mask contains an unknown physical register");
       }
       Word &= Word - 1;
     }
diff --git a/backend/unittests/CodeGen/CMakeLists.txt b/backend/unittests/CodeGen/CMakeLists.txt
index e7eed63..69f3646 100644
--- a/backend/unittests/CodeGen/CMakeLists.txt
+++ b/backend/unittests/CodeGen/CMakeLists.txt
@@ -16,6 +16,7 @@ set(
   MachineIRCoreTest.cpp
   MachineJumpTableInfoTest.cpp
   MachineMetadataTest.cpp
+  MachineOperandTest.cpp
   MachineVerifierTest.cpp
   MachineVerifierTargetTest.cpp
   MachineValueTypesTest.cpp
@@ -219,6 +220,18 @@ add_test(
   COMMAND RCCBackendCodeGenMachineMetadataTests
 )
 
+add_executable(RCCBackendCodeGenMachineOperandTests MachineOperandTest.cpp)
+rcc_backend_configure_target(RCCBackendCodeGenMachineOperandTests)
+target_link_libraries(
+  RCCBackendCodeGenMachineOperandTests
+  PRIVATE RCCBackend::CodeGen RCCBackend::TestTarget
+)
+
+add_test(
+  NAME RCCBackend.CodeGen.MachineOperand
+  COMMAND RCCBackendCodeGenMachineOperandTests
+)
+
 add_executable(RCCBackendCodeGenTargetConstantTests TargetConstantTest.cpp)
 rcc_backend_configure_target(RCCBackendCodeGenTargetConstantTests)
 target_link_libraries(RCCBackendCodeGenTargetConstantTests PRIVATE RCCBackend::CodeGen)
@@ -267,6 +280,7 @@ rcc_backend_register_format_files(
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineIRCoreTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineJumpTableInfoTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadataTest.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTargetTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineValueTypesTest.cpp"
@@ -289,6 +303,7 @@ rcc_backend_register_tidy_files(
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineIRCoreTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineJumpTableInfoTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadataTest.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTargetTest.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/MachineValueTypesTest.cpp"
diff --git a/backend/unittests/CodeGen/MachineOperandTest.cpp b/backend/unittests/CodeGen/MachineOperandTest.cpp
new file mode 100644
index 0000000..1b246ca
--- /dev/null
+++ b/backend/unittests/CodeGen/MachineOperandTest.cpp
@@ -0,0 +1,482 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/MachineOperand.h"
+
+#include "Target/TestTarget.h"
+#include "rcc/CodeGen/InstructionDescriptor.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineCFIInfo.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineFunctionAttributes.h"
+#include "rcc/CodeGen/MachineFunctionCloner.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineVerifier.h"
+#include "rcc/CodeGen/RegisterMask.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <iostream>
+#include <limits>
+#include <memory>
+#include <span>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace {
+
+constexpr std::uint32_t OperandOpcode = 1;
+
+class TestContext final {
+public:
+  void expect(bool Condition, std::string_view Message) {
+    if (Condition)
+      return;
+    std::cerr << Message << '\n';
+    ++FailureCount;
+  }
+
+  [[nodiscard]] int result() const noexcept {
+    return FailureCount == 0 ? 0 : 1;
+  }
+
+private:
+  int FailureCount = 0;
+};
+
+[[nodiscard]] rcc::OperandDescriptor operandDescriptor(rcc::MachineOperand::Kind Kind) {
+  rcc::OperandDescriptor Descriptor;
+  Descriptor.Kinds = {Kind};
+  return Descriptor;
+}
+
+[[nodiscard]] rcc::Expected<rcc::TargetDescription> createTarget() {
+  rcc::InstructionDescriptor Descriptor;
+  Descriptor.InstructionOpcode = rcc::Opcode::fromValue(OperandOpcode);
+  Descriptor.Name = "TEST_MIR_OPERANDS";
+  Descriptor.Flags = {rcc::InstructionFlag::ProducesCode};
+  Descriptor.Operands = {
+      operandDescriptor(rcc::MachineOperand::Kind::IntegerImmediate),
+      operandDescriptor(rcc::MachineOperand::Kind::LiveOutRegisterMask),
+      operandDescriptor(rcc::MachineOperand::Kind::CFIIndex),
+      operandDescriptor(rcc::MachineOperand::Kind::Intrinsic),
+      operandDescriptor(rcc::MachineOperand::Kind::Predicate),
+      operandDescriptor(rcc::MachineOperand::Kind::Predicate),
+      operandDescriptor(rcc::MachineOperand::Kind::ShuffleMask),
+      operandDescriptor(rcc::MachineOperand::Kind::DebugInstrRef),
+      operandDescriptor(rcc::MachineOperand::Kind::LaneMask),
+  };
+
+  auto Instructions = rcc::InstructionSet::create({std::move(Descriptor)});
+  if (!Instructions)
+    return std::unexpected(Instructions.error());
+  return rcc::test::createTestTargetDescription(std::move(*Instructions));
+}
+
+[[nodiscard]] std::unique_ptr<rcc::MachineFunction>
+createFunction(TestContext &Context, std::uint64_t ID, std::string Name) {
+  auto Function =
+      rcc::MachineFunction::create(std::move(Name), rcc::MachineFunctionStableID::fromValue(ID));
+  Context.expect(Function.has_value(), "machine function construction must succeed");
+  if (!Function)
+    return nullptr;
+  return std::move(*Function);
+}
+
+[[nodiscard]] rcc::MachineBasicBlock *appendBlock(TestContext &Context,
+                                                  rcc::MachineFunction &Function) {
+  auto Block = Function.createBasicBlock();
+  Context.expect(Block.has_value(), "machine basic block construction must succeed");
+  if (!Block)
+    return nullptr;
+  auto AppendResult = Function.appendBlock(**Block);
+  Context.expect(AppendResult.has_value(), "machine basic block insertion must succeed");
+  if (!AppendResult)
+    return nullptr;
+  return *Block;
+}
+
+[[nodiscard]] rcc::Expected<rcc::MachineInstr *>
+buildOperandInstruction(rcc::MachineFunction &Function, const rcc::TargetDescription &Target,
+                        rcc::MachineBasicBlock &Block, rcc::CFIInstructionIndex CFIIndex,
+                        std::shared_ptr<const rcc::RegisterMask> LiveOutMask) {
+  const std::array<std::uint64_t, 2> Words{
+      0x0123456789abcdefULL,
+      0xfedcba9876543210ULL,
+  };
+  auto Integer = rcc::IntegerImmediate::create(128, Words);
+  if (!Integer)
+    return std::unexpected(Integer.error());
+
+  rcc::MachineIRBuilder Builder(Function, Target);
+  auto InsertResult = Builder.setInsertPoint(Block);
+  if (!InsertResult)
+    return std::unexpected(InsertResult.error());
+  auto Pending = Builder.buildInstr(rcc::Opcode::fromValue(OperandOpcode));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Result = Pending->addIntegerImmediate(std::move(*Integer)); !Result)
+    return std::unexpected(Result.error());
+  if (auto Result = Pending->addLiveOutRegisterMask(std::move(LiveOutMask)); !Result)
+    return std::unexpected(Result.error());
+  if (auto Result = Pending->addCFIInstruction(CFIIndex); !Result)
+    return std::unexpected(Result.error());
+  if (auto Result = Pending->addIntrinsic(rcc::IntrinsicID::fromValue(7)); !Result)
+    return std::unexpected(Result.error());
+  if (auto Result = Pending->addIntegerPredicate(rcc::IntegerPredicate::SignedLessThan); !Result)
+    return std::unexpected(Result.error());
+  if (auto Result = Pending->addFloatingPredicate(rcc::FloatingPredicate::UnorderedGreaterEqual);
+      !Result) {
+    return std::unexpected(Result.error());
+  }
+  if (auto Result = Pending->addShuffleMask({1, rcc::ShuffleMaskOperand::Undef, 0}); !Result) {
+    return std::unexpected(Result.error());
+  }
+  if (auto Result = Pending->addDebugInstrRef(9, 3); !Result)
+    return std::unexpected(Result.error());
+  if (auto Result = Pending->addLaneMask(rcc::LaneMask::fromRaw(0x8000000000000001ULL)); !Result) {
+    return std::unexpected(Result.error());
+  }
+  return Pending->build();
+}
+
+void testIntegerImmediate(TestContext &Context) {
+  const std::array<std::uint64_t, 2> Words{
+      0x0123456789abcdefULL,
+      0x0000000000000001ULL,
+  };
+  auto Integer = rcc::IntegerImmediate::create(65, Words);
+  Context.expect(Integer.has_value() && Integer->bitWidth() == 65 && Integer->words().size() == 2 &&
+                     Integer->words()[0] == Words[0] && Integer->words()[1] == Words[1] &&
+                     Integer->isNegative() && !Integer->isZero(),
+                 "arbitrary-width integer must preserve exact little-endian words");
+
+  Context.expect(!rcc::IntegerImmediate::create(0, std::span<const std::uint64_t>{}) &&
+                     !rcc::IntegerImmediate::create(rcc::IntegerImmediate::MaxBitWidth + 1,
+                                                    std::span<const std::uint64_t>{}),
+                 "integer immediate must reject unsupported bit widths");
+  const std::array<std::uint64_t, 2> TwoWords{1, 0};
+  const std::array<std::uint64_t, 1> HighBits{0x100};
+  Context.expect(!rcc::IntegerImmediate::create(64, TwoWords) &&
+                     !rcc::IntegerImmediate::create(8, HighBits),
+                 "integer immediate must reject word-count and high-bit mismatches");
+
+  auto UnsignedMaximum = rcc::IntegerImmediate::fromUnsigned(8, 255);
+  auto UnsignedOverflow = rcc::IntegerImmediate::fromUnsigned(8, 256);
+  auto SignedMinimum = rcc::IntegerImmediate::fromSigned(8, -128);
+  auto SignedMaximum = rcc::IntegerImmediate::fromSigned(8, 127);
+  auto SignedUnderflow = rcc::IntegerImmediate::fromSigned(8, -129);
+  auto SignedOverflow = rcc::IntegerImmediate::fromSigned(8, 128);
+  Context.expect(UnsignedMaximum && UnsignedMaximum->words()[0] == 255 && !UnsignedOverflow &&
+                     SignedMinimum && SignedMinimum->words()[0] == 128 && SignedMaximum &&
+                     SignedMaximum->words()[0] == 127 && !SignedUnderflow && !SignedOverflow,
+                 "integer immediate scalar factories must enforce signedness and width");
+
+  auto SignExtended = rcc::IntegerImmediate::fromSigned(128, -1);
+  auto Zero = rcc::IntegerImmediate::fromUnsigned(128, 0);
+  Context.expect(SignExtended && SignExtended->isNegative() && SignExtended->words().size() == 2 &&
+                     SignExtended->words()[0] == std::numeric_limits<std::uint64_t>::max() &&
+                     SignExtended->words()[1] == std::numeric_limits<std::uint64_t>::max() &&
+                     Zero && Zero->isZero() && !Zero->isNegative(),
+                 "wide scalar factories must sign-extend and zero-extend exactly");
+
+  if (Integer) {
+    auto Operand = rcc::MachineOperand::integerImmediate(std::move(*Integer), 5);
+    const auto *Payload = Operand.getIf<rcc::IntegerImmediateOperand>();
+    Context.expect(Operand.kind() == rcc::MachineOperand::Kind::IntegerImmediate &&
+                       Operand.targetFlags() == 5 && Payload != nullptr &&
+                       Payload->Value.bitWidth() == 65,
+                   "typed integer operand must retain its distinct kind and target flags");
+  }
+}
+
+void testOperandFactories(TestContext &Context) {
+  auto Mask = rcc::test::createTestCallPreservedMask();
+  auto LiveOut = rcc::MachineOperand::liveOutRegisterMask(Mask, 3);
+  Context.expect(LiveOut && LiveOut->kind() == rcc::MachineOperand::Kind::LiveOutRegisterMask &&
+                     LiveOut->targetFlags() == 3 &&
+                     LiveOut->getIf<rcc::LiveOutRegisterMaskOperand>()->Mask == Mask &&
+                     !rcc::MachineOperand::liveOutRegisterMask(nullptr),
+                 "live-out mask must be distinct from a call-preserved register mask");
+
+  auto CFI = rcc::MachineOperand::cfiIndex(rcc::CFIInstructionIndex::fromValue(0), 4);
+  Context.expect(CFI.kind() == rcc::MachineOperand::Kind::CFIIndex &&
+                     CFI.getIf<rcc::CFIIndexOperand>()->Index.value() == 0 &&
+                     CFI.targetFlags() == 4,
+                 "CFI operand must preserve index zero and target flags");
+
+  auto Intrinsic = rcc::MachineOperand::intrinsic(rcc::IntrinsicID::fromValue(17), 6);
+  Context.expect(Intrinsic && Intrinsic->kind() == rcc::MachineOperand::Kind::Intrinsic &&
+                     Intrinsic->getIf<rcc::IntrinsicOperand>()->ID.value() == 17 &&
+                     Intrinsic->targetFlags() == 6 &&
+                     !rcc::MachineOperand::intrinsic(rcc::IntrinsicID{}),
+                 "intrinsic operand must use a nonzero strong ID");
+
+  auto IntegerPredicate =
+      rcc::MachineOperand::integerPredicate(rcc::IntegerPredicate::UnsignedGreaterEqual);
+  auto FloatingPredicate =
+      rcc::MachineOperand::floatingPredicate(rcc::FloatingPredicate::OrderedNotEqual);
+  Context.expect(
+      IntegerPredicate && FloatingPredicate &&
+          IntegerPredicate->getIf<rcc::PredicateOperand>()->integerPredicate() != nullptr &&
+          FloatingPredicate->getIf<rcc::PredicateOperand>()->floatingPredicate() != nullptr &&
+          !rcc::MachineOperand::integerPredicate(rcc::IntegerPredicate::Count) &&
+          !rcc::MachineOperand::floatingPredicate(rcc::FloatingPredicate::Count),
+      "integer and floating predicates must share one tagged operand without invalid sentinels");
+
+  auto Shuffle = rcc::MachineOperand::shuffleMask({1, rcc::ShuffleMaskOperand::Undef, -2, 0}, 7);
+  Context.expect(Shuffle && Shuffle->kind() == rcc::MachineOperand::Kind::ShuffleMask &&
+                     Shuffle->getIf<rcc::ShuffleMaskOperand>()->Elements ==
+                         std::vector<std::int32_t>({1, -1, -2, 0}) &&
+                     Shuffle->targetFlags() == 7 && !rcc::MachineOperand::shuffleMask({0}),
+                 "shuffle mask must preserve int32 elements and the canonical undef sentinel");
+
+  auto Reference = rcc::MachineOperand::debugInstrRef(9, 1'000'000, 8);
+  auto ZeroReference = rcc::MachineOperand::debugInstrRef(0, 0);
+  Context.expect(
+      Reference && Reference->kind() == rcc::MachineOperand::Kind::DebugInstrRef &&
+          Reference->getIf<rcc::DebugInstrRefOperand>()->InstructionNumber == 9 &&
+          Reference->getIf<rcc::DebugInstrRefOperand>()->OperandNumber == 1'000'000 &&
+          Reference->targetFlags() == 8 && ZeroReference &&
+          ZeroReference->getIf<rcc::DebugInstrRefOperand>()->InstructionNumber == 0 &&
+          !rcc::MachineOperand::debugInstrRef(
+              static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()) + 1, 0) &&
+          !rcc::MachineOperand::debugInstrRef(
+              1, static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()) + 1),
+      "debug instruction reference must preserve zero and enforce its two uint32 domains");
+
+  auto LaneMask = rcc::MachineOperand::laneMask(rcc::LaneMask::fromRaw(0xf00000000000000fULL), 9);
+  Context.expect(LaneMask.kind() == rcc::MachineOperand::Kind::LaneMask &&
+                     LaneMask.getIf<rcc::LaneMaskOperand>()->Mask.rawValue() ==
+                         0xf00000000000000fULL &&
+                     LaneMask.targetFlags() == 9,
+                 "lane-mask operand must preserve all 64 bits");
+
+  auto Clone = LaneMask.cloneDetached();
+  Context.expect(Clone.parent() == nullptr && Clone.kind() == LaneMask.kind() &&
+                     Clone.targetFlags() == LaneMask.targetFlags() &&
+                     Clone.getIf<rcc::LaneMaskOperand>()->Mask ==
+                         LaneMask.getIf<rcc::LaneMaskOperand>()->Mask,
+                 "detached operand cloning must preserve the new payload kinds");
+}
+
+void testCFIInfo(TestContext &Context) {
+  const auto R1 = rcc::DwarfRegisterNumber::fromValue(1);
+  const auto R2 = rcc::DwarfRegisterNumber::fromValue(2);
+  const auto R3 = rcc::DwarfRegisterNumber::fromValue(3);
+  rcc::MachineCFIInfo Info;
+
+  auto Add = [&Context, &Info](rcc::MachineCFIInstruction Instruction) {
+    auto Index = Info.addInstruction(std::move(Instruction));
+    Context.expect(Index.has_value(), "valid CFI instruction insertion must succeed");
+  };
+  auto AddExpected = [&Context, &Info](rcc::Expected<rcc::MachineCFIInstruction> Instruction) {
+    Context.expect(Instruction.has_value(), "valid structured CFI factory must succeed");
+    if (!Instruction)
+      return;
+    auto Index = Info.addInstruction(std::move(*Instruction));
+    Context.expect(Index.has_value(), "valid structured CFI insertion must succeed");
+  };
+
+  Add(rcc::MachineCFIInstruction::sameValue(R1));
+  Add(rcc::MachineCFIInstruction::rememberState());
+  Add(rcc::MachineCFIInstruction::restoreState());
+  Add(rcc::MachineCFIInstruction::offset(R1, -8));
+  Add(rcc::MachineCFIInstruction::llvmDefAddressSpaceCFA(R1, 16, 5));
+  Add(rcc::MachineCFIInstruction::defCFARegister(R2));
+  Add(rcc::MachineCFIInstruction::defCFAOffset(32));
+  Add(rcc::MachineCFIInstruction::defCFA(R2, 24));
+  Add(rcc::MachineCFIInstruction::relativeOffset(R3, -16));
+  Add(rcc::MachineCFIInstruction::adjustCFAOffset(8));
+  AddExpected(rcc::MachineCFIInstruction::escape({0x0c, 0x07, 0x08}));
+  Add(rcc::MachineCFIInstruction::restore(R1));
+  Add(rcc::MachineCFIInstruction::undefined(R2));
+  Add(rcc::MachineCFIInstruction::registerRule(R1, R3));
+  Add(rcc::MachineCFIInstruction::windowSave());
+  Add(rcc::MachineCFIInstruction::negateReturnAddressState());
+  Add(rcc::MachineCFIInstruction::negateReturnAddressStateWithPC());
+  Add(rcc::MachineCFIInstruction::gnuArgsSize(48));
+  AddExpected(rcc::MachineCFIInstruction::label("cfi.label"));
+  Add(rcc::MachineCFIInstruction::valueOffset(R3, 64));
+  Add(rcc::MachineCFIInstruction::llvmRegisterPair({
+      .Register = R1,
+      .FirstRegister = R2,
+      .FirstSizeInBits = 32,
+      .SecondRegister = R3,
+      .SecondSizeInBits = 32,
+  }));
+  AddExpected(rcc::MachineCFIInstruction::llvmVectorRegisters({
+      .Register = R1,
+      .VectorRegisters = {{R2, 0, 64}, {R3, 1, 64}},
+  }));
+  Add(rcc::MachineCFIInstruction::llvmVectorOffset({
+      .Register = R1,
+      .RegisterSizeInBits = 64,
+      .MaskRegister = R2,
+      .MaskRegisterSizeInBits = 8,
+      .Offset = -32,
+  }));
+  Add(rcc::MachineCFIInstruction::llvmVectorRegisterMask({
+      .Register = R1,
+      .SpillRegister = R2,
+      .SpillRegisterLaneSizeInBits = 64,
+      .MaskRegister = R3,
+      .MaskRegisterSizeInBits = 8,
+  }));
+
+  Context.expect(Info.instructions().size() == 24 && Info.verify().has_value() &&
+                     Info.instruction(rcc::CFIInstructionIndex::fromValue(0))->operation() ==
+                         rcc::MachineCFIOperation::SameValue &&
+                     Info.instruction(rcc::CFIInstructionIndex::fromValue(23))->operation() ==
+                         rcc::MachineCFIOperation::LLVMVectorRegisterMask &&
+                     Info.instruction(rcc::CFIInstructionIndex::fromValue(24)) == nullptr,
+                 "CFI table must preserve deterministic zero-based definition order");
+
+  Context.expect(!rcc::MachineCFIInstruction::escape({}) &&
+                     !rcc::MachineCFIInstruction::label("") &&
+                     !rcc::MachineCFIInstruction::llvmVectorRegisters({
+                         .Register = R1,
+                         .VectorRegisters = {},
+                     }),
+                 "CFI factories must reject structurally incomplete payloads");
+
+  auto ZeroWidthVectorRegisters = rcc::MachineCFIInstruction::llvmVectorRegisters({
+      .Register = R1,
+      .VectorRegisters = {{R2, 0, 0}},
+  });
+  Context.expect(
+      rcc::MachineCFIInstruction::llvmRegisterPair({
+                                                       .Register = R1,
+                                                       .FirstRegister = R2,
+                                                       .FirstSizeInBits = 0,
+                                                       .SecondRegister = R3,
+                                                       .SecondSizeInBits = 0,
+                                                   })
+              .verify() &&
+          ZeroWidthVectorRegisters && ZeroWidthVectorRegisters->verify() &&
+          rcc::MachineCFIInstruction::llvmVectorOffset({
+                                                           .Register = R1,
+                                                           .RegisterSizeInBits = 0,
+                                                           .MaskRegister = R2,
+                                                           .MaskRegisterSizeInBits = 0,
+                                                           .Offset = 0,
+                                                       })
+              .verify() &&
+          rcc::MachineCFIInstruction::llvmVectorRegisterMask({
+                                                                 .Register = R1,
+                                                                 .SpillRegister = R2,
+                                                                 .SpillRegisterLaneSizeInBits = 0,
+                                                                 .MaskRegister = R3,
+                                                                 .MaskRegisterSizeInBits = 0,
+                                                             })
+              .verify(),
+      "CFI unsigned size fields must preserve every value accepted by the fixed MIR grammar");
+
+  auto Clone = Info.clone();
+  Context.expect(Clone && *Clone == Info, "CFI information clone must preserve every rule");
+  if (Clone) {
+    auto Extra = Clone->addInstruction(rcc::MachineCFIInstruction::rememberState());
+    Context.expect(Extra && Clone->instructions().size() == 25 && Info.instructions().size() == 24,
+                   "CFI information clone must own independent storage");
+  }
+}
+
+void testBuilderVerifierAndClone(TestContext &Context) {
+  auto Target = createTarget();
+  Context.expect(Target.has_value(), "operand test target construction must succeed");
+  auto Function = createFunction(Context, 1, "operand-integration");
+  if (!Target || !Function)
+    return;
+  Function->attributes().set(rcc::MachineFunctionAttribute::DebugInstrRef);
+  auto CFIIndex = Function->cfiInfo().addInstruction(rcc::MachineCFIInstruction::defCFAOffset(16));
+  Context.expect(CFIIndex.has_value(), "function CFI instruction insertion must succeed");
+  rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+  if (!CFIIndex || Block == nullptr)
+    return;
+
+  auto Instruction = buildOperandInstruction(*Function, *Target, *Block, *CFIIndex,
+                                             rcc::test::createTestCallPreservedMask());
+  Context.expect(Instruction.has_value() &&
+                     rcc::MachineVerifier::verify(*Function, *Target).has_value(),
+                 "builder and verifier must accept every newly modeled operand kind");
+  if (!Instruction)
+    return;
+
+  const auto Operands = (*Instruction)->operands();
+  Context.expect(
+      Operands.size() == 9 &&
+          Operands[0].getIf<rcc::IntegerImmediateOperand>()->Value.bitWidth() == 128 &&
+          Operands[1].kind() == rcc::MachineOperand::Kind::LiveOutRegisterMask &&
+          Operands[2].getIf<rcc::CFIIndexOperand>()->Index == *CFIIndex &&
+          Operands[3].getIf<rcc::IntrinsicOperand>()->ID.value() == 7 &&
+          *Operands[4].getIf<rcc::PredicateOperand>()->integerPredicate() ==
+              rcc::IntegerPredicate::SignedLessThan &&
+          *Operands[5].getIf<rcc::PredicateOperand>()->floatingPredicate() ==
+              rcc::FloatingPredicate::UnorderedGreaterEqual &&
+          Operands[6].getIf<rcc::ShuffleMaskOperand>()->Elements[1] ==
+              rcc::ShuffleMaskOperand::Undef &&
+          Operands[7].getIf<rcc::DebugInstrRefOperand>()->InstructionNumber == 9 &&
+          Operands[8].getIf<rcc::LaneMaskOperand>()->Mask.rawValue() == 0x8000000000000001ULL,
+      "built instruction must expose all new payloads without string or immediate overloading");
+
+  auto Clone =
+      rcc::cloneMachineFunction(*Function, *Target,
+                                {
+                                    .NewStableID = rcc::MachineFunctionStableID::fromValue(2),
+                                    .NewName = std::string("operand-integration.clone"),
+                                });
+  Context.expect(Clone.has_value(), "operand-rich function must support explicit deep cloning");
+  if (Clone) {
+    const rcc::MachineInstr *ClonedInstruction =
+        Clone->ClonedFunction->firstBlock()->firstInstruction();
+    Context.expect(
+        Clone->ClonedFunction->cfiInfo() == Function->cfiInfo() && ClonedInstruction != nullptr &&
+            ClonedInstruction->operands().size() == 9 &&
+            ClonedInstruction->operands()[0].getIf<rcc::IntegerImmediateOperand>()->Value ==
+                Operands[0].getIf<rcc::IntegerImmediateOperand>()->Value &&
+            ClonedInstruction->operands()[6].getIf<rcc::ShuffleMaskOperand>()->Elements ==
+                Operands[6].getIf<rcc::ShuffleMaskOperand>()->Elements,
+        "deep clone must preserve CFI indices and owned variable-size operand data");
+  }
+
+  auto MissingCFIFunction = createFunction(Context, 3, "missing-cfi");
+  if (!MissingCFIFunction)
+    return;
+  rcc::MachineBasicBlock *MissingCFIBlock = appendBlock(Context, *MissingCFIFunction);
+  if (MissingCFIBlock == nullptr)
+    return;
+  auto MissingCFI = buildOperandInstruction(*MissingCFIFunction, *Target, *MissingCFIBlock,
+                                            rcc::CFIInstructionIndex::fromValue(0),
+                                            rcc::test::createTestCallPreservedMask());
+  Context.expect(!MissingCFI, "builder must reject an operand that references an unknown CFI rule");
+
+  auto InvalidMaskFunction = createFunction(Context, 4, "invalid-liveout");
+  if (!InvalidMaskFunction)
+    return;
+  auto ValidCFI =
+      InvalidMaskFunction->cfiInfo().addInstruction(rcc::MachineCFIInstruction::rememberState());
+  rcc::MachineBasicBlock *InvalidMaskBlock = appendBlock(Context, *InvalidMaskFunction);
+  if (!ValidCFI || InvalidMaskBlock == nullptr)
+    return;
+  auto InvalidMask = std::make_shared<const rcc::RegisterMask>(
+      std::vector<std::uint64_t>{std::uint64_t{1} << 63U});
+  auto InvalidLiveOut = buildOperandInstruction(*InvalidMaskFunction, *Target, *InvalidMaskBlock,
+                                                *ValidCFI, std::move(InvalidMask));
+  Context.expect(!InvalidLiveOut,
+                 "target validation must reject a live-out mask with unknown physical registers");
+}
+
+} // namespace
+
+int main() {
+  TestContext Context;
+  testIntegerImmediate(Context);
+  testOperandFactories(Context);
+  testCFIInfo(Context);
+  testBuilderVerifierAndClone(Context);
+  return Context.result();
+}