[CodeGen] Model complete MIR operands

5 files changed, 475 insertions, 4 deletions

diff --git a/backend/include/rcc/CodeGen/MachineCFIInfo.h b/backend/include/rcc/CodeGen/MachineCFIInfo.h
new file mode 100644
index 0000000..1819daf
--- /dev/null
+++ b/backend/include/rcc/CodeGen/MachineCFIInfo.h
@@ -0,0 +1,269 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef RCC_CODEGEN_MACHINECFIINFO_H
+#define RCC_CODEGEN_MACHINECFIINFO_H
+
+#include "rcc/CodeGen/MachineIRTypes.h"
+#include "rcc/Support/Error.h"
+
+#include <cstdint>
+#include <span>
+#include <string>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace rcc {
+
+class DwarfRegisterNumber final {
+public:
+  constexpr DwarfRegisterNumber() noexcept = default;
+
+  [[nodiscard]] static constexpr DwarfRegisterNumber fromValue(std::uint32_t Value) noexcept {
+    return DwarfRegisterNumber(Value);
+  }
+
+  [[nodiscard]] constexpr std::uint32_t value() const noexcept {
+    return Value;
+  }
+
+  friend constexpr bool operator==(DwarfRegisterNumber Left,
+                                   DwarfRegisterNumber Right) noexcept = default;
+
+private:
+  explicit constexpr DwarfRegisterNumber(std::uint32_t Value) noexcept : Value(Value) {}
+
+  std::uint32_t Value = 0;
+};
+
+enum class MachineCFIOperation : std::uint8_t {
+  SameValue,
+  RememberState,
+  RestoreState,
+  Offset,
+  LLVMDefAddressSpaceCFA,
+  DefCFARegister,
+  DefCFAOffset,
+  DefCFA,
+  RelativeOffset,
+  AdjustCFAOffset,
+  Escape,
+  Restore,
+  Undefined,
+  Register,
+  WindowSave,
+  NegateReturnAddressState,
+  NegateReturnAddressStateWithPC,
+  GNUArgsSize,
+  Label,
+  ValueOffset,
+  LLVMRegisterPair,
+  LLVMVectorRegisters,
+  LLVMVectorOffset,
+  LLVMVectorRegisterMask,
+  Count,
+};
+
+struct MachineCFINoOperands final {
+  friend constexpr bool operator==(MachineCFINoOperands Left,
+                                   MachineCFINoOperands Right) noexcept = default;
+};
+
+struct MachineCFIRegisterRule final {
+  DwarfRegisterNumber Register;
+
+  friend constexpr bool operator==(MachineCFIRegisterRule Left,
+                                   MachineCFIRegisterRule Right) noexcept = default;
+};
+
+struct MachineCFIOffsetRule final {
+  std::int64_t Offset = 0;
+
+  friend constexpr bool operator==(MachineCFIOffsetRule Left,
+                                   MachineCFIOffsetRule Right) noexcept = default;
+};
+
+struct MachineCFIRegisterOffsetRule final {
+  DwarfRegisterNumber Register;
+  std::int64_t Offset = 0;
+
+  friend constexpr bool operator==(MachineCFIRegisterOffsetRule Left,
+                                   MachineCFIRegisterOffsetRule Right) noexcept = default;
+};
+
+struct MachineCFIRegisterRegisterRule final {
+  DwarfRegisterNumber Register;
+  DwarfRegisterNumber SourceRegister;
+
+  friend constexpr bool operator==(MachineCFIRegisterRegisterRule Left,
+                                   MachineCFIRegisterRegisterRule Right) noexcept = default;
+};
+
+struct MachineCFIAddressSpaceCFARule final {
+  DwarfRegisterNumber Register;
+  std::int64_t Offset = 0;
+  std::uint32_t AddressSpace = 0;
+
+  friend constexpr bool operator==(MachineCFIAddressSpaceCFARule Left,
+                                   MachineCFIAddressSpaceCFARule Right) noexcept = default;
+};
+
+struct MachineCFIEscapeRule final {
+  std::vector<std::uint8_t> Bytes;
+
+  friend bool operator==(const MachineCFIEscapeRule &Left,
+                         const MachineCFIEscapeRule &Right) = default;
+};
+
+struct MachineCFILabelRule final {
+  std::string Symbol;
+
+  friend bool operator==(const MachineCFILabelRule &Left,
+                         const MachineCFILabelRule &Right) = default;
+};
+
+struct MachineCFIRegisterPairRule final {
+  DwarfRegisterNumber Register;
+  DwarfRegisterNumber FirstRegister;
+  std::uint32_t FirstSizeInBits = 0;
+  DwarfRegisterNumber SecondRegister;
+  std::uint32_t SecondSizeInBits = 0;
+
+  friend constexpr bool operator==(MachineCFIRegisterPairRule Left,
+                                   MachineCFIRegisterPairRule Right) noexcept = default;
+};
+
+struct MachineCFIVectorRegisterLane final {
+  DwarfRegisterNumber Register;
+  std::uint32_t Lane = 0;
+  std::uint32_t SizeInBits = 0;
+
+  friend constexpr bool operator==(MachineCFIVectorRegisterLane Left,
+                                   MachineCFIVectorRegisterLane Right) noexcept = default;
+};
+
+struct MachineCFIVectorRegistersRule final {
+  DwarfRegisterNumber Register;
+  std::vector<MachineCFIVectorRegisterLane> VectorRegisters;
+
+  friend bool operator==(const MachineCFIVectorRegistersRule &Left,
+                         const MachineCFIVectorRegistersRule &Right) = default;
+};
+
+struct MachineCFIVectorOffsetRule final {
+  DwarfRegisterNumber Register;
+  std::uint32_t RegisterSizeInBits = 0;
+  DwarfRegisterNumber MaskRegister;
+  std::uint32_t MaskRegisterSizeInBits = 0;
+  std::int64_t Offset = 0;
+
+  friend constexpr bool operator==(MachineCFIVectorOffsetRule Left,
+                                   MachineCFIVectorOffsetRule Right) noexcept = default;
+};
+
+struct MachineCFIVectorRegisterMaskRule final {
+  DwarfRegisterNumber Register;
+  DwarfRegisterNumber SpillRegister;
+  std::uint32_t SpillRegisterLaneSizeInBits = 0;
+  DwarfRegisterNumber MaskRegister;
+  std::uint32_t MaskRegisterSizeInBits = 0;
+
+  friend constexpr bool operator==(MachineCFIVectorRegisterMaskRule Left,
+                                   MachineCFIVectorRegisterMaskRule Right) noexcept = default;
+};
+
+class MachineCFIInstruction final {
+public:
+  using Payload =
+      std::variant<MachineCFINoOperands, MachineCFIRegisterRule, MachineCFIOffsetRule,
+                   MachineCFIRegisterOffsetRule, MachineCFIRegisterRegisterRule,
+                   MachineCFIAddressSpaceCFARule, MachineCFIEscapeRule, MachineCFILabelRule,
+                   MachineCFIRegisterPairRule, MachineCFIVectorRegistersRule,
+                   MachineCFIVectorOffsetRule, MachineCFIVectorRegisterMaskRule>;
+
+  [[nodiscard]] static MachineCFIInstruction sameValue(DwarfRegisterNumber Register);
+  [[nodiscard]] static MachineCFIInstruction rememberState();
+  [[nodiscard]] static MachineCFIInstruction restoreState();
+  [[nodiscard]] static MachineCFIInstruction offset(DwarfRegisterNumber Register,
+                                                    std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction llvmDefAddressSpaceCFA(DwarfRegisterNumber Register,
+                                                                    std::int64_t Offset,
+                                                                    std::uint32_t AddressSpace);
+  [[nodiscard]] static MachineCFIInstruction defCFARegister(DwarfRegisterNumber Register);
+  [[nodiscard]] static MachineCFIInstruction defCFAOffset(std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction defCFA(DwarfRegisterNumber Register,
+                                                    std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction relativeOffset(DwarfRegisterNumber Register,
+                                                            std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction adjustCFAOffset(std::int64_t Offset);
+  [[nodiscard]] static Expected<MachineCFIInstruction> escape(std::vector<std::uint8_t> Bytes);
+  [[nodiscard]] static MachineCFIInstruction restore(DwarfRegisterNumber Register);
+  [[nodiscard]] static MachineCFIInstruction undefined(DwarfRegisterNumber Register);
+  [[nodiscard]] static MachineCFIInstruction registerRule(DwarfRegisterNumber Register,
+                                                          DwarfRegisterNumber SourceRegister);
+  [[nodiscard]] static MachineCFIInstruction windowSave();
+  [[nodiscard]] static MachineCFIInstruction negateReturnAddressState();
+  [[nodiscard]] static MachineCFIInstruction negateReturnAddressStateWithPC();
+  [[nodiscard]] static MachineCFIInstruction gnuArgsSize(std::int64_t Size);
+  [[nodiscard]] static Expected<MachineCFIInstruction> label(std::string Symbol);
+  [[nodiscard]] static MachineCFIInstruction valueOffset(DwarfRegisterNumber Register,
+                                                         std::int64_t Offset);
+  [[nodiscard]] static MachineCFIInstruction llvmRegisterPair(MachineCFIRegisterPairRule Rule);
+  [[nodiscard]] static Expected<MachineCFIInstruction>
+  llvmVectorRegisters(MachineCFIVectorRegistersRule Rule);
+  [[nodiscard]] static MachineCFIInstruction llvmVectorOffset(MachineCFIVectorOffsetRule Rule);
+  [[nodiscard]] static MachineCFIInstruction
+  llvmVectorRegisterMask(MachineCFIVectorRegisterMaskRule Rule);
+
+  [[nodiscard]] MachineCFIOperation operation() const noexcept {
+    return Operation;
+  }
+
+  template <typename PayloadT> [[nodiscard]] const PayloadT *getIf() const noexcept {
+    return std::get_if<PayloadT>(&Data);
+  }
+
+  [[nodiscard]] Expected<void> verify() const;
+
+  friend bool operator==(const MachineCFIInstruction &Left,
+                         const MachineCFIInstruction &Right) = default;
+
+private:
+  MachineCFIInstruction(MachineCFIOperation Operation, Payload Data)
+      : Operation(Operation), Data(std::move(Data)) {}
+
+  MachineCFIOperation Operation;
+  Payload Data;
+};
+
+class MachineCFIInfo final {
+public:
+  MachineCFIInfo() = default;
+  MachineCFIInfo(const MachineCFIInfo &) = delete;
+  MachineCFIInfo &operator=(const MachineCFIInfo &) = delete;
+  MachineCFIInfo(MachineCFIInfo &&) noexcept = default;
+  MachineCFIInfo &operator=(MachineCFIInfo &&) noexcept = default;
+  ~MachineCFIInfo() = default;
+
+  [[nodiscard]] Expected<CFIInstructionIndex> addInstruction(MachineCFIInstruction Instruction);
+
+  [[nodiscard]] const MachineCFIInstruction *instruction(CFIInstructionIndex Index) const noexcept;
+
+  [[nodiscard]] std::span<const MachineCFIInstruction> instructions() const noexcept {
+    return Instructions;
+  }
+
+  [[nodiscard]] Expected<void> verify() const;
+  [[nodiscard]] Expected<MachineCFIInfo> clone() const;
+
+  friend bool operator==(const MachineCFIInfo &Left, const MachineCFIInfo &Right) {
+    return Left.Instructions == Right.Instructions;
+  }
+
+private:
+  std::vector<MachineCFIInstruction> Instructions;
+};
+
+} // namespace rcc
+
+#endif // RCC_CODEGEN_MACHINECFIINFO_H
diff --git a/backend/include/rcc/CodeGen/MachineFunction.h b/backend/include/rcc/CodeGen/MachineFunction.h
index 7b325d4..c188d4a 100644
--- a/backend/include/rcc/CodeGen/MachineFunction.h
+++ b/backend/include/rcc/CodeGen/MachineFunction.h
@@ -3,6 +3,7 @@
 #ifndef RCC_CODEGEN_MACHINEFUNCTION_H
 #define RCC_CODEGEN_MACHINEFUNCTION_H
 
+#include "rcc/CodeGen/MachineCFIInfo.h"
 #include "rcc/CodeGen/MachineCompilationStatus.h"
 #include "rcc/CodeGen/MachineConstantPool.h"
 #include "rcc/CodeGen/MachineFrameInfo.h"
@@ -117,6 +118,14 @@ public:
     return Metadata;
   }
 
+  [[nodiscard]] MachineCFIInfo &cfiInfo() noexcept {
+    return CFIInfo;
+  }
+
+  [[nodiscard]] const MachineCFIInfo &cfiInfo() const noexcept {
+    return CFIInfo;
+  }
+
   [[nodiscard]] const TargetFunctionState &targetState() const noexcept {
     return TargetState;
   }
@@ -199,6 +208,7 @@ private:
   MachineFunctionAttributes Attributes;
   MachineFunctionRecords Records;
   MachineMetadataInfo Metadata;
+  MachineCFIInfo CFIInfo;
   TargetFunctionState TargetState;
   MachineCompilationStatus CompilationStatus;
   MachineBasicBlock *FirstBlock = nullptr;
diff --git a/backend/include/rcc/CodeGen/MachineIRBuilder.h b/backend/include/rcc/CodeGen/MachineIRBuilder.h
index 918f389..2a5979b 100644
--- a/backend/include/rcc/CodeGen/MachineIRBuilder.h
+++ b/backend/include/rcc/CodeGen/MachineIRBuilder.h
@@ -52,6 +52,8 @@ public:
                                               std::uint32_t TargetFlags = 0);
 
   [[nodiscard]] Expected<void> addImmediate(std::int64_t Value, std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addIntegerImmediate(IntegerImmediate Value,
+                                                   std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addFloatingImmediate(FloatingPointBits Bits,
                                                     std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addBasicBlock(MachineBasicBlock &Block,
@@ -70,12 +72,27 @@ public:
                                                std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addRegisterMask(std::shared_ptr<const RegisterMask> Mask,
                                                std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addLiveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask,
+                                                      std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addMetadata(MachineMetadataValue Value,
                                            std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addTargetIndex(TargetIndex Index, std::int64_t Offset = 0,
                                               std::uint32_t TargetFlags = 0);
   [[nodiscard]] Expected<void> addMCSymbol(std::string Name, std::int64_t Offset = 0,
                                            std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addCFIInstruction(CFIInstructionIndex Index,
+                                                 std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addIntrinsic(IntrinsicID ID, std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addIntegerPredicate(IntegerPredicate Predicate,
+                                                   std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addFloatingPredicate(FloatingPredicate Predicate,
+                                                    std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addShuffleMask(std::vector<std::int32_t> Elements,
+                                              std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addDebugInstrRef(std::uint64_t InstructionNumber,
+                                                std::uint64_t OperandNumber,
+                                                std::uint32_t TargetFlags = 0);
+  [[nodiscard]] Expected<void> addLaneMask(LaneMask Mask, std::uint32_t TargetFlags = 0);
 
   [[nodiscard]] Expected<void> tieOperands(std::size_t DefOperand, std::size_t UseOperand);
   [[nodiscard]] Expected<void> addMemOperand(const MachineMemOperand &MemOperand);
@@ -102,6 +119,7 @@ private:
   [[nodiscard]] Expected<void> validateFrameReference(FrameIndex Index) const;
   [[nodiscard]] Expected<void> validateConstantPoolReference(ConstantPoolIndex Index) const;
   [[nodiscard]] Expected<void> validateJumpTableReference(JumpTableIndex Index) const;
+  [[nodiscard]] Expected<void> validateCFIReference(CFIInstructionIndex Index) const;
   [[nodiscard]] Expected<void> validateOperandOwnership(const MachineOperand &Operand) const;
   [[nodiscard]] std::vector<MachineOperandTie> collectOperandTies() const;
   [[nodiscard]] Expected<void> appendRegister(Register Reg, RegisterOperandFlags Flags,
diff --git a/backend/include/rcc/CodeGen/MachineIRTypes.h b/backend/include/rcc/CodeGen/MachineIRTypes.h
index 90aa2b7..f7095cc 100644
--- a/backend/include/rcc/CodeGen/MachineIRTypes.h
+++ b/backend/include/rcc/CodeGen/MachineIRTypes.h
@@ -43,6 +43,8 @@ struct FrameIndexTag;
 struct ConstantPoolIndexTag;
 struct JumpTableIndexTag;
 struct TargetIndexTag;
+struct CFIInstructionIndexTag;
+struct IntrinsicIDTag;
 struct SubRegIndexTag;
 struct RegisterUnitIDTag;
 struct RegisterBankIDTag;
@@ -66,6 +68,8 @@ using FrameIndex = MachineIndex<detail::FrameIndexTag>;
 using ConstantPoolIndex = MachineIndex<detail::ConstantPoolIndexTag>;
 using JumpTableIndex = MachineIndex<detail::JumpTableIndexTag>;
 using TargetIndex = MachineIndex<detail::TargetIndexTag>;
+using CFIInstructionIndex = MachineIndex<detail::CFIInstructionIndexTag>;
+using IntrinsicID = MachineIndex<detail::IntrinsicIDTag>;
 using SubRegIndex = MachineIndex<detail::SubRegIndexTag>;
 using RegisterUnitID = MachineIndex<detail::RegisterUnitIDTag>;
 using RegisterBankID = MachineIndex<detail::RegisterBankIDTag>;
diff --git a/backend/include/rcc/CodeGen/MachineOperand.h b/backend/include/rcc/CodeGen/MachineOperand.h
index ce5eb60..7344932 100644
--- a/backend/include/rcc/CodeGen/MachineOperand.h
+++ b/backend/include/rcc/CodeGen/MachineOperand.h
@@ -21,6 +21,7 @@
 #include <string_view>
 #include <utility>
 #include <variant>
+#include <vector>
 
 namespace rcc {
 
@@ -58,6 +59,39 @@ private:
   std::uint8_t WordCount;
 };
 
+class IntegerImmediate final {
+public:
+  static constexpr std::uint32_t MaxBitWidth = std::uint32_t{1} << 23U;
+
+  [[nodiscard]] static Expected<IntegerImmediate> create(std::uint32_t BitWidth,
+                                                         std::span<const std::uint64_t> Words);
+  [[nodiscard]] static Expected<IntegerImmediate> fromUnsigned(std::uint32_t BitWidth,
+                                                               std::uint64_t Value);
+  [[nodiscard]] static Expected<IntegerImmediate> fromSigned(std::uint32_t BitWidth,
+                                                             std::int64_t Value);
+
+  [[nodiscard]] std::uint32_t bitWidth() const noexcept {
+    return BitWidth;
+  }
+
+  [[nodiscard]] std::span<const std::uint64_t> words() const noexcept {
+    return Words;
+  }
+
+  [[nodiscard]] bool isZero() const noexcept;
+  [[nodiscard]] bool isNegative() const noexcept;
+  [[nodiscard]] Expected<void> verify() const;
+
+  friend bool operator==(const IntegerImmediate &Left, const IntegerImmediate &Right) = default;
+
+private:
+  IntegerImmediate(std::uint32_t BitWidth, std::vector<std::uint64_t> Words)
+      : BitWidth(BitWidth), Words(std::move(Words)) {}
+
+  std::uint32_t BitWidth;
+  std::vector<std::uint64_t> Words;
+};
+
 enum class RegisterOperandFlag : std::uint8_t {
   Define,
   Use,
@@ -115,6 +149,13 @@ struct FloatingImmediateOperand final {
                          const FloatingImmediateOperand &Right) = default;
 };
 
+struct IntegerImmediateOperand final {
+  IntegerImmediate Value;
+
+  friend bool operator==(const IntegerImmediateOperand &Left,
+                         const IntegerImmediateOperand &Right) = default;
+};
+
 struct BasicBlockOperand final {
   MachineBasicBlock *Block = nullptr;
 
@@ -176,6 +217,13 @@ struct RegisterMaskOperand final {
                          const RegisterMaskOperand &Right) = default;
 };
 
+struct LiveOutRegisterMaskOperand final {
+  std::shared_ptr<const RegisterMask> Mask;
+
+  friend bool operator==(const LiveOutRegisterMaskOperand &Left,
+                         const LiveOutRegisterMaskOperand &Right) = default;
+};
+
 struct MetadataOperand final {
   MachineMetadataValue Value;
 
@@ -197,11 +245,106 @@ struct MCSymbolOperand final {
   friend bool operator==(const MCSymbolOperand &Left, const MCSymbolOperand &Right) = default;
 };
 
+struct CFIIndexOperand final {
+  CFIInstructionIndex Index;
+
+  friend constexpr bool operator==(CFIIndexOperand Left, CFIIndexOperand Right) noexcept = default;
+};
+
+struct IntrinsicOperand final {
+  IntrinsicID ID;
+
+  friend constexpr bool operator==(IntrinsicOperand Left,
+                                   IntrinsicOperand Right) noexcept = default;
+};
+
+enum class IntegerPredicate : std::uint8_t {
+  Equal,
+  NotEqual,
+  SignedGreaterThan,
+  SignedGreaterEqual,
+  SignedLessThan,
+  SignedLessEqual,
+  UnsignedGreaterThan,
+  UnsignedGreaterEqual,
+  UnsignedLessThan,
+  UnsignedLessEqual,
+  Count,
+};
+
+enum class FloatingPredicate : std::uint8_t {
+  AlwaysFalse,
+  OrderedEqual,
+  OrderedGreaterThan,
+  OrderedGreaterEqual,
+  OrderedLessThan,
+  OrderedLessEqual,
+  OrderedNotEqual,
+  Ordered,
+  Unordered,
+  UnorderedEqual,
+  UnorderedGreaterThan,
+  UnorderedGreaterEqual,
+  UnorderedLessThan,
+  UnorderedLessEqual,
+  UnorderedNotEqual,
+  AlwaysTrue,
+  Count,
+};
+
+class PredicateOperand final {
+public:
+  explicit PredicateOperand(IntegerPredicate Predicate) : Value(Predicate) {}
+  explicit PredicateOperand(FloatingPredicate Predicate) : Value(Predicate) {}
+
+  [[nodiscard]] bool isInteger() const noexcept {
+    return std::holds_alternative<IntegerPredicate>(Value);
+  }
+
+  [[nodiscard]] const IntegerPredicate *integerPredicate() const noexcept {
+    return std::get_if<IntegerPredicate>(&Value);
+  }
+
+  [[nodiscard]] const FloatingPredicate *floatingPredicate() const noexcept {
+    return std::get_if<FloatingPredicate>(&Value);
+  }
+
+  [[nodiscard]] Expected<void> verify() const;
+
+  friend bool operator==(const PredicateOperand &Left, const PredicateOperand &Right) = default;
+
+private:
+  std::variant<IntegerPredicate, FloatingPredicate> Value;
+};
+
+struct ShuffleMaskOperand final {
+  static constexpr std::int32_t Undef = -1;
+
+  std::vector<std::int32_t> Elements;
+
+  friend bool operator==(const ShuffleMaskOperand &Left, const ShuffleMaskOperand &Right) = default;
+};
+
+struct DebugInstrRefOperand final {
+  std::uint32_t InstructionNumber = 0;
+  std::uint32_t OperandNumber = 0;
+
+  friend constexpr bool operator==(DebugInstrRefOperand Left,
+                                   DebugInstrRefOperand Right) noexcept = default;
+};
+
+struct LaneMaskOperand final {
+  LaneMask Mask;
+
+  friend constexpr bool operator==(LaneMaskOperand Left, LaneMaskOperand Right) noexcept = default;
+};
+
 class MachineOperand final {
 public:
   enum class Kind : std::uint8_t {
     Register,
     Immediate,
+    IntegerImmediate,
     FloatingImmediate,
     BasicBlock,
     FrameIndex,
@@ -211,9 +354,16 @@ public:
     ExternalSymbol,
     BlockAddress,
     RegisterMask,
+    LiveOutRegisterMask,
     Metadata,
     TargetIndex,
     MCSymbol,
+    CFIIndex,
+    Intrinsic,
+    Predicate,
+    ShuffleMask,
+    DebugInstrRef,
+    LaneMask,
     Count,
   };
 
@@ -228,6 +378,8 @@ public:
                                                                 SubRegIndex SubReg = {},
                                                                 std::uint32_t TargetFlags = 0);
   [[nodiscard]] static MachineOperand immediate(std::int64_t Value, std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static MachineOperand integerImmediate(IntegerImmediate Value,
+                                                       std::uint32_t TargetFlags = 0);
   [[nodiscard]] static MachineOperand floatingImmediate(FloatingPointBits Bits,
                                                         std::uint32_t TargetFlags = 0);
   [[nodiscard]] static MachineOperand basicBlock(MachineBasicBlock &Block,
@@ -248,12 +400,28 @@ public:
                                                              std::uint32_t TargetFlags = 0);
   [[nodiscard]] static Expected<MachineOperand>
   registerMask(std::shared_ptr<const RegisterMask> Mask, std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand>
+  liveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask, std::uint32_t TargetFlags = 0);
   [[nodiscard]] static Expected<MachineOperand> metadata(MachineMetadataValue Value,
                                                          std::uint32_t TargetFlags = 0);
   [[nodiscard]] static MachineOperand targetIndex(TargetIndex Index, std::int64_t Offset = 0,
                                                   std::uint32_t TargetFlags = 0);
   [[nodiscard]] static Expected<MachineOperand> mcSymbol(std::string Name, std::int64_t Offset = 0,
                                                          std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static MachineOperand cfiIndex(CFIInstructionIndex Index,
+                                               std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> intrinsic(IntrinsicID ID,
+                                                          std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> integerPredicate(IntegerPredicate Predicate,
+                                                                 std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> floatingPredicate(FloatingPredicate Predicate,
+                                                                  std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> shuffleMask(std::vector<std::int32_t> Elements,
+                                                            std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static Expected<MachineOperand> debugInstrRef(std::uint64_t InstructionNumber,
+                                                              std::uint64_t OperandNumber,
+                                                              std::uint32_t TargetFlags = 0);
+  [[nodiscard]] static MachineOperand laneMask(LaneMask Mask, std::uint32_t TargetFlags = 0);
 
   [[nodiscard]] Kind kind() const noexcept {
     return OperandKind;
@@ -307,10 +475,12 @@ public:
 
 private:
   using OperandStorage =
-      std::variant<Register, std::int64_t, FloatingImmediateOperand, BasicBlockOperand,
-                   FrameIndexOperand, ConstantPoolOperand, JumpTableOperand, GlobalSymbolOperand,
-                   ExternalSymbolOperand, BlockAddressOperand, RegisterMaskOperand, MetadataOperand,
-                   TargetIndexOperand, MCSymbolOperand>;
+      std::variant<Register, std::int64_t, IntegerImmediateOperand, FloatingImmediateOperand,
+                   BasicBlockOperand, FrameIndexOperand, ConstantPoolOperand, JumpTableOperand,
+                   GlobalSymbolOperand, ExternalSymbolOperand, BlockAddressOperand,
+                   RegisterMaskOperand, LiveOutRegisterMaskOperand, MetadataOperand,
+                   TargetIndexOperand, MCSymbolOperand, CFIIndexOperand, IntrinsicOperand,
+                   PredicateOperand, ShuffleMaskOperand, DebugInstrRefOperand, LaneMaskOperand>;
 
   MachineOperand(Kind OperandKind, OperandStorage Storage, std::uint32_t TargetFlags = 0)
       : OperandKind(OperandKind), Storage(std::move(Storage)), TargetFlags(TargetFlags) {}