[RISCV] Complete Zcmp frame lowering and optimization

8 files changed, 439 insertions, 210 deletions

diff --git a/backend/include/rcc/CodeGen/LiveRegUnits.h b/backend/include/rcc/CodeGen/LiveRegUnits.h
index 6eb06a6..2e18bc9 100644
--- a/backend/include/rcc/CodeGen/LiveRegUnits.h
+++ b/backend/include/rcc/CodeGen/LiveRegUnits.h
@@ -38,6 +38,13 @@ public:
   void unite(const LiveRegUnits &Other);
   void subtract(const LiveRegUnits &Other);
 
+  [[nodiscard]] Expected<bool> available(PhysicalRegister Reg,
+                                         const TargetDescription &Target) const;
+  [[nodiscard]] static Expected<void> accumulateUsedDefed(const MachineInstr &Instruction,
+                                                          LiveRegUnits &Modified,
+                                                          LiveRegUnits &Used,
+                                                          const TargetDescription &Target);
+
   [[nodiscard]] const std::unordered_map<RegisterUnitID, LaneMask> &entries() const noexcept {
     return Values;
   }
diff --git a/backend/include/rcc/CodeGen/MachineOptimizationPipeline.h b/backend/include/rcc/CodeGen/MachineOptimizationPipeline.h
index f60bbda..c47fc48 100644
--- a/backend/include/rcc/CodeGen/MachineOptimizationPipeline.h
+++ b/backend/include/rcc/CodeGen/MachineOptimizationPipeline.h
@@ -46,7 +46,7 @@ addSelectedSSAOptimizationPipeline(MachineFunctionPassManager &Pipeline,
 [[nodiscard]] Expected<void> addPostRegisterAllocationPipeline(
     MachineFunctionPassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo,
     MachineOptimizationPipelineOptions Options = {},
-    std::unique_ptr<MachineFunctionPass> PostRAPseudoExpansionTargetPass = nullptr);
+    MachineFunctionPassManager::PassList PostRAPseudoExpansionTargetPasses = {});
 
 } // namespace rcc
 
diff --git a/backend/lib/CodeGen/LiveRegUnits.cpp b/backend/lib/CodeGen/LiveRegUnits.cpp
index 7df55c0..e0aa992 100644
--- a/backend/lib/CodeGen/LiveRegUnits.cpp
+++ b/backend/lib/CodeGen/LiveRegUnits.cpp
@@ -3,6 +3,7 @@
 #include "rcc/CodeGen/LiveRegUnits.h"
 
 #include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
 #include "rcc/CodeGen/MachineOperand.h"
 #include "rcc/CodeGen/RegisterMask.h"
 #include "rcc/CodeGen/TargetDescription.h"
@@ -277,4 +278,54 @@ void LiveRegUnits::subtract(const LiveRegUnits &Other) {
   subtractLaneSets(Values, Other.Values);
 }
 
+[[nodiscard]] Expected<bool> LiveRegUnits::available(PhysicalRegister Reg,
+                                                     const TargetDescription &Target) const {
+  const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(Reg);
+  if (Description == nullptr || Description->Units.empty()) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "register availability query requires a physical register with units");
+  }
+  const LiveRegUnits Coverage = physicalRegisterCoverage(*Description);
+  return std::ranges::none_of(Coverage.entries(), [this](const auto &Entry) {
+    return intersects(Entry.first, Entry.second);
+  });
+}
+
+[[nodiscard]] Expected<void> LiveRegUnits::accumulateUsedDefed(const MachineInstr &Instruction,
+                                                               LiveRegUnits &Modified,
+                                                               LiveRegUnits &Used,
+                                                               const TargetDescription &Target) {
+  for (const MachineOperand &Operand : Instruction.operands()) {
+    if (Operand.isDebugUse())
+      continue;
+    if (Operand.isRegister()) {
+      const std::optional<Register> Reg = Operand.registerValue();
+      if (!Reg || !Reg->isPhysical() || Reg->isNoRegister())
+        continue;
+      auto Units = physicalOperandUnits(Operand, Target);
+      if (!Units)
+        return std::unexpected(Units.error());
+      for (const PhysicalUnitAccess &Access : *Units) {
+        if (Operand.isDef())
+          Modified.add(Access.Unit, Access.Lanes);
+        if (Operand.isUse())
+          Used.add(Access.Unit, Access.Lanes);
+      }
+      continue;
+    }
+    if (const auto *Mask = Operand.getIf<RegisterMaskOperand>();
+        Mask != nullptr && Mask->Mask != nullptr) {
+      if (auto Added = addRegisterMaskDefs(Modified, *Mask->Mask, Target); !Added)
+        return std::unexpected(Added.error());
+      continue;
+    }
+    if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>();
+        Mask != nullptr && Mask->Mask != nullptr) {
+      if (auto Added = addRegisterMaskUses(Used, *Mask->Mask, Target); !Added)
+        return std::unexpected(Added.error());
+    }
+  }
+  return {};
+}
+
 } // namespace rcc
diff --git a/backend/lib/CodeGen/MachineOptimizationPipeline.cpp b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp
index cbc4b1a..6fb6550 100644
--- a/backend/lib/CodeGen/MachineOptimizationPipeline.cpp
+++ b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp
@@ -80,7 +80,7 @@ Expected<void> addPreRegisterAllocationOptimizationPipeline(
 Expected<void> addPostRegisterAllocationPipeline(
     MachineFunctionPassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo,
     MachineOptimizationPipelineOptions Options,
-    std::unique_ptr<MachineFunctionPass> PostRAPseudoExpansionTargetPass) {
+    MachineFunctionPassManager::PassList PostRAPseudoExpansionTargetPasses) {
   auto Added = addPass<MachineStackSlotColoringPass>(Pipeline);
   if (!Added)
     return std::unexpected(Added.error());
@@ -112,8 +112,8 @@ Expected<void> addPostRegisterAllocationPipeline(
   Added = addPass<ExpandPostRAPseudosPass>(Pipeline, TargetInfo.PseudoExpansion);
   if (!Added)
     return std::unexpected(Added.error());
-  if (PostRAPseudoExpansionTargetPass != nullptr) {
-    Added = Pipeline.addPass(std::move(PostRAPseudoExpansionTargetPass));
+  for (std::unique_ptr<MachineFunctionPass> &TargetPass : PostRAPseudoExpansionTargetPasses) {
+    Added = Pipeline.addPass(std::move(TargetPass));
     if (!Added)
       return std::unexpected(Added.error());
   }
diff --git a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp
index 1c9ec62..206681a 100644
--- a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -15,6 +15,7 @@
 #include "rcc/CodeGen/RegisterMask.h"
 #include "rcc/CodeGen/RegisterScavenger.h"
 #include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVFunctionInfo.h"
 #include "rcc/Target/RISCV/RISCVInstructions.h"
 #include "rcc/Target/RISCV/RISCVRegisters.h"
 #include "rcc/Target/RISCV/RISCVTarget.h"
@@ -303,6 +304,12 @@ constexpr std::array SaveRestoreOrder{
     PhysicalRegisterID::X27,
 };
 
+struct ZcmpPushPopInfo final {
+  std::size_t RegisterCount = 0;
+  std::int64_t RegisterListEncoding = 0;
+  std::uint64_t FrameSize = 0;
+};
+
 [[nodiscard]] std::span<const PhysicalRegisterID> saveOrder(RISCVABI ABI) noexcept {
   return ABI == RISCVABI::LP64D ? std::span<const PhysicalRegisterID>(LP64DSaveOrder)
                                 : std::span<const PhysicalRegisterID>(LP64SaveOrder);
@@ -317,6 +324,64 @@ constexpr std::array SaveRestoreOrder{
   return std::nullopt;
 }
 
+[[nodiscard]] bool zcmpPushPopAllowed(const MachineFunction &Function,
+                                      const RISCVSubtargetFeatures &Features) noexcept {
+  if (!Features.has(RISCVExtension::Zcmp) || requiresFramePointer(Function))
+    return false;
+  const auto *Info = Function.getTargetInfo<RISCVFunctionInfo>();
+  return Info == nullptr || Info->varArgsSaveSize() == 0;
+}
+
+[[nodiscard]] std::optional<ZcmpPushPopInfo>
+zcmpPushPopInfo(const MachineFunction &Function, std::span<const PhysicalRegister> SavedRegisters,
+                const RISCVSubtargetFeatures &Features) noexcept {
+  if (!zcmpPushPopAllowed(Function, Features))
+    return std::nullopt;
+  std::size_t RegisterCount = 0;
+  for (const PhysicalRegister Reg : SavedRegisters) {
+    const auto Index = saveRestoreRegisterIndex(Reg);
+    if (Index)
+      RegisterCount = std::max(RegisterCount, *Index + 1U);
+  }
+  if (RegisterCount == 0)
+    return std::nullopt;
+  if (RegisterCount == 12)
+    RegisterCount = 13;
+  const std::uint64_t Bytes = static_cast<std::uint64_t>(RegisterCount) * 8U;
+  return ZcmpPushPopInfo{
+      .RegisterCount = RegisterCount,
+      .RegisterListEncoding =
+          RegisterCount == 13 ? 15 : static_cast<std::int64_t>(RegisterCount + 3U),
+      .FrameSize = (Bytes + StackAlignmentBytes - 1U) & ~(StackAlignmentBytes - 1U),
+  };
+}
+
+[[nodiscard]] bool zcmpManagesRegister(const ZcmpPushPopInfo &Info, PhysicalRegister Reg) noexcept {
+  const auto Index = saveRestoreRegisterIndex(Reg);
+  return Index && *Index < Info.RegisterCount;
+}
+
+[[nodiscard]] Expected<void>
+completeZcmpRegisterList(const MachineFunction &Function, const RISCVSubtargetFeatures &Features,
+                         std::vector<PhysicalRegister> &SavedRegisters) {
+  if (!zcmpPushPopAllowed(Function, Features))
+    return {};
+  auto X26 = physicalRegister(PhysicalRegisterID::X26);
+  auto X27 = physicalRegister(PhysicalRegisterID::X27);
+  if (!X26 || !X27)
+    return std::unexpected(!X26 ? X26.error() : X27.error());
+  if (std::ranges::find(SavedRegisters, *X26) == SavedRegisters.end() ||
+      std::ranges::find(SavedRegisters, *X27) != SavedRegisters.end()) {
+    return {};
+  }
+  const auto Position = std::ranges::find_if(SavedRegisters, [](PhysicalRegister Reg) {
+    const auto Index = saveRestoreRegisterIndex(Reg);
+    return !Index;
+  });
+  SavedRegisters.insert(Position, *X27);
+  return {};
+}
+
 [[nodiscard]] std::optional<std::size_t>
 saveRestoreLibCallID(std::span<const PhysicalRegister> SavedRegisters,
                      const RISCVSubtargetFeatures &Features) noexcept {
@@ -1007,7 +1072,8 @@ emitAddImmediateAdjustment(MachineIRBuilder &Builder, PhysicalRegister Destinati
 
 [[nodiscard]] std::uint64_t firstSPAdjustAmount(const MachineFunction &Function,
                                                 const RISCVSubtargetFeatures &Features) noexcept {
-  if (saveRestoreFrameSize(Function.records().calleeSavedRegisters(), Features) != 0)
+  if (saveRestoreFrameSize(Function.records().calleeSavedRegisters(), Features) != 0 ||
+      zcmpPushPopInfo(Function, Function.records().calleeSavedRegisters(), Features))
     return 0;
   const std::uint64_t StackSize = Function.frameInfo().state().StackSize;
   if (StackSize <= static_cast<std::uint64_t>(MaxSigned12) ||
@@ -1170,7 +1236,8 @@ emitAddImmediateAdjustment(MachineIRBuilder &Builder, PhysicalRegister Destinati
 [[nodiscard]] Expected<void>
 validateCalleeSaveObjects(const MachineFunction &Function,
                           std::span<const PhysicalRegister> SavedRegisters, RISCVABI ABI,
-                          const RISCVSubtargetFeatures &Features) {
+                          const RISCVSubtargetFeatures &Features,
+                          const std::optional<ZcmpPushPopInfo> &ZcmpInfo = std::nullopt) {
   std::unordered_set<PhysicalRegister> Slots;
   for (const StackObject &Object : Function.frameInfo().objects()) {
     if (Object.isTombstone() || Object.description()->IsDead ||
@@ -1184,17 +1251,21 @@ validateCalleeSaveObjects(const MachineFunction &Function,
                             "RISC-V callee-save object names a caller-saved register");
     }
     const auto SaveRestoreIndex = saveRestoreRegisterIndex(Reg);
-    const bool ManagedByLibCall =
-        Features.has(RISCVSubtargetOption::SaveRestore) && SaveRestoreIndex.has_value();
-    const StackObjectKind ExpectedKind =
-        ManagedByLibCall ? StackObjectKind::FixedSpillSlot : StackObjectKind::CalleeSaved;
-    const std::optional<std::int64_t> ExpectedOffset =
-        ManagedByLibCall
-            ? std::optional<std::int64_t>{-static_cast<std::int64_t>(*SaveRestoreIndex + 1U) * 8}
-            : std::nullopt;
+    const bool ManagedByZcmp = ZcmpInfo && zcmpManagesRegister(*ZcmpInfo, Reg);
+    const bool ManagedByLibCall = !ZcmpInfo && Features.has(RISCVSubtargetOption::SaveRestore) &&
+                                  SaveRestoreIndex.has_value();
+    const StackObjectKind ExpectedKind = ManagedByZcmp || ManagedByLibCall
+                                             ? StackObjectKind::FixedSpillSlot
+                                             : StackObjectKind::CalleeSaved;
+    std::optional<std::int64_t> ExpectedOffset;
+    if (ManagedByZcmp) {
+      ExpectedOffset = -static_cast<std::int64_t>(ZcmpInfo->RegisterCount - *SaveRestoreIndex) * 8;
+    } else if (ManagedByLibCall) {
+      ExpectedOffset = -static_cast<std::int64_t>(*SaveRestoreIndex + 1U) * 8;
+    }
     if (Description.Kind != ExpectedKind || Description.Size != std::optional<std::uint64_t>{8} ||
         Description.Align.bytes() < 8 || Description.Stack != StackID::defaultStack() ||
-        (ManagedByLibCall && Description.Offset != ExpectedOffset)) {
+        ((ManagedByZcmp || ManagedByLibCall) && Description.Offset != ExpectedOffset)) {
       return makeUnexpected(ErrorCode::VerificationError,
                             "RISC-V callee-save object has an invalid kind, size, or alignment");
     }
@@ -1347,11 +1418,16 @@ public:
     auto SavedRegisters = modifiedCalleeSavedRegisters(Function, Target, Subtarget.targetABI());
     if (!SavedRegisters)
       return std::unexpected(SavedRegisters.error());
+    auto CompletedZcmpList =
+        completeZcmpRegisterList(Function, Subtarget.features(), *SavedRegisters);
+    if (!CompletedZcmpList)
+      return std::unexpected(CompletedZcmpList.error());
+    const auto ZcmpInfo = zcmpPushPopInfo(Function, *SavedRegisters, Subtarget.features());
     auto Records = Function.records().setCalleeSavedRegisters(*SavedRegisters);
     if (!Records)
       return std::unexpected(Records.error());
     auto ExistingSlots = validateCalleeSaveObjects(Function, *SavedRegisters, Subtarget.targetABI(),
-                                                   Subtarget.features());
+                                                   Subtarget.features(), ZcmpInfo);
     if (!ExistingSlots)
       return std::unexpected(ExistingSlots.error());
     auto Align8 = Alignment::create(8);
@@ -1363,52 +1439,59 @@ public:
         continue;
       StackObjectDesc Description;
       const auto SaveRestoreIndex = saveRestoreRegisterIndex(Reg);
-      const bool ManagedByLibCall =
-          Subtarget.features().has(RISCVSubtargetOption::SaveRestore) && SaveRestoreIndex;
-      Description.Kind =
-          ManagedByLibCall ? StackObjectKind::FixedSpillSlot : StackObjectKind::CalleeSaved;
+      const bool ManagedByZcmp = ZcmpInfo && zcmpManagesRegister(*ZcmpInfo, Reg);
+      const bool ManagedByLibCall = !ZcmpInfo &&
+                                    Subtarget.features().has(RISCVSubtargetOption::SaveRestore) &&
+                                    SaveRestoreIndex;
+      Description.Kind = ManagedByZcmp || ManagedByLibCall ? StackObjectKind::FixedSpillSlot
+                                                           : StackObjectKind::CalleeSaved;
       Description.Size = 8;
       Description.Align =
           ManagedByLibCall && (*SaveRestoreIndex + 1U) % 2U == 0 ? *Align16 : *Align8;
-      if (ManagedByLibCall)
+      if (ManagedByZcmp) {
+        Description.Offset =
+            -static_cast<std::int64_t>(ZcmpInfo->RegisterCount - *SaveRestoreIndex) * 8;
+      } else if (ManagedByLibCall) {
         Description.Offset = -static_cast<std::int64_t>(*SaveRestoreIndex + 1U) * 8;
+      }
       Description.CalleeSavedRegister = Reg;
       auto Created = Frame.createObject(std::move(Description));
       if (!Created)
         return std::unexpected(Created.error());
     }
-    const std::uint64_t LibCallFrameSize =
-        saveRestoreFrameSize(*SavedRegisters, Subtarget.features());
-    if (LibCallFrameSize != 0) {
+    const std::uint64_t ReservedFrameSize =
+        ZcmpInfo ? ZcmpInfo->FrameSize
+                 : saveRestoreFrameSize(*SavedRegisters, Subtarget.features());
+    if (ReservedFrameSize != 0) {
       const bool HasCoveringObject =
           std::ranges::any_of(Frame.objects(), [&](const StackObject &O) {
             const StackObjectDesc *Description = O.description();
             return Description != nullptr && !Description->IsDead &&
                    Description->Kind == StackObjectKind::FixedSpillSlot &&
                    !Description->CalleeSavedRegister &&
-                   Description->Size == std::optional<std::uint64_t>{LibCallFrameSize} &&
+                   Description->Size == std::optional<std::uint64_t>{ReservedFrameSize} &&
                    Description->Offset ==
-                       std::optional<std::int64_t>{-static_cast<std::int64_t>(LibCallFrameSize)};
+                       std::optional<std::int64_t>{-static_cast<std::int64_t>(ReservedFrameSize)};
           });
       if (!HasCoveringObject) {
         StackObjectDesc Description;
         Description.Kind = StackObjectKind::FixedSpillSlot;
-        Description.Size = LibCallFrameSize;
+        Description.Size = ReservedFrameSize;
         Description.Align = *Align16;
-        Description.Offset = -static_cast<std::int64_t>(LibCallFrameSize);
+        Description.Offset = -static_cast<std::int64_t>(ReservedFrameSize);
         auto Created = Frame.createObject(std::move(Description));
         if (!Created)
           return std::unexpected(Created.error());
       }
     }
     auto CompleteSlots = validateCalleeSaveObjects(Function, *SavedRegisters, Subtarget.targetABI(),
-                                                   Subtarget.features());
+                                                   Subtarget.features(), ZcmpInfo);
     if (!CompleteSlots)
       return std::unexpected(CompleteSlots.error());
 
     const bool ReserveCallFrame = hasReservedCallFrame(Function);
     auto StackSize =
-        layoutFrame(Function, ReserveCallFrame, LibCallFrameSize, Options.EmitDebugOutput);
+        layoutFrame(Function, ReserveCallFrame, ReservedFrameSize, Options.EmitDebugOutput);
     if (!StackSize)
       return std::unexpected(StackSize.error());
 
@@ -1460,18 +1543,18 @@ public:
       if (!Emergency)
         return std::unexpected(Emergency.error());
       StackSize =
-          layoutFrame(Function, ReserveCallFrame, LibCallFrameSize, Options.EmitDebugOutput);
+          layoutFrame(Function, ReserveCallFrame, ReservedFrameSize, Options.EmitDebugOutput);
       if (!StackSize)
         return std::unexpected(StackSize.error());
     }
 
     State = Frame.state();
     State.StackSize = *StackSize;
-    if (*StackSize < LibCallFrameSize) {
+    if (*StackSize < ReservedFrameSize) {
       return makeUnexpected(ErrorCode::InternalError,
-                            "RISC-V save/restore frame exceeds the complete frame");
+                            "RISC-V managed callee-save frame exceeds the complete frame");
     }
-    State.AdjustsStack = State.AdjustsStack || *StackSize != LibCallFrameSize;
+    State.AdjustsStack = State.AdjustsStack || *StackSize != 0;
     State.IsCalleeSavedInfoValid = true;
     State.SavePoints.clear();
     State.RestorePoints.clear();
@@ -1559,8 +1642,30 @@ public:
                          : SaveBuilder.setInsertPoint(*SaveBlock);
     if (!SavePoint)
       return std::unexpected(SavePoint.error());
-    const auto LibCallID = saveRestoreLibCallID(Saved, Subtarget.features());
-    if (LibCallID) {
+    const auto ZcmpInfo = zcmpPushPopInfo(Function, Saved, Subtarget.features());
+    const auto LibCallID =
+        ZcmpInfo ? std::nullopt : saveRestoreLibCallID(Saved, Subtarget.features());
+    if (ZcmpInfo) {
+      auto Pending = SaveBuilder.buildInstr(instructionOpcode(InstructionOpcode::CM_PUSH));
+      if (!Pending)
+        return std::unexpected(Pending.error());
+      if (auto Result = Pending->addImmediate(ZcmpInfo->RegisterListEncoding); !Result)
+        return std::unexpected(Result.error());
+      if (auto Result = Pending->addImmediate(0); !Result)
+        return std::unexpected(Result.error());
+      for (std::size_t Index = 0; Index < ZcmpInfo->RegisterCount; ++Index) {
+        auto Reg = physicalRegister(SaveRestoreOrder[Index]);
+        if (!Reg)
+          return std::unexpected(Reg.error());
+        if (auto Result = Pending->addImplicitUse(Register::fromPhysical(*Reg)); !Result)
+          return std::unexpected(Result.error());
+      }
+      if (auto Result = setFrameFlag(*Pending, MachineInstrFlag::FrameSetup); !Result)
+        return std::unexpected(Result.error());
+      auto Built = Pending->build();
+      if (!Built)
+        return std::unexpected(Built.error());
+    } else if (LibCallID) {
       auto Link = physicalRegister(PhysicalRegisterID::X5);
       auto SP = physicalRegister(PhysicalRegisterID::X2);
       if (!Link || !SP)
@@ -1591,6 +1696,8 @@ public:
         return std::unexpected(Built.error());
     }
     for (const PhysicalRegister Reg : Saved) {
+      if (ZcmpInfo && zcmpManagesRegister(*ZcmpInfo, Reg))
+        continue;
       if (LibCallID && saveRestoreRegisterIndex(Reg))
         continue;
       const auto Slot = calleeSaveSlot(Function, Reg);
@@ -1621,6 +1728,8 @@ public:
         const auto EmitRestore = [&](PhysicalRegister Reg) -> Expected<void> {
           if (!calleeSaveIsRestored(Function, Reg))
             return {};
+          if (ZcmpInfo && zcmpManagesRegister(*ZcmpInfo, Reg))
+            return {};
           if (LibCallID && saveRestoreRegisterIndex(Reg))
             return {};
           const auto Slot = calleeSaveSlot(Function, Reg);
@@ -1650,6 +1759,27 @@ public:
           if (!Loaded)
             return std::unexpected(Loaded.error());
         }
+        if (ZcmpInfo) {
+          auto Pending = RestoreBuilder.buildInstr(instructionOpcode(InstructionOpcode::CM_POP));
+          if (!Pending)
+            return std::unexpected(Pending.error());
+          if (auto Result = Pending->addImmediate(ZcmpInfo->RegisterListEncoding); !Result)
+            return std::unexpected(Result.error());
+          if (auto Result = Pending->addImmediate(0); !Result)
+            return std::unexpected(Result.error());
+          for (std::size_t Index = 0; Index < ZcmpInfo->RegisterCount; ++Index) {
+            auto Reg = physicalRegister(SaveRestoreOrder[Index]);
+            if (!Reg)
+              return std::unexpected(Reg.error());
+            if (auto Result = Pending->addImplicitDef(Register::fromPhysical(*Reg)); !Result)
+              return std::unexpected(Result.error());
+          }
+          if (auto Result = setFrameFlag(*Pending, MachineInstrFlag::FrameDestroy); !Result)
+            return std::unexpected(Result.error());
+          auto Built = Pending->build();
+          if (!Built)
+            return std::unexpected(Built.error());
+        }
       }
     }
     return {};
@@ -1668,9 +1798,13 @@ public:
                                                         : Object.description()->Offset);
     }
     const std::uint64_t PreliminarySize = Frame.state().StackSize;
-    const std::uint64_t LibCallFrameSize =
-        saveRestoreFrameSize(Function.records().calleeSavedRegisters(), Subtarget.features());
-    auto StackSize = layoutFrame(Function, hasReservedCallFrame(Function), LibCallFrameSize);
+    const auto ZcmpInfo =
+        zcmpPushPopInfo(Function, Function.records().calleeSavedRegisters(), Subtarget.features());
+    const std::uint64_t ReservedFrameSize =
+        ZcmpInfo
+            ? ZcmpInfo->FrameSize
+            : saveRestoreFrameSize(Function.records().calleeSavedRegisters(), Subtarget.features());
+    auto StackSize = layoutFrame(Function, hasReservedCallFrame(Function), ReservedFrameSize);
     if (!StackSize)
       return std::unexpected(StackSize.error());
     if (*StackSize != PreliminarySize || Frame.objects().size() != PreliminaryOffsets.size()) {
@@ -1730,17 +1864,36 @@ public:
         calleeSaveOperations(*SaveBlock, Function, false);
     const std::span<const PhysicalRegister> SavedRegisters =
         Function.records().calleeSavedRegisters();
-    const auto LibCallID = saveRestoreLibCallID(SavedRegisters, Subtarget.features());
+    const auto ZcmpInfo = zcmpPushPopInfo(Function, SavedRegisters, Subtarget.features());
+    const auto LibCallID =
+        ZcmpInfo ? std::nullopt : saveRestoreLibCallID(SavedRegisters, Subtarget.features());
     const std::size_t ManualSaveCount =
         static_cast<std::size_t>(std::ranges::count_if(SavedRegisters, [&](PhysicalRegister Reg) {
-          return !LibCallID || !saveRestoreRegisterIndex(Reg);
+          return (!ZcmpInfo || !zcmpManagesRegister(*ZcmpInfo, Reg)) &&
+                 (!LibCallID || !saveRestoreRegisterIndex(Reg));
         }));
     if (Saves.size() != ManualSaveCount) {
       return makeUnexpected(ErrorCode::InternalError,
                             "RISC-V prologue has an incomplete callee-save sequence");
     }
+    MachineInstr *Push = nullptr;
     MachineInstr *SpillLibCall = nullptr;
-    if (LibCallID) {
+    if (ZcmpInfo) {
+      for (MachineInstr *Instruction = SaveBlock->firstInstruction(); Instruction != nullptr;
+           Instruction = Instruction->next()) {
+        if (Instruction->opcode() == instructionOpcode(InstructionOpcode::CM_PUSH) &&
+            Instruction->flags().has(MachineInstrFlag::FrameSetup)) {
+          if (Push != nullptr) {
+            return makeUnexpected(ErrorCode::InternalError,
+                                  "RISC-V prologue has duplicate Zcmp pushes");
+          }
+          Push = Instruction;
+        }
+      }
+      if (Push == nullptr) {
+        return makeUnexpected(ErrorCode::InternalError, "RISC-V prologue is missing its Zcmp push");
+      }
+    } else if (LibCallID) {
       for (MachineInstr *Instruction = SaveBlock->firstInstruction(); Instruction != nullptr;
            Instruction = Instruction->next()) {
         if (Instruction->opcode() == instructionOpcode(InstructionOpcode::PseudoCALLReg) &&
@@ -1759,10 +1912,11 @@ public:
     }
     MachineInstr *Anchor = SaveBlock->firstInstruction();
     MachineIRBuilder Builder(Function, Target);
-    auto Point = SpillLibCall != nullptr
-                     ? Builder.setInsertPointAfter(*SpillLibCall)
-                     : (Anchor != nullptr ? Builder.setInsertPointBefore(*Anchor)
-                                          : Builder.setInsertPoint(*SaveBlock));
+    auto Point = Push != nullptr ? Builder.setInsertPointAfter(*Push)
+                                 : (SpillLibCall != nullptr
+                                        ? Builder.setInsertPointAfter(*SpillLibCall)
+                                        : (Anchor != nullptr ? Builder.setInsertPointBefore(*Anchor)
+                                                             : Builder.setInsertPoint(*SaveBlock)));
     if (!Point)
       return std::unexpected(Point.error());
 
@@ -1779,18 +1933,32 @@ public:
     }
     PhysicalRegister Temporary = *DefaultTemporary;
     std::optional<ScavengedRegister> Scratch;
-    const std::uint64_t LibCallFrameSize =
-        saveRestoreFrameSize(SavedRegisters, Subtarget.features());
-    if (StackSize < LibCallFrameSize) {
+    const std::uint64_t BaseReservedFrameSize =
+        ZcmpInfo ? ZcmpInfo->FrameSize : saveRestoreFrameSize(SavedRegisters, Subtarget.features());
+    if (StackSize < BaseReservedFrameSize) {
       return makeUnexpected(ErrorCode::InternalError,
-                            "RISC-V save/restore frame exceeds the complete frame");
+                            "RISC-V managed callee-save frame exceeds the complete frame");
+    }
+    std::uint64_t ZcmpExtraAdjustment = 0;
+    if (Push != nullptr) {
+      ZcmpExtraAdjustment =
+          std::min((StackSize - BaseReservedFrameSize) & ~std::uint64_t{15}, std::uint64_t{48});
+      MachineOperand *Adjustment = Push->operandAt(1);
+      if (Adjustment == nullptr) {
+        return makeUnexpected(ErrorCode::InternalError,
+                              "RISC-V Zcmp push lacks its stack adjustment");
+      }
+      auto SetAdjustment = Adjustment->setImmediate(static_cast<std::int64_t>(ZcmpExtraAdjustment));
+      if (!SetAdjustment)
+        return std::unexpected(SetAdjustment.error());
     }
-    const std::uint64_t ManagedStackSize = StackSize - LibCallFrameSize;
+    const std::uint64_t ReservedFrameSize = BaseReservedFrameSize + ZcmpExtraAdjustment;
+    const std::uint64_t ManagedStackSize = StackSize - ReservedFrameSize;
     const std::uint64_t FirstAdjustment = firstSPAdjustAmount(Function, Subtarget.features());
     const std::uint64_t InitialAllocation =
         FirstAdjustment == 0 ? ManagedStackSize : FirstAdjustment;
     const std::uint64_t SecondAdjustment = ManagedStackSize - InitialAllocation;
-    const std::uint64_t FramePointerAdjustment = LibCallFrameSize + InitialAllocation;
+    const std::uint64_t FramePointerAdjustment = ReservedFrameSize + InitialAllocation;
     const std::int64_t InitialAmount = -static_cast<std::int64_t>(InitialAllocation);
     const std::int64_t SecondAmount = -static_cast<std::int64_t>(SecondAdjustment);
     const std::int64_t AlignmentMask = std::bit_cast<std::int64_t>(~(StackAlignment - 1U));
@@ -1816,8 +1984,26 @@ public:
       }
     }
 
-    if (LibCallFrameSize != 0) {
-      auto CFA = emitDefCFAOffset(Builder, static_cast<std::int64_t>(LibCallFrameSize),
+    if (ZcmpInfo) {
+      auto CFA = emitDefCFAOffset(Builder, static_cast<std::int64_t>(ReservedFrameSize),
+                                  MachineInstrFlag::FrameSetup);
+      if (!CFA)
+        return std::unexpected(CFA.error());
+      for (const PhysicalRegister Reg : SavedRegisters) {
+        if (!zcmpManagesRegister(*ZcmpInfo, Reg))
+          continue;
+        const auto Slot = calleeSaveSlot(Function, Reg);
+        const StackObject *Object = Slot ? Function.frameInfo().object(*Slot) : nullptr;
+        if (Object == nullptr || !Object->description()->Offset) {
+          return makeUnexpected(ErrorCode::InternalError, "RISC-V Zcmp stack slot is incomplete");
+        }
+        auto Saved = emitSavedRegisterCFI(Builder, Target, Reg, *Object->description()->Offset,
+                                          MachineInstrFlag::FrameSetup);
+        if (!Saved)
+          return std::unexpected(Saved.error());
+      }
+    } else if (BaseReservedFrameSize != 0) {
+      auto CFA = emitDefCFAOffset(Builder, static_cast<std::int64_t>(BaseReservedFrameSize),
                                   MachineInstrFlag::FrameSetup);
       if (!CFA)
         return std::unexpected(CFA.error());
@@ -1864,7 +2050,7 @@ public:
       }
       if (!Realign) {
         auto CFA =
-            emitDefCFAOffset(Builder, static_cast<std::int64_t>(LibCallFrameSize + AllocationSize),
+            emitDefCFAOffset(Builder, static_cast<std::int64_t>(ReservedFrameSize + AllocationSize),
                              MachineInstrFlag::FrameSetup);
         if (!CFA)
           return std::unexpected(CFA.error());
@@ -2089,21 +2275,54 @@ public:
     const bool Realign = requiresStackRealignment(Function);
     const std::span<const PhysicalRegister> SavedRegisters =
         Function.records().calleeSavedRegisters();
-    const auto LibCallID = saveRestoreLibCallID(SavedRegisters, Subtarget.features());
-    const std::uint64_t LibCallFrameSize =
-        saveRestoreFrameSize(SavedRegisters, Subtarget.features());
+    const auto ZcmpInfo = zcmpPushPopInfo(Function, SavedRegisters, Subtarget.features());
+    const auto LibCallID =
+        ZcmpInfo ? std::nullopt : saveRestoreLibCallID(SavedRegisters, Subtarget.features());
+    MachineInstr *Pop = nullptr;
+    if (ZcmpInfo) {
+      for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr;
+           Instruction = Instruction->next()) {
+        if (Instruction->opcode() == instructionOpcode(InstructionOpcode::CM_POP) &&
+            Instruction->flags().has(MachineInstrFlag::FrameDestroy)) {
+          if (Pop != nullptr) {
+            return makeUnexpected(ErrorCode::InternalError,
+                                  "RISC-V epilogue has duplicate Zcmp pops");
+          }
+          Pop = Instruction;
+        }
+      }
+      if (Pop == nullptr) {
+        return makeUnexpected(ErrorCode::InternalError, "RISC-V epilogue is missing its Zcmp pop");
+      }
+    }
+    const std::uint64_t BaseReservedFrameSize =
+        ZcmpInfo ? ZcmpInfo->FrameSize : saveRestoreFrameSize(SavedRegisters, Subtarget.features());
     const std::uint64_t TotalStackSize = Function.frameInfo().state().StackSize;
-    if (TotalStackSize < LibCallFrameSize) {
+    if (TotalStackSize < BaseReservedFrameSize) {
       return makeUnexpected(ErrorCode::InternalError,
-                            "RISC-V save/restore frame exceeds the complete frame");
+                            "RISC-V managed callee-save frame exceeds the complete frame");
+    }
+    std::uint64_t ZcmpExtraAdjustment = 0;
+    if (Pop != nullptr) {
+      ZcmpExtraAdjustment = std::min((TotalStackSize - BaseReservedFrameSize) & ~std::uint64_t{15},
+                                     std::uint64_t{48});
+      MachineOperand *Adjustment = Pop->operandAt(1);
+      if (Adjustment == nullptr) {
+        return makeUnexpected(ErrorCode::InternalError,
+                              "RISC-V Zcmp pop lacks its stack adjustment");
+      }
+      auto SetAdjustment = Adjustment->setImmediate(static_cast<std::int64_t>(ZcmpExtraAdjustment));
+      if (!SetAdjustment)
+        return std::unexpected(SetAdjustment.error());
     }
-    const std::uint64_t StackSize = TotalStackSize - LibCallFrameSize;
+    const std::uint64_t ReservedFrameSize = BaseReservedFrameSize + ZcmpExtraAdjustment;
+    const std::uint64_t StackSize = TotalStackSize - ReservedFrameSize;
 
     const std::vector<CalleeSaveOperation> BlockRestores =
         calleeSaveOperations(Block, Function, true);
     std::vector<CalleeSaveOperation> Restores;
-    for (MachineInstr *Cursor = Terminator.previous(); Cursor != nullptr;
-         Cursor = Cursor->previous()) {
+    for (MachineInstr *Cursor = Pop != nullptr ? Pop->previous() : Terminator.previous();
+         Cursor != nullptr; Cursor = Cursor->previous()) {
       const auto Position =
           std::ranges::find(BlockRestores, Cursor, &CalleeSaveOperation::Instruction);
       if (Position == BlockRestores.end())
@@ -2114,6 +2333,7 @@ public:
     const std::size_t ExpectedRestores =
         static_cast<std::size_t>(std::ranges::count_if(SavedRegisters, [&](PhysicalRegister Reg) {
           return calleeSaveIsRestored(Function, Reg) &&
+                 (!ZcmpInfo || !zcmpManagesRegister(*ZcmpInfo, Reg)) &&
                  (!LibCallID || !saveRestoreRegisterIndex(Reg));
         }));
     if (Restores.size() != ExpectedRestores) {
@@ -2222,7 +2442,8 @@ public:
         return makeUnexpected(ErrorCode::FrameLayoutFailed,
                               "RISC-V stack adjustment exceeds int64");
       MachineIRBuilder Teardown(Function, Target);
-      auto Point = Teardown.setInsertPointBefore(Terminator);
+      MachineInstr &DeallocationAnchor = Pop != nullptr ? *Pop : Terminator;
+      auto Point = Teardown.setInsertPointBefore(DeallocationAnchor);
       if (!Point)
         return std::unexpected(Point.error());
       const std::int64_t Amount = static_cast<std::int64_t>(StackSize);
@@ -2236,7 +2457,7 @@ public:
         auto Zero = physicalRegister(PhysicalRegisterID::X0);
         if (!Candidates || !Zero)
           return std::unexpected(!Candidates ? Candidates.error() : Zero.error());
-        auto Scratch = Scavenger.scavenge(Terminator, *Candidates);
+        auto Scratch = Scavenger.scavenge(DeallocationAnchor, *Candidates);
         if (!Scratch)
           return std::unexpected(Scratch.error());
         if (Scratch->RequiresSpill) {
@@ -2252,7 +2473,8 @@ public:
         if (!Adjust)
           return std::unexpected(Adjust.error());
       }
-      auto CFA = emitDefCFAOffset(Teardown, 0, MachineInstrFlag::FrameDestroy);
+      auto CFA = emitDefCFAOffset(Teardown, static_cast<std::int64_t>(ReservedFrameSize),
+                                  MachineInstrFlag::FrameDestroy);
       if (!CFA)
         return std::unexpected(CFA.error());
     }
diff --git a/backend/lib/Target/RISCV/RISCVMoveMerger.cpp b/backend/lib/Target/RISCV/RISCVMoveMerger.cpp
index 1accb2e..1e68187 100644
--- a/backend/lib/Target/RISCV/RISCVMoveMerger.cpp
+++ b/backend/lib/Target/RISCV/RISCVMoveMerger.cpp
@@ -2,6 +2,7 @@
 
 #include "RISCV.h"
 #include "RISCVInstrInfo.h"
+#include "rcc/CodeGen/LiveRegUnits.h"
 #include "rcc/CodeGen/MachineBasicBlock.h"
 #include "rcc/CodeGen/MachineFunction.h"
 #include "rcc/CodeGen/MachineIRBuilder.h"
@@ -79,86 +80,68 @@ struct CopyOperands final {
          isSavedRegister(Copy.DestinationPhysical, Target);
 }
 
-[[nodiscard]] bool operandOverlaps(const MachineOperand &Operand, PhysicalRegister RegisterValue,
-                                   const TargetDescription &Target) noexcept {
-  auto OperandRegister = Operand.registerValue();
-  if (!OperandRegister)
-    return false;
-  auto OperandPhysical = OperandRegister->physicalRegister();
-  return OperandPhysical && Target.registers().regsOverlap(*OperandPhysical, RegisterValue);
-}
-
-[[nodiscard]] bool instructionDefines(const MachineInstr &Instruction,
-                                      PhysicalRegister RegisterValue,
-                                      const TargetDescription &Target) noexcept {
-  for (const MachineOperand &Operand : Instruction.operands()) {
-    if (Operand.isDef() && operandOverlaps(Operand, RegisterValue, Target))
-      return true;
-  }
-  return false;
-}
-
-[[nodiscard]] bool instructionUses(const MachineInstr &Instruction, PhysicalRegister RegisterValue,
-                                   const TargetDescription &Target) noexcept {
-  for (const MachineOperand &Operand : Instruction.operands()) {
-    if (Operand.isUse() && !Operand.isDebugUse() && operandOverlaps(Operand, RegisterValue, Target))
-      return true;
-  }
-  return false;
-}
-
-[[nodiscard]] bool hasHazard(MachineInstr &First, MachineInstr &Second,
-                             const CopyOperands &SecondCopy,
-                             const TargetDescription &Target) noexcept {
+[[nodiscard]] Expected<bool> isReadBetween(MachineInstr &First, MachineInstr &Second,
+                                           PhysicalRegister RegisterValue,
+                                           const TargetDescription &Target) {
+  LiveRegUnits Modified;
+  LiveRegUnits Used;
   for (MachineInstr *Instruction = First.next(); Instruction != &Second;
        Instruction = Instruction->next()) {
     if (Instruction == nullptr)
-      return true;
+      return makeUnexpected(ErrorCode::InternalError,
+                            "RISC-V move pair is not in one machine basic block");
     if (Instruction->isDebugInstr())
       continue;
-    if (instructionDefines(*Instruction, SecondCopy.DestinationPhysical, Target) ||
-        instructionUses(*Instruction, SecondCopy.DestinationPhysical, Target) ||
-        instructionDefines(*Instruction, SecondCopy.SourcePhysical, Target)) {
-      return true;
+    if (auto Accumulated = LiveRegUnits::accumulateUsedDefed(*Instruction, Modified, Used, Target);
+        !Accumulated) {
+      return std::unexpected(Accumulated.error());
     }
   }
-  return false;
-}
-
-[[nodiscard]] bool isReadBetween(MachineInstr &First, MachineInstr &Second,
-                                 PhysicalRegister RegisterValue,
-                                 const TargetDescription &Target) noexcept {
-  for (MachineInstr *Instruction = First.next(); Instruction != &Second;
-       Instruction = Instruction->next()) {
-    if (Instruction == nullptr)
-      return true;
-    if (!Instruction->isDebugInstr() && instructionUses(*Instruction, RegisterValue, Target))
-      return true;
-  }
-  return false;
+  auto Available = Used.available(RegisterValue, Target);
+  if (!Available)
+    return std::unexpected(Available.error());
+  return !*Available;
 }
 
-[[nodiscard]] MachineInstr *findMatchingCopy(MachineInstr &First, const CopyOperands &FirstCopy,
-                                             bool MoveFromSavedToArgument,
-                                             const TargetDescription &Target) noexcept {
+[[nodiscard]] Expected<MachineInstr *> findMatchingCopy(MachineInstr &First,
+                                                        const CopyOperands &FirstCopy,
+                                                        bool MoveFromSavedToArgument,
+                                                        const TargetDescription &Target) {
+  LiveRegUnits Modified;
+  LiveRegUnits Used;
   for (MachineInstr *Instruction = First.next(); Instruction != nullptr;
        Instruction = Instruction->next()) {
     if (Instruction->isDebugInstr())
       continue;
     auto SecondCopy = copyOperands(*Instruction);
-    if (!SecondCopy)
-      continue;
-    const bool Candidate = MoveFromSavedToArgument ? isMoveFromSavedToArgument(*SecondCopy, Target)
-                                                   : isMoveFromArgumentToSaved(*SecondCopy, Target);
-    if (!Candidate)
-      continue;
-    if (FirstCopy.DestinationPhysical == SecondCopy->DestinationPhysical)
-      return nullptr;
-    if (!MoveFromSavedToArgument && FirstCopy.SourcePhysical == SecondCopy->SourcePhysical)
-      return nullptr;
-    if (hasHazard(First, *Instruction, *SecondCopy, Target))
-      return nullptr;
-    return Instruction;
+    if (SecondCopy) {
+      const bool Candidate = MoveFromSavedToArgument
+                                 ? isMoveFromSavedToArgument(*SecondCopy, Target)
+                                 : isMoveFromArgumentToSaved(*SecondCopy, Target);
+      if (Candidate) {
+        if (FirstCopy.DestinationPhysical == SecondCopy->DestinationPhysical)
+          return nullptr;
+        if (!MoveFromSavedToArgument && FirstCopy.SourcePhysical == SecondCopy->SourcePhysical) {
+          return nullptr;
+        }
+        auto DestinationNotModified = Modified.available(SecondCopy->DestinationPhysical, Target);
+        if (!DestinationNotModified)
+          return std::unexpected(DestinationNotModified.error());
+        auto DestinationNotUsed = Used.available(SecondCopy->DestinationPhysical, Target);
+        if (!DestinationNotUsed)
+          return std::unexpected(DestinationNotUsed.error());
+        auto SourceNotModified = Modified.available(SecondCopy->SourcePhysical, Target);
+        if (!SourceNotModified)
+          return std::unexpected(SourceNotModified.error());
+        if (!*DestinationNotModified || !*DestinationNotUsed || !*SourceNotModified)
+          return nullptr;
+        return Instruction;
+      }
+    }
+    if (auto Accumulated = LiveRegUnits::accumulateUsedDefed(*Instruction, Modified, Used, Target);
+        !Accumulated) {
+      return std::unexpected(Accumulated.error());
+    }
   }
   return nullptr;
 }
@@ -178,17 +161,23 @@ struct CopyOperands final {
       MoveFromSavedToArgument ? FirstCopy->Source : FirstCopy->Destination;
   MachineOperand *SecondOperand =
       MoveFromSavedToArgument ? SecondCopy->Source : SecondCopy->Destination;
+  RegisterOperandFlags PairedSourceFlags = SecondCopy->Source->registerFlags();
+  if (MoveFromSavedToArgument && PairedSourceFlags.has(RegisterOperandFlag::Kill)) {
+    auto ReadBetween = isReadBetween(First, Second, SecondCopy->SourcePhysical, Target);
+    if (!ReadBetween)
+      return std::unexpected(ReadBetween.error());
+    if (*ReadBetween)
+      PairedSourceFlags.clear(RegisterOperandFlag::Kill);
+  }
   if (MoveFromSavedToArgument ? !isRegister(FirstCopy->DestinationPhysical, PhysicalRegisterID::X10)
                               : !isRegister(FirstCopy->SourcePhysical, PhysicalRegisterID::X10)) {
     std::swap(FirstOperand, SecondOperand);
   }
 
-  RegisterOperandFlags FirstFlags = FirstOperand->registerFlags();
-  RegisterOperandFlags SecondFlags = SecondOperand->registerFlags();
-  if (MoveFromSavedToArgument && SecondFlags.has(RegisterOperandFlag::Kill) &&
-      isReadBetween(First, Second, *SecondOperand->registerValue()->physicalRegister(), Target)) {
-    SecondFlags.clear(RegisterOperandFlag::Kill);
-  }
+  const RegisterOperandFlags FirstFlags =
+      FirstOperand == SecondCopy->Source ? PairedSourceFlags : FirstOperand->registerFlags();
+  const RegisterOperandFlags SecondFlags =
+      SecondOperand == SecondCopy->Source ? PairedSourceFlags : SecondOperand->registerFlags();
 
   MachineIRBuilder Builder(Function, Target);
   auto InsertPoint = Builder.setInsertPointBefore(First);
@@ -199,6 +188,10 @@ struct CopyOperands final {
   auto Pending = Builder.buildInstr(instructionOpcode(Opcode));
   if (!Pending)
     return std::unexpected(Pending.error());
+  if (First.debugLocation()) {
+    if (auto Location = Pending->setDebugLocation(*First.debugLocation()); !Location)
+      return std::unexpected(Location.error());
+  }
   Expected<void> Added = MoveFromSavedToArgument
                              ? Pending->addUse(*FirstOperand->registerValue(), FirstFlags)
                              : Pending->addDef(*FirstOperand->registerValue(), FirstFlags);
@@ -231,24 +224,26 @@ struct CopyOperands final {
       Instruction = Next;
       continue;
     }
-    MachineInstr *Paired =
-        findMatchingCopy(*Instruction, *FirstCopy, MoveFromSavedToArgument, Target);
-    if (Paired == nullptr) {
+    auto Paired = findMatchingCopy(*Instruction, *FirstCopy, MoveFromSavedToArgument, Target);
+    if (!Paired)
+      return std::unexpected(Paired.error());
+    if (*Paired == nullptr) {
       Instruction = Next;
       continue;
     }
-    MachineInstr *AfterPair = Paired->next();
-    auto Merged = mergeCopies(Function, *Instruction, *Paired, MoveFromSavedToArgument, Target);
+    const bool Adjacent = Next == *Paired;
+    MachineInstr *AfterPair = (*Paired)->next();
+    auto Merged = mergeCopies(Function, *Instruction, **Paired, MoveFromSavedToArgument, Target);
     if (!Merged)
       return std::unexpected(Merged.error());
     auto RemovedFirst = Instruction->eraseFromParent();
     if (!RemovedFirst)
       return std::unexpected(RemovedFirst.error());
-    auto RemovedPaired = Paired->eraseFromParent();
+    auto RemovedPaired = (*Paired)->eraseFromParent();
     if (!RemovedPaired)
       return std::unexpected(RemovedPaired.error());
     Changed = true;
-    Instruction = Next == Paired ? AfterPair : Next;
+    Instruction = Adjacent ? AfterPair : Next;
   }
   return Changed;
 }
diff --git a/backend/lib/Target/RISCV/RISCVPipeline.cpp b/backend/lib/Target/RISCV/RISCVPipeline.cpp
index c34a590..3e7599d 100644
--- a/backend/lib/Target/RISCV/RISCVPipeline.cpp
+++ b/backend/lib/Target/RISCV/RISCVPipeline.cpp
@@ -102,6 +102,15 @@ template <typename PassT, typename... ArgumentT>
   return addPass<VirtualRegisterRewritePass>(Pipeline);
 }
 
+[[nodiscard]] MachineFunctionPassManager::PassList
+postRAPseudoExpansionPasses(const RISCVSubtarget &Subtarget) {
+  MachineFunctionPassManager::PassList Passes;
+  Passes.push_back(createMakeCompressiblePass(Subtarget));
+  Passes.push_back(createMoveMergePass(Subtarget));
+  Passes.push_back(createPushPopOptimizationPass(Subtarget));
+  return Passes;
+}
+
 [[nodiscard]] Expected<void> addCompleteRegistry(MachineFunctionPassManager &Pipeline,
                                                  const RISCVSubtarget &Subtarget,
                                                  TargetPipelineOptions Options) {
@@ -123,13 +132,7 @@ template <typename PassT, typename... ArgumentT>
                                Options.DebugPrologueEpilogue),
       MachineOptimizationPipelineOptions{.UseTargetCopyInstructions =
                                              Options.UseTargetCopyInstructions},
-      createPushPopOptimizationPass(Subtarget));
-  if (!Added)
-    return std::unexpected(Added.error());
-  Added = Pipeline.addPass(createMakeCompressiblePass(Subtarget));
-  if (!Added)
-    return std::unexpected(Added.error());
-  Added = Pipeline.addPass(createMoveMergePass(Subtarget));
+      postRAPseudoExpansionPasses(Subtarget));
   if (!Added)
     return std::unexpected(Added.error());
   return addPass<CFIInstrInserterPass>(Pipeline, Subtarget.frameLowering());
@@ -198,15 +201,10 @@ Expected<void> addCanonicalPipeline(MachineFunctionPassManager &Pipeline,
                                  Options.DebugPrologueEpilogue),
         MachineOptimizationPipelineOptions{.UseTargetCopyInstructions =
                                                Options.UseTargetCopyInstructions},
-        createPushPopOptimizationPass(Subtarget));
+        postRAPseudoExpansionPasses(Subtarget));
     if (!Added)
       return std::unexpected(Added.error());
   }
-  auto Added = Pipeline.addPass(createMakeCompressiblePass(Subtarget));
-  if (!Added)
-    return std::unexpected(Added.error());
-  if (Subtarget.features().has(RISCVExtension::Zcmp))
-    return Pipeline.addPass(createMoveMergePass(Subtarget));
   return {};
 }
 
diff --git a/backend/lib/Target/RISCV/RISCVPushPopOptimizer.cpp b/backend/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
index dc33ad1..77b0ee7 100644
--- a/backend/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
+++ b/backend/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
@@ -9,7 +9,6 @@
 #include "rcc/CodeGen/MachineIRBuilder.h"
 #include "rcc/CodeGen/MachineInstr.h"
 #include "rcc/CodeGen/MachineOperand.h"
-#include "rcc/CodeGen/RegisterMask.h"
 #include "rcc/CodeGen/TargetDescription.h"
 #include "rcc/Target/RISCV/RISCVRegisters.h"
 #include "rcc/Target/RISCV/RISCVTarget.h"
@@ -47,49 +46,6 @@ struct PushPopOpcodes final {
   return Contract;
 }
 
-[[nodiscard]] bool intersects(const LiveRegUnits &Left, const LiveRegUnits &Right) noexcept {
-  for (const auto &[Unit, Lanes] : Left.entries())
-    if (Right.intersects(Unit, Lanes))
-      return true;
-  return false;
-}
-
-[[nodiscard]] Expected<void> accumulateRegisterEffects(const MachineInstr &Instruction,
-                                                       const TargetDescription &Target,
-                                                       LiveRegUnits &Modified, LiveRegUnits &Used) {
-  for (const MachineOperand &Operand : Instruction.operands()) {
-    if (Operand.isDebugUse())
-      continue;
-    if (Operand.isRegister()) {
-      const std::optional<Register> Reg = Operand.registerValue();
-      if (!Reg || !Reg->isPhysical() || Reg->isNoRegister())
-        continue;
-      auto Units = physicalOperandUnits(Operand, Target);
-      if (!Units)
-        return std::unexpected(Units.error());
-      for (const PhysicalUnitAccess &Access : *Units) {
-        if (Operand.isDef())
-          Modified.add(Access.Unit, Access.Lanes);
-        if (Operand.isUse())
-          Used.add(Access.Unit, Access.Lanes);
-      }
-      continue;
-    }
-    if (const auto *Mask = Operand.getIf<RegisterMaskOperand>();
-        Mask != nullptr && Mask->Mask != nullptr) {
-      if (auto Added = addRegisterMaskDefs(Modified, *Mask->Mask, Target); !Added)
-        return std::unexpected(Added.error());
-      continue;
-    }
-    if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>();
-        Mask != nullptr && Mask->Mask != nullptr) {
-      if (auto Added = addRegisterMaskUses(Used, *Mask->Mask, Target); !Added)
-        return std::unexpected(Added.error());
-    }
-  }
-  return {};
-}
-
 [[nodiscard]] MachineInstr *previousNonDebug(MachineInstr &Instruction) noexcept {
   MachineInstr *Previous = Instruction.previous();
   while (Previous != nullptr && Previous->isDebugInstr())
@@ -112,12 +68,6 @@ struct PushPopOpcodes final {
                                                         const TargetDescription &Target,
                                                         PhysicalRegister ReturnRegister,
                                                         PhysicalRegister ZeroRegister) {
-  const PhysicalRegisterDesc *ReturnDescription =
-      Target.registers().physicalRegister(ReturnRegister);
-  if (ReturnDescription == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V return register has no register-unit description");
-  const LiveRegUnits ReturnUnits = physicalRegisterCoverage(*ReturnDescription);
   LiveRegUnits Modified;
   LiveRegUnits Used;
   for (MachineInstr *Instruction = previousNonDebug(Pop); Instruction != nullptr;
@@ -132,11 +82,17 @@ struct PushPopOpcodes final {
         return true;
       }
     }
-    if (auto Accumulated = accumulateRegisterEffects(*Instruction, Target, Modified, Used);
+    if (auto Accumulated = LiveRegUnits::accumulateUsedDefed(*Instruction, Modified, Used, Target);
         !Accumulated) {
       return std::unexpected(Accumulated.error());
     }
-    if (intersects(Modified, ReturnUnits) || intersects(Used, ReturnUnits))
+    auto ModifiedAvailable = Modified.available(ReturnRegister, Target);
+    if (!ModifiedAvailable)
+      return std::unexpected(ModifiedAvailable.error());
+    auto UsedAvailable = Used.available(ReturnRegister, Target);
+    if (!UsedAvailable)
+      return std::unexpected(UsedAvailable.error());
+    if (!*ModifiedAvailable || !*UsedAvailable)
       return false;
   }
   return false;