[RISCV] Implement make-compressible pass

9 files changed, 485 insertions, 4 deletions

diff --git a/backend/lib/CodeGen/MIRParser/MIRParser.cpp b/backend/lib/CodeGen/MIRParser/MIRParser.cpp
index c7e92a0..944d0c9 100644
--- a/backend/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/backend/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -276,6 +276,8 @@ bool MIRDocumentParser::parseFunction(const yaml::Node &Root, MIRDocument &Docum
     return importError(NameNode->range(), Created.error());
   MachineFunction &Function = **Created;
   ActiveFunction = &Function;
+  if (IRFunction != nullptr && IRFunction->hasMinSize())
+    Function.attributes().set(MachineFunctionAttribute::MinSize);
 
   MIRFunctionDocumentInfo DocumentInfo{
       .Function = Function.stableID(),
diff --git a/backend/lib/CodeGen/MachineFunctionAttributes.cpp b/backend/lib/CodeGen/MachineFunctionAttributes.cpp
index 799e701..afe18a9 100644
--- a/backend/lib/CodeGen/MachineFunctionAttributes.cpp
+++ b/backend/lib/CodeGen/MachineFunctionAttributes.cpp
@@ -31,6 +31,8 @@ std::string_view getMachineFunctionAttributeName(MachineFunctionAttribute Attrib
     return "DebugInstrRef";
   case MachineFunctionAttribute::TracksDebugUserValues:
     return "TracksDebugUserValues";
+  case MachineFunctionAttribute::MinSize:
+    return "MinSize";
   case MachineFunctionAttribute::Count:
     return "Invalid";
   }
diff --git a/backend/lib/CodeGen/MachineOperand.cpp b/backend/lib/CodeGen/MachineOperand.cpp
index f265c3f..3994286 100644
--- a/backend/lib/CodeGen/MachineOperand.cpp
+++ b/backend/lib/CodeGen/MachineOperand.cpp
@@ -300,6 +300,11 @@ Expected<void> MachineOperand::setRegister(Register Reg) {
     return makeUnexpected(ErrorCode::InvalidArgument, "setRegister requires a register operand");
   }
 
+  if (std::get<Register>(Storage) == Reg)
+    return {};
+
+  RegFlags.clear(RegisterOperandFlag::Renamable);
+
   MachineRegisterInfo *RegInfo = LinkedRegInfo;
   if (RegInfo == nullptr && Parent != nullptr)
     RegInfo = &Parent->function().registerInfo();
diff --git a/backend/lib/MIR/EmbeddedIR.cpp b/backend/lib/MIR/EmbeddedIR.cpp
index d936d66..67b91e2 100644
--- a/backend/lib/MIR/EmbeddedIR.cpp
+++ b/backend/lib/MIR/EmbeddedIR.cpp
@@ -9,6 +9,7 @@
 #include <optional>
 #include <string>
 #include <string_view>
+#include <unordered_set>
 #include <utility>
 
 namespace rcc::mir {
@@ -185,6 +186,7 @@ public:
       return failureResult();
     }
 
+    resolveFunctionAttributes();
     validateLocalReferences();
     if (Failed)
       return failureResult();
@@ -350,6 +352,10 @@ private:
       parseFunctionHeader(Line, LineOffset, Position, End, false);
       return;
     }
+    if (startsWord(Line, Position, End, "attributes")) {
+      parseAttributeGroup(Line, LineOffset, Position, End);
+      return;
+    }
     if (Line[Position] == '@') {
       parseGlobalObject(Line, LineOffset, Position, End);
       return;
@@ -531,6 +537,19 @@ private:
       }
       if (InQuote)
         continue;
+      if (HeaderParenthesisDepth == 0 && startsWord(Line, Position, End, "minsize")) {
+        Index.Functions[ActiveFunction.value()].HasMinSize = true;
+        Position += std::string_view("minsize").size() - 1U;
+        continue;
+      }
+      if (HeaderParenthesisDepth == 0 && Character == '#') {
+        auto Group = parseIdentifier(Line, LineOffset, Position, End, '#');
+        if (!Group || Failed)
+          return;
+        Index.Functions[ActiveFunction.value()].AttributeGroup = std::move(Group->Name);
+        Position = Group->End - 1U;
+        continue;
+      }
       if (Character == '%' && HeaderParenthesisDepth != 0 && ActiveFunction) {
         const std::size_t IdentifierBegin = Position;
         auto Name = parseIdentifier(Line, LineOffset, Position, End, '%');
@@ -801,6 +820,25 @@ private:
     }
   }
 
+  void parseAttributeGroup(std::string_view Line, std::size_t LineOffset, std::size_t Position,
+                           std::size_t End) {
+    Position += std::string_view("attributes").size();
+    Position = skipHorizontalSpace(Line, Position, End);
+    auto Group = parseIdentifier(Line, LineOffset, Position, End, '#');
+    if (!Group || Failed)
+      return;
+    if (containsWord(Line, End, "minsize"))
+      MinSizeAttributeGroups.insert(std::move(Group->Name));
+  }
+
+  void resolveFunctionAttributes() {
+    for (EmbeddedIRFunction &Function : Index.Functions) {
+      if (Function.AttributeGroup && MinSizeAttributeGroups.contains(*Function.AttributeGroup)) {
+        Function.HasMinSize = true;
+      }
+    }
+  }
+
   void parseGlobalObject(std::string_view Line, std::size_t LineOffset, std::size_t Position,
                          std::size_t End) {
     auto Name = parseIdentifier(Line, LineOffset, Position, End, '@');
@@ -893,6 +931,7 @@ private:
   EmbeddedIRReferenceIndex Index;
   std::vector<TextDiagnostic> Diagnostics;
   std::optional<std::size_t> ActiveFunction;
+  std::unordered_set<std::string> MinSizeAttributeGroups;
   std::size_t HeaderParenthesisDepth = 0;
   std::size_t FunctionBraceDepth = 0;
   bool Failed = false;
diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt
index 92a77a5..a9ad1c6 100644
--- a/backend/lib/Target/RISCV/CMakeLists.txt
+++ b/backend/lib/Target/RISCV/CMakeLists.txt
@@ -122,6 +122,7 @@ add_library(
   RISCVMCTargetLowering.cpp
   RISCVMachineCombiner.cpp
   RISCVMachineMemoryFolding.cpp
+  RISCVMakeCompressible.cpp
   RISCVMergeBaseOffset.cpp
   RISCVMoveMerger.cpp
   RISCVOptimizationPipeline.cpp
@@ -208,6 +209,7 @@ rcc_backend_register_format_files(
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMCTargetLowering.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineCombiner.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineMemoryFolding.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMakeCompressible.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMergeBaseOffset.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMoveMerger.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVOptimizationPipeline.cpp"
@@ -251,6 +253,7 @@ rcc_backend_register_tidy_files(
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMCTargetLowering.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineCombiner.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineMemoryFolding.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMakeCompressible.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVOptimizationPipeline.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVPipeline.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVPseudoExpansion.cpp"
diff --git a/backend/lib/Target/RISCV/RISCV.h b/backend/lib/Target/RISCV/RISCV.h
index 3a1f2e8..2e87be5 100644
--- a/backend/lib/Target/RISCV/RISCV.h
+++ b/backend/lib/Target/RISCV/RISCV.h
@@ -40,6 +40,8 @@ createSchedulingModel(const TargetSchedulingHooks &Hooks);
 [[nodiscard]] const TargetPseudoExpansionInfo &pseudoExpansionInfo() noexcept;
 [[nodiscard]] std::unique_ptr<MachineFunctionPass>
 createMoveMergePass(const RISCVSubtarget &Subtarget);
+[[nodiscard]] std::unique_ptr<MachineFunctionPass>
+createMakeCompressiblePass(const RISCVSubtarget &Subtarget);
 [[nodiscard]] std::unique_ptr<MachineFunctionPass> createMergeBaseOffsetPass();
 [[nodiscard]] MachineOptimizationPipelineTargetInfo
 optimizationPipelineInfo(const RISCVSubtarget &Subtarget, bool EmitSchedulerDebugOutput = false,
diff --git a/backend/lib/Target/RISCV/RISCVMakeCompressible.cpp b/backend/lib/Target/RISCV/RISCVMakeCompressible.cpp
new file mode 100644
index 0000000..e7fef9c
--- /dev/null
+++ b/backend/lib/Target/RISCV/RISCVMakeCompressible.cpp
@@ -0,0 +1,422 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCV.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineFunctionAttributes.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachinePassManager.h"
+#include "rcc/CodeGen/RegisterMask.h"
+#include "rcc/CodeGen/RegisterScavenger.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVInstructions.h"
+#include "rcc/Target/RISCV/RISCVRegisters.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <span>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+struct RegImmPair final {
+  Register Reg;
+  std::int64_t Imm = 0;
+};
+
+[[nodiscard]] constexpr RegisterClassID registerClassID(RegisterClass Value) noexcept {
+  return RegisterClassID::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+[[nodiscard]] PhysicalRegister physicalRegister(PhysicalRegisterID Value) noexcept {
+  const auto Created = PhysicalRegister::create(static_cast<std::uint32_t>(Value));
+  return *Created;
+}
+
+[[nodiscard]] bool isOpcode(const MachineInstr &Instruction, InstructionOpcode Value) noexcept {
+  return Instruction.opcode() == instructionOpcode(Value);
+}
+
+[[nodiscard]] std::optional<PhysicalRegister>
+physicalRegisterValue(const MachineOperand *Operand) noexcept {
+  const auto Reg = Operand == nullptr ? std::nullopt : Operand->registerValue();
+  return Reg ? Reg->physicalRegister() : std::nullopt;
+}
+
+[[nodiscard]] bool classContains(RegisterClass Class, Register Reg,
+                                 const TargetDescription &Target) noexcept {
+  const auto Physical = Reg.physicalRegister();
+  return Physical && Target.registers().registerClassContains(registerClassID(Class), *Physical);
+}
+
+[[nodiscard]] bool isCompressedRegister(Register Reg, const TargetDescription &Target) noexcept {
+  return classContains(RegisterClass::GPRC, Reg, Target) ||
+         classContains(RegisterClass::FPR32C, Reg, Target) ||
+         classContains(RegisterClass::FPR64C, Reg, Target);
+}
+
+[[nodiscard]] bool isCompressibleLoad(const MachineInstr &Instruction,
+                                      const RISCVSubtarget &Subtarget) noexcept {
+  if (!Subtarget.features().has(RISCVExtension::C))
+    return false;
+  if (isOpcode(Instruction, InstructionOpcode::LW) ||
+      isOpcode(Instruction, InstructionOpcode::LD)) {
+    return true;
+  }
+  return isOpcode(Instruction, InstructionOpcode::FLD) &&
+         Subtarget.features().has(RISCVExtension::D);
+}
+
+[[nodiscard]] bool isCompressibleStore(const MachineInstr &Instruction,
+                                       const RISCVSubtarget &Subtarget) noexcept {
+  if (!Subtarget.features().has(RISCVExtension::C))
+    return false;
+  if (isOpcode(Instruction, InstructionOpcode::SW) ||
+      isOpcode(Instruction, InstructionOpcode::SD)) {
+    return true;
+  }
+  return isOpcode(Instruction, InstructionOpcode::FSD) &&
+         Subtarget.features().has(RISCVExtension::D);
+}
+
+[[nodiscard]] std::optional<unsigned> loadStoreWidth(const MachineInstr &Instruction) noexcept {
+  if (isOpcode(Instruction, InstructionOpcode::LW) ||
+      isOpcode(Instruction, InstructionOpcode::SW)) {
+    return 2U;
+  }
+  if (isOpcode(Instruction, InstructionOpcode::LD) ||
+      isOpcode(Instruction, InstructionOpcode::SD) ||
+      isOpcode(Instruction, InstructionOpcode::FLD) ||
+      isOpcode(Instruction, InstructionOpcode::FSD)) {
+    return 3U;
+  }
+  return std::nullopt;
+}
+
+[[nodiscard]] std::int64_t compressedOffsetMask(const MachineInstr &Instruction) noexcept {
+  const std::optional<unsigned> Width = loadStoreWidth(Instruction);
+  if (!Width)
+    return 0;
+  return static_cast<std::int64_t>(std::uint64_t{0x1f} << *Width);
+}
+
+[[nodiscard]] bool isCompressibleStackOffset(std::int64_t Offset,
+                                             const MachineInstr &Instruction) noexcept {
+  const std::optional<unsigned> Width = loadStoreWidth(Instruction);
+  if (!Width || Offset < 0)
+    return false;
+  const std::uint64_t Value = static_cast<std::uint64_t>(Offset);
+  const std::uint64_t AlignmentMask = (std::uint64_t{1} << *Width) - 1U;
+  return (Value & AlignmentMask) == 0 && (Value >> *Width) < 64U;
+}
+
+[[nodiscard]] std::optional<RegImmPair>
+regImmPreventingCompression(MachineInstr &Instruction, const RISCVSubtarget &Subtarget,
+                            const TargetDescription &Target) noexcept {
+  const bool IsLoad = isCompressibleLoad(Instruction, Subtarget);
+  const bool IsStore = isCompressibleStore(Instruction, Subtarget);
+  if (!IsLoad && !IsStore)
+    return std::nullopt;
+
+  MachineOperand *ValueOperand = Instruction.operandAt(0);
+  MachineOperand *BaseOperand = Instruction.operandAt(1);
+  MachineOperand *OffsetOperand = Instruction.operandAt(2);
+  const auto Value = physicalRegisterValue(ValueOperand);
+  const auto Base = physicalRegisterValue(BaseOperand);
+  const auto *Offset = OffsetOperand == nullptr ? nullptr : OffsetOperand->getIf<std::int64_t>();
+  if (!Value || !Base || Offset == nullptr)
+    return std::nullopt;
+
+  const std::int64_t Mask = compressedOffsetMask(Instruction);
+  const std::int64_t BaseAdjustment = *Offset & ~Mask;
+  const Register BaseRegister = Register::fromPhysical(*Base);
+  if (*Base == physicalRegister(PhysicalRegisterID::X2)) {
+    if (!isCompressibleStackOffset(*Offset, Instruction) && BaseAdjustment != 0)
+      return RegImmPair{BaseRegister, BaseAdjustment};
+    return std::nullopt;
+  }
+
+  const Register ValueRegister = Register::fromPhysical(*Value);
+  const bool ValueCompressed = isCompressedRegister(ValueRegister, Target);
+  const bool BaseCompressed = isCompressedRegister(BaseRegister, Target);
+  if ((!BaseCompressed || BaseAdjustment != 0) && ValueCompressed)
+    return RegImmPair{BaseRegister, BaseAdjustment};
+  if (IsStore && !ValueCompressed && (BaseCompressed || *Value == *Base) && BaseAdjustment == 0) {
+    return RegImmPair{ValueRegister, 0};
+  }
+  return std::nullopt;
+}
+
+[[nodiscard]] bool maskClobbers(const RegisterMask &Mask, PhysicalRegister Reg,
+                                const TargetDescription &Target) noexcept {
+  for (const PhysicalRegisterDesc &Description : Target.registers().physicalRegisters()) {
+    if (Target.registers().regsOverlap(Description.Reg, Reg) && Mask.clobbers(Description.Reg))
+      return true;
+  }
+  return false;
+}
+
+[[nodiscard]] bool instructionModifies(const MachineInstr &Instruction, Register Reg,
+                                       const TargetDescription &Target) noexcept {
+  const auto Physical = Reg.physicalRegister();
+  if (!Physical)
+    return false;
+  for (const MachineOperand &Operand : Instruction.operands()) {
+    if (const auto OperandPhysical = physicalRegisterValue(&Operand);
+        OperandPhysical && Operand.isDef() &&
+        Target.registers().regsOverlap(*OperandPhysical, *Physical)) {
+      return true;
+    }
+    const auto *Mask = Operand.getIf<RegisterMaskOperand>();
+    if (Mask != nullptr && Mask->Mask && maskClobbers(*Mask->Mask, *Physical, Target))
+      return true;
+  }
+  return false;
+}
+
+[[nodiscard]] bool instructionTouches(const MachineInstr &Instruction, PhysicalRegister Reg,
+                                      const TargetDescription &Target) noexcept {
+  for (const MachineOperand &Operand : Instruction.operands()) {
+    if (const auto OperandPhysical = physicalRegisterValue(&Operand);
+        OperandPhysical && Target.registers().regsOverlap(*OperandPhysical, Reg)) {
+      return true;
+    }
+    const auto *Mask = Operand.getIf<RegisterMaskOperand>();
+    if (Mask != nullptr && Mask->Mask && maskClobbers(*Mask->Mask, Reg, Target))
+      return true;
+  }
+  return false;
+}
+
+[[nodiscard]] std::optional<RegisterClass>
+compressedClassFor(Register Reg, const TargetDescription &Target) noexcept {
+  if (classContains(RegisterClass::GPR, Reg, Target))
+    return RegisterClass::GPRC;
+  if (classContains(RegisterClass::FPR32, Reg, Target))
+    return RegisterClass::FPR32C;
+  if (classContains(RegisterClass::FPR64, Reg, Target))
+    return RegisterClass::FPR64C;
+  return std::nullopt;
+}
+
+[[nodiscard]] Expected<std::optional<PhysicalRegister>>
+scavengeCompressedRegister(MachineFunction &Function, MachineInstr &First, MachineInstr &Last,
+                           Register OldReg, const TargetDescription &Target) {
+  const auto Class = compressedClassFor(OldReg, Target);
+  if (!Class)
+    return std::optional<PhysicalRegister>{};
+  const RegisterClassDesc *Description = Target.registers().registerClass(registerClassID(*Class));
+  if (Description == nullptr)
+    return makeUnexpected(ErrorCode::InternalError,
+                          "RISC-V compressed register class is unavailable");
+
+  std::vector<PhysicalRegister> Candidates;
+  for (PhysicalRegister Candidate : Description->AllocationOrder) {
+    bool Touched = false;
+    for (MachineInstr *Instruction = &First; Instruction != nullptr;
+         Instruction = Instruction->next()) {
+      Touched |= instructionTouches(*Instruction, Candidate, Target);
+      if (Instruction == &Last)
+        break;
+    }
+    if (!Touched)
+      Candidates.push_back(Candidate);
+  }
+  if (Candidates.empty())
+    return std::optional<PhysicalRegister>{};
+
+  RegisterScavenger Scavenger(Function, Target);
+  auto Scavenged = Scavenger.scavenge(First, Candidates);
+  if (!Scavenged) {
+    if (Scavenged.error().code() == ErrorCode::RegisterAllocationFailed)
+      return std::optional<PhysicalRegister>{};
+    return std::unexpected(Scavenged.error());
+  }
+  if (Scavenged->RequiresSpill)
+    return std::optional<PhysicalRegister>{};
+  return std::optional<PhysicalRegister>(Scavenged->Reg);
+}
+
+[[nodiscard]] Expected<std::optional<PhysicalRegister>>
+analyzeCompressibleUses(MachineFunction &Function, MachineInstr &First, RegImmPair RegImm,
+                        std::vector<MachineInstr *> &Instructions, const RISCVSubtarget &Subtarget,
+                        const TargetDescription &Target) {
+  for (MachineInstr *Instruction = &First; Instruction != nullptr;
+       Instruction = Instruction->next()) {
+    const auto Candidate = regImmPreventingCompression(*Instruction, Subtarget, Target);
+    if (Candidate && Candidate->Reg == RegImm.Reg && Candidate->Imm == RegImm.Imm)
+      Instructions.push_back(Instruction);
+    if (instructionModifies(*Instruction, RegImm.Reg, Target))
+      break;
+  }
+
+  if (Instructions.size() <= 1U || (RegImm.Imm != 0 && Instructions.size() < 3U))
+    return std::optional<PhysicalRegister>{};
+  return scavengeCompressedRegister(Function, First, *Instructions.back(), RegImm.Reg, Target);
+}
+
+[[nodiscard]] Expected<MachineInstr *>
+insertRegisterCopy(MachineFunction &Function, MachineInstr &Before, PhysicalRegister Destination,
+                   RegImmPair Source, const TargetDescription &Target) {
+  MachineIRBuilder Builder(Function, Target);
+  auto InsertPoint = Builder.setInsertPointBefore(Before);
+  if (!InsertPoint)
+    return std::unexpected(InsertPoint.error());
+
+  InstructionOpcode Opcode = InstructionOpcode::ADDI;
+  const bool IsGPR = classContains(RegisterClass::GPR, Source.Reg, Target);
+  if (!IsGPR) {
+    if (Source.Imm != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V floating-point compressed copy has an offset");
+    if (classContains(RegisterClass::FPR32, Source.Reg, Target))
+      Opcode = InstructionOpcode::FSGNJ_S;
+    else if (classContains(RegisterClass::FPR64, Source.Reg, Target))
+      Opcode = InstructionOpcode::FSGNJ_D;
+    else
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V compressed copy has an unsupported register class");
+  } else if (Source.Imm < -2048 || Source.Imm > 2047) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V compressed base adjustment is outside ADDI range");
+  }
+
+  auto Pending = Builder.buildInstr(instructionOpcode(Opcode));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  auto Added = Pending->addDef(Register::fromPhysical(Destination));
+  if (!Added)
+    return std::unexpected(Added.error());
+  Added = Pending->addUse(Source.Reg);
+  if (!Added)
+    return std::unexpected(Added.error());
+  if (IsGPR)
+    Added = Pending->addImmediate(Source.Imm);
+  else
+    Added = Pending->addUse(Source.Reg);
+  if (!Added)
+    return std::unexpected(Added.error());
+  if (Before.debugLocation()) {
+    Added = Pending->setDebugLocation(*Before.debugLocation());
+    if (!Added)
+      return std::unexpected(Added.error());
+  }
+  return Pending->build();
+}
+
+[[nodiscard]] Expected<void> updateOperands(MachineInstr &Instruction, RegImmPair OldRegImm,
+                                            PhysicalRegister NewReg,
+                                            const RISCVSubtarget &Subtarget) {
+  const std::size_t FirstOperand =
+      isCompressibleStore(Instruction, Subtarget) && OldRegImm.Imm != 0 ? 1U : 0U;
+  for (std::size_t Index = FirstOperand; Index < Instruction.operandCount(); ++Index) {
+    MachineOperand *Operand = Instruction.operandAt(Index);
+    if (Operand == nullptr || Operand->isDef() || Operand->registerValue() != OldRegImm.Reg)
+      continue;
+    auto Updated = Operand->setRegister(Register::fromPhysical(NewReg));
+    if (!Updated)
+      return std::unexpected(Updated.error());
+  }
+
+  MachineOperand *Offset = Instruction.operandAt(2);
+  const auto *OldOffset = Offset == nullptr ? nullptr : Offset->getIf<std::int64_t>();
+  if (OldOffset == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V compressible load/store has no immediate offset");
+  return Offset->setImmediate(*OldOffset & compressedOffsetMask(Instruction));
+}
+
+[[nodiscard]] Expected<bool> makeBlockCompressible(MachineFunction &Function,
+                                                   MachineBasicBlock &Block,
+                                                   const RISCVSubtarget &Subtarget,
+                                                   const TargetDescription &Target) {
+  bool Changed = false;
+  for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr;
+       Instruction = Instruction->next()) {
+    const auto RegImm = regImmPreventingCompression(*Instruction, Subtarget, Target);
+    if (!RegImm)
+      continue;
+    std::vector<MachineInstr *> Uses;
+    auto NewReg = analyzeCompressibleUses(Function, *Instruction, *RegImm, Uses, Subtarget, Target);
+    if (!NewReg)
+      return std::unexpected(NewReg.error());
+    const PhysicalRegister Scavenged = NewReg->value_or(PhysicalRegister{});
+    if (!Scavenged.isValid())
+      continue;
+    auto Copy = insertRegisterCopy(Function, *Instruction, Scavenged, *RegImm, Target);
+    if (!Copy)
+      return std::unexpected(Copy.error());
+    for (MachineInstr *Use : Uses) {
+      auto Updated = updateOperands(*Use, *RegImm, Scavenged, Subtarget);
+      if (!Updated)
+        return std::unexpected(Updated.error());
+    }
+    Changed = true;
+  }
+  return Changed;
+}
+
+class RISCVMakeCompressiblePass final : public MachineFunctionPass {
+public:
+  explicit RISCVMakeCompressiblePass(const RISCVSubtarget &Subtarget) noexcept
+      : Subtarget(&Subtarget) {}
+
+  [[nodiscard]] const MachinePassContract &contract() const noexcept override {
+    static const MachinePassContract Contract = [] {
+      MachinePassContract Value{
+          .Name = "riscv-make-compressible",
+          .Requires = {},
+          .Sets = {},
+          .Clears = {},
+          .RequiredAnalyses = {},
+          .VerificationStage = std::nullopt,
+      };
+      Value.ThreadSafety = PassThreadSafety::ThreadSafe;
+      Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands,
+                               MachinePassChange::Registers};
+      return Value;
+    }();
+    return Contract;
+  }
+
+  [[nodiscard]] Expected<MachinePassResult> run(MachineFunction &Function,
+                                                const TargetDescription &Target,
+                                                MachineFunctionAnalysisManager &) override {
+    MachinePassResult Result;
+    Result.Preserved = PreservedAnalyses::all();
+    if (Subtarget == nullptr || !Subtarget->features().has(RISCVExtension::C) ||
+        !Function.attributes().has(MachineFunctionAttribute::MinSize)) {
+      return Result;
+    }
+    for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+         Block = Block->next()) {
+      auto BlockResult = makeBlockCompressible(Function, *Block, *Subtarget, Target);
+      if (!BlockResult)
+        return std::unexpected(BlockResult.error());
+      Result.Changed |= *BlockResult;
+    }
+    if (Result.Changed) {
+      Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands,
+                        MachinePassChange::Registers};
+      Result.Preserved = PreservedAnalyses::none();
+    }
+    return Result;
+  }
+
+private:
+  const RISCVSubtarget *Subtarget = nullptr;
+};
+
+} // namespace
+
+std::unique_ptr<MachineFunctionPass> createMakeCompressiblePass(const RISCVSubtarget &Subtarget) {
+  return std::make_unique<RISCVMakeCompressiblePass>(Subtarget);
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/RISCVPipeline.cpp b/backend/lib/Target/RISCV/RISCVPipeline.cpp
index c2f28f2..854417a 100644
--- a/backend/lib/Target/RISCV/RISCVPipeline.cpp
+++ b/backend/lib/Target/RISCV/RISCVPipeline.cpp
@@ -109,6 +109,9 @@ template <typename PassT, typename... ArgumentT>
                                              Options.UseTargetCopyInstructions});
   if (!Added)
     return std::unexpected(Added.error());
+  Added = Pipeline.addPass(createMakeCompressiblePass(Subtarget));
+  if (!Added)
+    return std::unexpected(Added.error());
   Added = Pipeline.addPass(createMoveMergePass(Subtarget));
   if (!Added)
     return std::unexpected(Added.error());
@@ -172,6 +175,9 @@ Expected<void> addCanonicalPipeline(MachineFunctionPassManager &Pipeline,
     if (!Added)
       return std::unexpected(Added.error());
   }
+  auto Added = Pipeline.addPass(createMakeCompressiblePass(Subtarget));
+  if (!Added)
+    return std::unexpected(Added.error());
   if (Subtarget.features().has(RISCVExtension::Zcmp))
     return Pipeline.addPass(createMoveMergePass(Subtarget));
   return {};
diff --git a/backend/lib/Target/RISCV/RISCVRegisterInfo.rccdesc b/backend/lib/Target/RISCV/RISCVRegisterInfo.rccdesc
index 4806ec4..975aa56 100644
--- a/backend/lib/Target/RISCV/RISCVRegisterInfo.rccdesc
+++ b/backend/lib/Target/RISCV/RISCVRegisterInfo.rccdesc
@@ -270,7 +270,7 @@ register_classes:
     spill_alignment_bits: 64
     lanes: 1
     registers: [F0_D, F1_D, F2_D, F3_D, F4_D, F5_D, F6_D, F7_D, F8_D, F9_D, F10_D, F11_D, F12_D, F13_D, F14_D, F15_D, F16_D, F17_D, F18_D, F19_D, F20_D, F21_D, F22_D, F23_D, F24_D, F25_D, F26_D, F27_D, F28_D, F29_D, F30_D, F31_D]
-    allocation_order: [F10_D, F11_D, F12_D, F13_D, F14_D, F15_D, F16_D, F17_D, F0_D, F1_D, F2_D, F3_D, F4_D, F5_D, F6_D, F7_D, F28_D, F29_D, F30_D, F31_D, F8_D, F9_D, F18_D, F19_D, F20_D, F21_D, F22_D, F23_D, F24_D, F25_D, F26_D, F27_D]
+    allocation_order: [F15_D, F14_D, F13_D, F12_D, F11_D, F10_D, F0_D, F1_D, F2_D, F3_D, F4_D, F5_D, F6_D, F7_D, F16_D, F17_D, F28_D, F29_D, F30_D, F31_D, F8_D, F9_D, F18_D, F19_D, F20_D, F21_D, F22_D, F23_D, F24_D, F25_D, F26_D, F27_D]
   -
     id: 4
     symbol: FPR64C
@@ -282,7 +282,7 @@ register_classes:
     spill_alignment_bits: 64
     lanes: 1
     registers: [F8_D, F9_D, F10_D, F11_D, F12_D, F13_D, F14_D, F15_D]
-    allocation_order: [F10_D, F11_D, F12_D, F13_D, F14_D, F15_D, F8_D, F9_D]
+    allocation_order: [F15_D, F14_D, F13_D, F12_D, F11_D, F10_D, F8_D, F9_D]
   -
     id: 5
     symbol: FPR32
@@ -293,7 +293,7 @@ register_classes:
     spill_alignment_bits: 32
     lanes: 1
     registers: [F0_F, F1_F, F2_F, F3_F, F4_F, F5_F, F6_F, F7_F, F8_F, F9_F, F10_F, F11_F, F12_F, F13_F, F14_F, F15_F, F16_F, F17_F, F18_F, F19_F, F20_F, F21_F, F22_F, F23_F, F24_F, F25_F, F26_F, F27_F, F28_F, F29_F, F30_F, F31_F]
-    allocation_order: [F10_F, F11_F, F12_F, F13_F, F14_F, F15_F, F16_F, F17_F, F0_F, F1_F, F2_F, F3_F, F4_F, F5_F, F6_F, F7_F, F28_F, F29_F, F30_F, F31_F, F8_F, F9_F, F18_F, F19_F, F20_F, F21_F, F22_F, F23_F, F24_F, F25_F, F26_F, F27_F]
+    allocation_order: [F15_F, F14_F, F13_F, F12_F, F11_F, F10_F, F0_F, F1_F, F2_F, F3_F, F4_F, F5_F, F6_F, F7_F, F16_F, F17_F, F28_F, F29_F, F30_F, F31_F, F8_F, F9_F, F18_F, F19_F, F20_F, F21_F, F22_F, F23_F, F24_F, F25_F, F26_F, F27_F]
   -
     id: 6
     symbol: FPR32C
@@ -305,7 +305,7 @@ register_classes:
     spill_alignment_bits: 32
     lanes: 1
     registers: [F8_F, F9_F, F10_F, F11_F, F12_F, F13_F, F14_F, F15_F]
-    allocation_order: [F10_F, F11_F, F12_F, F13_F, F14_F, F15_F, F8_F, F9_F]
+    allocation_order: [F15_F, F14_F, F13_F, F12_F, F11_F, F10_F, F8_F, F9_F]
   -
     id: 7
     symbol: GPR_NX0