[RISCV][GlobalISel] Legalize floating-point remainder

17 files changed, 342 insertions, 36 deletions

diff --git a/backend/include/rcc/CodeGen/CommandFlags.h b/backend/include/rcc/CodeGen/CommandFlags.h
index 0ee2f0d..aa7ed0d 100644
--- a/backend/include/rcc/CodeGen/CommandFlags.h
+++ b/backend/include/rcc/CodeGen/CommandFlags.h
@@ -41,6 +41,12 @@ enum class CodeGenFileType : std::uint8_t {
   Null,
 };
 
+enum class GlobalISelAbortMode : std::uint8_t {
+  Disable,
+  Enable,
+  DisableWithDiagnostic,
+};
+
 struct CodeGenCommandFlags final {
   std::optional<std::string> Triple;
   std::optional<std::string> CPU;
@@ -63,6 +69,7 @@ struct CodeGenCommandFlags final {
 [[nodiscard]] Expected<CodeModel> parseCodeModel(std::string_view Value);
 [[nodiscard]] Expected<OptimizationLevel> parseOptimizationLevel(std::string_view Value);
 [[nodiscard]] Expected<CodeGenFileType> parseCodeGenFileType(std::string_view Value);
+[[nodiscard]] Expected<GlobalISelAbortMode> parseGlobalISelAbortMode(std::string_view Value);
 [[nodiscard]] Expected<bool> parseCodeGenCommandFlag(std::span<const std::string_view> Arguments,
                                                      std::size_t &Index,
                                                      CodeGenCommandFlags &Flags);
diff --git a/backend/include/rcc/CodeGen/GlobalISel/Legalizer.h b/backend/include/rcc/CodeGen/GlobalISel/Legalizer.h
index 2ba0012..e90d09c 100644
--- a/backend/include/rcc/CodeGen/GlobalISel/Legalizer.h
+++ b/backend/include/rcc/CodeGen/GlobalISel/Legalizer.h
@@ -15,15 +15,18 @@ namespace rcc {
 class Legalizer final : public MachineFunctionPass {
 public:
   Legalizer(const LegalizerInfo *Info, OptimizationLevel Optimization,
-            LibcallLoweringInfo Libcalls = {})
-      : Info(Info), Optimization(Optimization), Libcalls(Libcalls) {}
+            LibcallLoweringInfo Libcalls = {},
+            GlobalISelAbortMode AbortMode = GlobalISelAbortMode::Enable)
+      : Info(Info), Optimization(Optimization), Libcalls(Libcalls), AbortMode(AbortMode) {}
   Legalizer(const LegalizerInfo &Info, OptimizationLevel Optimization,
-            LibcallLoweringInfo Libcalls = {})
-      : Info(&Info), Optimization(Optimization), Libcalls(Libcalls) {}
+            LibcallLoweringInfo Libcalls = {},
+            GlobalISelAbortMode AbortMode = GlobalISelAbortMode::Enable)
+      : Info(&Info), Optimization(Optimization), Libcalls(Libcalls), AbortMode(AbortMode) {}
   Legalizer(std::unique_ptr<LegalizerInfo> Info, OptimizationLevel Optimization,
-            LibcallLoweringInfo Libcalls = {})
+            LibcallLoweringInfo Libcalls = {},
+            GlobalISelAbortMode AbortMode = GlobalISelAbortMode::Enable)
       : OwnedInfo(std::move(Info)), Info(OwnedInfo.get()), Optimization(Optimization),
-        Libcalls(Libcalls) {}
+        Libcalls(Libcalls), AbortMode(AbortMode) {}
 
   [[nodiscard]] const MachinePassContract &contract() const noexcept override;
   [[nodiscard]] Expected<MachinePassResult> run(MachineFunction &Function,
@@ -39,6 +42,7 @@ private:
   const LegalizerInfo *Info = nullptr;
   OptimizationLevel Optimization;
   LibcallLoweringInfo Libcalls;
+  GlobalISelAbortMode AbortMode;
 };
 
 } // namespace rcc
diff --git a/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h b/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h
index 3e14311..aff6443 100644
--- a/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h
@@ -103,6 +103,8 @@ private:
                                                                    const APInt &Amount,
                                                                    LowLevelType TargetType,
                                                                    LowLevelType AmountType);
+  [[nodiscard]] Expected<void> fewerElements(MachineInstr &Instruction, std::size_t TypeIndex,
+                                             LowLevelType NarrowType);
 
   struct ShiftParameters final {
     Register WordShift;
diff --git a/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h b/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h
index 3433e3a..78e6a32 100644
--- a/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h
@@ -139,6 +139,7 @@ public:
 
   LegalizeRuleSet &custom();
   LegalizeRuleSet &widenScalarToNextPow2(std::size_t TypeIndex, std::uint32_t MinimumBitWidth = 0);
+  LegalizeRuleSet &scalarize(std::size_t TypeIndex);
   LegalizeRuleSet &minScalar(std::size_t TypeIndex, LowLevelType MinimumType);
   LegalizeRuleSet &minScalar(bool Enabled, std::size_t TypeIndex, LowLevelType MinimumType);
   LegalizeRuleSet &maxScalar(std::size_t TypeIndex, LowLevelType MaximumType);
diff --git a/backend/include/rcc/Target/RISCV/RISCVPipeline.h b/backend/include/rcc/Target/RISCV/RISCVPipeline.h
index 07f3ece..262f7c1 100644
--- a/backend/include/rcc/Target/RISCV/RISCVPipeline.h
+++ b/backend/include/rcc/Target/RISCV/RISCVPipeline.h
@@ -23,6 +23,7 @@ namespace riscv {
 struct RISCVPipelineOptions final {
   OptimizationLevel Optimization = OptimizationLevel::O2;
   bool UseTargetCopyInstructions = false;
+  GlobalISelAbortMode GlobalISelAbort = GlobalISelAbortMode::Enable;
 };
 
 [[nodiscard]] Expected<void> addCanonicalPipeline(MachineFunctionPassManager &Pipeline,
diff --git a/backend/lib/CodeGen/CommandFlags.cpp b/backend/lib/CodeGen/CommandFlags.cpp
index dfbaadc..2a659eb 100644
--- a/backend/lib/CodeGen/CommandFlags.cpp
+++ b/backend/lib/CodeGen/CommandFlags.cpp
@@ -148,6 +148,16 @@ Expected<CodeGenFileType> parseCodeGenFileType(std::string_view Value) {
   return makeUnexpected(ErrorCode::InvalidInvocation, "unknown file type");
 }
 
+Expected<GlobalISelAbortMode> parseGlobalISelAbortMode(std::string_view Value) {
+  if (Value == "0")
+    return GlobalISelAbortMode::Disable;
+  if (Value == "1")
+    return GlobalISelAbortMode::Enable;
+  if (Value == "2")
+    return GlobalISelAbortMode::DisableWithDiagnostic;
+  return makeUnexpected(ErrorCode::InvalidInvocation, "-global-isel-abort requires 0, 1, or 2");
+}
+
 Expected<bool> parseCodeGenCommandFlag(std::span<const std::string_view> Arguments,
                                        std::size_t &Index, CodeGenCommandFlags &Flags) {
   const std::string_view Argument = Arguments[Index];
diff --git a/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index c001ac5..09b2b66 100644
--- a/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -190,6 +190,12 @@ const MachinePassContract &InstructionSelect::contract() const noexcept {
 Expected<MachinePassResult> InstructionSelect::run(MachineFunction &Function,
                                                    const TargetDescription &Target,
                                                    MachineFunctionAnalysisManager &Analyses) {
+  if (Function.properties().has(MachineFunctionProperty::FailedISel))
+    return MachinePassResult{
+        .Changed = false,
+        .Changes = {},
+        .Preserved = PreservedAnalyses::all(),
+    };
   if (Info == nullptr)
     return std::unexpected(passFailure("instruction select has no target behavior interface",
                                        ErrorCode::InvalidArgument));
diff --git a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
index 877777e..8f3f06f 100644
--- a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
+++ b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
@@ -258,6 +258,12 @@ const MachinePassContract &Legalizer::contract() const noexcept {
 Expected<MachinePassResult> Legalizer::run(MachineFunction &Function,
                                            const TargetDescription &Target,
                                            MachineFunctionAnalysisManager &Analyses) {
+  if (Function.properties().has(MachineFunctionProperty::FailedISel))
+    return MachinePassResult{
+        .Changed = false,
+        .Changes = {},
+        .Preserved = PreservedAnalyses::all(),
+    };
   if (Info == nullptr)
     return std::unexpected(
         passFailure("legalizer has no target behavior interface", ErrorCode::InvalidArgument));
@@ -265,6 +271,24 @@ Expected<MachinePassResult> Legalizer::run(MachineFunction &Function,
   if (!Required)
     return std::unexpected(Required.error());
 
+  if (AbortMode != GlobalISelAbortMode::Enable) {
+    if (AbortMode != GlobalISelAbortMode::Disable &&
+        AbortMode != GlobalISelAbortMode::DisableWithDiagnostic) {
+      return std::unexpected(passFailure("legalizer has an unknown GlobalISel abort mode",
+                                         ErrorCode::InvalidArgument));
+    }
+    MachineActionReplay Actions(Analyses);
+    auto Result = runImplementation(Function, Target, Actions);
+    if (Result || Result.error().code() != ErrorCode::Unsupported)
+      return Result;
+    Function.properties().set(MachineFunctionProperty::FailedISel);
+    return MachinePassResult{
+        .Changed = true,
+        .Changes = {MachinePassChange::Instructions},
+        .Preserved = PreservedAnalyses::none(),
+    };
+  }
+
   auto Trial = cloneMachineFunction(
       Function, Target,
       MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt});
diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index b95f113..69396f4 100644
--- a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -336,6 +336,25 @@ buildFlaggedBinaryInstruction(MachineIRBuilder &Builder, GenericOpcode Generic,
   }
 }
 
+[[nodiscard]] Expected<std::string> floatingBinaryLibcallName(GenericOpcode Generic,
+                                                              std::uint32_t BitWidth) {
+  if (Generic != GenericOpcode::G_FREM) {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "floating libcall has no runtime function mapping");
+  }
+  switch (BitWidth) {
+  case 32:
+    return "fmodf";
+  case 64:
+    return "fmod";
+  case 128:
+    return "fmodl";
+  default:
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "floating libcall requires a 32, 64, or 128-bit scalar");
+  }
+}
+
 [[nodiscard]] Expected<std::string> conversionLibcallName(GenericOpcode Generic,
                                                           std::uint32_t DestinationWidth,
                                                           std::uint32_t SourceWidth) {
@@ -1698,6 +1717,57 @@ Expected<void> LegalizerHelper::widenScalar(MachineInstr &Instruction, std::size
     break;
   }
 
+  case GenericOpcode::G_FADD:
+  case GenericOpcode::G_FMUL:
+  case GenericOpcode::G_FSUB:
+  case GenericOpcode::G_FMA:
+  case GenericOpcode::G_FMAD:
+  case GenericOpcode::G_FNEG:
+  case GenericOpcode::G_FABS:
+  case GenericOpcode::G_FCANONICALIZE:
+  case GenericOpcode::G_FMINNUM:
+  case GenericOpcode::G_FMAXNUM:
+  case GenericOpcode::G_FMINNUM_IEEE:
+  case GenericOpcode::G_FMAXNUM_IEEE:
+  case GenericOpcode::G_FMINIMUM:
+  case GenericOpcode::G_FMAXIMUM:
+  case GenericOpcode::G_FMINIMUMNUM:
+  case GenericOpcode::G_FMAXIMUMNUM:
+  case GenericOpcode::G_FDIV:
+  case GenericOpcode::G_FREM:
+  case GenericOpcode::G_FCEIL:
+  case GenericOpcode::G_FFLOOR:
+  case GenericOpcode::G_FCOS:
+  case GenericOpcode::G_FSIN:
+  case GenericOpcode::G_FTAN:
+  case GenericOpcode::G_FACOS:
+  case GenericOpcode::G_FASIN:
+  case GenericOpcode::G_FATAN:
+  case GenericOpcode::G_FATAN2:
+  case GenericOpcode::G_FCOSH:
+  case GenericOpcode::G_FSINH:
+  case GenericOpcode::G_FTANH:
+  case GenericOpcode::G_FLOG10:
+  case GenericOpcode::G_FLOG:
+  case GenericOpcode::G_FLOG2:
+  case GenericOpcode::G_FRINT:
+  case GenericOpcode::G_FNEARBYINT:
+  case GenericOpcode::G_FSQRT:
+  case GenericOpcode::G_FEXP:
+  case GenericOpcode::G_FEXP2:
+  case GenericOpcode::G_FEXP10:
+  case GenericOpcode::G_FPOW:
+    if (TypeIndex != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "floating operation widening has an invalid type index");
+    for (std::size_t OperandIndex = 1; OperandIndex < Instruction.operandCount(); ++OperandIndex) {
+      if (auto Result = Apply(WidenSource(OperandIndex, GenericOpcode::G_FPEXT)); !Result)
+        return Result;
+    }
+    if (auto Result = Apply(WidenDestination(0, GenericOpcode::G_FPTRUNC)); !Result)
+      return Result;
+    break;
+
   case GenericOpcode::G_FPEXT:
     if (TypeIndex != 1)
       return makeUnexpected(ErrorCode::Unsupported,
@@ -3252,6 +3322,127 @@ Expected<void> LegalizerHelper::narrowScalar(MachineInstr &Instruction, std::siz
   }
 }
 
+Expected<void> LegalizerHelper::fewerElements(MachineInstr &Instruction, std::size_t TypeIndex,
+                                              LowLevelType NarrowType) {
+  auto Query = LegalizerInfo::buildQuery(Instruction, Function, MIRBuilder.target());
+  if (!Query)
+    return std::unexpected(Query.error());
+  if (TypeIndex >= Query->Types.size() || TypeIndex >= Query->TypeOperands.size()) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "vector legalization has an invalid type index");
+  }
+
+  const LowLevelType VectorType = Query->Types[TypeIndex];
+  const std::optional<LowLevelType> ElementType = VectorType.scalarElementType();
+  if (!VectorType.isVector() || VectorType.isScalableVector() || !ElementType ||
+      *ElementType != NarrowType || !NarrowType.isCategoryNeutralScalar()) {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "vector legalization requires fixed vectors split into scalar elements");
+  }
+
+  std::size_t ExplicitOperandCount = 0;
+  while (ExplicitOperandCount < Instruction.operandCount() &&
+         !Instruction.operandAt(ExplicitOperandCount)->isImplicit()) {
+    ++ExplicitOperandCount;
+  }
+  const std::vector<std::size_t> &TypedOperands = Query->TypeOperands[TypeIndex];
+  if (TypedOperands.size() != ExplicitOperandCount || TypedOperands.empty()) {
+    return makeUnexpected(
+        ErrorCode::Unsupported,
+        "vector legalization requires all explicit operands to share one vector type");
+  }
+
+  std::vector<std::size_t> DefinitionIndices;
+  std::vector<std::size_t> SourceIndices;
+  DefinitionIndices.reserve(TypedOperands.size());
+  SourceIndices.reserve(TypedOperands.size());
+  for (const std::size_t OperandIndex : TypedOperands) {
+    const MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+    const Register *RegisterOperand = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+    if (RegisterOperand == nullptr || !RegisterOperand->isVirtual()) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "vector legalization requires virtual-register operands");
+    }
+    auto OperandType = MRI.getType(*RegisterOperand);
+    if (!OperandType)
+      return std::unexpected(OperandType.error());
+    if (*OperandType != VectorType) {
+      return makeUnexpected(
+          ErrorCode::VerificationError,
+          "vector legalization type operands do not match the requested vector type");
+    }
+    if (Operand->isDef() && !Operand->isUse()) {
+      DefinitionIndices.push_back(OperandIndex);
+    } else if (Operand->isUse() && !Operand->isDef()) {
+      SourceIndices.push_back(OperandIndex);
+    } else {
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "vector legalization does not support tied or non-register operands");
+    }
+  }
+  if (DefinitionIndices.empty() || SourceIndices.empty()) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "vector legalization requires register definitions and sources");
+  }
+
+  const std::size_t ElementCount = VectorType.minimumElementCount();
+  std::vector<std::vector<Register>> SourceParts;
+  SourceParts.reserve(SourceIndices.size());
+  for (const std::size_t SourceIndex : SourceIndices) {
+    const Register Source = *Instruction.operandAt(SourceIndex)->getIf<Register>();
+    std::vector<Register> Parts;
+    auto Extracted =
+        extractParts(Source, NarrowType, static_cast<int>(ElementCount), Parts, MIRBuilder, MRI);
+    if (!Extracted)
+      return std::unexpected(Extracted.error());
+    SourceParts.push_back(std::move(Parts));
+  }
+
+  std::vector<std::vector<Register>> DestinationParts(DefinitionIndices.size());
+  for (std::vector<Register> &Parts : DestinationParts)
+    Parts.reserve(ElementCount);
+  for (std::size_t ElementIndex = 0; ElementIndex < ElementCount; ++ElementIndex) {
+    auto ScalarInstruction = MIRBuilder.buildInstr(Instruction.opcode());
+    if (!ScalarInstruction)
+      return std::unexpected(ScalarInstruction.error());
+    if (auto SetFlags = ScalarInstruction->setFlags(Instruction.flags()); !SetFlags)
+      return std::unexpected(SetFlags.error());
+    for (std::size_t DefinitionNumber = 0; DefinitionNumber < DefinitionIndices.size();
+         ++DefinitionNumber) {
+      auto Destination = MRI.createVirtualRegister(NarrowType);
+      if (!Destination)
+        return std::unexpected(Destination.error());
+      if (auto Added = ScalarInstruction->addDef(*Destination); !Added)
+        return std::unexpected(Added.error());
+      DestinationParts[DefinitionNumber].push_back(*Destination);
+    }
+    for (const std::vector<Register> &Parts : SourceParts) {
+      if (auto Added = ScalarInstruction->addUse(Parts[ElementIndex]); !Added)
+        return std::unexpected(Added.error());
+    }
+    if (auto Built = ScalarInstruction->build(); !Built)
+      return std::unexpected(Built.error());
+  }
+
+  for (std::size_t DefinitionNumber = 0; DefinitionNumber < DefinitionIndices.size();
+       ++DefinitionNumber) {
+    const Register Destination =
+        *Instruction.operandAt(DefinitionIndices[DefinitionNumber])->getIf<Register>();
+    if (auto Merged =
+            MIRBuilder.buildMergeLikeInstr(Destination, DestinationParts[DefinitionNumber]);
+        !Merged) {
+      return std::unexpected(Merged.error());
+    }
+  }
+
+  MachineBasicBlock *Block = Instruction.parent();
+  if (Block == nullptr) {
+    return makeUnexpected(ErrorCode::InternalError,
+                          "vector instruction became detached during legalization");
+  }
+  return Block->removeInstruction(Instruction);
+}
+
 Expected<void> LegalizerHelper::lowerVAArg(MachineInstr &Instruction) {
   if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 ||
       !Instruction.memOperands().empty()) {
@@ -4723,6 +4914,7 @@ Expected<void> LegalizerHelper::libcall(MachineInstr &Instruction) {
       Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC ||
       Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI ||
       Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP;
+  const bool IsFloatingBinary = Generic == GenericOpcode::G_FREM;
   if (Instruction.operandCount() < 2 || (IsConversion && Instruction.operandCount() != 2)) {
     return makeUnexpected(ErrorCode::VerificationError,
                           "libcall has an invalid number of operands");
@@ -4744,18 +4936,19 @@ Expected<void> LegalizerHelper::libcall(MachineInstr &Instruction) {
   }
 
   Expected<std::string> Name =
-      IsConversion ? conversionLibcallName(Generic, DestinationType->scalarBitWidth(),
-                                           SourceType->scalarBitWidth())
-                   : integerLibcallName(Generic, DestinationType->scalarBitWidth());
+      IsConversion       ? conversionLibcallName(Generic, DestinationType->scalarBitWidth(),
+                                                 SourceType->scalarBitWidth())
+      : IsFloatingBinary ? floatingBinaryLibcallName(Generic, DestinationType->scalarBitWidth())
+                         : integerLibcallName(Generic, DestinationType->scalarBitWidth());
   if (!Name)
     return std::unexpected(Name.error());
 
   const bool DestinationIsFloat =
       Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC ||
-      Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP;
+      Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP || IsFloatingBinary;
   const bool SourceIsFloat =
       Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC ||
-      Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI;
+      Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI || IsFloatingBinary;
   const ABIValueCategory DestinationCategory =
       DestinationIsFloat ? ABIValueCategory::FloatingPoint : ABIValueCategory::Integer;
   const ABIValueCategory SourceCategory =
@@ -4843,7 +5036,7 @@ Expected<void> LegalizerHelper::libcall(MachineInstr &Instruction) {
     }
     if (!IsConversion && *ArgumentType != *DestinationType) {
       return makeUnexpected(ErrorCode::VerificationError,
-                            "integer libcall source and destination types must match");
+                            "binary libcall source and destination types must match");
     }
     const ABIExtensionKind Extension =
         ExtensionFor(SourceCategory, false, ArgumentType->scalarBitWidth());
@@ -4974,11 +5167,12 @@ Expected<void> LegalizerHelper::legalize(MachineInstr &Instruction,
     return lower(Instruction, Decision);
   case LegalizationAction::Libcall:
     return libcall(Instruction);
+  case LegalizationAction::FewerElements:
+    return fewerElements(Instruction, Decision.TypeIndex, Decision.NewType);
   case LegalizationAction::Legal:
     return {};
   case LegalizationAction::Unsupported:
   case LegalizationAction::MoreElements:
-  case LegalizationAction::FewerElements:
   case LegalizationAction::Bitcast:
   case LegalizationAction::ChangeElementType:
   case LegalizationAction::ChangeAddressSpace:
diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
index ee7f856..bdc783b 100644
--- a/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
+++ b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -256,6 +256,16 @@ LegalizeRuleSet &LegalizeRuleSet::widenScalarToNextPow2(std::size_t TypeIndex,
       });
 }
 
+LegalizeRuleSet &LegalizeRuleSet::scalarize(std::size_t TypeIndex) {
+  return fewerElementsIf(
+      [TypeIndex](const LegalizationQuery &Query) {
+        return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isVector();
+      },
+      [TypeIndex](const LegalizationQuery &Query) {
+        return std::pair{TypeIndex, *Query.Types[TypeIndex].scalarElementType()};
+      });
+}
+
 LegalizeRuleSet &LegalizeRuleSet::minScalar(std::size_t TypeIndex, LowLevelType MinimumType) {
   return widenScalarIf(
       [TypeIndex, MinimumType](const LegalizationQuery &Query) {
diff --git a/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp
index ef29333..a5747c3 100644
--- a/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp
+++ b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp
@@ -290,6 +290,12 @@ const MachinePassContract &RegBankSelect::contract() const noexcept {
 Expected<MachinePassResult> RegBankSelect::run(MachineFunction &Function,
                                                const TargetDescription &Target,
                                                MachineFunctionAnalysisManager &) {
+  if (Function.properties().has(MachineFunctionProperty::FailedISel))
+    return MachinePassResult{
+        .Changed = false,
+        .Changes = {},
+        .Preserved = PreservedAnalyses::all(),
+    };
   if (Info == nullptr)
     return std::unexpected(passFailure("register-bank select has no target behavior interface",
                                        ErrorCode::InvalidArgument));
diff --git a/backend/lib/CodeGen/MachineVerifier.cpp b/backend/lib/CodeGen/MachineVerifier.cpp
index 14ce8ef..68a28db 100644
--- a/backend/lib/CodeGen/MachineVerifier.cpp
+++ b/backend/lib/CodeGen/MachineVerifier.cpp
@@ -71,6 +71,8 @@ public:
       : Function(Function), Target(Target), Options(Options) {}
 
   [[nodiscard]] Expected<void> run() {
+    if (Function.properties().has(MachineFunctionProperty::FailedISel))
+      return {};
     if (!Function.stableID().isValid())
       return fail("function identity is invalid");
 
diff --git a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
index 050de5b..66b4d87 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
+++ b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
@@ -839,9 +839,14 @@ createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std::
   return *Extended;
 }
 
-[[nodiscard]] Expected<void> emitOutgoingRegisterAssignment(MachineIRBuilder &Builder,
-                                                            const CCAssignment &Assignment,
-                                                            Register Source) {
+struct OutgoingRegisterValue final {
+  PhysicalRegister Location;
+  Register Value;
+};
+
+[[nodiscard]] Expected<OutgoingRegisterValue>
+prepareOutgoingRegisterAssignment(MachineIRBuilder &Builder, const CCAssignment &Assignment,
+                                  Register Source) {
   if (!Assignment.LocationRegister) {
     return makeUnexpected(ErrorCode::InternalError,
                           "RISC-V register assignment has no location register");
@@ -849,9 +854,20 @@ createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std::
   auto Extended = extendOutgoingRegister(Builder, Source, Assignment);
   if (!Extended)
     return std::unexpected(Extended.error());
-  auto Copy = Builder.buildCopy(Register::fromPhysical(*Assignment.LocationRegister), *Extended);
-  if (!Copy)
-    return std::unexpected(Copy.error());
+  return OutgoingRegisterValue{
+      .Location = *Assignment.LocationRegister,
+      .Value = *Extended,
+  };
+}
+
+[[nodiscard]] Expected<void>
+emitOutgoingRegisterAssignments(MachineIRBuilder &Builder,
+                                std::span<const OutgoingRegisterValue> Assignments) {
+  for (const OutgoingRegisterValue &Assignment : Assignments) {
+    auto Copy = Builder.buildCopy(Register::fromPhysical(Assignment.Location), Assignment.Value);
+    if (!Copy)
+      return std::unexpected(Copy.error());
+  }
   return {};
 }
 
@@ -1348,15 +1364,18 @@ emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan,
   MachineFunction &Function = Builder.function();
   MachineFrameInfo &Frame = Function.frameInfo();
   const auto Assignments = Plan.argumentAssignments();
+  std::vector<OutgoingRegisterValue> RegisterAssignments;
+  RegisterAssignments.reserve(Assignments.size());
   for (std::size_t Index = 0; Index < Assignments.size(); ++Index) {
     const CCAssignment &Assignment = Assignments[Index];
     const auto Replacement = Replacements.find(Index);
     const Register Source =
         Replacement == Replacements.end() ? Assignment.OriginalValue : Replacement->second;
     if (Assignment.LocationRegister) {
-      auto Copy = emitOutgoingRegisterAssignment(Builder, Assignment, Source);
-      if (!Copy)
-        return std::unexpected(Copy.error());
+      auto Prepared = prepareOutgoingRegisterAssignment(Builder, Assignment, Source);
+      if (!Prepared)
+        return std::unexpected(Prepared.error());
+      RegisterAssignments.push_back(*Prepared);
       continue;
     }
 
@@ -1388,7 +1407,8 @@ emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan,
     if (!Store)
       return std::unexpected(Store.error());
   }
-  return {};
+  return emitOutgoingRegisterAssignments(
+      Builder, std::span<const OutgoingRegisterValue>(RegisterAssignments));
 }
 
 [[nodiscard]] Expected<void> addCallTarget(MachineInstrBuilder &Instruction,
@@ -1505,11 +1525,19 @@ emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan,
 }
 
 [[nodiscard]] Expected<void> emitReturn(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+  std::vector<OutgoingRegisterValue> RegisterAssignments;
+  RegisterAssignments.reserve(Plan.returnAssignments().size());
   for (const CCAssignment &Assignment : Plan.returnAssignments()) {
-    auto Copy = emitOutgoingRegisterAssignment(Builder, Assignment, Assignment.OriginalValue);
-    if (!Copy)
-      return std::unexpected(Copy.error());
-  }
+    auto Prepared =
+        prepareOutgoingRegisterAssignment(Builder, Assignment, Assignment.OriginalValue);
+    if (!Prepared)
+      return std::unexpected(Prepared.error());
+    RegisterAssignments.push_back(*Prepared);
+  }
+  auto Copies = emitOutgoingRegisterAssignments(
+      Builder, std::span<const OutgoingRegisterValue>(RegisterAssignments));
+  if (!Copies)
+    return std::unexpected(Copies.error());
 
   auto Return = Builder.buildInstr(instructionOpcode(InstructionOpcode::PseudoRET));
   if (!Return)
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 3f8f4cb..243c6de 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -576,7 +576,8 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) : Subtar
       .lowerFor({S32, S64, S128});
   getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREM))
       .libcallFor({S32, S64, S128})
-      .minScalar(0, S32);
+      .minScalar(0, S32)
+      .scalarize(0);
   getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCOPYSIGN))
       .legalFor(HasF, {{S32, S32}})
       .legalFor(HasD, {{S64, S64}, {S32, S64}, {S64, S32}})
diff --git a/backend/lib/Target/RISCV/RISCVPipeline.cpp b/backend/lib/Target/RISCV/RISCVPipeline.cpp
index a8d0a20..f1de092 100644
--- a/backend/lib/Target/RISCV/RISCVPipeline.cpp
+++ b/backend/lib/Target/RISCV/RISCVPipeline.cpp
@@ -45,7 +45,7 @@ template <typename PassT, typename... ArgumentT>
   if (!Added)
     return std::unexpected(Added.error());
   Added = addPass<Legalizer>(Pipeline, Subtarget.getLegalizerInfo(), Options.Optimization,
-                             libcallLoweringInfo(Subtarget));
+                             libcallLoweringInfo(Subtarget), Options.GlobalISelAbort);
   if (!Added)
     return std::unexpected(Added.error());
   Added = addPass<RegBankSelect>(Pipeline, Subtarget.getRegBankInfo());
@@ -100,7 +100,7 @@ Expected<void> addCanonicalPipeline(MachineFunctionPassManager &Pipeline,
     if (!Added)
       return std::unexpected(Added.error());
     Added = addPass<Legalizer>(Pipeline, Subtarget.getLegalizerInfo(), Options.Optimization,
-                               libcallLoweringInfo(Subtarget));
+                               libcallLoweringInfo(Subtarget), Options.GlobalISelAbort);
     if (!Added)
       return std::unexpected(Added.error());
   }
diff --git a/backend/lib/Tooling/PipelineDriver.cpp b/backend/lib/Tooling/PipelineDriver.cpp
index 7f252a6..42c33e7 100644
--- a/backend/lib/Tooling/PipelineDriver.cpp
+++ b/backend/lib/Tooling/PipelineDriver.cpp
@@ -98,12 +98,22 @@ struct FunctionRunResult final {
 [[nodiscard]] Expected<MachineFunctionPassManager::PassList>
 createPasses(const MachineFunction &Function, const BackendInvocation &Invocation,
              const riscv::RISCVSubtarget &Subtarget, MachinePassManagerOptions Options) {
+  const auto GlobalISelAbortOption = std::ranges::find(
+      Invocation.TargetOptions, std::string_view{"global-isel-abort"}, &BackendOption::Name);
+  GlobalISelAbortMode GlobalISelAbort = GlobalISelAbortMode::Enable;
+  if (GlobalISelAbortOption != Invocation.TargetOptions.end()) {
+    auto Parsed = parseGlobalISelAbortMode(GlobalISelAbortOption->Value);
+    if (!Parsed)
+      return std::unexpected(Parsed.error());
+    GlobalISelAbort = *Parsed;
+  }
   MachineFunctionPassManager Pipeline(Options);
   if (Invocation.PassPipeline.size() == 1 && Invocation.PassPipeline.front() == "default") {
     auto Added = riscv::addCanonicalPipeline(
         Pipeline, Function, Subtarget,
         riscv::RISCVPipelineOptions{.Optimization = Invocation.Optimization,
-                                    .UseTargetCopyInstructions = true});
+                                    .UseTargetCopyInstructions = true,
+                                    .GlobalISelAbort = GlobalISelAbort});
     if (!Added)
       return std::unexpected(Added.error());
   } else {
@@ -119,7 +129,8 @@ createPasses(const MachineFunction &Function, const BackendInvocation &Invocatio
       auto Pass = riscv::createCanonicalPipelinePass(
           Name, Subtarget,
           riscv::RISCVPipelineOptions{.Optimization = Invocation.Optimization,
-                                      .UseTargetCopyInstructions = UseTargetCopyInstructions});
+                                      .UseTargetCopyInstructions = UseTargetCopyInstructions,
+                                      .GlobalISelAbort = GlobalISelAbort});
       if (!Pass)
         return std::unexpected(Pass.error());
       auto Added = Pipeline.addPass(std::move(*Pass));
diff --git a/backend/tools/MIRTool/MIRToolMain.cpp b/backend/tools/MIRTool/MIRToolMain.cpp
index 5528fbb..d6e7d9e 100644
--- a/backend/tools/MIRTool/MIRToolMain.cpp
+++ b/backend/tools/MIRTool/MIRToolMain.cpp
@@ -371,10 +371,9 @@ void setBackendOption(std::vector<rcc::tooling::BackendOption> &Options, std::st
     if (!GlobalISelAbort)
       return std::unexpected(GlobalISelAbort.error());
     if (*GlobalISelAbort) {
-      if (**GlobalISelAbort != "0" && **GlobalISelAbort != "1" && **GlobalISelAbort != "2") {
-        return rcc::makeUnexpected(ErrorCode::InvalidInvocation,
-                                   "-global-isel-abort requires 0, 1, or 2");
-      }
+      auto Mode = rcc::parseGlobalISelAbortMode(**GlobalISelAbort);
+      if (!Mode)
+        return std::unexpected(Mode.error());
       setBackendOption(Command.Invocation.TargetOptions, "global-isel-abort",
                        std::string(**GlobalISelAbort));
       continue;