[RISCV] Legalize va_copy intrinsics

4 files changed, 191 insertions, 43 deletions

diff --git a/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h b/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h
index d3515f3..9162526 100644
--- a/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h
@@ -4,8 +4,8 @@
 #define RCC_CODEGEN_GLOBALISEL_LEGALIZERINFO_H
 
 #include "rcc/CodeGen/LowLevelType.h"
-#include "rcc/CodeGen/MachineMemOperand.h"
 #include "rcc/CodeGen/MachineIRTypes.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
 #include "rcc/Support/Error.h"
 
 #include <cstddef>
@@ -98,14 +98,12 @@ class LegalizeRuleSet final {
 public:
   LegalizeRuleSet &actionIf(LegalizationAction Action, LegalityPredicate Predicate,
                             LegalizeMutation Mutation = {});
+  LegalizeRuleSet &actionFor(LegalizationAction Action, std::initializer_list<LowLevelType> Types);
   LegalizeRuleSet &actionFor(LegalizationAction Action,
-                             std::initializer_list<LowLevelType> Types);
-  LegalizeRuleSet &actionFor(
-      LegalizationAction Action,
-      std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
-  LegalizeRuleSet &actionFor(
-      LegalizationAction Action,
-      std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types);
+                             std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
+  LegalizeRuleSet &
+  actionFor(LegalizationAction Action,
+            std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types);
   LegalizeRuleSet &legalIf(LegalityPredicate Predicate);
   LegalizeRuleSet &customIf(LegalityPredicate Predicate);
   LegalizeRuleSet &lowerIf(LegalityPredicate Predicate);
@@ -119,38 +117,30 @@ public:
 
   LegalizeRuleSet &legalFor(std::initializer_list<LowLevelType> Types);
   LegalizeRuleSet &legalFor(bool Enabled, std::initializer_list<LowLevelType> Types);
-  LegalizeRuleSet &legalFor(
-      std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
-  LegalizeRuleSet &legalFor(
-      bool Enabled, std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
-  LegalizeRuleSet &legalFor(
-      std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types);
+  LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
+  LegalizeRuleSet &legalFor(bool Enabled,
+                            std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
+  LegalizeRuleSet &
+  legalFor(std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types);
   LegalizeRuleSet &customFor(std::initializer_list<LowLevelType> Types);
   LegalizeRuleSet &customFor(bool Enabled, std::initializer_list<LowLevelType> Types);
-  LegalizeRuleSet &customFor(
-      std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
-  LegalizeRuleSet &customFor(
-      bool Enabled, std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
+  LegalizeRuleSet &customFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
+  LegalizeRuleSet &customFor(bool Enabled,
+                             std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
   LegalizeRuleSet &lowerFor(std::initializer_list<LowLevelType> Types);
-  LegalizeRuleSet &lowerFor(
-      std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
+  LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
   LegalizeRuleSet &libcallFor(std::initializer_list<LowLevelType> Types);
   LegalizeRuleSet &libcallFor(bool Enabled, std::initializer_list<LowLevelType> Types);
-  LegalizeRuleSet &libcallFor(
-      std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
-  LegalizeRuleSet &libcallFor(
-      bool Enabled, std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
-  LegalizeRuleSet &legalForTypesWithMemDesc(
-      std::initializer_list<LegalizeMemoryRule> Rules);
-
-  LegalizeRuleSet &widenScalarToNextPow2(std::size_t TypeIndex,
-                                         std::uint32_t MinimumBitWidth = 0);
+  LegalizeRuleSet &libcallFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
+  LegalizeRuleSet &libcallFor(bool Enabled,
+                              std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types);
+  LegalizeRuleSet &legalForTypesWithMemDesc(std::initializer_list<LegalizeMemoryRule> Rules);
+
+  LegalizeRuleSet &widenScalarToNextPow2(std::size_t TypeIndex, std::uint32_t MinimumBitWidth = 0);
   LegalizeRuleSet &minScalar(std::size_t TypeIndex, LowLevelType MinimumType);
-  LegalizeRuleSet &minScalar(bool Enabled, std::size_t TypeIndex,
-                             LowLevelType MinimumType);
+  LegalizeRuleSet &minScalar(bool Enabled, std::size_t TypeIndex, LowLevelType MinimumType);
   LegalizeRuleSet &maxScalar(std::size_t TypeIndex, LowLevelType MaximumType);
-  LegalizeRuleSet &maxScalar(bool Enabled, std::size_t TypeIndex,
-                             LowLevelType MaximumType);
+  LegalizeRuleSet &maxScalar(bool Enabled, std::size_t TypeIndex, LowLevelType MaximumType);
   LegalizeRuleSet &clampScalar(std::size_t TypeIndex, LowLevelType MinimumType,
                                LowLevelType MaximumType);
   LegalizeRuleSet &scalarSameSizeAs(std::size_t TypeIndex, std::size_t ReferenceTypeIndex);
@@ -159,8 +149,7 @@ public:
   LegalizeRuleSet &lower();
   LegalizeRuleSet &libcall();
 
-  [[nodiscard]] std::optional<LegalizationDecision>
-  apply(const LegalizationQuery &Query) const;
+  [[nodiscard]] std::optional<LegalizationDecision> apply(const LegalizationQuery &Query) const;
 
 private:
   std::vector<LegalizeRule> Rules;
@@ -170,16 +159,15 @@ class LegalizerInfo {
 public:
   virtual ~LegalizerInfo() = default;
 
-  [[nodiscard]] static Expected<LegalizationQuery>
-  buildQuery(const MachineInstr &Instruction, const MachineFunction &Function,
-             const TargetDescription &Target);
+  [[nodiscard]] static Expected<LegalizationQuery> buildQuery(const MachineInstr &Instruction,
+                                                              const MachineFunction &Function,
+                                                              const TargetDescription &Target);
 
-  [[nodiscard]] Expected<LegalizationDecision>
-  getDecision(const MachineInstr &Instruction, const MachineFunction &Function,
-              const TargetDescription &Target) const;
+  [[nodiscard]] Expected<LegalizationDecision> getDecision(const MachineInstr &Instruction,
+                                                           const MachineFunction &Function,
+                                                           const TargetDescription &Target) const;
 
-  [[nodiscard]] Expected<LegalizationDecision>
-  getDecision(const LegalizationQuery &Query) const;
+  [[nodiscard]] Expected<LegalizationDecision> getDecision(const LegalizationQuery &Query) const;
 
   [[nodiscard]] Expected<bool> isLegal(const LegalizationQuery &Query) const;
   [[nodiscard]] Expected<bool> isUnsupported(const LegalizationQuery &Query) const;
@@ -188,6 +176,11 @@ public:
   legalizeCustom(LegalizerHelper &Helper, MachineInstr &Instruction,
                  const LegalizationDecision &Decision) const = 0;
 
+  // Target intrinsic lowering runs before the ordinary legality action.
+  [[nodiscard]] virtual Expected<bool> legalizeIntrinsic(LegalizerHelper &, MachineInstr &) const {
+    return true;
+  }
+
   // This hook performs read-only target checks before a lowering mutation is committed.
   [[nodiscard]] virtual Expected<void> validateLowering(const MachineInstr &,
                                                         const MachineFunction &,
diff --git a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
index c749039..dfa4523 100644
--- a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
+++ b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
@@ -70,6 +70,18 @@ namespace {
   }
 }
 
+[[nodiscard]] bool isIntrinsicOpcode(Opcode InstructionOpcode) noexcept {
+  switch (static_cast<GenericOpcode>(InstructionOpcode.value())) {
+  default:
+    return false;
+  case GenericOpcode::G_INTRINSIC:
+  case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
+  case GenericOpcode::G_INTRINSIC_CONVERGENT:
+  case GenericOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
+    return true;
+  }
+}
+
 using InstList = GISelWorkList<256>;
 using ArtifactList = GISelWorkList<128>;
 
@@ -329,6 +341,27 @@ Expected<MachinePassResult> Legalizer::runImplementation(MachineFunction &Functi
             withErrorContext(Validation.error(), "legalizer input is invalid: "));
       }
 
+      if (isIntrinsicOpcode(Instruction->opcode())) {
+        auto Accepted = Actions.request("legalize.intrinsic", Function, Instruction->stableID());
+        if (!Accepted)
+          return std::unexpected(Accepted.error());
+        if (!*Accepted)
+          return std::unexpected(passFailure("required intrinsic legalization action was rejected",
+                                             ErrorCode::VerificationError));
+        auto IntrinsicResult = Info->legalizeIntrinsic(Helper, *Instruction);
+        if (!IntrinsicResult)
+          return std::unexpected(IntrinsicResult.error());
+        if (!*IntrinsicResult) {
+          const std::string_view Name = Descriptor->Name;
+          return std::unexpected(passFailure("target legalizer rejected intrinsic instruction '" +
+                                                 std::string(Name) + "'",
+                                             ErrorCode::Unsupported));
+        }
+        Result.Changed = true;
+        Result.Changes.add(MachinePassChange::Instructions);
+        continue;
+      }
+
       auto Decision = Info->getDecision(*Instruction, Function, Target);
       if (!Decision)
         return std::unexpected(Decision.error());
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index f1d2fac..88ee56a 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -15,6 +15,7 @@
 #include "rcc/CodeGen/MachineOperand.h"
 #include "rcc/CodeGen/MachineRegisterInfo.h"
 #include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/IR/DataLayout.h"
 #include "rcc/Target/RISCV/RISCVSubtarget.h"
 #include "rcc/Target/RISCV/RISCVTarget.h"
 
@@ -1186,4 +1187,123 @@ Expected<void> RISCVLegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
                         "RISC-V legalizer has no lowering rule for this instruction");
 }
 
+Expected<bool> RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
+                                                     MachineInstr &Instruction) const {
+  const MachineOperand *IntrinsicMachineOperand = Instruction.operandAt(0);
+  const auto *Intrinsic = IntrinsicMachineOperand == nullptr
+                              ? nullptr
+                              : IntrinsicMachineOperand->getIf<rcc::IntrinsicOperand>();
+  if (Intrinsic == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V intrinsic legalization requires an intrinsic operand");
+  if (Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy))
+    return false;
+  if (Instruction.operandCount() != 3)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "llvm.va_copy requires destination and source operands");
+
+  const MachineOperand *DestinationOperand = Instruction.operandAt(1);
+  const MachineOperand *SourceOperand = Instruction.operandAt(2);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+  if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() ||
+      !Source->isVirtual()) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "llvm.va_copy requires virtual-register pointer operands");
+  }
+
+  MachineFunction &Function = Helper.MIRBuilder.function();
+  auto DestinationType = Function.registerInfo().getType(*Destination);
+  if (!DestinationType)
+    return std::unexpected(DestinationType.error());
+  auto SourceType = Function.registerInfo().getType(*Source);
+  if (!SourceType)
+    return std::unexpected(SourceType.error());
+  if (!DestinationType->isPointer() || !SourceType->isPointer() ||
+      *DestinationType != *SourceType || DestinationType->scalarBitWidth() == 0) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "llvm.va_copy operands must have the same scalar pointer type");
+  }
+
+  const auto AddressSpace = DestinationType->addressSpace();
+  if (!AddressSpace)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "llvm.va_copy pointer type has no address space");
+  auto Layout = DataLayout::parse(TargetDataLayout);
+  if (!Layout)
+    return std::unexpected(Layout.error());
+  const GenericConstantType PointerType = GenericConstantType::pointer(*AddressSpace);
+  auto PointerWidth = Layout->getPointerSizeInBits(*AddressSpace);
+  if (!PointerWidth)
+    return std::unexpected(PointerWidth.error());
+  if (*PointerWidth != DestinationType->scalarBitWidth())
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "llvm.va_copy pointer type disagrees with the target data layout");
+  auto ABIAlignment = Layout->getABITypeAlign(PointerType);
+  if (!ABIAlignment)
+    return std::unexpected(ABIAlignment.error());
+  auto Size = MemorySize::fixed((static_cast<std::uint64_t>(*PointerWidth) + 7U) / 8U);
+  auto Alignment = rcc::Alignment::create(*ABIAlignment);
+  if (!Size || !Alignment)
+    return makeUnexpected(ErrorCode::InternalError,
+                          "llvm.va_copy pointer type has an invalid memory layout");
+
+  MachineMemOperandDesc LoadDescription;
+  LoadDescription.Accesses = {MemoryAccessFlag::Load};
+  LoadDescription.Size = *Size;
+  LoadDescription.MemoryType = *DestinationType;
+  LoadDescription.Align = *Alignment;
+  auto LoadMemory = Function.createMemOperand(std::move(LoadDescription));
+  if (!LoadMemory)
+    return std::unexpected(LoadMemory.error());
+
+  MachineMemOperandDesc StoreDescription;
+  StoreDescription.Accesses = {MemoryAccessFlag::Store};
+  StoreDescription.Size = *Size;
+  StoreDescription.MemoryType = *DestinationType;
+  StoreDescription.Align = *Alignment;
+  auto StoreMemory = Function.createMemOperand(std::move(StoreDescription));
+  if (!StoreMemory)
+    return std::unexpected(StoreMemory.error());
+
+  auto Temporary = Function.registerInfo().createVirtualRegister(*DestinationType);
+  if (!Temporary)
+    return std::unexpected(Temporary.error());
+
+  if (auto Point = Helper.MIRBuilder.setInsertPointBefore(Instruction); !Point)
+    return std::unexpected(Point.error());
+  auto LoadBuilder = Helper.MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD));
+  if (!LoadBuilder)
+    return std::unexpected(LoadBuilder.error());
+  if (auto Added = LoadBuilder->addDef(*Temporary); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = LoadBuilder->addUse(*Source); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = LoadBuilder->addMemOperand(**LoadMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = LoadBuilder->build(); !Built)
+    return std::unexpected(Built.error());
+
+  auto StoreBuilder = Helper.MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE));
+  if (!StoreBuilder)
+    return std::unexpected(StoreBuilder.error());
+  if (auto Added = StoreBuilder->addUse(*Temporary); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = StoreBuilder->addUse(*Destination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = StoreBuilder->addMemOperand(**StoreMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = StoreBuilder->build(); !Built)
+    return std::unexpected(Built.error());
+
+  MachineBasicBlock *Block = Instruction.parent();
+  if (Block == nullptr)
+    return makeUnexpected(ErrorCode::InternalError,
+                          "llvm.va_copy intrinsic became detached during lowering");
+  if (auto Removed = Block->removeInstruction(Instruction); !Removed)
+    return std::unexpected(Removed.error());
+  return true;
+}
+
 } // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
index d8e5e3e..b6625ea 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
@@ -14,6 +14,8 @@ public:
   explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget);
   [[nodiscard]] Expected<void> legalizeCustom(LegalizerHelper &Helper, MachineInstr &Instruction,
                                               const LegalizationDecision &Decision) const override;
+  [[nodiscard]] Expected<bool> legalizeIntrinsic(LegalizerHelper &Helper,
+                                                 MachineInstr &Instruction) const override;
 
 private:
 };