[RISCV] Legalize llvm.va_copy

5 files changed, 200 insertions, 3 deletions

diff --git a/backend/include/rcc/CodeGen/MachineIRTypes.h b/backend/include/rcc/CodeGen/MachineIRTypes.h
index e71b80e..922adef 100644
--- a/backend/include/rcc/CodeGen/MachineIRTypes.h
+++ b/backend/include/rcc/CodeGen/MachineIRTypes.h
@@ -73,6 +73,15 @@ using JumpTableIndex = MachineIndex<detail::JumpTableIndexTag>;
 using TargetIndex = MachineIndex<detail::TargetIndexTag>;
 using CFIInstructionIndex = MachineIndex<detail::CFIInstructionIndexTag>;
 using IntrinsicID = MachineIndex<detail::IntrinsicIDTag>;
+
+enum class GenericIntrinsic : std::uint32_t {
+  VACopy = 1,
+};
+
+[[nodiscard]] constexpr IntrinsicID intrinsicID(GenericIntrinsic Intrinsic) noexcept {
+  return IntrinsicID::fromValue(static_cast<std::uint32_t>(Intrinsic));
+}
+
 using SubRegIndex = MachineIndex<detail::SubRegIndexTag>;
 using RegisterUnitID = MachineIndex<detail::RegisterUnitIDTag>;
 using RegisterBankID = MachineIndex<detail::RegisterBankIDTag>;
diff --git a/backend/lib/CodeGen/TargetMIRTextInfo.cpp b/backend/lib/CodeGen/TargetMIRTextInfo.cpp
index 5d54ea6..938f513 100644
--- a/backend/lib/CodeGen/TargetMIRTextInfo.cpp
+++ b/backend/lib/CodeGen/TargetMIRTextInfo.cpp
@@ -137,6 +137,7 @@ Expected<void> MIRTargetNode::verify() const {
 
 Expected<TargetMIRTextInfo> TargetMIRTextInfo::create(TargetMIRTextInfoDesc Description,
                                                       const TargetRegisterInfo &Registers) {
+  Description.Intrinsics.push_back({"llvm.va_copy", intrinsicID(GenericIntrinsic::VACopy)});
   TargetMIRTextInfo Result(std::move(Description));
 
   for (std::size_t Index = 0; Index < Result.Description.RegisterMasks.size(); ++Index) {
diff --git a/backend/lib/Target/RISCV/RISCVCodeGen.cpp b/backend/lib/Target/RISCV/RISCVCodeGen.cpp
index 076e030..1129f89 100644
--- a/backend/lib/Target/RISCV/RISCVCodeGen.cpp
+++ b/backend/lib/Target/RISCV/RISCVCodeGen.cpp
@@ -2975,6 +2975,102 @@ buildIncomingConversion(MachineFunction &Function, const TargetDescription &Targ
   return true;
 }
 
+struct VACopyPlan final {
+  Register Destination;
+  Register Source;
+  LLT PointerType;
+};
+
+[[nodiscard]] std::optional<VACopyPlan> vacopyPlan(const MachineInstr &Instruction,
+                                                   const MachineFunction &Function) {
+  if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) ||
+      Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) {
+    return std::nullopt;
+  }
+  const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0);
+  const MachineOperand *DestinationOperand = Instruction.operandAt(1);
+  const MachineOperand *SourceOperand = Instruction.operandAt(2);
+  const auto *Intrinsic =
+      IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf<IntrinsicOperand>();
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+  const std::optional<LLT> DestinationType = operandType(Function, Instruction, 1);
+  const std::optional<LLT> SourceType = operandType(Function, Instruction, 2);
+  if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) ||
+      Destination == nullptr || Source == nullptr || !Destination->isVirtual() ||
+      !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() ||
+      !DestinationType || !SourceType || *DestinationType != *SourceType ||
+      !DestinationType->isPointer() || DestinationType->addressSpace() != 0 ||
+      DestinationType->scalarBitWidth() != 64) {
+    return std::nullopt;
+  }
+  return VACopyPlan{*Destination, *Source, *DestinationType};
+}
+
+[[nodiscard]] Expected<void> lowerVACopy(MachineInstr &Instruction, MachineFunction &Function,
+                                         const TargetDescription &Target) {
+  const auto Plan = vacopyPlan(Instruction, Function);
+  if (!Plan) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V va_copy legalization requires two p0 operands");
+  }
+  auto Size = MemorySize::fixed(8);
+  auto Align = Alignment::create(8);
+  if (!Size || !Align)
+    return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid");
+
+  MachineMemOperandDesc LoadDescription;
+  LoadDescription.Accesses = {MemoryAccessFlag::Load};
+  LoadDescription.Size = *Size;
+  LoadDescription.MemoryType = Plan->PointerType;
+  LoadDescription.Align = *Align;
+  auto LoadMemory = Function.createMemOperand(std::move(LoadDescription));
+  if (!LoadMemory)
+    return std::unexpected(LoadMemory.error());
+
+  MachineMemOperandDesc StoreDescription;
+  StoreDescription.Accesses = {MemoryAccessFlag::Store};
+  StoreDescription.Size = *Size;
+  StoreDescription.MemoryType = Plan->PointerType;
+  StoreDescription.Align = *Align;
+  auto StoreMemory = Function.createMemOperand(std::move(StoreDescription));
+  if (!StoreMemory)
+    return std::unexpected(StoreMemory.error());
+
+  auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType);
+  if (!Value)
+    return std::unexpected(Value.error());
+  MachineIRBuilder Builder(Function, Target);
+  if (auto Point = Builder.setInsertPointBefore(Instruction); !Point)
+    return std::unexpected(Point.error());
+
+  auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
+  if (!Load)
+    return std::unexpected(Load.error());
+  if (auto Added = Load->addDef(*Value); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Load->addUse(Plan->Source); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Load->addMemOperand(**LoadMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = Load->build(); !Built)
+    return std::unexpected(Built.error());
+
+  auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE));
+  if (!Store)
+    return std::unexpected(Store.error());
+  if (auto Added = Store->addUse(*Value); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Store->addUse(Plan->Destination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Store->addMemOperand(**StoreMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = Store->build(); !Built)
+    return std::unexpected(Built.error());
+  return eraseInstruction(Instruction);
+}
+
 class RISCVLegalizerInfo final : public TargetLegalizerInfo {
 public:
   explicit RISCVLegalizerInfo(RISCVSubtargetFeatures Features) : Features(Features) {}
@@ -2992,6 +3088,9 @@ public:
     const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
     const std::optional<LLT> Type = resultType(Function, Instruction);
     switch (Generic) {
+    case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
+      return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom
+                                               : LegalizationAction::Unsupported;
     case GenericOpcode::G_BR:
       return LegalizationAction::Legal;
     case GenericOpcode::G_BRCOND: {
@@ -3617,6 +3716,10 @@ public:
   [[nodiscard]] Expected<void> lower(MachineInstr &Instruction, MachineFunction &Function,
                                      const TargetDescription &Target,
                                      LegalizationAction Action) const override {
+    if (Action == LegalizationAction::Custom &&
+        Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) {
+      return lowerVACopy(Instruction, Function, Target);
+    }
     if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) && !Features.has(RISCVExtension::M)) {
       if (Action == LegalizationAction::Custom)
         return widenNarrowMultiply(Instruction, Function, Target);
diff --git a/backend/unittests/CodeGen/TargetMIRTextInfoTest.cpp b/backend/unittests/CodeGen/TargetMIRTextInfoTest.cpp
index d7e2f13..24945bd 100644
--- a/backend/unittests/CodeGen/TargetMIRTextInfoTest.cpp
+++ b/backend/unittests/CodeGen/TargetMIRTextInfoTest.cpp
@@ -109,9 +109,12 @@ void testTextSchemaQueries(TestContext &Context) {
           !Text->machineMemOperandFlagNames(0x4) && !Text->virtualRegisterFlagNames(0x4),
       "MMO and virtual-register flags must reject every unregistered bit");
 
-  Context.expect(Text->intrinsic("llvm.test.seven") == rcc::IntrinsicID::fromValue(7) &&
-                     Text->intrinsicName(rcc::IntrinsicID::fromValue(17)) == "llvm.test.seventeen",
-                 "intrinsic names and strong IDs must round-trip");
+  Context.expect(
+      Text->intrinsic("llvm.test.seven") == rcc::IntrinsicID::fromValue(7) &&
+          Text->intrinsicName(rcc::IntrinsicID::fromValue(17)) == "llvm.test.seventeen" &&
+          Text->intrinsic("llvm.va_copy") == rcc::intrinsicID(rcc::GenericIntrinsic::VACopy) &&
+          Text->intrinsicName(rcc::intrinsicID(rcc::GenericIntrinsic::VACopy)) == "llvm.va_copy",
+      "intrinsic names and strong IDs must round-trip");
 
   const auto G0 = phys(rcc::test::TestRegister::G0);
   Context.expect(Text->dwarfRegister(G0) == rcc::test::TestRegister::G0 &&
diff --git a/backend/unittests/Target/RISCVCodeGenTest.cpp b/backend/unittests/Target/RISCVCodeGenTest.cpp
index fcf0e57..4f675a4 100644
--- a/backend/unittests/Target/RISCVCodeGenTest.cpp
+++ b/backend/unittests/Target/RISCVCodeGenTest.cpp
@@ -15,6 +15,7 @@
 #include <cstdint>
 #include <iostream>
 #include <memory>
+#include <span>
 #include <string>
 #include <string_view>
 #include <vector>
@@ -1458,6 +1459,85 @@ void testUnsupportedTypes(TestContext &Context) {
                  "unsupported selection diagnostics must name the rejected opcode");
 }
 
+void testVACopyLegalization(TestContext &Context) {
+  auto Target = rcc::riscv::createTargetDescription();
+  Context.expect(Target.has_value(), "RV64GC target must be available for va_copy legalization");
+  if (!Target)
+    return;
+  Context.expect(Target->mirText().intrinsic("llvm.va_copy") ==
+                     rcc::intrinsicID(rcc::GenericIntrinsic::VACopy),
+                 "RV64GC MIR text must expose the generic va_copy intrinsic identity");
+
+  auto Function =
+      rcc::MachineFunction::create("riscv_vacopy", rcc::MachineFunctionStableID::fromValue(102));
+  if (!Function)
+    return;
+  auto Block = (*Function)->createBasicBlock();
+  auto Pointer = rcc::LLT::makePointer(0, 64);
+  if (!Block || !Pointer || !(*Function)->appendBlock(**Block))
+    return;
+  auto Destination = (*Function)->registerInfo().createVirtualRegister(*Pointer);
+  auto Source = (*Function)->registerInfo().createVirtualRegister(*Pointer);
+  if (!Destination || !Source)
+    return;
+
+  rcc::MachineIRBuilder Builder(**Function, *Target);
+  if (!Builder.setInsertPoint(**Block))
+    return;
+  auto Pending = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS));
+  if (!Pending || !Pending->addIntrinsic(rcc::intrinsicID(rcc::GenericIntrinsic::VACopy)) ||
+      !Pending->addUse(*Destination) || !Pending->addUse(*Source)) {
+    return;
+  }
+  auto Built = Pending->build();
+  if (!Built)
+    return;
+  auto Action = rcc::riscv::legalizerInfo().classify(**Built, **Function, *Target);
+  Context.expect(Action && *Action == rcc::LegalizationAction::Custom,
+                 "RV64GC must custom-legalize va_copy");
+  if (!Action)
+    return;
+  auto Lowered = rcc::riscv::legalizerInfo().lower(**Built, **Function, *Target, *Action);
+  Context.expect(Lowered.has_value(), "RV64GC va_copy legalization must succeed");
+  if (!Lowered)
+    return;
+
+  rcc::MachineInstr *Load = (*Function)->firstBlock()->firstInstruction();
+  rcc::MachineInstr *Store = Load == nullptr ? nullptr : Load->next();
+  const rcc::Register *Loaded = Load == nullptr || Load->operandAt(0) == nullptr
+                                    ? nullptr
+                                    : Load->operandAt(0)->getIf<rcc::Register>();
+  const rcc::Register *LoadAddress = Load == nullptr || Load->operandAt(1) == nullptr
+                                         ? nullptr
+                                         : Load->operandAt(1)->getIf<rcc::Register>();
+  const rcc::Register *Stored = Store == nullptr || Store->operandAt(0) == nullptr
+                                    ? nullptr
+                                    : Store->operandAt(0)->getIf<rcc::Register>();
+  const rcc::Register *StoreAddress = Store == nullptr || Store->operandAt(1) == nullptr
+                                          ? nullptr
+                                          : Store->operandAt(1)->getIf<rcc::Register>();
+  Context.expect(
+      (*Built)->parent() == nullptr && Load != nullptr && Store != nullptr &&
+          Store->next() == nullptr && Load->opcode() == rcc::opcode(rcc::GenericOpcode::G_LOAD) &&
+          Store->opcode() == rcc::opcode(rcc::GenericOpcode::G_STORE) && Loaded != nullptr &&
+          Stored != nullptr && *Loaded == *Stored && LoadAddress != nullptr &&
+          *LoadAddress == *Source && StoreAddress != nullptr && *StoreAddress == *Destination,
+      "va_copy must become a pointer load followed by a store to the destination list");
+  const auto LoadMemory =
+      Load == nullptr ? std::span<const rcc::MachineMemOperand *const>{} : Load->memOperands();
+  const auto StoreMemory =
+      Store == nullptr ? std::span<const rcc::MachineMemOperand *const>{} : Store->memOperands();
+  Context.expect(
+      LoadMemory.size() == 1 && StoreMemory.size() == 1 &&
+          LoadMemory.front()->description().Accesses.has(rcc::MemoryAccessFlag::Load) &&
+          StoreMemory.front()->description().Accesses.has(rcc::MemoryAccessFlag::Store) &&
+          LoadMemory.front()->description().Size.minimumBytes() == 8 &&
+          StoreMemory.front()->description().Size.minimumBytes() == 8 &&
+          LoadMemory.front()->description().MemoryType == *Pointer &&
+          StoreMemory.front()->description().MemoryType == *Pointer,
+      "va_copy expansion must attach complete pointer-sized load/store memory operands");
+}
+
 void testMemoryOffsetSelection(TestContext &Context) {
   auto Target = rcc::riscv::createTargetDescription();
   Context.expect(Target.has_value(), "RV64GC target must be available for memory selection");
@@ -1541,6 +1621,7 @@ int main() {
   testMultiplyLibcallWithoutM(Context);
   testAtomicSelection(Context);
   testUnsupportedTypes(Context);
+  testVACopyLegalization(Context);
   testMemoryOffsetSelection(Context);
   return Context.result();
 }