diff options
| -rw-r--r-- | backend/include/rcc/CodeGen/MachineIRTypes.h | 9 | ||||
| -rw-r--r-- | backend/lib/CodeGen/TargetMIRTextInfo.cpp | 1 | ||||
| -rw-r--r-- | backend/lib/Target/RISCV/RISCVCodeGen.cpp | 103 | ||||
| -rw-r--r-- | backend/unittests/CodeGen/TargetMIRTextInfoTest.cpp | 9 | ||||
| -rw-r--r-- | backend/unittests/Target/RISCVCodeGenTest.cpp | 81 |
5 files changed, 200 insertions, 3 deletions
diff --git a/backend/include/rcc/CodeGen/MachineIRTypes.h b/backend/include/rcc/CodeGen/MachineIRTypes.h index e71b80e..922adef 100644 --- a/backend/include/rcc/CodeGen/MachineIRTypes.h +++ b/backend/include/rcc/CodeGen/MachineIRTypes.h @@ -73,6 +73,15 @@ using JumpTableIndex = MachineIndex<detail::JumpTableIndexTag>; using TargetIndex = MachineIndex<detail::TargetIndexTag>; using CFIInstructionIndex = MachineIndex<detail::CFIInstructionIndexTag>; using IntrinsicID = MachineIndex<detail::IntrinsicIDTag>; + +enum class GenericIntrinsic : std::uint32_t { + VACopy = 1, +}; + +[[nodiscard]] constexpr IntrinsicID intrinsicID(GenericIntrinsic Intrinsic) noexcept { + return IntrinsicID::fromValue(static_cast<std::uint32_t>(Intrinsic)); +} + using SubRegIndex = MachineIndex<detail::SubRegIndexTag>; using RegisterUnitID = MachineIndex<detail::RegisterUnitIDTag>; using RegisterBankID = MachineIndex<detail::RegisterBankIDTag>; diff --git a/backend/lib/CodeGen/TargetMIRTextInfo.cpp b/backend/lib/CodeGen/TargetMIRTextInfo.cpp index 5d54ea6..938f513 100644 --- a/backend/lib/CodeGen/TargetMIRTextInfo.cpp +++ b/backend/lib/CodeGen/TargetMIRTextInfo.cpp @@ -137,6 +137,7 @@ Expected<void> MIRTargetNode::verify() const { Expected<TargetMIRTextInfo> TargetMIRTextInfo::create(TargetMIRTextInfoDesc Description, const TargetRegisterInfo &Registers) { + Description.Intrinsics.push_back({"llvm.va_copy", intrinsicID(GenericIntrinsic::VACopy)}); TargetMIRTextInfo Result(std::move(Description)); for (std::size_t Index = 0; Index < Result.Description.RegisterMasks.size(); ++Index) { diff --git a/backend/lib/Target/RISCV/RISCVCodeGen.cpp b/backend/lib/Target/RISCV/RISCVCodeGen.cpp index 076e030..1129f89 100644 --- a/backend/lib/Target/RISCV/RISCVCodeGen.cpp +++ b/backend/lib/Target/RISCV/RISCVCodeGen.cpp @@ -2975,6 +2975,102 @@ buildIncomingConversion(MachineFunction &Function, const TargetDescription &Targ return true; } +struct VACopyPlan final { + Register Destination; + Register Source; + LLT PointerType; +}; + +[[nodiscard]] std::optional<VACopyPlan> vacopyPlan(const MachineInstr &Instruction, + const MachineFunction &Function) { + if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) || + Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) { + return std::nullopt; + } + const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0); + const MachineOperand *DestinationOperand = Instruction.operandAt(1); + const MachineOperand *SourceOperand = Instruction.operandAt(2); + const auto *Intrinsic = + IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf<IntrinsicOperand>(); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + const std::optional<LLT> DestinationType = operandType(Function, Instruction, 1); + const std::optional<LLT> SourceType = operandType(Function, Instruction, 2); + if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) || + Destination == nullptr || Source == nullptr || !Destination->isVirtual() || + !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() || + !DestinationType || !SourceType || *DestinationType != *SourceType || + !DestinationType->isPointer() || DestinationType->addressSpace() != 0 || + DestinationType->scalarBitWidth() != 64) { + return std::nullopt; + } + return VACopyPlan{*Destination, *Source, *DestinationType}; +} + +[[nodiscard]] Expected<void> lowerVACopy(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const auto Plan = vacopyPlan(Instruction, Function); + if (!Plan) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V va_copy legalization requires two p0 operands"); + } + auto Size = MemorySize::fixed(8); + auto Align = Alignment::create(8); + if (!Size || !Align) + return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid"); + + MachineMemOperandDesc LoadDescription; + LoadDescription.Accesses = {MemoryAccessFlag::Load}; + LoadDescription.Size = *Size; + LoadDescription.MemoryType = Plan->PointerType; + LoadDescription.Align = *Align; + auto LoadMemory = Function.createMemOperand(std::move(LoadDescription)); + if (!LoadMemory) + return std::unexpected(LoadMemory.error()); + + MachineMemOperandDesc StoreDescription; + StoreDescription.Accesses = {MemoryAccessFlag::Store}; + StoreDescription.Size = *Size; + StoreDescription.MemoryType = Plan->PointerType; + StoreDescription.Align = *Align; + auto StoreMemory = Function.createMemOperand(std::move(StoreDescription)); + if (!StoreMemory) + return std::unexpected(StoreMemory.error()); + + auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType); + if (!Value) + return std::unexpected(Value.error()); + MachineIRBuilder Builder(Function, Target); + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!Load) + return std::unexpected(Load.error()); + if (auto Added = Load->addDef(*Value); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addUse(Plan->Source); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addMemOperand(**LoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Load->build(); !Built) + return std::unexpected(Built.error()); + + auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!Store) + return std::unexpected(Store.error()); + if (auto Added = Store->addUse(*Value); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addUse(Plan->Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addMemOperand(**StoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Store->build(); !Built) + return std::unexpected(Built.error()); + return eraseInstruction(Instruction); +} + class RISCVLegalizerInfo final : public TargetLegalizerInfo { public: explicit RISCVLegalizerInfo(RISCVSubtargetFeatures Features) : Features(Features) {} @@ -2992,6 +3088,9 @@ public: const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); const std::optional<LLT> Type = resultType(Function, Instruction); switch (Generic) { + case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS: + return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom + : LegalizationAction::Unsupported; case GenericOpcode::G_BR: return LegalizationAction::Legal; case GenericOpcode::G_BRCOND: { @@ -3617,6 +3716,10 @@ public: [[nodiscard]] Expected<void> lower(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, LegalizationAction Action) const override { + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) { + return lowerVACopy(Instruction, Function, Target); + } if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) && !Features.has(RISCVExtension::M)) { if (Action == LegalizationAction::Custom) return widenNarrowMultiply(Instruction, Function, Target); diff --git a/backend/unittests/CodeGen/TargetMIRTextInfoTest.cpp b/backend/unittests/CodeGen/TargetMIRTextInfoTest.cpp index d7e2f13..24945bd 100644 --- a/backend/unittests/CodeGen/TargetMIRTextInfoTest.cpp +++ b/backend/unittests/CodeGen/TargetMIRTextInfoTest.cpp @@ -109,9 +109,12 @@ void testTextSchemaQueries(TestContext &Context) { !Text->machineMemOperandFlagNames(0x4) && !Text->virtualRegisterFlagNames(0x4), "MMO and virtual-register flags must reject every unregistered bit"); - Context.expect(Text->intrinsic("llvm.test.seven") == rcc::IntrinsicID::fromValue(7) && - Text->intrinsicName(rcc::IntrinsicID::fromValue(17)) == "llvm.test.seventeen", - "intrinsic names and strong IDs must round-trip"); + Context.expect( + Text->intrinsic("llvm.test.seven") == rcc::IntrinsicID::fromValue(7) && + Text->intrinsicName(rcc::IntrinsicID::fromValue(17)) == "llvm.test.seventeen" && + Text->intrinsic("llvm.va_copy") == rcc::intrinsicID(rcc::GenericIntrinsic::VACopy) && + Text->intrinsicName(rcc::intrinsicID(rcc::GenericIntrinsic::VACopy)) == "llvm.va_copy", + "intrinsic names and strong IDs must round-trip"); const auto G0 = phys(rcc::test::TestRegister::G0); Context.expect(Text->dwarfRegister(G0) == rcc::test::TestRegister::G0 && diff --git a/backend/unittests/Target/RISCVCodeGenTest.cpp b/backend/unittests/Target/RISCVCodeGenTest.cpp index fcf0e57..4f675a4 100644 --- a/backend/unittests/Target/RISCVCodeGenTest.cpp +++ b/backend/unittests/Target/RISCVCodeGenTest.cpp @@ -15,6 +15,7 @@ #include <cstdint> #include <iostream> #include <memory> +#include <span> #include <string> #include <string_view> #include <vector> @@ -1458,6 +1459,85 @@ void testUnsupportedTypes(TestContext &Context) { "unsupported selection diagnostics must name the rejected opcode"); } +void testVACopyLegalization(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "RV64GC target must be available for va_copy legalization"); + if (!Target) + return; + Context.expect(Target->mirText().intrinsic("llvm.va_copy") == + rcc::intrinsicID(rcc::GenericIntrinsic::VACopy), + "RV64GC MIR text must expose the generic va_copy intrinsic identity"); + + auto Function = + rcc::MachineFunction::create("riscv_vacopy", rcc::MachineFunctionStableID::fromValue(102)); + if (!Function) + return; + auto Block = (*Function)->createBasicBlock(); + auto Pointer = rcc::LLT::makePointer(0, 64); + if (!Block || !Pointer || !(*Function)->appendBlock(**Block)) + return; + auto Destination = (*Function)->registerInfo().createVirtualRegister(*Pointer); + auto Source = (*Function)->registerInfo().createVirtualRegister(*Pointer); + if (!Destination || !Source) + return; + + rcc::MachineIRBuilder Builder(**Function, *Target); + if (!Builder.setInsertPoint(**Block)) + return; + auto Pending = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)); + if (!Pending || !Pending->addIntrinsic(rcc::intrinsicID(rcc::GenericIntrinsic::VACopy)) || + !Pending->addUse(*Destination) || !Pending->addUse(*Source)) { + return; + } + auto Built = Pending->build(); + if (!Built) + return; + auto Action = rcc::riscv::legalizerInfo().classify(**Built, **Function, *Target); + Context.expect(Action && *Action == rcc::LegalizationAction::Custom, + "RV64GC must custom-legalize va_copy"); + if (!Action) + return; + auto Lowered = rcc::riscv::legalizerInfo().lower(**Built, **Function, *Target, *Action); + Context.expect(Lowered.has_value(), "RV64GC va_copy legalization must succeed"); + if (!Lowered) + return; + + rcc::MachineInstr *Load = (*Function)->firstBlock()->firstInstruction(); + rcc::MachineInstr *Store = Load == nullptr ? nullptr : Load->next(); + const rcc::Register *Loaded = Load == nullptr || Load->operandAt(0) == nullptr + ? nullptr + : Load->operandAt(0)->getIf<rcc::Register>(); + const rcc::Register *LoadAddress = Load == nullptr || Load->operandAt(1) == nullptr + ? nullptr + : Load->operandAt(1)->getIf<rcc::Register>(); + const rcc::Register *Stored = Store == nullptr || Store->operandAt(0) == nullptr + ? nullptr + : Store->operandAt(0)->getIf<rcc::Register>(); + const rcc::Register *StoreAddress = Store == nullptr || Store->operandAt(1) == nullptr + ? nullptr + : Store->operandAt(1)->getIf<rcc::Register>(); + Context.expect( + (*Built)->parent() == nullptr && Load != nullptr && Store != nullptr && + Store->next() == nullptr && Load->opcode() == rcc::opcode(rcc::GenericOpcode::G_LOAD) && + Store->opcode() == rcc::opcode(rcc::GenericOpcode::G_STORE) && Loaded != nullptr && + Stored != nullptr && *Loaded == *Stored && LoadAddress != nullptr && + *LoadAddress == *Source && StoreAddress != nullptr && *StoreAddress == *Destination, + "va_copy must become a pointer load followed by a store to the destination list"); + const auto LoadMemory = + Load == nullptr ? std::span<const rcc::MachineMemOperand *const>{} : Load->memOperands(); + const auto StoreMemory = + Store == nullptr ? std::span<const rcc::MachineMemOperand *const>{} : Store->memOperands(); + Context.expect( + LoadMemory.size() == 1 && StoreMemory.size() == 1 && + LoadMemory.front()->description().Accesses.has(rcc::MemoryAccessFlag::Load) && + StoreMemory.front()->description().Accesses.has(rcc::MemoryAccessFlag::Store) && + LoadMemory.front()->description().Size.minimumBytes() == 8 && + StoreMemory.front()->description().Size.minimumBytes() == 8 && + LoadMemory.front()->description().MemoryType == *Pointer && + StoreMemory.front()->description().MemoryType == *Pointer, + "va_copy expansion must attach complete pointer-sized load/store memory operands"); +} + void testMemoryOffsetSelection(TestContext &Context) { auto Target = rcc::riscv::createTargetDescription(); Context.expect(Target.has_value(), "RV64GC target must be available for memory selection"); @@ -1541,6 +1621,7 @@ int main() { testMultiplyLibcallWithoutM(Context); testAtomicSelection(Context); testUnsupportedTypes(Context); + testVACopyLegalization(Context); testMemoryOffsetSelection(Context); return Context.result(); } |