From 1e0003ec107dff77cd579d245f6314cbf7d4f3fc Mon Sep 17 00:00:00 2001 From: Richard-Qin-X Date: Tue, 11 Aug 2026 17:59:16 +0800 Subject: [RISCV] Legalize narrow atomic compare exchange --- backend/lib/CodeGen/GenericOpcodes.cpp | 4 +- backend/lib/Target/RISCV/RISCVCodeGen.cpp | 140 ++++++++++++++++++++++++++ backend/unittests/Target/RISCVCodeGenTest.cpp | 86 ++++++++++++++++ 3 files changed, 228 insertions(+), 2 deletions(-) (limited to 'backend') diff --git a/backend/lib/CodeGen/GenericOpcodes.cpp b/backend/lib/CodeGen/GenericOpcodes.cpp index 0ffb26a..4c60675 100644 --- a/backend/lib/CodeGen/GenericOpcodes.cpp +++ b/backend/lib/CodeGen/GenericOpcodes.cpp @@ -891,12 +891,12 @@ private: if (MemoryOperands.size() != 1 || MemoryOperands.front()->description().AtomicKind != AtomicOperation::CompareExchange) return semanticError("G_ATOMIC_CMPXCHG_WITH_SUCCESS requires one compare-exchange MMO"); - return VerifySingleAccess(2, 0, true, true, SizeRelation::Equal); + return VerifySingleAccess(2, 0, true, true, SizeRelation::NoLarger); case GO::G_ATOMIC_CMPXCHG: if (MemoryOperands.size() != 1 || MemoryOperands.front()->description().AtomicKind != AtomicOperation::CompareExchange) return semanticError("G_ATOMIC_CMPXCHG requires one compare-exchange MMO"); - return VerifySingleAccess(1, 0, true, true, SizeRelation::Equal); + return VerifySingleAccess(1, 0, true, true, SizeRelation::NoLarger); case GO::G_ATOMICRMW_XCHG: case GO::G_ATOMICRMW_ADD: case GO::G_ATOMICRMW_SUB: diff --git a/backend/lib/Target/RISCV/RISCVCodeGen.cpp b/backend/lib/Target/RISCV/RISCVCodeGen.cpp index bfbf9a6..44f2e7e 100644 --- a/backend/lib/Target/RISCV/RISCVCodeGen.cpp +++ b/backend/lib/Target/RISCV/RISCVCodeGen.cpp @@ -386,6 +386,11 @@ buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LLT Type, Register Left, Register Right); +[[nodiscard]] Expected buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LLT Type, Register Source); + +[[nodiscard]] Expected eraseInstruction(MachineInstr &Instruction); + struct NarrowAnyExtBinaryPlan final { MachineInstr *Binary = nullptr; GenericOpcode Generic = GenericOpcode::Invalid; @@ -501,6 +506,107 @@ narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Fun return {}; } +[[nodiscard]] Expected widenAtomicCompareExchangeSource(MachineInstr &Instruction, + MachineFunction &Function, + MachineIRBuilder &Builder, + Register Source, LLT WideType) { + const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo || !SourceInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V compare-exchange source has no scalar type"); + if (SourceInfo->get().Type->scalarBitWidth() == WideType.scalarBitWidth()) + return Source; + + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast(Definition->parent()); + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && + singleRegisterUse(Function, Source) == &Instruction) { + auto Widened = widenConstantToRV64(*Def, Function); + if (!Widened) + return std::unexpected(Widened.error()); + return Source; + } + return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, WideType, Source); +} + +[[nodiscard]] Expected lowerAtomicCompareExchange(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const std::optional OriginalType = resultType(Function, Instruction); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *CompareOperand = Instruction.operandAt(2); + const MachineOperand *NewValueOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf(); + const Register *NewValue = + NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf(); + if (!OriginalType || Destination == nullptr || Compare == nullptr || NewValue == nullptr || + OriginalType->scalarBitWidth() >= 64) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow compare-exchange has invalid operands"); + + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + auto WideCompare = + widenAtomicCompareExchangeSource(Instruction, Function, Builder, *Compare, *WideType); + if (!WideCompare) + return std::unexpected(WideCompare.error()); + auto WideNewValue = + widenAtomicCompareExchangeSource(Instruction, Function, Builder, *NewValue, *WideType); + if (!WideNewValue) + return std::unexpected(WideNewValue.error()); + + Register WideDestination; + MachineInstr *FoldedExtension = singleRegisterUse(Function, *Destination); + if (FoldedExtension != nullptr && FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) && + operandType(Function, *FoldedExtension, 0) + .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false)) { + const MachineOperand *WideDestinationOperand = FoldedExtension->operandAt(0); + const Register *FoldedDestination = + WideDestinationOperand == nullptr ? nullptr : WideDestinationOperand->getIf(); + if (FoldedDestination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V compare-exchange extension has no destination"); + WideDestination = *FoldedDestination; + } else { + auto Created = Function.registerInfo().createVirtualRegister(*WideType); + if (!Created) + return std::unexpected(Created.error()); + WideDestination = *Created; + FoldedExtension = nullptr; + } + + if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) + return std::unexpected(Replaced.error()); + if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideCompare); !Replaced) + return std::unexpected(Replaced.error()); + if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideNewValue); !Replaced) + return std::unexpected(Replaced.error()); + + if (FoldedExtension != nullptr) + return eraseInstruction(*FoldedExtension); + + auto Point = Builder.setInsertPointAfter(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); + if (!Truncation) + return std::unexpected(Truncation.error()); + if (auto Added = Truncation->addDef(*Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Truncation->addUse(WideDestination); !Added) + return std::unexpected(Added.error()); + auto Built = Truncation->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + [[nodiscard]] Expected foldNarrowAnyExtBinary(MachineInstr &Extension, MachineFunction &Function, const TargetDescription &Target) { @@ -737,6 +843,21 @@ narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Fun Description.SyncScope.value() == 0; } +[[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction, + AtomicOperation Operation, + std::uint64_t ContainerBytes) noexcept { + if (Instruction.memOperands().size() != 1 || ContainerBytes == 0) + return false; + const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); + if (Description.AtomicKind != Operation || Description.AddressSpace != 0 || + !Description.Size.isKnown() || Description.Size.isScalable() || + Description.SyncScope.value() != 0) + return false; + const std::uint64_t MemoryBytes = Description.Size.minimumBytes(); + return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) && + MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes; +} + [[nodiscard]] std::optional memoryAccessBytes(const MachineInstr &Instruction) noexcept { if (Instruction.memOperands().size() != 1) @@ -2911,6 +3032,22 @@ public: ? LegalizationAction::Legal : LegalizationAction::Unsupported; } + case GenericOpcode::G_ATOMIC_CMPXCHG: { + const std::optional Address = operandType(Function, Instruction, 1); + const std::optional Compare = operandType(Function, Instruction, 2); + const std::optional NewValue = operandType(Function, Instruction, 3); + if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type || + (!Type->isInteger() && !Type->isAnyScalar()) || + (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 && + Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) || + !Address->isPointer() || Address->addressSpace() != 0 || + Address->scalarBitWidth() != 64 || + !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, + (Type->scalarBitWidth() + 7U) / 8U)) { + return LegalizationAction::Unsupported; + } + return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; + } case GenericOpcode::G_FENCE: { const MachineOperand *OrderingOperand = Instruction.operandAt(0); const MachineOperand *ScopeOperand = Instruction.operandAt(1); @@ -3071,6 +3208,9 @@ public: if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_PHI)) return lowerNarrowPhi(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG)) + return lowerAtomicCompareExchange(Instruction, Function, Target); if (Action == LegalizationAction::Custom && (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) || Instruction.opcode() == opcode(GenericOpcode::G_SEXT) || diff --git a/backend/unittests/Target/RISCVCodeGenTest.cpp b/backend/unittests/Target/RISCVCodeGenTest.cpp index efb8eda..3a08f92 100644 --- a/backend/unittests/Target/RISCVCodeGenTest.cpp +++ b/backend/unittests/Target/RISCVCodeGenTest.cpp @@ -824,6 +824,91 @@ void testOverflowSelection(TestContext &Context) { } } +void testAtomicCompareExchangeLegalization(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Function = rcc::MachineFunction::create("riscv_atomic_cmpxchg", + rcc::MachineFunctionStableID::fromValue(1299)); + if (!Target || !Function) + return; + auto Block = (*Function)->createBasicBlock(); + auto I8 = rcc::LLT::makeAnyScalar(8); + auto I64 = rcc::LLT::makeAnyScalar(64); + auto Pointer = rcc::LLT::makePointer(0, 64); + auto Size = rcc::MemorySize::fixed(1); + auto Align = rcc::Alignment::create(1); + if (!Block || !(*Function)->appendBlock(**Block) || !I8 || !I64 || !Pointer || !Size || !Align) + return; + auto Address = (*Function)->registerInfo().createVirtualRegister(*Pointer); + auto Compare = (*Function)->registerInfo().createVirtualRegister(*I8); + auto NewValue = (*Function)->registerInfo().createVirtualRegister(*I8); + auto Result = (*Function)->registerInfo().createVirtualRegister(*I8); + auto Extended = (*Function)->registerInfo().createVirtualRegister(*I64); + if (!Address || !Compare || !NewValue || !Result || !Extended) + return; + + rcc::MachineIRBuilder Builder(**Function, *Target); + if (!Builder.setInsertPoint(**Block)) + return; + auto AddressDefinition = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_IMPLICIT_DEF)); + if (!AddressDefinition || !AddressDefinition->addDef(*Address) || !AddressDefinition->build()) + return; + const auto BuildConstant = [&](rcc::Register Destination, std::uint64_t Value) -> bool { + auto Immediate = rcc::IntegerImmediate::fromUnsigned(8, Value); + auto Pending = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_CONSTANT)); + return Immediate && Pending && Pending->addDef(Destination) && + Pending->addIntegerImmediate(std::move(*Immediate)) && Pending->build(); + }; + if (!BuildConstant(*Compare, 0) || !BuildConstant(*NewValue, 1)) + return; + + rcc::MachineMemOperandDesc Description; + Description.Accesses = {rcc::MemoryAccessFlag::Load, rcc::MemoryAccessFlag::Store}; + Description.AtomicKind = rcc::AtomicOperation::CompareExchange; + Description.SuccessOrdering = rcc::AtomicOrdering::Monotonic; + Description.Size = *Size; + Description.MemoryType = *I8; + Description.Align = *Align; + Description.Base = rcc::MemoryBase::stack(); + auto Memory = (*Function)->createMemOperand(std::move(Description)); + auto Atomic = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_ATOMIC_CMPXCHG)); + if (!Memory || !Atomic || !Atomic->addDef(*Result) || !Atomic->addUse(*Address) || + !Atomic->addUse(*Compare) || !Atomic->addUse(*NewValue) || !Atomic->addMemOperand(**Memory)) + return; + auto BuiltAtomic = Atomic->build(); + auto Extension = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_ANYEXT)); + if (!BuiltAtomic || !Extension || !Extension->addDef(*Extended) || !Extension->addUse(*Result)) + return; + auto BuiltExtension = Extension->build(); + if (!BuiltExtension) + return; + + auto Action = rcc::riscv::legalizerInfo().classify(**BuiltAtomic, **Function, *Target); + auto Lowered = + Action ? rcc::riscv::legalizerInfo().lower(**BuiltAtomic, **Function, *Target, *Action) + : rcc::Expected(std::unexpected(Action.error())); + Context.expect(Action && *Action == rcc::LegalizationAction::Custom && Lowered.has_value(), + "narrow RV64 compare-exchange must use the widening legalizer rule"); + if (!Lowered) + return; + + const auto CompareInfo = (*Function)->registerInfo().virtualRegisterInfo(*Compare); + const auto NewValueInfo = (*Function)->registerInfo().virtualRegisterInfo(*NewValue); + const rcc::Register *AtomicResult = (*BuiltAtomic)->operandAt(0)->getIf(); + const auto AtomicResultInfo = + AtomicResult == nullptr + ? rcc::Expected>(std::unexpected( + rcc::Error(rcc::ErrorCode::InvalidArgument, "missing compare-exchange result"))) + : (*Function)->registerInfo().virtualRegisterInfo(*AtomicResult); + auto LegalAction = rcc::riscv::legalizerInfo().classify(**BuiltAtomic, **Function, *Target); + Context.expect(CompareInfo && CompareInfo->get().Type == *I64 && NewValueInfo && + NewValueInfo->get().Type == *I64 && AtomicResultInfo && + AtomicResultInfo->get().Type == *I64 && + (*BuiltExtension)->parent() == nullptr && + Target->validateInstruction(**BuiltAtomic, (*Function)->registerInfo()) && + LegalAction && *LegalAction == rcc::LegalizationAction::Legal, + "compare-exchange widening must preserve the narrow MMO in a legal i64 container"); +} + void testAtomicSelection(TestContext &Context) { auto Target = rcc::riscv::createTargetDescription(); if (!Target) @@ -1101,6 +1186,7 @@ int main() { testComparisonsAndControl(Context); testCompositeScalarSelection(Context); testOverflowSelection(Context); + testAtomicCompareExchangeLegalization(Context); testAtomicSelection(Context); testUnsupportedTypes(Context); testMemoryOffsetSelection(Context); -- cgit v1.2.3