diff options
Diffstat (limited to 'backend/lib/CodeGen')
115 files changed, 54145 insertions, 0 deletions
diff --git a/backend/lib/CodeGen/CFIInstrInserter.cpp b/backend/lib/CodeGen/CFIInstrInserter.cpp new file mode 100644 index 0000000..45db108 --- /dev/null +++ b/backend/lib/CodeGen/CFIInstrInserter.cpp @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/CFIInstrInserter.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFIInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetFrameLowering.h" + +#include <algorithm> +#include <cstdio> +#include <cstdlib> +#include <iterator> +#include <set> +#include <string_view> +#include <unordered_map> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct SavedLocation final { + enum class Kind : std::uint8_t { + Register, + CFAOffset, + }; + + Kind LocationKind = Kind::Register; + std::int64_t Value = 0; + + friend bool operator==(SavedLocation Left, SavedLocation Right) noexcept = default; +}; + +struct BlockCFIInfo final { + MachineBasicBlock *Block = nullptr; + std::int64_t IncomingCFAOffset = 0; + std::int64_t OutgoingCFAOffset = 0; + DwarfRegisterNumber IncomingCFARegister; + DwarfRegisterNumber OutgoingCFARegister; + std::set<std::uint32_t> IncomingSavedRegisters; + std::set<std::uint32_t> OutgoingSavedRegisters; + bool Processed = false; +}; + +using BlockInfoMap = std::unordered_map<MachineBasicBlock *, BlockCFIInfo>; +using SavedLocationMap = std::unordered_map<std::uint32_t, SavedLocation>; + +[[noreturn]] void reportFatalCFIError(std::string_view Message) { + std::fputs("LLVM ERROR: ", stderr); + std::fwrite(Message.data(), sizeof(char), Message.size(), stderr); + std::fputc('\n', stderr); + std::fflush(stderr); + std::abort(); +} + +[[nodiscard]] Expected<const MachineCFIInstruction *> +referencedCFI(const MachineInstr &Instruction, const MachineFunction &Function) { + const MachineOperand *Operand = Instruction.operandAt(0); + const auto *Index = Operand == nullptr ? nullptr : Operand->getIf<CFIIndexOperand>(); + if (Index == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "CFI instruction has no CFI index operand"); + } + const MachineCFIInstruction *CFI = Function.cfiInfo().instruction(Index->Index); + if (CFI == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "CFI instruction index is out of range"); + return CFI; +} + +[[nodiscard]] Expected<void> recordSavedLocation(SavedLocationMap &Locations, + std::set<std::uint32_t> &SavedRegisters, + DwarfRegisterNumber Register, + SavedLocation Location) { + const std::uint32_t Number = Register.value(); + const auto [Position, Inserted] = Locations.emplace(Number, Location); + if (!Inserted && Position->second != Location) + reportFatalCFIError("Different saved locations for the same CSR"); + SavedRegisters.insert(Number); + return {}; +} + +[[nodiscard]] Expected<void> calculateOutgoingCFI(BlockCFIInfo &Info, MachineFunction &Function, + SavedLocationMap &Locations) { + std::int64_t CFAOffset = Info.IncomingCFAOffset; + DwarfRegisterNumber CFARegister = Info.IncomingCFARegister; + std::set<std::uint32_t> SavedRegisters; + std::set<std::uint32_t> RestoredRegisters; + std::int64_t RememberStateDepth = 0; + + for (MachineInstr *Instruction = Info.Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Instruction->opcode() != opcode(GenericOpcode::CFI_INSTRUCTION)) + continue; + auto CFIResult = referencedCFI(*Instruction, Function); + if (!CFIResult) + return std::unexpected(CFIResult.error()); + const MachineCFIInstruction &CFI = **CFIResult; + switch (CFI.operation()) { + case MachineCFIOperation::DefCFARegister: { + const auto *Rule = CFI.getIf<MachineCFIRegisterRule>(); + if (Rule == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "invalid def_cfa_register rule"); + CFARegister = Rule->Register; + break; + } + case MachineCFIOperation::DefCFAOffset: { + const auto *Rule = CFI.getIf<MachineCFIOffsetRule>(); + if (Rule == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "invalid def_cfa_offset rule"); + CFAOffset = Rule->Offset; + break; + } + case MachineCFIOperation::AdjustCFAOffset: { + const auto *Rule = CFI.getIf<MachineCFIOffsetRule>(); + if (Rule == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "invalid adjust_cfa_offset rule"); + CFAOffset += Rule->Offset; + break; + } + case MachineCFIOperation::DefCFA: { + const auto *Rule = CFI.getIf<MachineCFIRegisterOffsetRule>(); + if (Rule == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "invalid def_cfa rule"); + CFARegister = Rule->Register; + CFAOffset = Rule->Offset; + break; + } + case MachineCFIOperation::Offset: { + const auto *Rule = CFI.getIf<MachineCFIRegisterOffsetRule>(); + if (Rule == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "invalid CFI offset rule"); + auto Recorded = + recordSavedLocation(Locations, SavedRegisters, Rule->Register, + SavedLocation{SavedLocation::Kind::CFAOffset, Rule->Offset}); + if (!Recorded) + return std::unexpected(Recorded.error()); + break; + } + case MachineCFIOperation::Register: { + const auto *Rule = CFI.getIf<MachineCFIRegisterRegisterRule>(); + if (Rule == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "invalid CFI register rule"); + auto Recorded = recordSavedLocation( + Locations, SavedRegisters, Rule->Register, + SavedLocation{SavedLocation::Kind::Register, + static_cast<std::int64_t>(Rule->SourceRegister.value())}); + if (!Recorded) + return std::unexpected(Recorded.error()); + break; + } + case MachineCFIOperation::RelativeOffset: { + const auto *Rule = CFI.getIf<MachineCFIRegisterOffsetRule>(); + if (Rule == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "invalid CFI rel_offset rule"); + auto Recorded = recordSavedLocation( + Locations, SavedRegisters, Rule->Register, + SavedLocation{SavedLocation::Kind::CFAOffset, Rule->Offset - CFAOffset}); + if (!Recorded) + return std::unexpected(Recorded.error()); + break; + } + case MachineCFIOperation::Restore: { + const auto *Rule = CFI.getIf<MachineCFIRegisterRule>(); + if (Rule == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "invalid CFI restore rule"); + RestoredRegisters.insert(Rule->Register.value()); + break; + } + case MachineCFIOperation::RememberState: + ++RememberStateDepth; + break; + case MachineCFIOperation::RestoreState: + --RememberStateDepth; + break; + case MachineCFIOperation::SameValue: + case MachineCFIOperation::LLVMDefAddressSpaceCFA: + case MachineCFIOperation::Escape: + case MachineCFIOperation::Undefined: + case MachineCFIOperation::WindowSave: + case MachineCFIOperation::NegateReturnAddressState: + case MachineCFIOperation::NegateReturnAddressStateWithPC: + case MachineCFIOperation::GNUArgsSize: + case MachineCFIOperation::Label: + case MachineCFIOperation::ValueOffset: + case MachineCFIOperation::LLVMRegisterPair: + case MachineCFIOperation::LLVMVectorRegisters: + case MachineCFIOperation::LLVMVectorOffset: + case MachineCFIOperation::LLVMVectorRegisterMask: + break; + case MachineCFIOperation::Count: + return makeUnexpected(ErrorCode::VerificationError, "invalid CFI operation"); + } + } + + if (RememberStateDepth != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "cfi_remember_state and cfi_restore_state must be paired in one block"); + } + Info.Processed = true; + Info.OutgoingCFAOffset = CFAOffset; + Info.OutgoingCFARegister = CFARegister; + Info.OutgoingSavedRegisters = Info.IncomingSavedRegisters; + Info.OutgoingSavedRegisters.insert(SavedRegisters.begin(), SavedRegisters.end()); + for (const std::uint32_t Register : RestoredRegisters) + Info.OutgoingSavedRegisters.erase(Register); + return {}; +} + +[[nodiscard]] Expected<void> calculateCFI(BlockInfoMap &BlockInfos, SavedLocationMap &Locations, + MachineFunction &Function) { + MachineBasicBlock *Entry = Function.firstBlock(); + if (Entry == nullptr) + return {}; + std::vector<MachineBasicBlock *> Stack{Entry}; + while (!Stack.empty()) { + MachineBasicBlock *Block = Stack.back(); + Stack.pop_back(); + BlockCFIInfo &Info = BlockInfos.at(Block); + auto Calculated = calculateOutgoingCFI(Info, Function, Locations); + if (!Calculated) + return std::unexpected(Calculated.error()); + for (const MachineSuccessor &Successor : Block->successors()) { + BlockCFIInfo &SuccessorInfo = BlockInfos.at(Successor.Block); + if (SuccessorInfo.Processed) + continue; + SuccessorInfo.IncomingCFAOffset = Info.OutgoingCFAOffset; + SuccessorInfo.IncomingCFARegister = Info.OutgoingCFARegister; + SuccessorInfo.IncomingSavedRegisters = Info.OutgoingSavedRegisters; + Stack.push_back(Successor.Block); + } + } + return {}; +} + +[[nodiscard]] Expected<void> emitCFI(MachineFunction &Function, MachineBasicBlock &Block, + MachineInstr *Anchor, const TargetDescription &Target, + MachineCFIInstruction CFI) { + auto Index = Function.cfiInfo().addInstruction(std::move(CFI)); + if (!Index) + return std::unexpected(Index.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = + Anchor == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::CFI_INSTRUCTION)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addCFIInstruction(*Index); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] std::vector<std::uint32_t> setDifference(const std::set<std::uint32_t> &Left, + const std::set<std::uint32_t> &Right) { + std::vector<std::uint32_t> Difference; + std::set_difference(Left.begin(), Left.end(), Right.begin(), Right.end(), + std::back_inserter(Difference)); + return Difference; +} + +[[nodiscard]] Expected<void> emitSavedLocation(MachineFunction &Function, MachineBasicBlock &Block, + MachineInstr *Anchor, + const TargetDescription &Target, + std::uint32_t Register, + const SavedLocationMap &Locations) { + const auto Position = Locations.find(Register); + if (Position == Locations.end()) { + return makeUnexpected(ErrorCode::InternalError, "saved CFI register has no recorded location"); + } + const DwarfRegisterNumber DwarfRegister = DwarfRegisterNumber::fromValue(Register); + const SavedLocation &Location = Position->second; + if (Location.LocationKind == SavedLocation::Kind::CFAOffset) { + return emitCFI(Function, Block, Anchor, Target, + MachineCFIInstruction::offset(DwarfRegister, Location.Value)); + } + return emitCFI(Function, Block, Anchor, Target, + MachineCFIInstruction::registerRule( + DwarfRegister, + DwarfRegisterNumber::fromValue(static_cast<std::uint32_t>(Location.Value)))); +} + +[[nodiscard]] Expected<bool> insertCFIInstructions(BlockInfoMap &BlockInfos, + const SavedLocationMap &Locations, + MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Entry = Function.firstBlock(); + if (Entry == nullptr) + return false; + const BlockCFIInfo *PreviousInfo = &BlockInfos.at(Entry); + bool Changed = false; + for (MachineBasicBlock *Block = Entry->next(); Block != nullptr; Block = Block->next()) { + const BlockCFIInfo &Info = BlockInfos.at(Block); + MachineInstr *Anchor = Block->firstInstruction(); + const MachineBasicBlock *PreviousBlock = Block->previous(); + const bool ForceFullCFA = PreviousBlock != nullptr && PreviousBlock->attributes().sectionID() != + Block->attributes().sectionID(); + + if ((PreviousInfo->OutgoingCFAOffset != Info.IncomingCFAOffset && + PreviousInfo->OutgoingCFARegister != Info.IncomingCFARegister) || + ForceFullCFA) { + auto Emitted = + emitCFI(Function, *Block, Anchor, Target, + MachineCFIInstruction::defCFA(Info.IncomingCFARegister, Info.IncomingCFAOffset)); + if (!Emitted) + return std::unexpected(Emitted.error()); + Changed = true; + } else if (PreviousInfo->OutgoingCFAOffset != Info.IncomingCFAOffset) { + auto Emitted = emitCFI(Function, *Block, Anchor, Target, + MachineCFIInstruction::defCFAOffset(Info.IncomingCFAOffset)); + if (!Emitted) + return std::unexpected(Emitted.error()); + Changed = true; + } else if (PreviousInfo->OutgoingCFARegister != Info.IncomingCFARegister) { + auto Emitted = emitCFI(Function, *Block, Anchor, Target, + MachineCFIInstruction::defCFARegister(Info.IncomingCFARegister)); + if (!Emitted) + return std::unexpected(Emitted.error()); + Changed = true; + } + + if (ForceFullCFA) { + for (const std::uint32_t Register : Info.IncomingSavedRegisters) { + auto Emitted = emitSavedLocation(Function, *Block, Anchor, Target, Register, Locations); + if (!Emitted) + return std::unexpected(Emitted.error()); + } + Changed = true; + PreviousInfo = &Info; + continue; + } + + for (const std::uint32_t Register : + setDifference(PreviousInfo->OutgoingSavedRegisters, Info.IncomingSavedRegisters)) { + auto Emitted = + emitCFI(Function, *Block, Anchor, Target, + MachineCFIInstruction::restore(DwarfRegisterNumber::fromValue(Register))); + if (!Emitted) + return std::unexpected(Emitted.error()); + Changed = true; + } + for (const std::uint32_t Register : + setDifference(Info.IncomingSavedRegisters, PreviousInfo->OutgoingSavedRegisters)) { + auto Emitted = emitSavedLocation(Function, *Block, Anchor, Target, Register, Locations); + if (!Emitted) + return std::unexpected(Emitted.error()); + Changed = true; + } + PreviousInfo = &Info; + } + return Changed; +} + +} // namespace + +const MachinePassContract &CFIInstrInserterPass::contract() const noexcept { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "cfi-instr-inserter", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = std::nullopt, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +Expected<MachinePassResult> CFIInstrInserterPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + if (FrameLowering == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "CFIInstrInserter requires target frame lowering"); + } + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + if (Function.firstBlock() == nullptr || Function.cfiInfo().instructions().empty()) + return Result; + + const PhysicalRegister InitialRegister = FrameLowering->initialCFARegister(Function, Target); + const auto InitialDwarfRegister = Target.mirText().dwarfRegister(InitialRegister); + if (!InitialDwarfRegister) { + return makeUnexpected(ErrorCode::InternalError, + "initial CFA register has no DWARF register number"); + } + const std::int64_t InitialOffset = FrameLowering->initialCFAOffset(Function); + BlockInfoMap BlockInfos; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + BlockInfos.emplace( + Block, BlockCFIInfo{ + .Block = Block, + .IncomingCFAOffset = InitialOffset, + .OutgoingCFAOffset = InitialOffset, + .IncomingCFARegister = DwarfRegisterNumber::fromValue(*InitialDwarfRegister), + .OutgoingCFARegister = DwarfRegisterNumber::fromValue(*InitialDwarfRegister), + .IncomingSavedRegisters = {}, + .OutgoingSavedRegisters = {}, + .Processed = false, + }); + } + SavedLocationMap Locations; + auto Calculated = calculateCFI(BlockInfos, Locations, Function); + if (!Calculated) + return std::unexpected(Calculated.error()); + auto Inserted = insertCFIInstructions(BlockInfos, Locations, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + Result.Changed = *Inserted; + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Frame}; + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt new file mode 100644 index 0000000..ccbf131 --- /dev/null +++ b/backend/lib/CodeGen/CMakeLists.txt @@ -0,0 +1,250 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +set( + RCCBackendCodeGenSources + CFIInstrInserter.cpp + CompilationUnit.cpp + CommandFlags.cpp + GlobalObject.cpp + GenericOpcodes.cpp + FrameIndexElimination.cpp + FinalizeISel.cpp + InstructionDescriptor.cpp + LowLevelType.cpp + MachineBasicBlockAttributes.cpp + MachineCFIInfo.cpp + MachineCompilationStatus.cpp + MachineCombiner.cpp + MachineCombinerPattern.cpp + MachineBasicBlock.cpp + MachineBlockPlacement.cpp + MachineBranchOptimization.cpp + MachineBranchRelaxation.cpp + MachineCFGUpdater.cpp + MachineConstantPool.cpp + MachineCopyPropagation.cpp + MachineCSE.cpp + MachineDebugInfo.cpp + MachineStripDebug.cpp + MachineDCE.cpp + MachineFrameInfo.cpp + MachineFinalization.cpp + MachineFunction.cpp + MachineFunctionAttributes.cpp + MachineFunctionCloner.cpp + MachineIRTypes.cpp + MachineIRBuilder.cpp + MachineAnalyses.cpp + MachineInstr.cpp + MachineJumpTableInfo.cpp + MachineLICM.cpp + SlotIndexes.cpp + LiveVariables.cpp + LiveInterval.cpp + LiveIntervals.cpp + LivePhysRegs.cpp + LiveRegUnits.cpp + MachineMemoryFolding.cpp + MachinePeepholeOptimizer.cpp + MachineMemOperand.cpp + MachineMetadata.cpp + MachineOperand.cpp + MachineOptimizationPipeline.cpp + MachineOutliner.cpp + MachinePassInstrumentation.cpp + MachinePassManager.cpp + MachinePseudoExpansion.cpp + MachineRegisterInfo.cpp + MachineScheduler.cpp + MachineShrinkWrapping.cpp + MachineSink.cpp + MachineStackSlotColoring.cpp + MachineTailDuplication.cpp + PrologueEpilogueInsertion.cpp + PostRAScheduler.cpp + RegisterPressure.cpp + ReachingDefAnalysis.cpp + RegisterScavenger.cpp + MacroFusion.cpp + ScheduleDAG.cpp + ScheduleHazardRecognizer.cpp + InlineSpiller.cpp + LiveRangeEdit.cpp + LiveRegMatrix.cpp + RegAllocBase.cpp + RegAllocBasic.cpp + RegAllocFast.cpp + RegAllocGreedy.cpp + RegAllocPBQP.cpp + VirtRegMap.cpp + PHIElimination.cpp + MachineVerifier.cpp + MachineFunctionProperties.cpp + MachineFunctionRecords.cpp + Register.cpp + RegisterMask.cpp + TargetConstant.cpp + TargetCallLowering.cpp + TargetInstrInfo.cpp + TargetDescription.cpp + TargetFunctionState.cpp + TargetMIRTextInfo.cpp + TargetRegisterInfo.cpp + TargetSchedulingModel.cpp + TwoAddressInstructionPass.cpp +) +set( + RCCBackendCodeGenHeaders + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/CFIInstrInserter.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/CompilationUnit.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/CommandFlags.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/CSEConfigBase.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalObject.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GenericOpcodes.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/FrameIndexElimination.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/FinalizeISel.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/InstructionDescriptor.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/InlineAsm.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LowLevelType.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MacroFusion.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlockAttributes.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFIInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCompilationStatus.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCombiner.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCombinerPattern.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBasicBlock.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBlockPlacement.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBranchOptimization.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineBranchRelaxation.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCFGUpdater.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineConstantPool.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCopyPropagation.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineCSE.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineDebugInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineStripDebug.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineDCE.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFrameInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFinalization.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunction.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionAttributes.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionCloner.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineIRTypes.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineIRBuilder.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineAnalyses.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineInstr.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineJumpTableInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineLICM.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/SlotIndexes.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LiveVariables.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LiveInterval.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LiveIntervals.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LivePhysRegs.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LiveRegUnits.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMemoryFolding.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePeepholeOptimizer.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMemOperand.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMetadata.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOperand.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOptimizationPipeline.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOutliner.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/IR/PassManager.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionAnalysisManager.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePassInstrumentation.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePassManager.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePseudoExpansion.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PrologueEpilogueInsertion.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterPressure.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/ReachingDefAnalysis.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterScavenger.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/ScheduleDAG.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/ScheduleHazardRecognizer.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetFrameLowering.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineRegisterInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineScheduler.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineShrinkWrapping.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineSink.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineStackSlotColoring.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineTailDuplication.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PostRAScheduler.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Graph.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Math.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/ReductionRules.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Solution.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PBQP/Solver.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LiveRangeEdit.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/LiveRegMatrix.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegAllocCommon.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegAllocFast.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegAllocPBQP.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/Spiller.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetRegisterAllocationInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/VirtRegMap.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/PHIElimination.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineVerifier.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionProperties.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionRecords.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/Register.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterBankInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/RegisterMask.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/StableID.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetConstant.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetCallLowering.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetInstrInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetDescription.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetFunctionState.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetMIRTextInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetMemoryFolding.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetRegisterInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetSchedulingModel.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TwoAddressInstructionPass.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetPseudoExpansion.h" +) + +add_library(RCCBackendCodeGen ${RCCBackendCodeGenSources}) +add_library(RCCBackend::CodeGen ALIAS RCCBackendCodeGen) + +add_subdirectory(GlobalISel) + +rcc_backend_configure_target(RCCBackendCodeGen) + +target_include_directories( + RCCBackendCodeGen + PUBLIC + "$<BUILD_INTERFACE:${RCC_BACKEND_SOURCE_DIR}/include>" + "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>" +) +target_link_libraries(RCCBackendCodeGen PUBLIC RCCBackend::IR) + +set_target_properties( + RCCBackendCodeGen + PROPERTIES + EXPORT_NAME CodeGen + OUTPUT_NAME RCCBackendCodeGen +) + +install( + TARGETS RCCBackendCodeGen + EXPORT RCCBackendTargets + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" +) + +list( + TRANSFORM RCCBackendCodeGenSources + PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/" + OUTPUT_VARIABLE RCCBackendCodeGenSourcePaths +) +set( + RCCBackendCodeGenPrivateHeaders + "${CMAKE_CURRENT_SOURCE_DIR}/MachineSchedulerInternal.h" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBase.h" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocBasic.h" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocGreedy.h" + "${CMAKE_CURRENT_SOURCE_DIR}/RegAllocInternal.h" +) +rcc_backend_register_format_files( + ${RCCBackendCodeGenSourcePaths} + ${RCCBackendCodeGenHeaders} + ${RCCBackendCodeGenPrivateHeaders} +) diff --git a/backend/lib/CodeGen/CommandFlags.cpp b/backend/lib/CodeGen/CommandFlags.cpp new file mode 100644 index 0000000..2a659eb --- /dev/null +++ b/backend/lib/CodeGen/CommandFlags.cpp @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/CommandFlags.h" + +#include <optional> +#include <string> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] Expected<std::optional<std::string_view>> +takeValue(std::span<const std::string_view> Arguments, std::size_t &Index, std::string_view Name) { + const std::string_view Argument = Arguments[Index]; + const std::string LongName = + Name.starts_with('-') && !Name.starts_with("--") ? "-" + std::string(Name) : std::string(); + if (Argument == Name || (!LongName.empty() && Argument == LongName)) { + if (Index + 1U >= Arguments.size()) { + return makeUnexpected(ErrorCode::InvalidInvocation, + "missing value for command-line option '" + std::string(Name) + "'"); + } + return Arguments[++Index]; + } + std::string Prefix(Name); + Prefix.push_back('='); + if (Argument.starts_with(Prefix)) + return Argument.substr(Prefix.size()); + if (!LongName.empty()) { + std::string LongPrefix(LongName); + LongPrefix.push_back('='); + if (Argument.starts_with(LongPrefix)) + return Argument.substr(LongPrefix.size()); + } + return std::nullopt; +} + +template <typename ValueT, typename ParserT> +[[nodiscard]] Expected<bool> parseValueFlag(std::span<const std::string_view> Arguments, + std::size_t &Index, std::string_view Name, + std::optional<ValueT> &Destination, ParserT Parser) { + auto Value = takeValue(Arguments, Index, Name); + if (!Value) + return std::unexpected(Value.error()); + if (!*Value) + return false; + auto Parsed = Parser(**Value); + if (!Parsed) + return std::unexpected(Parsed.error()); + Destination = std::move(*Parsed); + return true; +} + +} // namespace + +std::string_view spelling(RelocationModel Value) noexcept { + switch (Value) { + case RelocationModel::Static: + return "static"; + case RelocationModel::PositionIndependent: + return "pic"; + } + return "invalid"; +} + +std::string_view spelling(CodeModel Value) noexcept { + switch (Value) { + case CodeModel::Small: + return "small"; + case CodeModel::Medium: + return "medium"; + case CodeModel::Large: + return "large"; + } + return "invalid"; +} + +std::string_view spelling(OptimizationLevel Value) noexcept { + switch (Value) { + case OptimizationLevel::O0: + return "O0"; + case OptimizationLevel::O1: + return "O1"; + case OptimizationLevel::O2: + return "O2"; + case OptimizationLevel::O3: + return "O3"; + case OptimizationLevel::Os: + return "Os"; + case OptimizationLevel::Oz: + return "Oz"; + } + return "invalid"; +} + +std::string_view spelling(CodeGenFileType Value) noexcept { + switch (Value) { + case CodeGenFileType::AssemblyFile: + return "asm"; + case CodeGenFileType::ObjectFile: + return "obj"; + case CodeGenFileType::Null: + return "null"; + } + return "invalid"; +} + +Expected<RelocationModel> parseRelocationModel(std::string_view Value) { + if (Value == "static") + return RelocationModel::Static; + if (Value == "pic") + return RelocationModel::PositionIndependent; + return makeUnexpected(ErrorCode::InvalidInvocation, "unknown relocation model"); +} + +Expected<CodeModel> parseCodeModel(std::string_view Value) { + if (Value == "small") + return CodeModel::Small; + if (Value == "medium") + return CodeModel::Medium; + if (Value == "large") + return CodeModel::Large; + return makeUnexpected(ErrorCode::InvalidInvocation, "unknown code model"); +} + +Expected<OptimizationLevel> parseOptimizationLevel(std::string_view Value) { + if (Value == "O0") + return OptimizationLevel::O0; + if (Value == "O1") + return OptimizationLevel::O1; + if (Value == "O2") + return OptimizationLevel::O2; + if (Value == "O3") + return OptimizationLevel::O3; + if (Value == "Os") + return OptimizationLevel::Os; + if (Value == "Oz") + return OptimizationLevel::Oz; + return makeUnexpected(ErrorCode::InvalidInvocation, "unknown optimization level"); +} + +Expected<CodeGenFileType> parseCodeGenFileType(std::string_view Value) { + if (Value == "asm") + return CodeGenFileType::AssemblyFile; + if (Value == "obj") + return CodeGenFileType::ObjectFile; + if (Value == "null") + return CodeGenFileType::Null; + return makeUnexpected(ErrorCode::InvalidInvocation, "unknown file type"); +} + +Expected<GlobalISelAbortMode> parseGlobalISelAbortMode(std::string_view Value) { + if (Value == "0") + return GlobalISelAbortMode::Disable; + if (Value == "1") + return GlobalISelAbortMode::Enable; + if (Value == "2") + return GlobalISelAbortMode::DisableWithDiagnostic; + return makeUnexpected(ErrorCode::InvalidInvocation, "-global-isel-abort requires 0, 1, or 2"); +} + +Expected<bool> parseCodeGenCommandFlag(std::span<const std::string_view> Arguments, + std::size_t &Index, CodeGenCommandFlags &Flags) { + const std::string_view Argument = Arguments[Index]; + const std::string_view Optimization = Argument.starts_with("--") ? Argument.substr(2) + : Argument.starts_with('-') ? Argument.substr(1) + : std::string_view{}; + if ((Argument.starts_with('-') && !Argument.starts_with("---")) && + (Optimization == "O0" || Optimization == "O1" || Optimization == "O2" || + Optimization == "O3" || Optimization == "Os" || Optimization == "Oz")) { + auto Level = parseOptimizationLevel(Optimization); + if (!Level) + return std::unexpected(Level.error()); + Flags.Optimization = *Level; + return true; + } + + const auto ParseString = [&](std::string_view Name, + std::optional<std::string> &Destination) -> Expected<bool> { + auto Value = takeValue(Arguments, Index, Name); + if (!Value) + return std::unexpected(Value.error()); + if (!*Value) + return false; + Destination = std::string(**Value); + return true; + }; + for (const auto &[Name, Destination] : + {std::pair{"-mtriple", &Flags.Triple}, std::pair{"-mcpu", &Flags.CPU}, + std::pair{"-mtune", &Flags.TuneCPU}, std::pair{"-target-abi", &Flags.ABI}}) { + auto Parsed = ParseString(Name, *Destination); + if (!Parsed) + return std::unexpected(Parsed.error()); + if (*Parsed) + return true; + } + + auto Features = takeValue(Arguments, Index, "-mattr"); + if (!Features) + return std::unexpected(Features.error()); + if (*Features) { + std::size_t Begin = 0; + while (Begin <= (**Features).size()) { + const std::size_t Comma = (**Features).find(',', Begin); + const std::size_t End = Comma == std::string_view::npos ? (**Features).size() : Comma; + if (End == Begin) { + return makeUnexpected(ErrorCode::InvalidInvocation, + "target feature list contains an empty element"); + } + Flags.FeatureOverrides.emplace_back((**Features).substr(Begin, End - Begin)); + if (Comma == std::string_view::npos) + break; + Begin = Comma + 1U; + } + return true; + } + + auto Relocation = + parseValueFlag(Arguments, Index, "-relocation-model", Flags.Relocation, parseRelocationModel); + if (!Relocation) + return std::unexpected(Relocation.error()); + if (*Relocation) + return true; + auto Code = parseValueFlag(Arguments, Index, "-code-model", Flags.Code, parseCodeModel); + if (!Code) + return std::unexpected(Code.error()); + if (*Code) + return true; + + auto FileType = takeValue(Arguments, Index, "-filetype"); + if (!FileType) + return std::unexpected(FileType.error()); + if (*FileType) { + auto Parsed = parseCodeGenFileType(**FileType); + if (!Parsed) + return std::unexpected(Parsed.error()); + Flags.FileType = *Parsed; + Flags.FileTypeWasSpecified = true; + return true; + } + return false; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/CompilationUnit.cpp b/backend/lib/CodeGen/CompilationUnit.cpp new file mode 100644 index 0000000..a017851 --- /dev/null +++ b/backend/lib/CodeGen/CompilationUnit.cpp @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/CompilationUnit.h" + +#include "rcc/CodeGen/MachineFunction.h" + +#include <cstddef> +#include <memory> +#include <string> +#include <string_view> +#include <unordered_set> +#include <utility> + +namespace rcc { + +CompilationUnit::CompilationUnit() = default; + +CompilationUnit::CompilationUnit(CompilationUnit &&) noexcept = default; + +CompilationUnit &CompilationUnit::operator=(CompilationUnit &&) noexcept = default; + +CompilationUnit::~CompilationUnit() = default; + +Expected<MachineFunction *> CompilationUnit::createMachineFunction(std::string Name) { + if (!Name.empty() && FunctionNameToIndex.contains(Name)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine function name is already present in the compilation unit"); + } + if (!Name.empty() && GlobalNameToIndex.contains(Name)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine function name conflicts with a global object"); + } + + auto StableID = FunctionIDGenerator.allocate(); + if (!StableID) + return std::unexpected(StableID.error()); + auto Function = MachineFunction::create(std::move(Name), *StableID); + if (!Function) + return std::unexpected(Function.error()); + + MachineFunction *Result = Function->get(); + const std::size_t Index = Functions.size(); + if (!Result->name().empty()) + FunctionNameToIndex.emplace(std::string(Result->name()), Index); + Functions.push_back(std::move(*Function)); + return Result; +} + +Expected<GlobalObjectDesc *> CompilationUnit::addGlobalObject(GlobalObjectDesc Object) { + auto Verification = Object.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + if (FunctionNameToIndex.contains(Object.Symbol)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "global object symbol conflicts with a machine function"); + } + if (GlobalNameToIndex.contains(Object.Symbol)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "global object symbol is already present in the compilation unit"); + } + + const std::size_t Index = GlobalObjects.size(); + auto Storage = std::make_unique<GlobalObjectDesc>(std::move(Object)); + GlobalObjectDesc *Result = Storage.get(); + GlobalNameToIndex.emplace(Result->Symbol, Index); + GlobalObjects.push_back(std::move(Storage)); + return Result; +} + +MachineFunction *CompilationUnit::functionAt(std::size_t Index) noexcept { + if (Index >= Functions.size()) + return nullptr; + return Functions[Index].get(); +} + +const MachineFunction *CompilationUnit::functionAt(std::size_t Index) const noexcept { + if (Index >= Functions.size()) + return nullptr; + return Functions[Index].get(); +} + +MachineFunction *CompilationUnit::function(std::string_view Name) noexcept { + const auto Position = FunctionNameToIndex.find(Name); + if (Position == FunctionNameToIndex.end()) + return nullptr; + return Functions[Position->second].get(); +} + +const MachineFunction *CompilationUnit::function(std::string_view Name) const noexcept { + const auto Position = FunctionNameToIndex.find(Name); + if (Position == FunctionNameToIndex.end()) + return nullptr; + return Functions[Position->second].get(); +} + +MachineFunction *CompilationUnit::function(MachineFunctionStableID ID) noexcept { + for (const std::unique_ptr<MachineFunction> &Function : Functions) { + if (Function->stableID() == ID) + return Function.get(); + } + return nullptr; +} + +const MachineFunction *CompilationUnit::function(MachineFunctionStableID ID) const noexcept { + for (const std::unique_ptr<MachineFunction> &Function : Functions) { + if (Function->stableID() == ID) + return Function.get(); + } + return nullptr; +} + +GlobalObjectDesc *CompilationUnit::globalObjectAt(std::size_t Index) noexcept { + return Index < GlobalObjects.size() ? GlobalObjects[Index].get() : nullptr; +} + +const GlobalObjectDesc *CompilationUnit::globalObjectAt(std::size_t Index) const noexcept { + return Index < GlobalObjects.size() ? GlobalObjects[Index].get() : nullptr; +} + +GlobalObjectDesc *CompilationUnit::globalObject(std::string_view Name) noexcept { + const auto Position = GlobalNameToIndex.find(Name); + return Position == GlobalNameToIndex.end() ? nullptr : GlobalObjects[Position->second].get(); +} + +const GlobalObjectDesc *CompilationUnit::globalObject(std::string_view Name) const noexcept { + const auto Position = GlobalNameToIndex.find(Name); + return Position == GlobalNameToIndex.end() ? nullptr : GlobalObjects[Position->second].get(); +} + +Expected<void> CompilationUnit::verify() const { + if (GlobalNameToIndex.size() != GlobalObjects.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit global object name index has an inconsistent size"); + } + + std::unordered_set<MachineFunctionStableID> StableIDs; + std::size_t NamedFunctionCount = 0; + for (std::size_t Index = 0; Index < Functions.size(); ++Index) { + const std::unique_ptr<MachineFunction> &Function = Functions[Index]; + if (Function == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit contains a null machine function"); + } + if (!Function->stableID().isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit contains an invalid machine function identity"); + } + if (!StableIDs.insert(Function->stableID()).second) { + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit contains duplicate machine function StableIDs"); + } + + if (!Function->name().empty()) { + ++NamedFunctionCount; + const auto Position = FunctionNameToIndex.find(Function->name()); + if (Position == FunctionNameToIndex.end() || Position->second != Index) { + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit function name index is inconsistent"); + } + if (GlobalNameToIndex.contains(Function->name())) { + return makeUnexpected(ErrorCode::VerificationError, + "function name conflicts with a global object symbol"); + } + } + } + if (FunctionNameToIndex.size() != NamedFunctionCount) { + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit function name index has an inconsistent size"); + } + + for (std::size_t Index = 0; Index < GlobalObjects.size(); ++Index) { + const std::unique_ptr<GlobalObjectDesc> &Object = GlobalObjects[Index]; + if (Object == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit contains a null global object"); + } + auto ObjectVerification = Object->verify(); + if (!ObjectVerification) + return std::unexpected(ObjectVerification.error()); + const auto Position = GlobalNameToIndex.find(Object->Symbol); + if (Position == GlobalNameToIndex.end() || Position->second != Index) { + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit global object name index is inconsistent"); + } + if (FunctionNameToIndex.contains(Object->Symbol)) { + return makeUnexpected(ErrorCode::VerificationError, + "global object symbol conflicts with a function name"); + } + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/FinalizeISel.cpp b/backend/lib/CodeGen/FinalizeISel.cpp new file mode 100644 index 0000000..9b5c11f --- /dev/null +++ b/backend/lib/CodeGen/FinalizeISel.cpp @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/FinalizeISel.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &finalizeISelContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "finalize-isel", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout, + MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +} // namespace + +const MachinePassContract &FinalizeISelPass::contract() const noexcept { + return finalizeISelContract(); +} + +Expected<MachinePassResult> FinalizeISelPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + MachinePassResult Result; + MachineBasicBlock *Block = Function.firstBlock(); + while (Block != nullptr) { + MachineInstr *Instruction = Block->firstInstruction(); + while (Instruction != nullptr) { + MachineInstr *Next = Instruction->next(); + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "finalize-isel encountered an unknown opcode"); + } + + if (Instruction->flags().has(MachineInstrFlag::FrameSetup) || + Instruction->flags().has(MachineInstrFlag::FrameDestroy)) { + MachineFrameState State = Function.frameInfo().state(); + if (!State.AdjustsStack) { + State.AdjustsStack = true; + auto Updated = Function.frameInfo().setState(std::move(State)); + if (!Updated) + return std::unexpected(Updated.error()); + Result.Changes.add(MachinePassChange::Frame); + } + } + + if (!Descriptor->Flags.has(InstructionFlag::UsesCustomInserter)) { + Instruction = Next; + continue; + } + + auto Expanded = Info->emitInstrWithCustomInserter(*Instruction, *Block, Function, Target); + if (!Expanded) + return std::unexpected(Expanded.error()); + if (*Expanded == nullptr || &(*Expanded)->function() != &Function) { + return makeUnexpected(ErrorCode::InternalError, + "custom inserter returned an invalid basic block"); + } + + Result.Changes.add(MachinePassChange::Instructions); + Result.Changes.add(MachinePassChange::Operands); + if (*Expanded != Block) { + Result.Changes.add(MachinePassChange::CFG); + Result.Changes.add(MachinePassChange::BlockLayout); + Block = *Expanded; + Instruction = Block->firstInstruction(); + } else { + Instruction = Next; + } + } + Block = Block->next(); + } + + Result.Changed = !Result.Changes.empty(); + Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/FrameIndexElimination.cpp b/backend/lib/CodeGen/FrameIndexElimination.cpp new file mode 100644 index 0000000..acc502d --- /dev/null +++ b/backend/lib/CodeGen/FrameIndexElimination.cpp @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/FrameIndexElimination.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterScavenger.h" +#include "rcc/CodeGen/TargetFrameLowering.h" + +#include <cstddef> + +namespace rcc { + +Expected<bool> eliminateFrameIndices(MachineFunction &Function, const TargetDescription &Target, + const TargetFrameLowering &FrameLowering, + RegisterScavenger &Scavenger) { + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + if (Instruction->operandAt(Index)->kind() != MachineOperand::Kind::FrameIndex) + continue; + auto Eliminated = + FrameLowering.eliminateFrameIndex(*Instruction, Index, Function, Target, Scavenger); + if (!Eliminated) + return std::unexpected(Eliminated.error()); + Changed = true; + } + + Instruction = Next; + } + } + return Changed; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GenericOpcodes.cpp b/backend/lib/CodeGen/GenericOpcodes.cpp new file mode 100644 index 0000000..f45db39 --- /dev/null +++ b/backend/lib/CodeGen/GenericOpcodes.cpp @@ -0,0 +1,1517 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GenericOpcodes.h" + +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <exception> +#include <initializer_list> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +using GO = GenericOpcode; + +[[nodiscard]] Expected<void> verifyGenericExplicitOperands(Opcode InstructionOpcode, + std::span<const MachineOperand> Operands, + const MachineRegisterInfo &RegInfo); +[[nodiscard]] Expected<void> verifyGenericInstructionSemantics(const MachineInstr &Instruction, + const MachineRegisterInfo &RegInfo); + +[[nodiscard]] LowLevelTypeKindSet anyValueKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Pointer, + LowLevelType::Kind::Vector, LowLevelType::Kind::VectorPointer}; +} + +[[nodiscard]] LowLevelTypeKindSet integerKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Vector}; +} + +[[nodiscard]] LowLevelTypeKindSet integerOrPointerKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Pointer, + LowLevelType::Kind::Vector, LowLevelType::Kind::VectorPointer}; +} + +[[nodiscard]] LowLevelTypeKindSet scalarIntegerKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar}; +} + +[[nodiscard]] LowLevelTypeKindSet scalarIntegerOrPointerKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Pointer}; +} + +[[nodiscard]] LowLevelTypeKindSet scalarValueKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Pointer}; +} + +[[nodiscard]] LowLevelTypeKindSet floatingKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar, LowLevelType::Kind::Vector}; +} + +[[nodiscard]] LowLevelTypeKindSet scalarFloatingKinds() { + return {LowLevelType::Kind::CategoryNeutralScalar}; +} + +[[nodiscard]] LowLevelTypeKindSet pointerKinds() { + return {LowLevelType::Kind::Pointer, LowLevelType::Kind::VectorPointer}; +} + +[[nodiscard]] LowLevelTypeKindSet scalarPointerKinds() { + return {LowLevelType::Kind::Pointer}; +} + +[[nodiscard]] LowLevelTypeKindSet vectorKinds() { + return {LowLevelType::Kind::Vector, LowLevelType::Kind::VectorPointer}; +} + +[[nodiscard]] OperandKindSet allOperandKinds() { + return {MachineOperand::Kind::Register, + MachineOperand::Kind::Immediate, + MachineOperand::Kind::IntegerImmediate, + MachineOperand::Kind::FloatingImmediate, + MachineOperand::Kind::BasicBlock, + MachineOperand::Kind::FrameIndex, + MachineOperand::Kind::ConstantPoolIndex, + MachineOperand::Kind::JumpTableIndex, + MachineOperand::Kind::GlobalSymbol, + MachineOperand::Kind::ExternalSymbol, + MachineOperand::Kind::BlockAddress, + MachineOperand::Kind::RegisterMask, + MachineOperand::Kind::LiveOutRegisterMask, + MachineOperand::Kind::Metadata, + MachineOperand::Kind::TargetIndex, + MachineOperand::Kind::MCSymbol, + MachineOperand::Kind::CFIIndex, + MachineOperand::Kind::Intrinsic, + MachineOperand::Kind::Predicate, + MachineOperand::Kind::ShuffleMask, + MachineOperand::Kind::DebugInstrRef, + MachineOperand::Kind::LaneMask}; +} + +[[nodiscard]] OperandDescriptor typedRegister(RegisterOperandRole Role, LowLevelTypeKindSet Kinds, + std::uint16_t Group) { + OperandDescriptor Result; + Result.Kinds = {MachineOperand::Kind::Register}; + Result.RegisterRole = Role; + Result.TypeConstraint = OperandTypeConstraint{Kinds, Group}; + return Result; +} + +[[nodiscard]] OperandDescriptor untypedRegister(RegisterOperandRole Role, bool AllowSubReg = true) { + OperandDescriptor Result; + Result.Kinds = {MachineOperand::Kind::Register}; + Result.RegisterRole = Role; + Result.AllowSubReg = AllowSubReg; + return Result; +} + +[[nodiscard]] OperandDescriptor operand(OperandKindSet Kinds) { + OperandDescriptor Result; + Result.Kinds = Kinds; + if (Kinds.contains(MachineOperand::Kind::Register)) { + Result.RegisterRole = RegisterOperandRole::Any; + Result.AllowSubReg = true; + } + return Result; +} + +[[nodiscard]] OperandDescriptor kind(MachineOperand::Kind Kind) { + return operand(OperandKindSet{Kind}); +} + +[[nodiscard]] OperandDescriptor immediate() { + return operand({MachineOperand::Kind::Immediate, MachineOperand::Kind::IntegerImmediate}); +} + +[[nodiscard]] OperandDescriptor symbol() { + return operand({MachineOperand::Kind::GlobalSymbol, MachineOperand::Kind::ExternalSymbol, + MachineOperand::Kind::MCSymbol}); +} + +[[nodiscard]] OperandDescriptor anyOperand() { + return operand(allOperandKinds()); +} + +[[nodiscard]] OperandDescriptor repeated(OperandDescriptor Descriptor) { + Descriptor.VariadicGroup = true; + return Descriptor; +} + +[[nodiscard]] OperandDescriptor registeredTargetFlags(OperandDescriptor Descriptor) { + Descriptor.AllowAnyTargetFlags = true; + return Descriptor; +} + +class DescriptorRegistry final { +public: + DescriptorRegistry() { + Descriptors.reserve(CorePseudoOpcodeCount + GenericOpcodeCount); + Configured.reserve(CorePseudoOpcodeCount + GenericOpcodeCount); +#define RCC_CORE_OPCODE(Name) add(GO::Name, #Name, false); +#define RCC_GENERIC_OPCODE(Name) add(GO::Name, #Name, true); +#include "rcc/CodeGen/GenericOpcode.def" +#undef RCC_GENERIC_OPCODE +#undef RCC_CORE_OPCODE + } + + void configure(std::initializer_list<GO> Opcodes, const std::vector<OperandDescriptor> &Operands, + std::size_t NumDefs, std::initializer_list<InstructionFlag> Flags = {}, + bool Variadic = false, std::size_t MinimumRepetitions = 0, + bool RequiresMemoryOperands = false, PseudoOwner Owner = PseudoOwner::None, + PipelineStage Deadline = PipelineStage::Emission, + bool AllowExtraImplicitOperands = false) { + for (const GO OpcodeValue : Opcodes) { + const std::size_t Index = index(OpcodeValue); + if (Index >= Descriptors.size() || Configured[Index]) + std::terminate(); + Configured[Index] = true; + InstructionDescriptor &Descriptor = Descriptors[Index]; + Descriptor.Operands = Operands; + Descriptor.NumExplicitDefs = NumDefs; + Descriptor.Variadic = Variadic; + Descriptor.MinimumVariadicRepetitions = MinimumRepetitions; + Descriptor.Flags = {}; + Descriptor.Flags.set(isGenericOpcode(Descriptor.InstructionOpcode) ? InstructionFlag::Generic + : InstructionFlag::Pseudo); + Descriptor.Flags.set(InstructionFlag::AllowedPreLegalizer); + for (const InstructionFlag Flag : Flags) + Descriptor.Flags.set(Flag); + Descriptor.RequiresMemoryOperands = RequiresMemoryOperands; + Descriptor.Owner = Owner; + Descriptor.Deadline = Deadline; + Descriptor.AllowExtraImplicitOperands = AllowExtraImplicitOperands; + if (isGenericOpcode(Descriptor.InstructionOpcode)) { + Descriptor.VerifyExplicitOperands = &verifyGenericExplicitOperands; + Descriptor.VerifyInstructionSemantics = &verifyGenericInstructionSemantics; + } + } + } + + void addFlag(std::initializer_list<GO> Opcodes, InstructionFlag Flag) { + for (const GO OpcodeValue : Opcodes) { + const std::size_t Index = index(OpcodeValue); + if (Index >= Descriptors.size() || !Configured[Index]) + std::terminate(); + Descriptors[Index].Flags.set(Flag); + } + } + + void tie(GO OpcodeValue, std::size_t Left, std::size_t Right) { + InstructionDescriptor &Descriptor = Descriptors[index(OpcodeValue)]; + if (!Configured[index(OpcodeValue)] || Left >= Descriptor.Operands.size() || + Right >= Descriptor.Operands.size()) { + std::terminate(); + } + Descriptor.Operands[Left].DefaultTiedOperand = Right; + Descriptor.Operands[Right].DefaultTiedOperand = Left; + } + + [[nodiscard]] std::vector<InstructionDescriptor> take() { + if (!std::ranges::all_of(Configured, [](bool Value) { return Value; })) + std::terminate(); + return std::move(Descriptors); + } + +private: + static std::size_t index(GO OpcodeValue) { + const std::size_t Value = static_cast<std::size_t>(OpcodeValue); + if (Value < static_cast<std::size_t>(GO::CORE_PSEUDO_END)) + return Value - static_cast<std::size_t>(GO::PHI); + return CorePseudoOpcodeCount + Value - static_cast<std::size_t>(GO::G_ASSERT_SEXT); + } + + void add(GO OpcodeValue, std::string_view Name, bool IsGeneric) { + InstructionDescriptor Descriptor; + Descriptor.InstructionOpcode = opcode(OpcodeValue); + Descriptor.Name = Name; + if (IsGeneric != isGenericOpcode(Descriptor.InstructionOpcode)) + std::terminate(); + Descriptors.push_back(std::move(Descriptor)); + Configured.push_back(false); + } + + std::vector<InstructionDescriptor> Descriptors; + std::vector<bool> Configured; +}; + +[[nodiscard]] Expected<void> semanticError(std::string_view Message) { + return makeUnexpected(ErrorCode::VerificationError, std::string(Message)); +} + +[[nodiscard]] std::optional<std::int64_t> plainImmediate(const MachineOperand &Operand) noexcept { + const auto *Value = Operand.getIf<std::int64_t>(); + if (Operand.kind() != MachineOperand::Kind::Immediate || Value == nullptr) + return std::nullopt; + return *Value; +} + +[[nodiscard]] std::uint64_t minimumBitWidth(LowLevelType Type) noexcept { + const std::uint64_t Elements = Type.isVector() ? Type.minimumElementCount() : 1; + return Elements * Type.scalarBitWidth(); +} + +[[nodiscard]] std::uint64_t minimumStoreBytes(LowLevelType Type) noexcept { + return (minimumBitWidth(Type) + 7) / 8; +} + +[[nodiscard]] LowLevelType scalarElementType(LowLevelType Type) noexcept { + if (const auto Element = Type.scalarElementType()) + return *Element; + return Type; +} + +[[nodiscard]] bool hasSameShape(LowLevelType Left, LowLevelType Right) noexcept { + if (Left.isVector() != Right.isVector()) + return false; + if (!Left.isVector()) + return true; + return Left.isScalableVector() == Right.isScalableVector() && + Left.minimumElementCount() == Right.minimumElementCount(); +} + +[[nodiscard]] Expected<void> requireSameShape(LowLevelType Left, LowLevelType Right, + std::string_view Message) { + if (!hasSameShape(Left, Right)) + return semanticError(Message); + return {}; +} + +[[nodiscard]] Expected<void> requireI1WithShape(LowLevelType Condition, LowLevelType Value, + bool AllowScalarCondition) { + if ((!Condition.isCategoryNeutralScalar() && !Condition.isVectorValue()) || + Condition.scalarBitWidth() == 0) + return semanticError("generic condition or status result must use an integer LowLevelType"); + if (AllowScalarCondition && !Condition.isVector()) + return {}; + return requireSameShape(Condition, Value, + "generic condition or status vector shape does not match its value"); +} + +[[nodiscard]] Expected<void> verifyGenericExplicitOperands(Opcode InstructionOpcode, + std::span<const MachineOperand> Operands, + const MachineRegisterInfo &RegInfo) { + std::vector<std::optional<LowLevelType>> Types(Operands.size()); + bool SawNonDefinition = false; + for (std::size_t Index = 0; Index < Operands.size(); ++Index) { + const MachineOperand &Operand = Operands[Index]; + if (!Operand.isRegister()) { + SawNonDefinition = true; + continue; + } + + if (Operand.isDef() == Operand.isUse()) + return semanticError("generic register operand must be either a pure definition or use"); + if (Operand.isDef()) { + if (SawNonDefinition) + return semanticError("generic register definitions must form the explicit operand prefix"); + } else { + SawNonDefinition = true; + if (Operand.registerFlags().has(RegisterOperandFlag::Undef)) + return semanticError("generic virtual-register uses cannot carry the undef flag"); + } + + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return semanticError("generic instruction register operands must be virtual registers"); + auto Info = RegInfo.virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + const std::optional<LowLevelType> &Type = Info->get().Type; + if (!Type) + return semanticError("generic virtual register is missing its LowLevelType"); + Types[Index] = Type.value(); + } + + const auto TypeAt = [&Types](std::size_t Index) -> LowLevelType { return *Types[Index]; }; + const GO Generic = static_cast<GO>(InstructionOpcode.value()); + + const auto VerifyWidthChange = [&](bool IsExtension) -> Expected<void> { + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); + if (auto Shape = + requireSameShape(Dst, Src, "generic width conversion must preserve vector shape"); + !Shape) { + return Shape; + } + if ((IsExtension && Dst.scalarBitWidth() <= Src.scalarBitWidth()) || + (!IsExtension && Dst.scalarBitWidth() >= Src.scalarBitWidth())) { + return semanticError(IsExtension + ? "generic extension destination must be wider than its source" + : "generic truncation destination must be narrower than its source"); + } + return {}; + }; + + const auto VerifyImmediateBitWidth = [&](std::size_t ImmediateIndex, std::size_t TypeIndex, + bool StrictlySmaller) -> Expected<void> { + const auto Value = plainImmediate(Operands[ImmediateIndex]); + if (!Value || *Value <= 0) + return semanticError("generic bit-width immediate must be a positive machine immediate"); + const std::uint64_t Width = TypeAt(TypeIndex).scalarBitWidth(); + if ((StrictlySmaller && static_cast<std::uint64_t>(*Value) >= Width) || + (!StrictlySmaller && static_cast<std::uint64_t>(*Value) > Width)) { + return semanticError("generic bit-width immediate exceeds its register element width"); + } + return {}; + }; + + switch (Generic) { + case GO::G_ASSERT_SEXT: + case GO::G_ASSERT_ZEXT: + case GO::G_SEXT_INREG: + return VerifyImmediateBitWidth(2, 1, true); + case GO::G_ASSERT_ALIGN: { + const auto AlignmentValue = plainImmediate(Operands[2]); + if (!AlignmentValue || *AlignmentValue <= 0 || + !std::has_single_bit(static_cast<std::uint64_t>(*AlignmentValue))) { + return semanticError("G_ASSERT_ALIGN requires a positive power-of-two alignment"); + } + return {}; + } + case GO::G_CONSTANT: { + if (const auto *Payload = Operands[1].getIf<IntegerImmediateOperand>(); + Payload != nullptr && Payload->Value.bitWidth() != TypeAt(0).scalarBitWidth()) { + return semanticError("G_CONSTANT integer width must match its destination LowLevelType"); + } + return {}; + } + case GO::G_FCONSTANT: { + const auto *Payload = Operands[1].getIf<FloatingImmediateOperand>(); + if (Payload == nullptr) + return semanticError("G_FCONSTANT requires a floating-point immediate"); + const LowLevelType DestinationType = TypeAt(0); + if (Payload->Bits.bitWidth() != DestinationType.scalarBitWidth()) + return semanticError("G_FCONSTANT width must match its destination LowLevelType"); + return {}; + } + case GO::G_BITCAST: { + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); + if (Dst.isPointerOrPointerVector() != Src.isPointerOrPointerVector()) + return semanticError("G_BITCAST cannot cross the pointer/non-pointer boundary"); + if (minimumBitWidth(Dst) != minimumBitWidth(Src) || + Dst.isScalableVector() != Src.isScalableVector()) { + return semanticError("G_BITCAST source and destination sizes must match"); + } + if (Dst == Src) + return semanticError("G_BITCAST must change the LowLevelType"); + return {}; + } + case GO::G_INTTOPTR: + case GO::G_PTRTOINT: + return requireSameShape(TypeAt(0), TypeAt(1), + "pointer/integer conversion must preserve vector shape"); + case GO::G_ADDRSPACE_CAST: { + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); + if (auto Shape = requireSameShape(Dst, Src, "G_ADDRSPACE_CAST must preserve vector shape"); + !Shape) { + return Shape; + } + if (Dst.addressSpace() == Src.addressSpace()) + return semanticError("G_ADDRSPACE_CAST must change the address space"); + return {}; + } + case GO::G_ANYEXT: + case GO::G_SEXT: + case GO::G_ZEXT: + case GO::G_FPEXT: + return VerifyWidthChange(true); + case GO::G_TRUNC: + case GO::G_TRUNC_SSAT_S: + case GO::G_TRUNC_SSAT_U: + case GO::G_TRUNC_USAT_U: + case GO::G_FPTRUNC: + return VerifyWidthChange(false); + case GO::G_PTR_ADD: + case GO::G_PTRMASK: + return requireSameShape(TypeAt(1), TypeAt(2), + "pointer arithmetic operands must have matching scalar/vector shape"); + case GO::G_ICMP: + case GO::G_FCMP: + case GO::G_STRICT_FCMP: + case GO::G_STRICT_FCMPS: { + const PredicateOperand *Predicate = Operands[1].getIf<PredicateOperand>(); + if (Predicate == nullptr || ((Generic == GO::G_ICMP) != Predicate->isInteger())) { + return semanticError("generic comparison predicate kind does not match its opcode"); + } + const LowLevelType Result = TypeAt(0); + const LowLevelType Compared = TypeAt(2); + if (!Compared.isVector()) + return Result.isCategoryNeutralScalar() + ? Expected<void>{} + : semanticError("scalar generic comparison requires a scalar integer result"); + return requireI1WithShape(Result, Compared, false); + } + case GO::G_SCMP: + case GO::G_UCMP: + if (TypeAt(0).scalarBitWidth() < 2) + return semanticError("G_SCMP/G_UCMP result elements must be at least two bits wide"); + return requireSameShape(TypeAt(0), TypeAt(1), + "G_SCMP/G_UCMP must preserve scalar/vector shape"); + case GO::G_SELECT: + return requireI1WithShape(TypeAt(1), TypeAt(0), true); + case GO::G_UADDO: + case GO::G_USUBO: + case GO::G_SADDO: + case GO::G_SSUBO: + case GO::G_UMULO: + case GO::G_SMULO: + case GO::G_UADDE: + case GO::G_USUBE: + case GO::G_SADDE: + case GO::G_SSUBE: + return requireI1WithShape(TypeAt(1), TypeAt(0), false); + case GO::G_CTTZ: + case GO::G_CTTZ_ZERO_POISON: + case GO::G_CTLZ: + case GO::G_CTLZ_ZERO_POISON: + case GO::G_CTLS: + case GO::G_CTPOP: + return requireSameShape(TypeAt(0), TypeAt(1), + "generic bit-count operation must preserve scalar/vector shape"); + case GO::G_BSWAP: + if ((TypeAt(0).scalarBitWidth() % 16) != 0) + return semanticError("G_BSWAP element width must be a multiple of 16 bits"); + return {}; + case GO::G_FPTOSI: + case GO::G_FPTOUI: + case GO::G_FPTOSI_SAT: + case GO::G_FPTOUI_SAT: + case GO::G_SITOFP: + case GO::G_UITOFP: + case GO::G_LROUND: + case GO::G_LLROUND: + return requireSameShape(TypeAt(0), TypeAt(1), + "generic numeric conversion must preserve scalar/vector shape"); + case GO::G_FCOPYSIGN: + return requireSameShape(TypeAt(0), TypeAt(2), + "G_FCOPYSIGN operands must have matching scalar/vector shape"); + case GO::G_IS_FPCLASS: { + if (auto Shape = requireI1WithShape(TypeAt(0), TypeAt(1), false); !Shape) + return Shape; + const auto Classes = plainImmediate(Operands[2]); + if (!Classes || *Classes < 0 || *Classes > 0x3ff) + return semanticError("G_IS_FPCLASS class mask must fit the supported ten-bit mask"); + return {}; + } + case GO::G_FPOWI: + case GO::G_FLDEXP: + case GO::G_STRICT_FLDEXP: + return requireSameShape(TypeAt(0), TypeAt(2), + "floating operation exponent must match the result vector shape"); + case GO::G_FFREXP: + return requireSameShape(TypeAt(0), TypeAt(1), + "G_FFREXP mantissa and exponent results must have matching shape"); + case GO::G_MERGE_VALUES: { + if (TypeAt(0).isVector() || TypeAt(1).isVector()) + return semanticError("G_MERGE_VALUES only merges scalar values"); + if (minimumBitWidth(TypeAt(0)) != minimumBitWidth(TypeAt(1)) * (Operands.size() - 1)) + return semanticError("G_MERGE_VALUES source widths do not fill the destination"); + return {}; + } + case GO::G_UNMERGE_VALUES: { + const std::size_t DestinationCount = Operands.size() - 1; + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(DestinationCount); + if (minimumBitWidth(Src) != minimumBitWidth(Dst) * DestinationCount) + return semanticError("G_UNMERGE_VALUES destination widths do not cover the source"); + if (Dst.isVector() && (!Src.isVector() || Dst.isScalableVector() != Src.isScalableVector() || + scalarElementType(Dst) != scalarElementType(Src))) { + return semanticError("vector G_UNMERGE_VALUES operands have incompatible element types"); + } + return {}; + } + case GO::G_EXTRACT: { + const auto Offset = plainImmediate(Operands[2]); + const std::uint64_t DstBits = minimumBitWidth(TypeAt(0)); + const std::uint64_t SrcBits = minimumBitWidth(TypeAt(1)); + if (!Offset || *Offset < 0 || DstBits >= SrcBits || + static_cast<std::uint64_t>(*Offset) > SrcBits - DstBits) { + return semanticError("G_EXTRACT offset and result width must stay inside its source"); + } + return {}; + } + case GO::G_INSERT: { + const auto Offset = plainImmediate(Operands[3]); + const std::uint64_t DstBits = minimumBitWidth(TypeAt(0)); + const std::uint64_t InsertBits = minimumBitWidth(TypeAt(2)); + if (!Offset || *Offset < 0 || InsertBits >= DstBits || + static_cast<std::uint64_t>(*Offset) > DstBits - InsertBits) { + return semanticError("G_INSERT offset and inserted width must stay inside its result"); + } + return {}; + } + case GO::G_BUILD_VECTOR: + case GO::G_BUILD_VECTOR_TRUNC: { + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); + if (!Dst.isVector() || Dst.isScalableVector() || Src.isVector()) + return semanticError("G_BUILD_VECTOR requires a fixed vector result and scalar sources"); + if (Dst.minimumElementCount() != Operands.size() - 1) + return semanticError("G_BUILD_VECTOR source count must equal the result lane count"); + const LowLevelType Element = scalarElementType(Dst); + if (Generic == GO::G_BUILD_VECTOR && Element != Src) + return semanticError( + "G_BUILD_VECTOR source LowLevelType must equal the result element LowLevelType"); + if (Generic == GO::G_BUILD_VECTOR_TRUNC && Src.scalarBitWidth() <= Element.scalarBitWidth()) + return semanticError("G_BUILD_VECTOR_TRUNC sources must be wider than result elements"); + return {}; + } + case GO::G_CONCAT_VECTORS: { + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); + if (!Dst.isVector() || !Src.isVector() || Dst.isScalableVector() != Src.isScalableVector() || + scalarElementType(Dst) != scalarElementType(Src) || + Dst.minimumElementCount() != Src.minimumElementCount() * (Operands.size() - 1)) { + return semanticError("G_CONCAT_VECTORS operands do not exactly tile the result vector"); + } + return {}; + } + case GO::G_VSCALE: { + const auto *Scale = Operands[1].getIf<IntegerImmediateOperand>(); + if (TypeAt(0).isVector() || Scale == nullptr || Scale->Value.isZero() || + Scale->Value.bitWidth() != TypeAt(0).scalarBitWidth()) { + return semanticError("G_VSCALE requires a nonzero typed constant matching a scalar result"); + } + return {}; + } + case GO::G_SHL: + case GO::G_LSHR: + case GO::G_ASHR: + case GO::G_ROTR: + case GO::G_ROTL: + case GO::G_USHLSAT: + case GO::G_SSHLSAT: + case GO::G_FSHL: + case GO::G_FSHR: + return requireSameShape(TypeAt(1), TypeAt(Operands.size() - 1), + "shift amount must match the shifted value's scalar/vector shape"); + case GO::G_INSERT_SUBVECTOR: + case GO::G_EXTRACT_SUBVECTOR: { + const bool IsInsert = Generic == GO::G_INSERT_SUBVECTOR; + const LowLevelType Outer = TypeAt(IsInsert ? 0 : 1); + const LowLevelType Inner = TypeAt(IsInsert ? 2 : 0); + const auto Index = plainImmediate(Operands[IsInsert ? 3 : 2]); + if (!Outer.isVector() || !Inner.isVector() || + scalarElementType(Outer) != scalarElementType(Inner) || !Index || *Index < 0) { + return semanticError("subvector operation has incompatible vector types or index"); + } + if ((!Outer.isScalableVector() && Inner.isScalableVector()) || + Inner.minimumElementCount() > Outer.minimumElementCount()) { + return semanticError("subvector operation cannot place a larger vector inside a smaller one"); + } + const std::uint64_t Start = static_cast<std::uint64_t>(*Index); + const std::uint64_t InnerLength = Inner.minimumElementCount(); + const std::uint64_t OuterLength = Outer.minimumElementCount(); + const bool MixedScalability = Outer.isScalableVector() && !Inner.isScalableVector(); + if ((Start % InnerLength) != 0 || Start >= OuterLength || + (!MixedScalability && Start + InnerLength > OuterLength)) { + return semanticError("subvector index is misaligned or outside the containing vector"); + } + return {}; + } + case GO::G_SHUFFLE_VECTOR: { + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); + const auto *Mask = Operands[3].getIf<ShuffleMaskOperand>(); + if (!Dst.isVector() || !Src.isVector() || Dst.isScalableVector() || Src.isScalableVector() || + scalarElementType(Dst) != scalarElementType(Src) || Mask == nullptr || + Mask->Elements.size() != Dst.minimumElementCount()) { + return semanticError("G_SHUFFLE_VECTOR types and mask length are inconsistent"); + } + const std::uint64_t SourceLanes = Src.minimumElementCount(); + for (const std::int32_t Element : Mask->Elements) { + if (Element < ShuffleMaskOperand::Undef || + (Element >= 0 && static_cast<std::uint64_t>(Element) >= 2 * SourceLanes)) { + return semanticError("G_SHUFFLE_VECTOR mask contains an invalid lane index"); + } + } + return {}; + } + case GO::G_SPLAT_VECTOR: { + const LowLevelType Dst = TypeAt(0); + const LowLevelType Src = TypeAt(1); + if (!Dst.isScalableVector() || Src.isVector() || + scalarElementType(Dst).scalarBitWidth() > Src.scalarBitWidth()) { + return semanticError("G_SPLAT_VECTOR requires a scalable result with no wider elements"); + } + return {}; + } + case GO::G_STEP_VECTOR: { + const LowLevelType Dst = TypeAt(0); + const auto *Step = Operands[1].getIf<IntegerImmediateOperand>(); + if (!Dst.isScalableVector() || Step == nullptr || Step->Value.isZero() || + Step->Value.isNegative() || Step->Value.bitWidth() != Dst.scalarBitWidth()) { + return semanticError("G_STEP_VECTOR step must be a positive typed element-width constant"); + } + return {}; + } + case GO::G_INSERT_VECTOR_ELT: { + const LowLevelType Vector = TypeAt(0); + if (scalarElementType(Vector) != TypeAt(2) || TypeAt(3).isVector()) + return semanticError("G_INSERT_VECTOR_ELT element or index LowLevelType is incompatible"); + return {}; + } + case GO::G_EXTRACT_VECTOR_ELT: { + const LowLevelType Vector = TypeAt(1); + if (scalarElementType(Vector) != TypeAt(0) || TypeAt(2).isVector()) + return semanticError("G_EXTRACT_VECTOR_ELT result or index LowLevelType is incompatible"); + return {}; + } + case GO::G_VECTOR_COMPRESS: + return requireI1WithShape(TypeAt(2), TypeAt(0), false); + case GO::G_VECREDUCE_SEQ_FADD: + case GO::G_VECREDUCE_SEQ_FMUL: + if (TypeAt(0).isVector() || TypeAt(1).isVector() || !TypeAt(2).isVector() || + scalarElementType(TypeAt(2)) != TypeAt(0) || TypeAt(0) != TypeAt(1)) { + return semanticError("sequential vector reduction accumulator and element LLTs must match"); + } + return {}; + case GO::G_VECREDUCE_FADD: + case GO::G_VECREDUCE_FMUL: + case GO::G_VECREDUCE_FMAX: + case GO::G_VECREDUCE_FMIN: + case GO::G_VECREDUCE_FMAXIMUM: + case GO::G_VECREDUCE_FMINIMUM: + case GO::G_VECREDUCE_ADD: + case GO::G_VECREDUCE_MUL: + case GO::G_VECREDUCE_AND: + case GO::G_VECREDUCE_OR: + case GO::G_VECREDUCE_XOR: + case GO::G_VECREDUCE_SMAX: + case GO::G_VECREDUCE_SMIN: + case GO::G_VECREDUCE_UMAX: + case GO::G_VECREDUCE_UMIN: + if (TypeAt(0).isVector() || !TypeAt(1).isVector() || + scalarElementType(TypeAt(1)) != TypeAt(0)) { + return semanticError("vector reduction result must equal the source element LowLevelType"); + } + return {}; + case GO::G_SBFX: + case GO::G_UBFX: + if (TypeAt(0).isVector() || TypeAt(2).isVector() || TypeAt(3).isVector()) + return semanticError("generic bit-field extraction only supports scalar operands"); + return {}; + case GO::G_BRCOND: + if (!TypeAt(0).isScalarValue()) + return semanticError("G_BRCOND condition must be a scalar value"); + return {}; + case GO::G_BRJT: + if (TypeAt(0).isVector() || TypeAt(2).isVector()) + return semanticError("G_BRJT table pointer and index must be scalar values"); + return {}; + case GO::G_INTRINSIC: + case GO::G_INTRINSIC_W_SIDE_EFFECTS: + case GO::G_INTRINSIC_CONVERGENT: + case GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: { + const auto FirstUse = std::ranges::find_if_not(Operands, [](const MachineOperand &Operand) { + return Operand.isRegister() && Operand.isDef(); + }); + if (FirstUse == Operands.end() || FirstUse->kind() != MachineOperand::Kind::Intrinsic) + return semanticError("generic intrinsic must place its intrinsic ID after all definitions"); + return {}; + } + case GO::G_INDEXED_LOAD: + case GO::G_INDEXED_SEXTLOAD: + case GO::G_INDEXED_ZEXTLOAD: + if (const auto Mode = plainImmediate(Operands[4]); !Mode || (*Mode != 0 && *Mode != 1)) + return semanticError("generic indexed-load address mode must be immediate 0 or 1"); + return {}; + case GO::G_INDEXED_STORE: + if (const auto Mode = plainImmediate(Operands[4]); !Mode || (*Mode != 0 && *Mode != 1)) + return semanticError("G_INDEXED_STORE address mode must be immediate 0 or 1"); + return {}; + case GO::G_ATOMIC_CMPXCHG_WITH_SUCCESS: + return requireI1WithShape(TypeAt(1), TypeAt(0), false); + case GO::G_FENCE: { + const auto Ordering = plainImmediate(Operands[0]); + const auto Scope = plainImmediate(Operands[1]); + if (!Ordering || *Ordering < 4 || *Ordering > 7 || !Scope || *Scope < 0 || + static_cast<std::uint64_t>(*Scope) > std::numeric_limits<std::uint32_t>::max()) { + return semanticError("G_FENCE requires acquire-or-stronger ordering and a uint32 scope"); + } + return {}; + } + case GO::G_PREFETCH: { + const auto ReadWrite = plainImmediate(Operands[1]); + const auto Locality = plainImmediate(Operands[2]); + const auto CacheType = plainImmediate(Operands[3]); + if (!ReadWrite || *ReadWrite < 0 || *ReadWrite > 1 || !Locality || *Locality < 0 || + *Locality > 3 || !CacheType || *CacheType < 0 || *CacheType > 1) { + return semanticError("G_PREFETCH control operands are outside their defined ranges"); + } + return {}; + } + case GO::G_VAARG: { + const auto Align = plainImmediate(Operands[2]); + if (!Align || *Align <= 0 || !std::has_single_bit(static_cast<std::uint64_t>(*Align))) + return semanticError("G_VAARG alignment must be a positive power of two"); + return {}; + } + case GO::G_DYN_STACKALLOC: { + const auto Align = plainImmediate(Operands[2]); + if (!Align || *Align <= 0 || !std::has_single_bit(static_cast<std::uint64_t>(*Align))) + return semanticError("G_DYN_STACKALLOC alignment must be a positive power of two"); + return {}; + } + case GO::G_MEMCPY: + case GO::G_MEMMOVE: + if (const auto Tail = plainImmediate(Operands[3]); !Tail || (*Tail != 0 && *Tail != 1)) + return semanticError("generic memory-transfer tail flag must be immediate 0 or 1"); + return {}; + case GO::G_MEMSET: + if (TypeAt(1).isVector() || TypeAt(1).scalarBitWidth() != 8) + return semanticError("G_MEMSET fill value must be a scalar i8 value"); + if (const auto Tail = plainImmediate(Operands[3]); !Tail || (*Tail != 0 && *Tail != 1)) + return semanticError("G_MEMSET tail flag must be immediate 0 or 1"); + return {}; + case GO::G_BZERO: + if (const auto Tail = plainImmediate(Operands[2]); !Tail || (*Tail != 0 && *Tail != 1)) + return semanticError("G_BZERO tail flag must be immediate 0 or 1"); + return {}; + case GO::G_UBSANTRAP: { + const auto Kind = plainImmediate(Operands[0]); + if (!Kind || *Kind < -128 || *Kind > 127) + return semanticError("G_UBSANTRAP kind must fit in a signed eight-bit immediate"); + return {}; + } + case GO::G_INTRINSIC_FPTRUNC_ROUND: { + const auto Rounding = plainImmediate(Operands[2]); + if (!Rounding || *Rounding < 0 || (*Rounding > 4 && *Rounding != 7)) + return semanticError("floating truncation rounding mode is invalid"); + return VerifyWidthChange(false); + } + default: + return {}; + } +} + +[[nodiscard]] Expected<void> verifyGenericInstructionSemantics(const MachineInstr &Instruction, + const MachineRegisterInfo &RegInfo) { + const auto Operands = Instruction.operands(); + const auto MemoryOperands = Instruction.memOperands(); + const GO Generic = static_cast<GO>(Instruction.opcode().value()); + + const auto TypeAt = [&](std::size_t Index) -> Expected<LowLevelType> { + if (Index >= Operands.size()) + return makeUnexpected(ErrorCode::InternalError, + "generic semantic verifier operand index is out of range"); + const Register *Reg = Operands[Index].getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return makeUnexpected(ErrorCode::InternalError, + "generic semantic verifier expected a virtual register"); + auto Info = RegInfo.virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Type) + return makeUnexpected(ErrorCode::InternalError, + "generic semantic verifier found a register without an LowLevelType"); + return *Info->get().Type; + }; + + const auto VerifyAddress = [&](std::size_t OperandIndex, + const MachineMemOperand &Memory) -> Expected<void> { + auto Type = TypeAt(OperandIndex); + if (!Type) + return std::unexpected(Type.error()); + if (!Type->isPointer() || Type->addressSpace() != Memory.description().AddressSpace) + return semanticError( + "generic memory address LowLevelType and MMO address space do not match"); + return {}; + }; + + enum class SizeRelation : std::uint8_t { + Equal, + NoLarger, + StrictlySmaller, + }; + const auto VerifySize = [&](LowLevelType ValueType, const MachineMemOperand &Memory, + SizeRelation Relation) -> Expected<void> { + const MemorySize Size = Memory.description().Size; + if (!Size.isKnown()) + return semanticError("typed generic memory access requires a known MMO size"); + if (Size.isScalable() != ValueType.isScalableVector()) + return semanticError("generic value and MMO scalability do not match"); + const std::uint64_t MemoryBytes = Size.minimumBytes(); + const std::uint64_t ValueBytes = minimumStoreBytes(ValueType); + const bool Valid = Relation == SizeRelation::Equal + ? MemoryBytes == ValueBytes + : (Relation == SizeRelation::NoLarger ? MemoryBytes <= ValueBytes + : MemoryBytes < ValueBytes); + if (!Valid) + return semanticError( + "generic memory access size is incompatible with its value LowLevelType"); + return {}; + }; + + const auto VerifySingleAccess = [&](std::size_t AddressIndex, std::size_t ValueIndex, + bool MustLoad, bool MustStore, + SizeRelation Relation) -> Expected<void> { + if (MemoryOperands.size() != 1) + return semanticError("generic scalar memory instruction requires exactly one MMO"); + const MachineMemOperand &Memory = *MemoryOperands.front(); + if (Memory.isLoad() != MustLoad || Memory.isStore() != MustStore) + return semanticError("generic instruction and MMO access directions do not match"); + if (auto Address = VerifyAddress(AddressIndex, Memory); !Address) + return Address; + auto ValueType = TypeAt(ValueIndex); + if (!ValueType) + return std::unexpected(ValueType.error()); + return VerifySize(*ValueType, Memory, Relation); + }; + + switch (Generic) { + case GO::G_LOAD: + return VerifySingleAccess(1, 0, true, false, SizeRelation::NoLarger); + case GO::G_SEXTLOAD: + case GO::G_ZEXTLOAD: + case GO::G_FPEXTLOAD: + return VerifySingleAccess(1, 0, true, false, SizeRelation::StrictlySmaller); + case GO::G_STORE: + return VerifySingleAccess(1, 0, false, true, SizeRelation::NoLarger); + case GO::G_FPTRUNCSTORE: + return VerifySingleAccess(1, 0, false, true, SizeRelation::StrictlySmaller); + case GO::G_INDEXED_LOAD: + return VerifySingleAccess(2, 0, true, false, SizeRelation::NoLarger); + case GO::G_INDEXED_SEXTLOAD: + case GO::G_INDEXED_ZEXTLOAD: + return VerifySingleAccess(2, 0, true, false, SizeRelation::StrictlySmaller); + case GO::G_INDEXED_STORE: + return VerifySingleAccess(2, 1, false, true, SizeRelation::NoLarger); + case GO::G_ATOMIC_CMPXCHG_WITH_SUCCESS: + if (MemoryOperands.size() != 1 || + MemoryOperands.front()->description().AtomicKind != AtomicOperation::CompareExchange) + return semanticError("G_ATOMIC_CMPXCHG_WITH_SUCCESS requires one compare-exchange MMO"); + return VerifySingleAccess(2, 0, true, true, SizeRelation::NoLarger); + case GO::G_ATOMIC_CMPXCHG: + if (MemoryOperands.size() != 1 || + MemoryOperands.front()->description().AtomicKind != AtomicOperation::CompareExchange) + return semanticError("G_ATOMIC_CMPXCHG requires one compare-exchange MMO"); + return VerifySingleAccess(1, 0, true, true, SizeRelation::NoLarger); + case GO::G_ATOMICRMW_XCHG: + case GO::G_ATOMICRMW_ADD: + case GO::G_ATOMICRMW_SUB: + case GO::G_ATOMICRMW_AND: + case GO::G_ATOMICRMW_NAND: + case GO::G_ATOMICRMW_OR: + case GO::G_ATOMICRMW_XOR: + case GO::G_ATOMICRMW_MAX: + case GO::G_ATOMICRMW_MIN: + case GO::G_ATOMICRMW_UMAX: + case GO::G_ATOMICRMW_UMIN: + case GO::G_ATOMICRMW_FADD: + case GO::G_ATOMICRMW_FSUB: + case GO::G_ATOMICRMW_FMAX: + case GO::G_ATOMICRMW_FMIN: + case GO::G_ATOMICRMW_FMAXIMUM: + case GO::G_ATOMICRMW_FMINIMUM: + case GO::G_ATOMICRMW_FMAXIMUMNUM: + case GO::G_ATOMICRMW_FMINIMUMNUM: + case GO::G_ATOMICRMW_UINC_WRAP: + case GO::G_ATOMICRMW_UDEC_WRAP: + case GO::G_ATOMICRMW_USUB_COND: + case GO::G_ATOMICRMW_USUB_SAT: + if (MemoryOperands.size() != 1 || + MemoryOperands.front()->description().AtomicKind != AtomicOperation::ReadModifyWrite) + return semanticError("generic atomic RMW requires one read-modify-write MMO"); + return VerifySingleAccess(1, 0, true, true, SizeRelation::NoLarger); + case GO::G_PREFETCH: { + if (MemoryOperands.size() != 1) + return semanticError("G_PREFETCH requires exactly one MMO"); + const auto ReadWrite = plainImmediate(Operands[1]); + if (!ReadWrite) + return semanticError("G_PREFETCH is missing its validated read/write selector"); + const MachineMemOperand &Memory = *MemoryOperands.front(); + if ((*ReadWrite == 0 && (!Memory.isLoad() || Memory.isStore())) || + (*ReadWrite == 1 && (Memory.isLoad() || !Memory.isStore()))) { + return semanticError("G_PREFETCH selector and MMO access direction do not match"); + } + return VerifyAddress(0, Memory); + } + case GO::G_VASTART: + if (MemoryOperands.size() != 1 || MemoryOperands.front()->isLoad() || + !MemoryOperands.front()->isStore()) { + return semanticError("G_VASTART requires exactly one store-only MMO"); + } + return VerifyAddress(0, *MemoryOperands.front()); + case GO::G_VAARG: { + return MemoryOperands.empty() + ? Expected<void>{} + : semanticError("G_VAARG must not carry MMOs before legalization"); + } + case GO::G_MEMCPY: + case GO::G_MEMCPY_INLINE: + case GO::G_MEMMOVE: { + if (MemoryOperands.size() != 2 || MemoryOperands[0]->isLoad() || + !MemoryOperands[0]->isStore() || !MemoryOperands[1]->isLoad() || + MemoryOperands[1]->isStore()) { + return semanticError("generic memory transfer requires store and load MMOs in that order"); + } + if (MemoryOperands[0]->description().Size != MemoryOperands[1]->description().Size) + return semanticError("generic memory-transfer MMOs must have identical sizes"); + if (auto Destination = VerifyAddress(0, *MemoryOperands[0]); !Destination) + return Destination; + return VerifyAddress(1, *MemoryOperands[1]); + } + case GO::G_MEMSET: + case GO::G_BZERO: + if (MemoryOperands.size() != 1 || MemoryOperands.front()->isLoad() || + !MemoryOperands.front()->isStore()) { + return semanticError("generic memory fill requires exactly one store-only MMO"); + } + return VerifyAddress(0, *MemoryOperands.front()); + default: + return {}; + } +} + +void configureCoreOpcodes(DescriptorRegistry &Registry) { + const auto RegDef = untypedRegister(RegisterOperandRole::Def); + const auto RegUse = untypedRegister(RegisterOperandRole::Use); + const auto TransferDef = registeredTargetFlags(RegDef); + const auto TransferUse = registeredTargetFlags(RegUse); + const auto Any = anyOperand(); + const auto VariadicAny = repeated(Any); + const auto VariadicUse = repeated(RegUse); + + Registry.configure( + {GO::PHI}, + {TransferDef, repeated(TransferUse), repeated(kind(MachineOperand::Kind::BasicBlock))}, 1, + {InstructionFlag::Phi}, true, 1, false, PseudoOwner::PhiElimination, + PipelineStage::SSAEliminated); + Registry.configure({GO::INLINEASM}, {VariadicAny}, 0, + {InstructionFlag::ProducesCode, InstructionFlag::TerminalEmission, + InstructionFlag::UnmodeledSideEffects}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::INLINEASM_BR}, {VariadicAny}, 0, + {InstructionFlag::ProducesCode, InstructionFlag::TerminalEmission, + InstructionFlag::UnmodeledSideEffects, InstructionFlag::HasSideEffects}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::CFI_INSTRUCTION}, {kind(MachineOperand::Kind::CFIIndex)}, 0, + {InstructionFlag::Meta, InstructionFlag::NotDuplicable}); + Registry.configure({GO::EH_LABEL, GO::GC_LABEL, GO::ANNOTATION_LABEL}, + {kind(MachineOperand::Kind::MCSymbol)}, 0, + {InstructionFlag::Meta, InstructionFlag::NotDuplicable}); + Registry.configure({GO::KILL}, {VariadicUse}, 0, {InstructionFlag::Meta}, true); + Registry.configure({GO::EXTRACT_SUBREG}, {RegDef, RegUse, immediate()}, 1, {}, false, 0, false, + PseudoOwner::PostRAExpansion, PipelineStage::PostRAFinalized); + Registry.configure({GO::INSERT_SUBREG}, {TransferDef, TransferUse, TransferUse, immediate()}, 1, + {}, false, 0, false, PseudoOwner::TwoAddressRewrite, + PipelineStage::TiedOpsRewritten); + Registry.tie(GO::INSERT_SUBREG, 0, 1); + Registry.configure({GO::IMPLICIT_DEF}, {TransferDef}, 1, + {InstructionFlag::Meta, InstructionFlag::Rematerializable}); + Registry.configure({GO::INIT_UNDEF}, {RegDef}, 1, {}, false, 0, false, + PseudoOwner::RegisterAllocator, PipelineStage::RegisterAllocated); + Registry.configure({GO::SUBREG_TO_REG}, {RegDef, RegUse, immediate()}, 1, {}, false, 0, false, + PseudoOwner::PostRAExpansion, PipelineStage::PostRAFinalized); + Registry.configure({GO::COPY_TO_REGCLASS}, {RegDef, RegUse, immediate()}, 1, {}, false, 0, false, + PseudoOwner::RegisterAllocator, PipelineStage::RegisterAllocated); + Registry.configure({GO::DBG_VALUE, GO::DBG_VALUE_LIST, GO::DBG_INSTR_REF, GO::DBG_PHI}, + {VariadicAny}, 0, {InstructionFlag::Meta}, true); + Registry.configure({GO::DBG_LABEL}, {kind(MachineOperand::Kind::Metadata)}, 0, + {InstructionFlag::Meta}); + Registry.configure({GO::REG_SEQUENCE}, + {TransferDef, repeated(TransferUse), repeated(immediate())}, 1, {}, true, 1, + false, PseudoOwner::TwoAddressRewrite, PipelineStage::TiedOpsRewritten); + Registry.configure({GO::COPY}, {TransferDef, TransferUse}, 1, {}, false, 0, false, + PseudoOwner::PostRAExpansion, PipelineStage::PostRAFinalized, true); + Registry.configure({GO::COPY_LANEMASK}, {RegDef, RegUse, kind(MachineOperand::Kind::LaneMask)}, 1, + {}, false, 0, false, PseudoOwner::PostRAExpansion, + PipelineStage::PostRAFinalized); + Registry.configure({GO::BUNDLE}, {VariadicAny}, 0, {InstructionFlag::Meta}, true, 0, false, + PseudoOwner::None, PipelineStage::Emission, true); + Registry.configure({GO::LIFETIME_START, GO::LIFETIME_END}, {immediate()}, 0, + {InstructionFlag::Meta}); + Registry.configure({GO::PSEUDO_PROBE}, {immediate(), immediate(), immediate(), immediate()}, 0, + {InstructionFlag::Meta, InstructionFlag::HasSideEffects}); + Registry.configure({GO::ARITH_FENCE}, {RegDef, RegUse}, 1, {InstructionFlag::Meta}); + Registry.tie(GO::ARITH_FENCE, 0, 1); + Registry.configure({GO::STACKMAP}, {immediate(), immediate(), VariadicAny}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::FENTRY_CALL}, {}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + false, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure( + {GO::PATCHPOINT}, + {RegDef, immediate(), immediate(), Any, immediate(), immediate(), VariadicAny}, 1, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::HasSideEffects, + InstructionFlag::ProducesCode, InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::LOAD_STACK_GUARD}, {RegDef}, 1, + {InstructionFlag::MayLoad, InstructionFlag::Rematerializable}, false, 0, false, + PseudoOwner::InstructionSelect, PipelineStage::Selected); + Registry.configure({GO::PREALLOCATED_SETUP}, {immediate()}, 0, {InstructionFlag::HasSideEffects}, + false, 0, false, PseudoOwner::InstructionSelect, PipelineStage::Selected); + Registry.configure({GO::PREALLOCATED_ARG}, {RegDef, immediate(), immediate()}, 1, + {InstructionFlag::HasSideEffects}, false, 0, false, + PseudoOwner::InstructionSelect, PipelineStage::Selected); + Registry.configure({GO::STATEPOINT}, {VariadicAny}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::LOCAL_ESCAPE}, {symbol(), kind(MachineOperand::Kind::FrameIndex)}, 0, + {InstructionFlag::Meta}); + Registry.configure({GO::FAULTING_OP}, {RegDef, VariadicAny}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::Terminator, + InstructionFlag::Branch, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::PATCHABLE_OP}, {VariadicAny}, 0, + {InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::PATCHABLE_FUNCTION_ENTER, GO::PATCHABLE_FUNCTION_EXIT}, {}, 0, + {InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + false, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission); + Registry.configure({GO::PATCHABLE_RET, GO::PATCHABLE_TAIL_CALL}, {VariadicAny}, 0, + {InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission, InstructionFlag::Terminator, + InstructionFlag::Return}, + true, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::PATCHABLE_EVENT_CALL}, {RegUse, Any}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + false, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::PATCHABLE_TYPED_EVENT_CALL}, {Any, RegUse, Any}, 0, + {InstructionFlag::Call, InstructionFlag::MayLoad, InstructionFlag::MayStore, + InstructionFlag::HasSideEffects, InstructionFlag::ProducesCode, + InstructionFlag::TerminalEmission}, + false, 0, false, PseudoOwner::AsmPrinter, PipelineStage::Emission, true); + Registry.configure({GO::ICALL_BRANCH_FUNNEL}, {VariadicAny}, 0, {InstructionFlag::HasSideEffects}, + true, 0, false, PseudoOwner::PostRAExpansion, PipelineStage::PostRAFinalized, + true); + Registry.configure({GO::FAKE_USE}, {VariadicUse}, 0, + {InstructionFlag::Meta, InstructionFlag::FakeUse}, true); + Registry.configure({GO::MEMBARRIER}, {}, 0, + {InstructionFlag::Meta, InstructionFlag::HasSideEffects}); + Registry.configure({GO::JUMP_TABLE_DEBUG_INFO}, {kind(MachineOperand::Kind::JumpTableIndex)}, 0, + {InstructionFlag::Meta}); + Registry.configure({GO::RELOC_NONE}, {symbol()}, 0, + {InstructionFlag::Meta, InstructionFlag::HasSideEffects}); + Registry.configure({GO::CONVERGENCECTRL_ENTRY, GO::CONVERGENCECTRL_ANCHOR}, {RegDef}, 1, + {InstructionFlag::Meta, InstructionFlag::Convergent}); + Registry.configure({GO::CONVERGENCECTRL_LOOP}, {RegDef, RegUse}, 1, + {InstructionFlag::Meta, InstructionFlag::Convergent}); + Registry.configure({GO::CONVERGENCECTRL_GLUE}, {RegUse}, 0, + {InstructionFlag::Meta, InstructionFlag::Convergent}); +} + +void configureGenericOpcodes(DescriptorRegistry &Registry) { + const LowLevelTypeKindSet AnyTypes = anyValueKinds(); + const LowLevelTypeKindSet IntTypes = integerKinds(); + const LowLevelTypeKindSet IntegerOrPointers = integerOrPointerKinds(); + const LowLevelTypeKindSet ScalarValues = scalarValueKinds(); + const LowLevelTypeKindSet ScalarInts = scalarIntegerKinds(); + const LowLevelTypeKindSet ScalarIntegerOrPointers = scalarIntegerOrPointerKinds(); + const LowLevelTypeKindSet FloatTypes = floatingKinds(); + const LowLevelTypeKindSet ScalarFloats = scalarFloatingKinds(); + const LowLevelTypeKindSet PointerTypes = pointerKinds(); + const LowLevelTypeKindSet ScalarPointers = scalarPointerKinds(); + const LowLevelTypeKindSet VectorTypes = vectorKinds(); + + const auto AnyDef0 = typedRegister(RegisterOperandRole::Def, AnyTypes, 0); + const auto AnyUse0 = typedRegister(RegisterOperandRole::Use, AnyTypes, 0); + const auto AnyUse1 = typedRegister(RegisterOperandRole::Use, AnyTypes, 1); + const auto ScalarDef0 = typedRegister(RegisterOperandRole::Def, ScalarValues, 0); + const auto ScalarUse0 = typedRegister(RegisterOperandRole::Use, ScalarValues, 0); + const auto IntDef0 = typedRegister(RegisterOperandRole::Def, IntTypes, 0); + const auto IntDef1 = typedRegister(RegisterOperandRole::Def, IntTypes, 1); + const auto IntUse0 = typedRegister(RegisterOperandRole::Use, IntTypes, 0); + const auto IntUse1 = typedRegister(RegisterOperandRole::Use, IntTypes, 1); + const auto IntegerOrPointerUse1 = typedRegister(RegisterOperandRole::Use, IntegerOrPointers, 1); + const auto ScalarIntDef0 = typedRegister(RegisterOperandRole::Def, ScalarInts, 0); + const auto ScalarIntDef1 = typedRegister(RegisterOperandRole::Def, ScalarInts, 1); + const auto ScalarIntUse0 = typedRegister(RegisterOperandRole::Use, ScalarInts, 0); + const auto ScalarIntUse1 = typedRegister(RegisterOperandRole::Use, ScalarInts, 1); + const auto ScalarIntUse2 = typedRegister(RegisterOperandRole::Use, ScalarInts, 2); + const auto FloatDef0 = typedRegister(RegisterOperandRole::Def, FloatTypes, 0); + const auto FloatUse0 = typedRegister(RegisterOperandRole::Use, FloatTypes, 0); + const auto FloatUse1 = typedRegister(RegisterOperandRole::Use, FloatTypes, 1); + const auto ScalarFloatDef0 = typedRegister(RegisterOperandRole::Def, ScalarFloats, 0); + const auto ScalarFloatUse0 = typedRegister(RegisterOperandRole::Use, ScalarFloats, 0); + const auto PointerDef0 = typedRegister(RegisterOperandRole::Def, PointerTypes, 0); + const auto PointerUse0 = typedRegister(RegisterOperandRole::Use, PointerTypes, 0); + const auto PointerUse1 = typedRegister(RegisterOperandRole::Use, PointerTypes, 1); + const auto ScalarPointerDef0 = typedRegister(RegisterOperandRole::Def, ScalarPointers, 0); + const auto ScalarPointerDef1 = typedRegister(RegisterOperandRole::Def, ScalarPointers, 1); + const auto ScalarPointerUse0 = typedRegister(RegisterOperandRole::Use, ScalarPointers, 0); + const auto ScalarPointerUse1 = typedRegister(RegisterOperandRole::Use, ScalarPointers, 1); + const auto ScalarPointerUse2 = typedRegister(RegisterOperandRole::Use, ScalarPointers, 2); + + Registry.configure({GO::G_ASSERT_SEXT, GO::G_ASSERT_ZEXT}, {IntDef0, IntUse0, immediate()}, 1); + Registry.configure({GO::G_ASSERT_ALIGN}, {AnyDef0, AnyUse0, immediate()}, 1); + + Registry.configure( + {GO::G_ADD, GO::G_SUB, GO::G_MUL, GO::G_SDIV, GO::G_UDIV, GO::G_SREM, + GO::G_UREM, GO::G_AND, GO::G_OR, GO::G_XOR, GO::G_ABDS, GO::G_ABDU, + GO::G_UAVGFLOOR, GO::G_UAVGCEIL, GO::G_SAVGFLOOR, GO::G_SAVGCEIL, GO::G_UMULH, GO::G_SMULH, + GO::G_UADDSAT, GO::G_SADDSAT, GO::G_USUBSAT, GO::G_SSUBSAT, GO::G_SMIN, GO::G_SMAX, + GO::G_UMIN, GO::G_UMAX, GO::G_CLMUL}, + {IntDef0, IntUse0, IntUse0}, 1); + Registry.addFlag({GO::G_ADD, GO::G_MUL, GO::G_AND, GO::G_OR, GO::G_XOR, GO::G_ABDS, GO::G_ABDU, + GO::G_UMULH, GO::G_SMULH, GO::G_UADDSAT, GO::G_SADDSAT, GO::G_SMIN, GO::G_SMAX, + GO::G_UMIN, GO::G_UMAX, GO::G_CLMUL}, + InstructionFlag::Commutable); + + Registry.configure({GO::G_SDIVREM, GO::G_UDIVREM}, {IntDef0, IntDef0, IntUse0, IntUse0}, 2); + const auto TransferDef0 = registeredTargetFlags(AnyDef0); + const auto TransferUse0 = registeredTargetFlags(AnyUse0); + Registry.configure({GO::G_IMPLICIT_DEF}, {TransferDef0}, 1); + Registry.configure( + {GO::G_PHI}, + {TransferDef0, repeated(TransferUse0), repeated(kind(MachineOperand::Kind::BasicBlock))}, 1, + {InstructionFlag::Phi}, true, 1); + Registry.configure({GO::G_FRAME_INDEX}, + {ScalarPointerDef0, kind(MachineOperand::Kind::FrameIndex)}, 1); + Registry.configure({GO::G_GLOBAL_VALUE}, + {ScalarPointerDef0, kind(MachineOperand::Kind::GlobalSymbol)}, 1); + Registry.configure( + {GO::G_PTRAUTH_GLOBAL_VALUE}, + {ScalarPointerDef0, ScalarPointerUse1, immediate(), ScalarIntUse2, immediate()}, 1); + Registry.configure({GO::G_CONSTANT_POOL}, + {ScalarPointerDef0, kind(MachineOperand::Kind::ConstantPoolIndex)}, 1); + Registry.configure({GO::G_EXTRACT}, {AnyDef0, AnyUse1, immediate()}, 1); + Registry.configure({GO::G_UNMERGE_VALUES}, {AnyDef0, repeated(AnyDef0), AnyUse1}, 2, {}, true, 1); + Registry.configure({GO::G_INSERT}, {AnyDef0, AnyUse0, AnyUse1, immediate()}, 1); + Registry.configure({GO::G_MERGE_VALUES}, {AnyDef0, AnyUse1, repeated(AnyUse1)}, 1, {}, true, 1); + + const auto VectorDef0 = typedRegister(RegisterOperandRole::Def, VectorTypes, 0); + Registry.configure({GO::G_BUILD_VECTOR, GO::G_BUILD_VECTOR_TRUNC}, + {VectorDef0, AnyUse1, repeated(AnyUse1)}, 1, {}, true); + Registry.configure({GO::G_CONCAT_VECTORS}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1), + repeated(typedRegister(RegisterOperandRole::Use, VectorTypes, 1))}, + 1, {}, true, 1); + + Registry.configure({GO::G_PTRTOINT}, {IntDef0, PointerUse1}, 1); + Registry.configure({GO::G_INTTOPTR}, {PointerDef0, IntUse1}, 1); + Registry.configure({GO::G_BITCAST}, {AnyDef0, AnyUse1}, 1); + Registry.configure({GO::G_FREEZE, GO::G_CONSTANT_FOLD_BARRIER}, {AnyDef0, AnyUse0}, 1); + Registry.configure({GO::G_INTRINSIC_FPTRUNC_ROUND}, {FloatDef0, FloatUse1, immediate()}, 1); + Registry.configure({GO::G_INTRINSIC_TRUNC, GO::G_INTRINSIC_ROUND, GO::G_INTRINSIC_ROUNDEVEN}, + {FloatDef0, FloatUse0}, 1); + Registry.configure({GO::G_INTRINSIC_LRINT, GO::G_INTRINSIC_LLRINT}, {IntDef0, FloatUse1}, 1); + Registry.configure({GO::G_READCYCLECOUNTER, GO::G_READSTEADYCOUNTER}, + {typedRegister(RegisterOperandRole::Def, ScalarInts, 0)}, 1, + {InstructionFlag::HasSideEffects}); + + Registry.configure({GO::G_LOAD}, {AnyDef0, ScalarPointerUse1}, 1, {InstructionFlag::MayLoad}, + false, 0, true); + Registry.configure({GO::G_SEXTLOAD, GO::G_ZEXTLOAD}, {IntDef0, ScalarPointerUse1}, 1, + {InstructionFlag::MayLoad}, false, 0, true); + Registry.configure({GO::G_FPEXTLOAD}, {FloatDef0, ScalarPointerUse1}, 1, + {InstructionFlag::MayLoad}, false, 0, true); + Registry.configure({GO::G_INDEXED_LOAD}, + {AnyDef0, ScalarPointerDef1, ScalarPointerUse1, ScalarIntUse2, immediate()}, 2, + {InstructionFlag::MayLoad}, false, 0, true); + Registry.configure({GO::G_INDEXED_SEXTLOAD, GO::G_INDEXED_ZEXTLOAD}, + {IntDef0, ScalarPointerDef1, ScalarPointerUse1, ScalarIntUse2, immediate()}, 2, + {InstructionFlag::MayLoad}, false, 0, true); + Registry.configure({GO::G_STORE}, {AnyUse0, ScalarPointerUse1}, 0, {InstructionFlag::MayStore}, + false, 0, true); + Registry.configure({GO::G_FPTRUNCSTORE}, {FloatUse0, ScalarPointerUse1}, 0, + {InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_INDEXED_STORE}, + {ScalarPointerDef0, AnyUse1, ScalarPointerUse0, ScalarIntUse2, immediate()}, 1, + {InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMIC_CMPXCHG_WITH_SUCCESS}, + {ScalarDef0, ScalarIntDef1, ScalarPointerUse2, ScalarUse0, ScalarUse0}, 2, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMIC_CMPXCHG}, + {ScalarDef0, ScalarPointerUse1, ScalarUse0, ScalarUse0}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMICRMW_XCHG}, {ScalarDef0, ScalarPointerUse1, ScalarUse0}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMICRMW_ADD, GO::G_ATOMICRMW_SUB, GO::G_ATOMICRMW_AND, + GO::G_ATOMICRMW_NAND, GO::G_ATOMICRMW_OR, GO::G_ATOMICRMW_XOR, + GO::G_ATOMICRMW_MAX, GO::G_ATOMICRMW_MIN, GO::G_ATOMICRMW_UMAX, + GO::G_ATOMICRMW_UMIN, GO::G_ATOMICRMW_UINC_WRAP, GO::G_ATOMICRMW_UDEC_WRAP, + GO::G_ATOMICRMW_USUB_COND, GO::G_ATOMICRMW_USUB_SAT}, + {ScalarIntDef0, ScalarPointerUse1, ScalarIntUse0}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_ATOMICRMW_FADD, GO::G_ATOMICRMW_FSUB, GO::G_ATOMICRMW_FMAX, + GO::G_ATOMICRMW_FMIN, GO::G_ATOMICRMW_FMAXIMUM, GO::G_ATOMICRMW_FMINIMUM, + GO::G_ATOMICRMW_FMAXIMUMNUM, GO::G_ATOMICRMW_FMINIMUMNUM}, + {ScalarFloatDef0, ScalarPointerUse1, ScalarFloatUse0}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_FENCE}, {immediate(), immediate()}, 0, + {InstructionFlag::HasSideEffects}); + Registry.configure( + {GO::G_PREFETCH}, {ScalarPointerUse0, immediate(), immediate(), immediate()}, 0, + {InstructionFlag::MayLoad, InstructionFlag::MayStore, InstructionFlag::HasSideEffects}, false, + 0, true); + Registry.configure({GO::G_BRCOND}, {ScalarIntUse0, kind(MachineOperand::Kind::BasicBlock)}, 0, + {InstructionFlag::Terminator, InstructionFlag::Branch}); + Registry.configure({GO::G_BRINDIRECT}, {ScalarPointerUse0}, 0, + {InstructionFlag::Terminator, InstructionFlag::Branch, + InstructionFlag::IndirectBranch, InstructionFlag::Barrier}); + Registry.configure({GO::G_INVOKE_REGION_START}, {}, 0, {InstructionFlag::Terminator}); + Registry.configure({GO::G_INTRINSIC, GO::G_INTRINSIC_W_SIDE_EFFECTS, GO::G_INTRINSIC_CONVERGENT, + GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + {repeated(anyOperand())}, 0, {}, true, 1, false, PseudoOwner::None, + PipelineStage::Emission, true); + Registry.addFlag({GO::G_INTRINSIC_W_SIDE_EFFECTS, GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + InstructionFlag::HasSideEffects); + Registry.addFlag({GO::G_INTRINSIC_W_SIDE_EFFECTS, GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + InstructionFlag::MayLoad); + Registry.addFlag({GO::G_INTRINSIC_W_SIDE_EFFECTS, GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + InstructionFlag::MayStore); + Registry.addFlag({GO::G_INTRINSIC_CONVERGENT, GO::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS}, + InstructionFlag::Convergent); + + Registry.configure({GO::G_ANYEXT, GO::G_SEXT, GO::G_ZEXT}, {IntDef0, IntUse1}, 1); + Registry.configure({GO::G_TRUNC, GO::G_TRUNC_SSAT_S, GO::G_TRUNC_SSAT_U, GO::G_TRUNC_USAT_U}, + {IntDef0, IntUse1}, 1); + Registry.configure( + {GO::G_CONSTANT}, + {typedRegister(RegisterOperandRole::Def, ScalarIntegerOrPointers, 0), + operand({MachineOperand::Kind::Immediate, MachineOperand::Kind::IntegerImmediate})}, + 1); + Registry.configure({GO::G_FCONSTANT}, + {ScalarFloatDef0, kind(MachineOperand::Kind::FloatingImmediate)}, 1); + Registry.configure({GO::G_VASTART}, {ScalarPointerUse0}, 0, {InstructionFlag::MayStore}, false, 0, + true); + Registry.configure({GO::G_VAARG}, {AnyDef0, ScalarPointerUse1, immediate()}, 1, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, false); + Registry.configure({GO::G_SEXT_INREG}, {IntDef0, IntUse0, immediate()}, 1); + Registry.configure( + {GO::G_SHL, GO::G_LSHR, GO::G_ASHR, GO::G_ROTR, GO::G_ROTL, GO::G_USHLSAT, GO::G_SSHLSAT}, + {IntDef0, IntUse0, IntUse1}, 1); + Registry.configure({GO::G_FSHL, GO::G_FSHR}, {IntDef0, IntUse0, IntUse0, IntUse1}, 1); + Registry.configure( + {GO::G_ICMP}, + {IntDef0, kind(MachineOperand::Kind::Predicate), IntegerOrPointerUse1, IntegerOrPointerUse1}, + 1); + Registry.configure({GO::G_FCMP}, + {IntDef0, kind(MachineOperand::Kind::Predicate), FloatUse1, FloatUse1}, 1); + Registry.configure({GO::G_SCMP, GO::G_UCMP}, {IntDef0, IntUse1, IntUse1}, 1); + Registry.configure({GO::G_SELECT}, {AnyDef0, IntUse1, AnyUse0, AnyUse0}, 1); + + Registry.configure({GO::G_UADDO, GO::G_USUBO, GO::G_SADDO, GO::G_SSUBO, GO::G_UMULO, GO::G_SMULO}, + {IntDef0, IntDef1, IntUse0, IntUse0}, 2); + Registry.configure({GO::G_UADDE, GO::G_USUBE, GO::G_SADDE, GO::G_SSUBE}, + {IntDef0, IntDef1, IntUse0, IntUse0, IntUse1}, 2); + Registry.addFlag({GO::G_UADDO, GO::G_SADDO, GO::G_UMULO, GO::G_SMULO}, + InstructionFlag::Commutable); + Registry.configure({GO::G_SMULFIX, GO::G_UMULFIX, GO::G_SMULFIXSAT, GO::G_UMULFIXSAT, + GO::G_SDIVFIX, GO::G_UDIVFIX, GO::G_SDIVFIXSAT, GO::G_UDIVFIXSAT}, + {IntDef0, IntUse0, IntUse0, immediate()}, 1); + Registry.addFlag({GO::G_SMULFIX, GO::G_UMULFIX, GO::G_SMULFIXSAT, GO::G_UMULFIXSAT}, + InstructionFlag::Commutable); + + Registry.configure({GO::G_FNEG, GO::G_FABS, GO::G_FCANONICALIZE, GO::G_FEXP, GO::G_FEXP2, + GO::G_FEXP10, GO::G_FLOG, GO::G_FLOG2, GO::G_FLOG10, GO::G_FCEIL, + GO::G_FCOS, GO::G_FSIN, GO::G_FTAN, GO::G_FACOS, GO::G_FASIN, + GO::G_FATAN, GO::G_FCOSH, GO::G_FSINH, GO::G_FTANH, GO::G_FSQRT, + GO::G_FFLOOR, GO::G_FRINT, GO::G_FNEARBYINT}, + {FloatDef0, FloatUse0}, 1); + Registry.configure({GO::G_FPEXT, GO::G_FPTRUNC}, {FloatDef0, FloatUse1}, 1); + Registry.configure({GO::G_FPTOSI, GO::G_FPTOUI, GO::G_FPTOSI_SAT, GO::G_FPTOUI_SAT}, + {IntDef0, FloatUse1}, 1); + Registry.configure({GO::G_SITOFP, GO::G_UITOFP}, {FloatDef0, IntUse1}, 1); + Registry.configure({GO::G_FCOPYSIGN}, {FloatDef0, FloatUse0, FloatUse1}, 1); + Registry.configure({GO::G_IS_FPCLASS}, {IntDef0, FloatUse1, immediate()}, 1); + Registry.configure({GO::G_FADD, GO::G_FSUB, GO::G_FMUL, GO::G_FDIV, GO::G_FREM, GO::G_FPOW, + GO::G_FATAN2, GO::G_FMINNUM, GO::G_FMAXNUM, GO::G_FMINNUM_IEEE, + GO::G_FMAXNUM_IEEE, GO::G_FMINIMUM, GO::G_FMAXIMUM, GO::G_FMINIMUMNUM, + GO::G_FMAXIMUMNUM}, + {FloatDef0, FloatUse0, FloatUse0}, 1); + Registry.addFlag({GO::G_FADD, GO::G_FMUL, GO::G_FMINNUM, GO::G_FMAXNUM, GO::G_FMINNUM_IEEE, + GO::G_FMAXNUM_IEEE, GO::G_FMINIMUM, GO::G_FMAXIMUM, GO::G_FMINIMUMNUM, + GO::G_FMAXIMUMNUM}, + InstructionFlag::Commutable); + Registry.configure({GO::G_FMA, GO::G_FMAD}, {FloatDef0, FloatUse0, FloatUse0, FloatUse0}, 1); + Registry.configure({GO::G_FMODF, GO::G_FSINCOS}, {FloatDef0, FloatDef0, FloatUse0}, 2); + Registry.configure({GO::G_FPOWI, GO::G_FLDEXP}, {FloatDef0, FloatUse0, IntUse1}, 1); + Registry.configure({GO::G_FFREXP}, {FloatDef0, IntDef1, FloatUse0}, 2); + + Registry.configure({GO::G_GET_FPENV, GO::G_GET_FPMODE}, {AnyDef0}, 1, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_SET_FPENV, GO::G_SET_FPMODE}, {AnyUse0}, 0, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_RESET_FPENV, GO::G_RESET_FPMODE}, {}, 0, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_GET_ROUNDING}, {IntDef0}, 1, {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_SET_ROUNDING}, {IntUse0}, 0, {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_PTR_ADD}, {PointerDef0, PointerUse0, IntUse1}, 1); + Registry.configure({GO::G_PTRMASK}, {PointerDef0, PointerUse0, IntUse1}, 1); + Registry.configure({GO::G_ABS}, {IntDef0, IntUse0}, 1); + Registry.configure({GO::G_LROUND, GO::G_LLROUND}, {IntDef0, FloatUse1}, 1); + Registry.configure( + {GO::G_BR}, {kind(MachineOperand::Kind::BasicBlock)}, 0, + {InstructionFlag::Terminator, InstructionFlag::Branch, InstructionFlag::Barrier}); + Registry.configure({GO::G_BRJT}, + {ScalarPointerUse0, kind(MachineOperand::Kind::JumpTableIndex), ScalarIntUse1}, + 0, + {InstructionFlag::Terminator, InstructionFlag::Branch, + InstructionFlag::IndirectBranch, InstructionFlag::Barrier}); + Registry.configure({GO::G_VSCALE}, {ScalarIntDef0, immediate()}, 1); + + Registry.configure({GO::G_INSERT_SUBVECTOR}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 0), + typedRegister(RegisterOperandRole::Use, VectorTypes, 1), immediate()}, + 1); + Registry.configure( + {GO::G_EXTRACT_SUBVECTOR}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1), immediate()}, 1); + Registry.configure( + {GO::G_INSERT_VECTOR_ELT}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 0), AnyUse1, ScalarIntUse2}, + 1); + Registry.configure( + {GO::G_EXTRACT_VECTOR_ELT}, + {AnyDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1), ScalarIntUse2}, 1); + Registry.configure({GO::G_SHUFFLE_VECTOR}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1), + typedRegister(RegisterOperandRole::Use, VectorTypes, 1), + kind(MachineOperand::Kind::ShuffleMask)}, + 1); + Registry.configure({GO::G_SPLAT_VECTOR}, {VectorDef0, AnyUse1}, 1); + Registry.configure({GO::G_STEP_VECTOR}, {VectorDef0, immediate()}, 1); + Registry.configure({GO::G_VECTOR_COMPRESS}, + {VectorDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 0), + typedRegister(RegisterOperandRole::Use, VectorTypes, 1), + typedRegister(RegisterOperandRole::Use, VectorTypes, 0)}, + 1); + + Registry.configure({GO::G_CTTZ, GO::G_CTTZ_ZERO_POISON, GO::G_CTLZ, GO::G_CTLZ_ZERO_POISON, + GO::G_CTLS, GO::G_CTPOP}, + {IntDef0, IntUse1}, 1); + Registry.configure({GO::G_BSWAP, GO::G_BITREVERSE}, {IntDef0, IntUse0}, 1); + Registry.configure({GO::G_ADDRSPACE_CAST}, {PointerDef0, PointerUse1}, 1); + Registry.configure({GO::G_BLOCK_ADDR}, + {ScalarPointerDef0, kind(MachineOperand::Kind::BlockAddress)}, 1); + Registry.configure({GO::G_JUMP_TABLE}, + {ScalarPointerDef0, kind(MachineOperand::Kind::JumpTableIndex)}, 1); + Registry.configure({GO::G_DYN_STACKALLOC}, {ScalarPointerDef0, ScalarIntUse1, immediate()}, 1, + {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_STACKSAVE}, {ScalarPointerDef0}, 1, {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_STACKRESTORE}, {ScalarPointerUse0}, 0, + {InstructionFlag::HasSideEffects}); + + Registry.configure({GO::G_STRICT_FADD, GO::G_STRICT_FSUB, GO::G_STRICT_FMUL, GO::G_STRICT_FDIV, + GO::G_STRICT_FREM}, + {FloatDef0, FloatUse0, FloatUse0}, 1, + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); + Registry.configure({GO::G_STRICT_FMA}, {FloatDef0, FloatUse0, FloatUse0, FloatUse0}, 1, + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); + Registry.configure({GO::G_STRICT_FSQRT}, {FloatDef0, FloatUse0}, 1, + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); + Registry.configure({GO::G_STRICT_FLDEXP}, {FloatDef0, FloatUse0, IntUse1}, 1, + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); + Registry.configure({GO::G_STRICT_FCMP, GO::G_STRICT_FCMPS}, + {IntDef0, kind(MachineOperand::Kind::Predicate), FloatUse1, FloatUse1}, 1, + {InstructionFlag::HasSideEffects, InstructionFlag::MayRaiseFPException}); + Registry.configure({GO::G_READ_REGISTER}, {AnyDef0, kind(MachineOperand::Kind::Metadata)}, 1, + {InstructionFlag::HasSideEffects, InstructionFlag::Convergent}); + Registry.configure({GO::G_WRITE_REGISTER}, {kind(MachineOperand::Kind::Metadata), AnyUse0}, 0, + {InstructionFlag::HasSideEffects, InstructionFlag::Convergent}); + Registry.configure({GO::G_MEMCPY, GO::G_MEMMOVE}, + {ScalarPointerUse0, ScalarPointerUse1, ScalarIntUse2, immediate()}, 0, + {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_MEMCPY_INLINE}, {ScalarPointerUse0, ScalarPointerUse1, ScalarIntUse2}, + 0, {InstructionFlag::MayLoad, InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_MEMSET}, {ScalarPointerUse0, ScalarIntUse1, ScalarIntUse2, immediate()}, + 0, {InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_BZERO}, {ScalarPointerUse0, ScalarIntUse1, immediate()}, 0, + {InstructionFlag::MayStore}, false, 0, true); + Registry.configure({GO::G_TRAP}, {}, 0, + {InstructionFlag::HasSideEffects, InstructionFlag::MayStore}); + Registry.configure({GO::G_DEBUGTRAP}, {}, 0, {InstructionFlag::HasSideEffects}); + Registry.configure({GO::G_UBSANTRAP}, {immediate()}, 0, {InstructionFlag::HasSideEffects}); + + Registry.configure({GO::G_VECREDUCE_SEQ_FADD, GO::G_VECREDUCE_SEQ_FMUL}, + {ScalarFloatDef0, typedRegister(RegisterOperandRole::Use, ScalarFloats, 1), + typedRegister(RegisterOperandRole::Use, VectorTypes, 2)}, + 1); + Registry.configure({GO::G_VECREDUCE_FADD, GO::G_VECREDUCE_FMUL, GO::G_VECREDUCE_FMAX, + GO::G_VECREDUCE_FMIN, GO::G_VECREDUCE_FMAXIMUM, GO::G_VECREDUCE_FMINIMUM}, + {ScalarFloatDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1)}, 1); + Registry.configure({GO::G_VECREDUCE_ADD, GO::G_VECREDUCE_MUL, GO::G_VECREDUCE_AND, + GO::G_VECREDUCE_OR, GO::G_VECREDUCE_XOR, GO::G_VECREDUCE_SMAX, + GO::G_VECREDUCE_SMIN, GO::G_VECREDUCE_UMAX, GO::G_VECREDUCE_UMIN}, + {ScalarIntDef0, typedRegister(RegisterOperandRole::Use, VectorTypes, 1)}, 1); + Registry.configure({GO::G_SBFX, GO::G_UBFX}, + {ScalarIntDef0, ScalarIntUse0, ScalarIntUse1, ScalarIntUse1}, 1); +} + +[[nodiscard]] std::vector<InstructionDescriptor> buildDescriptors() { + DescriptorRegistry Registry; + configureCoreOpcodes(Registry); + configureGenericOpcodes(Registry); + return Registry.take(); +} + +[[nodiscard]] const std::vector<InstructionDescriptor> &descriptors() { + static const std::vector<InstructionDescriptor> Value = buildDescriptors(); + return Value; +} + +} // namespace + +std::string_view getGenericOpcodeName(GenericOpcode OpcodeValue) noexcept { + const Opcode Value = opcode(OpcodeValue); + const auto Position = + std::ranges::find_if(descriptors(), [Value](const InstructionDescriptor &Descriptor) { + return Descriptor.InstructionOpcode == Value; + }); + return Position == descriptors().end() ? std::string_view{} : std::string_view(Position->Name); +} + +std::span<const InstructionDescriptor> genericInstructionDescriptors() noexcept { + return descriptors(); +} + +std::span<const ImmediatePredicate> genericImmediatePredicates() noexcept { + static const std::vector<ImmediatePredicate> Predicates; + return Predicates; +} + +Expected<void> appendGenericInstructionInfo(std::vector<InstructionDescriptor> &Descriptors, + std::vector<ImmediatePredicate> &Predicates) { + const auto ExistingOpcode = [&Descriptors](Opcode Value) { + return std::ranges::any_of(Descriptors, [Value](const InstructionDescriptor &Descriptor) { + return Descriptor.InstructionOpcode == Value; + }); + }; + for (const InstructionDescriptor &Descriptor : descriptors()) { + if (ExistingOpcode(Descriptor.InstructionOpcode)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target instruction set already contains a target-independent opcode"); + } + Descriptors.push_back(Descriptor); + } + for (const ImmediatePredicate &Predicate : genericImmediatePredicates()) + Predicates.push_back(Predicate); + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/CMakeLists.txt b/backend/lib/CodeGen/GlobalISel/CMakeLists.txt new file mode 100644 index 0000000..cd336cb --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/CMakeLists.txt @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +set( + RCCBackendGlobalISelSources + CSEInfo.cpp + CSEMIRBuilder.cpp + Combiner.cpp + GIMatchTableExecutor.cpp + GISelChangeObserver.cpp + GISelValueTracking.cpp + InstructionSelect.cpp + Legalizer.cpp + LegalizerHelper.cpp + LegalizerInfo.cpp + RegBankSelect.cpp + Utils.cpp +) + +set( + RCCBackendGlobalISelHeaders + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/CSEInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/CSEMIRBuilder.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/Combiner.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GIMatchTableExecutor.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelChangeObserver.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelValueTracking.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelWorkList.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GenericMachineInstrs.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/InstructionSelect.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/InstructionSelector.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/Legalizer.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/RegBankSelect.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/Utils.h" +) + +target_sources(RCCBackendCodeGen PRIVATE ${RCCBackendGlobalISelSources}) + +rcc_backend_register_format_files( + "${CMAKE_CURRENT_SOURCE_DIR}/CSEInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/CSEMIRBuilder.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Combiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GIMatchTableExecutor.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GISelChangeObserver.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/GISelValueTracking.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/InstructionSelect.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Legalizer.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerHelper.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerInfo.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RegBankSelect.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Utils.cpp" + ${RCCBackendGlobalISelHeaders} +) diff --git a/backend/lib/CodeGen/GlobalISel/CSEInfo.cpp b/backend/lib/CodeGen/GlobalISel/CSEInfo.cpp new file mode 100644 index 0000000..201dedd --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/CSEInfo.cpp @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/CSEInfo.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <bit> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <memory> +#include <utility> + +namespace rcc { +namespace { + +enum class ProfileTag : std::uint64_t { + Definition, + Use, + RegisterNumber, + PhysicalRegister, + Type, + RegisterBank, + NoRegisterBank, + RegisterClass, + NoRegisterClass, + Immediate, + IntegerImmediate, + FloatingImmediate, + IntegerPredicate, + FloatingPredicate, + Flags, +}; + +void appendTypeProfile(std::vector<std::uint64_t> &Profile, LowLevelType Type, + std::optional<RegisterBankID> Bank, + std::optional<RegisterClassID> RegClass) { + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Type)); + Profile.push_back(Type.rawValue()); + if (Bank) { + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::RegisterBank)); + Profile.push_back(Bank->value()); + } else { + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::NoRegisterBank)); + } + if (RegClass) { + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::RegisterClass)); + Profile.push_back(RegClass->value()); + } else { + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::NoRegisterClass)); + } +} + +void appendAPIntProfile(std::vector<std::uint64_t> &Profile, const APInt &Value) { + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::IntegerImmediate)); + Profile.push_back(Value.bitWidth()); + Profile.push_back(Value.words().size()); + Profile.insert(Profile.end(), Value.words().begin(), Value.words().end()); +} + +void appendAPFloatProfile(std::vector<std::uint64_t> &Profile, const APFloat &Value) { + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::FloatingImmediate)); + Profile.push_back(static_cast<std::uint64_t>(Value.semantics())); + Profile.push_back(Value.words().size()); + Profile.insert(Profile.end(), Value.words().begin(), Value.words().end()); +} + +void hashCombine(std::size_t &Seed, std::size_t Value) noexcept { + Seed ^= Value + 0x9e3779b97f4a7c15ULL + (Seed << 6U) + (Seed >> 2U); +} + +} // namespace + +bool CSEConfigFull::shouldCSEOpcode(Opcode InstructionOpcode) const noexcept { + switch (static_cast<GenericOpcode>(InstructionOpcode.value())) { + default: + return false; + case GenericOpcode::G_ADD: + case GenericOpcode::G_AND: + case GenericOpcode::G_ASHR: + case GenericOpcode::G_LSHR: + case GenericOpcode::G_MUL: + case GenericOpcode::G_OR: + case GenericOpcode::G_SHL: + case GenericOpcode::G_SUB: + case GenericOpcode::G_XOR: + case GenericOpcode::G_UDIV: + case GenericOpcode::G_SDIV: + case GenericOpcode::G_UREM: + case GenericOpcode::G_SREM: + case GenericOpcode::G_CONSTANT: + case GenericOpcode::G_FCONSTANT: + case GenericOpcode::G_IMPLICIT_DEF: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_UNMERGE_VALUES: + case GenericOpcode::G_TRUNC: + case GenericOpcode::G_PTR_ADD: + case GenericOpcode::G_EXTRACT: + case GenericOpcode::G_SELECT: + case GenericOpcode::G_BUILD_VECTOR: + case GenericOpcode::G_BUILD_VECTOR_TRUNC: + case GenericOpcode::G_SEXT_INREG: + case GenericOpcode::G_FADD: + case GenericOpcode::G_FSUB: + case GenericOpcode::G_FMUL: + case GenericOpcode::G_FDIV: + case GenericOpcode::G_FABS: + case GenericOpcode::G_FMAXNUM: + case GenericOpcode::G_FMINNUM: + case GenericOpcode::G_FMAXNUM_IEEE: + case GenericOpcode::G_FMINNUM_IEEE: + return true; + } +} + +bool CSEConfigConstantOnly::shouldCSEOpcode(Opcode InstructionOpcode) const noexcept { + return InstructionOpcode == opcode(GenericOpcode::G_CONSTANT) || + InstructionOpcode == opcode(GenericOpcode::G_FCONSTANT) || + InstructionOpcode == opcode(GenericOpcode::G_IMPLICIT_DEF); +} + +std::unique_ptr<CSEConfigBase> getStandardCSEConfigForOpt(OptimizationLevel Level) { + if (Level == OptimizationLevel::O0) + return std::make_unique<CSEConfigConstantOnly>(); + return std::make_unique<CSEConfigFull>(); +} + +std::size_t GISelCSEInfo::CSEKeyHash::operator()(const CSEKey &Key) const noexcept { + std::size_t Result = std::hash<const MachineBasicBlock *>{}(Key.Block); + hashCombine(Result, std::hash<std::uint32_t>{}(Key.InstructionOpcode.value())); + for (std::uint64_t Value : Key.Profile) + hashCombine(Result, std::hash<std::uint64_t>{}(Value)); + return Result; +} + +void GISelCSEInfo::setCSEConfig(std::unique_ptr<CSEConfigBase> NewConfig) noexcept { + Config = std::move(NewConfig); +} + +bool GISelCSEInfo::shouldCSE(Opcode InstructionOpcode) const noexcept { + return Config != nullptr && isGenericOpcode(InstructionOpcode) && + Config->shouldCSEOpcode(InstructionOpcode); +} + +Expected<void> GISelCSEInfo::appendRegisterProfile(std::vector<std::uint64_t> &Profile, + Register Reg, bool IsDefinition) const { + Profile.push_back( + static_cast<std::uint64_t>(IsDefinition ? ProfileTag::Definition : ProfileTag::Use)); + if (!IsDefinition) { + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::RegisterNumber)); + Profile.push_back(Reg.opaqueValue()); + } + if (!Reg.isVirtual()) { + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::PhysicalRegister)); + Profile.push_back(Reg.opaqueValue()); + return {}; + } + if (Function == nullptr) + return makeUnexpected(ErrorCode::InternalError, "GlobalISel CSE has no machine function"); + auto Info = Function->registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + const std::optional<LowLevelType> &Type = Info->get().Type; + if (!Type.has_value()) { + return makeUnexpected(ErrorCode::VerificationError, + "GlobalISel CSE requires typed virtual registers"); + } + appendTypeProfile(Profile, Type.value(), Info->get().Bank, Info->get().RegClass); + return {}; +} + +Expected<void> GISelCSEInfo::appendResultProfile(std::vector<std::uint64_t> &Profile, + const MachineIRBuilderResult &Result) const { + if (Result.kind() == MachineIRBuilderResult::Kind::Register) + return appendRegisterProfile(Profile, Result.reg(), true); + Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Definition)); + appendTypeProfile(Profile, Result.type(), std::nullopt, std::nullopt); + return {}; +} + +Expected<GISelCSEInfo::CSEKey> +GISelCSEInfo::profileInstruction(const MachineInstr &Instruction) const { + if (Instruction.parent() == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "cannot profile a detached instruction"); + if (!Instruction.memOperands().empty()) { + return makeUnexpected(ErrorCode::Unsupported, + "GlobalISel CSE does not profile memory operands"); + } + + CSEKey Key{ + .Block = Instruction.parent(), .InstructionOpcode = Instruction.opcode(), .Profile = {}}; + for (const MachineOperand &Operand : Instruction.operands()) { + switch (Operand.kind()) { + case MachineOperand::Kind::Register: { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Operand.isImplicit()) { + return makeUnexpected(ErrorCode::Unsupported, + "GlobalISel CSE does not profile implicit register operands"); + } + auto Appended = appendRegisterProfile(Key.Profile, *Reg, Operand.isDef()); + if (!Appended) + return std::unexpected(Appended.error()); + break; + } + case MachineOperand::Kind::Immediate: { + const std::int64_t *Value = Operand.getIf<std::int64_t>(); + if (Value == nullptr) + return makeUnexpected(ErrorCode::InternalError, "invalid immediate operand storage"); + Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Immediate)); + Key.Profile.push_back(std::bit_cast<std::uint64_t>(*Value)); + break; + } + case MachineOperand::Kind::IntegerImmediate: { + const auto *Value = Operand.getIf<IntegerImmediateOperand>(); + if (Value == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "invalid integer immediate operand storage"); + appendAPIntProfile(Key.Profile, Value->Value); + break; + } + case MachineOperand::Kind::FloatingImmediate: { + const auto *Value = Operand.getIf<FloatingImmediateOperand>(); + if (Value == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "invalid floating immediate operand storage"); + appendAPFloatProfile(Key.Profile, Value->Bits); + break; + } + case MachineOperand::Kind::Predicate: { + const auto *Predicate = Operand.getIf<PredicateOperand>(); + if (Predicate == nullptr) + return makeUnexpected(ErrorCode::InternalError, "invalid predicate operand storage"); + if (const IntegerPredicate *Integer = Predicate->integerPredicate()) { + Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::IntegerPredicate)); + Key.Profile.push_back(static_cast<std::uint64_t>(*Integer)); + } else if (const FloatingPredicate *Floating = Predicate->floatingPredicate()) { + Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::FloatingPredicate)); + Key.Profile.push_back(static_cast<std::uint64_t>(*Floating)); + } else { + return makeUnexpected(ErrorCode::InternalError, "empty predicate operand"); + } + break; + } + default: + return makeUnexpected(ErrorCode::Unsupported, + "GlobalISel CSE encountered an unsupported operand kind"); + } + } + Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Flags)); + Key.Profile.push_back(Instruction.flags().rawValue()); + return Key; +} + +Expected<GISelCSEInfo::CSEKey> GISelCSEInfo::profileBuilderInstruction( + Opcode InstructionOpcode, std::span<const MachineIRBuilderResult> Destinations, + std::span<const Register> Sources, std::span<const std::int64_t> Immediates, + MachineInstrFlags Flags, const MachineBasicBlock &Block) const { + CSEKey Key{.Block = &Block, .InstructionOpcode = InstructionOpcode, .Profile = {}}; + for (const MachineIRBuilderResult &Destination : Destinations) { + auto Appended = appendResultProfile(Key.Profile, Destination); + if (!Appended) + return std::unexpected(Appended.error()); + } + for (Register Source : Sources) { + auto Appended = appendRegisterProfile(Key.Profile, Source, false); + if (!Appended) + return std::unexpected(Appended.error()); + } + for (std::int64_t Immediate : Immediates) { + Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Immediate)); + Key.Profile.push_back(std::bit_cast<std::uint64_t>(Immediate)); + } + Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Flags)); + Key.Profile.push_back(Flags.rawValue()); + return Key; +} + +Expected<GISelCSEInfo::CSEKey> +GISelCSEInfo::profileConstant(const MachineIRBuilderResult &Destination, const APInt &Value, + const MachineBasicBlock &Block) const { + CSEKey Key{ + .Block = &Block, .InstructionOpcode = opcode(GenericOpcode::G_CONSTANT), .Profile = {}}; + auto Appended = appendResultProfile(Key.Profile, Destination); + if (!Appended) + return std::unexpected(Appended.error()); + appendAPIntProfile(Key.Profile, Value); + Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Flags)); + Key.Profile.push_back(0); + return Key; +} + +Expected<void> GISelCSEInfo::insertInstruction(MachineInstr &Instruction) { + TemporaryInstructions.remove(&Instruction); + auto Key = profileInstruction(Instruction); + if (!Key) + return std::unexpected(Key.error()); + auto [Position, Inserted] = CSEMap.emplace(*Key, &Instruction); + if (Inserted) + InstructionMapping.emplace(&Instruction, std::move(*Key)); + return {}; +} + +void GISelCSEInfo::recordNewInstruction(MachineInstr &Instruction) { + if (shouldCSE(Instruction.opcode())) + TemporaryInstructions.insert(&Instruction); +} + +Expected<void> GISelCSEInfo::handleRecordedInstruction(MachineInstr &Instruction) { + TemporaryInstructions.remove(&Instruction); + handleRemoveInstruction(Instruction); + if (Instruction.parent() == nullptr || !shouldCSE(Instruction.opcode())) + return {}; + return insertInstruction(Instruction); +} + +Expected<void> GISelCSEInfo::handleRecordedInstructions() { + if (HandlingRecordedInstructions) + return {}; + HandlingRecordedInstructions = true; + while (!TemporaryInstructions.empty()) { + MachineInstr *Instruction = TemporaryInstructions.popBack(); + if (Instruction == nullptr) + continue; + auto Handled = handleRecordedInstruction(*Instruction); + if (!Handled) { + HandlingRecordedInstructions = false; + return std::unexpected(Handled.error()); + } + } + HandlingRecordedInstructions = false; + return {}; +} + +void GISelCSEInfo::handleRemoveInstruction(MachineInstr &Instruction) noexcept { + const auto Mapping = InstructionMapping.find(&Instruction); + if (Mapping != InstructionMapping.end()) { + const auto Position = CSEMap.find(Mapping->second); + if (Position != CSEMap.end() && Position->second == &Instruction) + CSEMap.erase(Position); + InstructionMapping.erase(Mapping); + } + TemporaryInstructions.remove(&Instruction); +} + +Expected<MachineInstr *> GISelCSEInfo::getMachineInstrIfExists( + Opcode InstructionOpcode, std::span<const MachineIRBuilderResult> Destinations, + std::span<const Register> Sources, std::span<const std::int64_t> Immediates, + MachineInstrFlags Flags, MachineBasicBlock &Block) { + auto Handled = handleRecordedInstructions(); + if (!Handled) + return std::unexpected(Handled.error()); + auto Key = + profileBuilderInstruction(InstructionOpcode, Destinations, Sources, Immediates, Flags, Block); + if (!Key) + return std::unexpected(Key.error()); + const auto Position = CSEMap.find(*Key); + if (Position == CSEMap.end()) + return nullptr; + countOpcodeHit(InstructionOpcode); + return Position->second; +} + +Expected<MachineInstr *> +GISelCSEInfo::getConstantIfExists(const MachineIRBuilderResult &Destination, const APInt &Value, + MachineBasicBlock &Block) { + auto Handled = handleRecordedInstructions(); + if (!Handled) + return std::unexpected(Handled.error()); + auto Key = profileConstant(Destination, Value, Block); + if (!Key) + return std::unexpected(Key.error()); + const auto Position = CSEMap.find(*Key); + if (Position == CSEMap.end()) + return nullptr; + countOpcodeHit(opcode(GenericOpcode::G_CONSTANT)); + return Position->second; +} + +void GISelCSEInfo::countOpcodeHit(Opcode InstructionOpcode) noexcept { + ++OpcodeHitTable[InstructionOpcode.value()]; +} + +Expected<void> GISelCSEInfo::analyze(MachineFunction &NewFunction) { + if (Config == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "GlobalISel CSE configuration is not set"); + CSEMap.clear(); + InstructionMapping.clear(); + TemporaryInstructions.clear(); + OpcodeHitTable.clear(); + HandlingRecordedInstructions = false; + Function = &NewFunction; + for (MachineBasicBlock *Block = NewFunction.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (!shouldCSE(Instruction->opcode())) + continue; + auto Inserted = insertInstruction(*Instruction); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + } + return {}; +} + +Expected<void> GISelCSEInfo::verify() { + auto Handled = handleRecordedInstructions(); + if (!Handled) + return std::unexpected(Handled.error()); + for (const auto &[Instruction, Key] : InstructionMapping) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + return makeUnexpected(ErrorCode::InternalError, "GlobalISel CSE maps a detached instruction"); + auto CurrentKey = profileInstruction(*Instruction); + if (!CurrentKey) + return std::unexpected(CurrentKey.error()); + const auto Position = CSEMap.find(*CurrentKey); + if (*CurrentKey != Key || Position == CSEMap.end() || Position->second != Instruction) { + return makeUnexpected(ErrorCode::InternalError, + "GlobalISel CSE instruction mapping is inconsistent"); + } + } + for (const auto &[Key, Instruction] : CSEMap) { + const auto Mapping = InstructionMapping.find(Instruction); + if (Mapping == InstructionMapping.end() || Mapping->second != Key) { + return makeUnexpected(ErrorCode::InternalError, + "GlobalISel CSE table has no instruction mapping"); + } + } + return {}; +} + +void GISelCSEInfo::releaseMemory() noexcept { + CSEMap.clear(); + InstructionMapping.clear(); + TemporaryInstructions.clear(); + OpcodeHitTable.clear(); + Config.reset(); + Function = nullptr; + HandlingRecordedInstructions = false; +} + +void GISelCSEInfo::erasingInstr(MachineInstr &Instruction) { + handleRemoveInstruction(Instruction); +} + +void GISelCSEInfo::createdInstr(MachineInstr &Instruction) { + recordNewInstruction(Instruction); +} + +void GISelCSEInfo::changingInstr(MachineInstr &Instruction) { + erasingInstr(Instruction); + createdInstr(Instruction); +} + +void GISelCSEInfo::changedInstr(MachineInstr &Instruction) { + changingInstr(Instruction); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/backend/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp new file mode 100644 index 0000000..caf9652 --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/CSEMIRBuilder.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/CSEInfo.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <algorithm> +#include <array> +#include <cstdint> +#include <limits> +#include <span> +#include <utility> + +namespace rcc { + +bool CSEMIRBuilder::canPerformCSE(Opcode InstructionOpcode) const noexcept { + const GISelCSEInfo *Info = getCSEInfo(); + return Info != nullptr && Info->shouldCSE(InstructionOpcode); +} + +bool CSEMIRBuilder::dominatesInsertionPoint(const MachineInstr &Instruction) const noexcept { + const MachineBasicBlock *Block = insertionBlock(); + const MachineInstr *InsertBefore = insertionInstruction(); + if (Block == nullptr || Instruction.parent() != Block) + return false; + if (InsertBefore == nullptr) + return true; + for (const MachineInstr *Current = Block->firstInstruction(); Current != nullptr; + Current = Current->next()) { + if (Current == &Instruction) + return true; + if (Current == InsertBefore) + return false; + } + return false; +} + +Expected<void> CSEMIRBuilder::makeInstructionDominate(MachineInstr &Instruction) const { + MachineBasicBlock *Block = insertionBlock(); + MachineInstr *InsertBefore = insertionInstruction(); + if (Block == nullptr || Instruction.parent() != Block) { + return makeUnexpected(ErrorCode::InternalError, + "GlobalISel CSE candidate is not in the insertion block"); + } + if (&Instruction == InsertBefore) { + return const_cast<CSEMIRBuilder *>(this)->setInsertPointAfter(Instruction); + } + if (InsertBefore == nullptr || dominatesInsertionPoint(Instruction)) + return {}; + return Block->spliceInstructionBefore(*InsertBefore, Instruction); +} + +Expected<MachineInstr *> +CSEMIRBuilder::generateCopyIfRequired(const MachineIRBuilderResult &Destination, + MachineInstr &Instruction) const { + if (Destination.kind() == MachineIRBuilderResult::Kind::Type) + return &Instruction; + const MachineOperand *Definition = Instruction.operandAt(0); + const Register *Source = Definition == nullptr ? nullptr : Definition->getIf<Register>(); + if (Source == nullptr || !Definition->isDef()) { + return makeUnexpected(ErrorCode::InternalError, + "GlobalISel CSE candidate has no register definition"); + } + if (Destination.reg() == *Source) + return &Instruction; + return MachineIRBuilder::buildCopy(Destination, *Source); +} + +Expected<MachineInstr *> CSEMIRBuilder::memoize(MachineInstr &Instruction) const { + GISelCSEInfo *Info = getCSEInfo(); + if (Info == nullptr) + return &Instruction; + auto Inserted = Info->handleRecordedInstruction(Instruction); + if (!Inserted) + return std::unexpected(Inserted.error()); + return &Instruction; +} + +Expected<MachineInstr *> +CSEMIRBuilder::buildInstr(Opcode InstructionOpcode, + std::span<const MachineIRBuilderResult> Destinations, + std::span<const Register> Sources) const { + if (!canPerformCSE(InstructionOpcode)) + return MachineIRBuilder::buildInstr(InstructionOpcode, Destinations, Sources); + MachineBasicBlock *Block = insertionBlock(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point"); + + GISelCSEInfo *Info = getCSEInfo(); + const bool CanGenerateCopies = + Destinations.size() == 1 || std::ranges::all_of(Destinations, [](const auto &Destination) { + return Destination.kind() == MachineIRBuilderResult::Kind::Type; + }); + if (!CanGenerateCopies) { + auto Built = MachineIRBuilder::buildInstr(InstructionOpcode, Destinations, Sources); + if (!Built) + return std::unexpected(Built.error()); + Info->handleRemoveInstruction(**Built); + return *Built; + } + + auto Existing = + Info->getMachineInstrIfExists(InstructionOpcode, Destinations, Sources, + std::span<const std::int64_t>(), MachineInstrFlags(), *Block); + if (!Existing) + return std::unexpected(Existing.error()); + if (*Existing != nullptr) { + auto Dominating = makeInstructionDominate(**Existing); + if (!Dominating) + return std::unexpected(Dominating.error()); + if (Destinations.size() == 1) + return generateCopyIfRequired(Destinations.front(), **Existing); + return *Existing; + } + + auto Built = MachineIRBuilder::buildInstr(InstructionOpcode, Destinations, Sources); + if (!Built) + return std::unexpected(Built.error()); + return memoize(**Built); +} + +Expected<MachineInstr *> CSEMIRBuilder::buildConstant(MachineIRBuilderResult Destination, + APInt Value) const { + const Opcode ConstantOpcode = opcode(GenericOpcode::G_CONSTANT); + if (!canPerformCSE(ConstantOpcode)) + return MachineIRBuilder::buildConstant(Destination, std::move(Value)); + MachineBasicBlock *Block = insertionBlock(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point"); + + GISelCSEInfo *Info = getCSEInfo(); + auto Existing = Info->getConstantIfExists(Destination, Value, *Block); + if (!Existing) + return std::unexpected(Existing.error()); + if (*Existing != nullptr) { + auto Dominating = makeInstructionDominate(**Existing); + if (!Dominating) + return std::unexpected(Dominating.error()); + return generateCopyIfRequired(Destination, **Existing); + } + + auto Built = MachineIRBuilder::buildConstant(Destination, std::move(Value)); + if (!Built) + return std::unexpected(Built.error()); + return memoize(**Built); +} + +Expected<MachineInstr *> CSEMIRBuilder::buildSExtInReg(MachineIRBuilderResult Destination, + Register Source, std::uint32_t Width) const { + const Opcode InstructionOpcode = opcode(GenericOpcode::G_SEXT_INREG); + if (!canPerformCSE(InstructionOpcode)) + return MachineIRBuilder::buildSExtInReg(Destination, Source, Width); + MachineBasicBlock *Block = insertionBlock(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point"); + + GISelCSEInfo *Info = getCSEInfo(); + const std::array Destinations{Destination}; + const std::array Sources{Source}; + const std::array Immediates{static_cast<std::int64_t>(Width)}; + auto Existing = Info->getMachineInstrIfExists( + InstructionOpcode, std::span<const MachineIRBuilderResult>(Destinations), + std::span<const Register>(Sources), std::span<const std::int64_t>(Immediates), + MachineInstrFlags(), *Block); + if (!Existing) + return std::unexpected(Existing.error()); + if (*Existing != nullptr) { + auto Dominating = makeInstructionDominate(**Existing); + if (!Dominating) + return std::unexpected(Dominating.error()); + return generateCopyIfRequired(Destination, **Existing); + } + + auto Built = MachineIRBuilder::buildSExtInReg(Destination, Source, Width); + if (!Built) + return std::unexpected(Built.error()); + return memoize(**Built); +} + +Expected<MachineInstr *> CSEMIRBuilder::buildExtract(MachineIRBuilderResult Destination, + Register Source, std::uint64_t Offset) const { + if (Offset > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return MachineIRBuilder::buildExtract(Destination, Source, Offset); + const Opcode InstructionOpcode = opcode(GenericOpcode::G_EXTRACT); + if (!canPerformCSE(InstructionOpcode)) + return MachineIRBuilder::buildExtract(Destination, Source, Offset); + MachineBasicBlock *Block = insertionBlock(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point"); + + GISelCSEInfo *Info = getCSEInfo(); + const std::array Destinations{Destination}; + const std::array Sources{Source}; + const std::array Immediates{static_cast<std::int64_t>(Offset)}; + auto Existing = Info->getMachineInstrIfExists( + InstructionOpcode, std::span<const MachineIRBuilderResult>(Destinations), + std::span<const Register>(Sources), std::span<const std::int64_t>(Immediates), + MachineInstrFlags(), *Block); + if (!Existing) + return std::unexpected(Existing.error()); + if (*Existing != nullptr) { + auto Dominating = makeInstructionDominate(**Existing); + if (!Dominating) + return std::unexpected(Dominating.error()); + return generateCopyIfRequired(Destination, **Existing); + } + + auto Built = MachineIRBuilder::buildExtract(Destination, Source, Offset); + if (!Built) + return std::unexpected(Built.error()); + return memoize(**Built); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/Combiner.cpp b/backend/lib/CodeGen/GlobalISel/Combiner.cpp new file mode 100644 index 0000000..c0a92cc --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/Combiner.cpp @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/Combiner.h" + +#include "rcc/CodeGen/GlobalISel/Utils.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" + +#include <string> + +namespace rcc { + +const MachinePassContract &GlobalISelCombinerPass::contract() const noexcept { + return Contract; +} + +Expected<MachinePassResult> GlobalISelCombinerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +GlobalISelCombinerPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) const { + if (MaximumRewrites == 0 || MaximumIterations == 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "GlobalISel combiner iteration limits must be nonzero"); + MachinePassResult Result; + std::size_t Rewrites = 0; + for (std::size_t Iteration = 0; Iteration < MaximumIterations; ++Iteration) { + bool IterationChanged = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Matches = Executor->match(*Table, *Instruction, Function, Target); + if (!Matches) + return std::unexpected(Matches.error()); + if (*Matches && Rewrites < MaximumRewrites) { + auto Accepted = + Analyses.requestAction("gisel-combiner", Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (*Accepted) { + auto Combined = Executor->combine(*Table, *Instruction, Function, Target); + if (!Combined) + return std::unexpected(Combined.error()); + if (*Combined) { + ++Rewrites; + IterationChanged = true; + Result.Changed = true; + } + } + } + Instruction = Next; + } + } + if (!IterationChanged || Rewrites == MaximumRewrites) + break; + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp b/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp new file mode 100644 index 0000000..88fcc24 --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp @@ -0,0 +1,1611 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/GIMatchTableExecutor.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/Utils.h" +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <charconv> +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <limits> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] Expected<std::optional<LowLevelType>> operandType(const MachineInstr &Instruction, + const MachineFunction &Function, + std::size_t OperandIndex) { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return std::optional<LowLevelType>{}; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + return Info->get().Type; +} + +[[nodiscard]] Expected<bool> matchesType(const MachineInstr &Instruction, + const MachineFunction &Function, + const GIMatchTableInstruction &Step) { + auto Type = operandType(Instruction, Function, Step.OperandIndex); + if (!Type) + return std::unexpected(Type.error()); + if (!*Type) + return false; + + const auto Kind = static_cast<GIMatchTypeKind>(Step.Auxiliary); + switch (Kind) { + case GIMatchTypeKind::Any: + return Step.Value == 0 || (*Type)->scalarBitWidth() == Step.Value; + case GIMatchTypeKind::Scalar: + return (*Type)->isCategoryNeutralScalar() && (*Type)->scalarBitWidth() == Step.Value; + case GIMatchTypeKind::Pointer: + return (*Type)->isPointer() && (*Type)->scalarBitWidth() == Step.Value; + } + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid type predicate"); +} + +[[nodiscard]] bool matchesInstructionFlags(const MachineInstr &Instruction, + const GIMatchTableInstruction &Step) noexcept { + return (Instruction.flags().rawValue() & Step.Value) == Step.Value; +} + +[[nodiscard]] Expected<bool> matchesMemorySize(const MachineInstr &Instruction, + const GIMatchTableInstruction &Step) { + if (Step.Value == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid memory size"); + } + if (Step.OperandIndex >= Instruction.memOperands().size()) + return false; + const MachineMemOperand *Memory = Instruction.memOperands()[Step.OperandIndex]; + if (Memory == nullptr) + return false; + const MemorySize Size = Memory->description().Size; + return Size.isKnown() && !Size.isScalable() && Size.minimumBytes() == Step.Value; +} + +[[nodiscard]] Expected<bool> matchesBank(const MachineInstr &Instruction, + const MachineFunction &Function, + const GIMatchTableInstruction &Step) { + const MachineOperand *Operand = Instruction.operandAt(Step.OperandIndex); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return false; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + const VirtualRegisterInfo &RegisterInfo = Info->get(); + if (!RegisterInfo.Bank) + return false; + return RegisterInfo.Bank->value() == Step.Value; +} + +[[nodiscard]] Expected<bool> matchesClass(const MachineInstr &Instruction, + const MachineFunction &Function, + const GIMatchTableInstruction &Step) { + const MachineOperand *Operand = Instruction.operandAt(Step.OperandIndex); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return false; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + const VirtualRegisterInfo &RegisterInfo = Info->get(); + if (!RegisterInfo.RegClass) + return false; + return RegisterInfo.RegClass->value() == Step.Value; +} + +[[nodiscard]] Expected<bool> matchesIntegerPredicate(const MachineInstr &Instruction, + std::size_t OperandIndex, + std::uint32_t PredicateValue) { + if (PredicateValue >= static_cast<std::uint32_t>(IntegerPredicate::Count)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid integer predicate"); + } + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const auto *Predicate = Operand == nullptr ? nullptr : Operand->getIf<PredicateOperand>(); + const IntegerPredicate *Integer = Predicate == nullptr ? nullptr : Predicate->integerPredicate(); + return Integer != nullptr && static_cast<std::uint32_t>(*Integer) == PredicateValue; +} + +[[nodiscard]] Expected<bool> matchesFloatingPredicate(const MachineInstr &Instruction, + std::size_t OperandIndex, + std::uint32_t PredicateValue) { + if (PredicateValue >= static_cast<std::uint32_t>(FloatingPredicate::Count)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid floating predicate"); + } + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const auto *Predicate = Operand == nullptr ? nullptr : Operand->getIf<PredicateOperand>(); + const FloatingPredicate *Floating = + Predicate == nullptr ? nullptr : Predicate->floatingPredicate(); + return Floating != nullptr && static_cast<std::uint32_t>(*Floating) == PredicateValue; +} + +[[nodiscard]] std::optional<std::int64_t> immediateValue(const MachineOperand &Operand) { + if (const std::int64_t *Immediate = Operand.getIf<std::int64_t>()) + return *Immediate; + const auto *Integer = Operand.getIf<IntegerImmediateOperand>(); + if (Integer == nullptr) + return std::optional<std::int64_t>{}; + const std::string Text = Integer->Value.toSignedDecimal(); + std::int64_t Parsed = 0; + const auto [End, Error] = std::from_chars(Text.data(), Text.data() + Text.size(), Parsed); + if (Error != std::errc{} || End != Text.data() + Text.size()) + return std::optional<std::int64_t>{}; + return Parsed; +} + +[[nodiscard]] Expected<std::optional<std::int64_t>> +operandDefConstantValue(const MachineInstr &Instruction, const MachineFunction &Function, + std::size_t OperandIndex) { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return std::optional<std::int64_t>{}; + const MachineInstr *Definition = Function.registerInfo().getVRegDef(*Reg); + if (Definition == nullptr || Definition->opcode() != opcode(GenericOpcode::G_CONSTANT)) + return std::optional<std::int64_t>{}; + const MachineOperand *Value = Definition->operandAt(1); + if (Value == nullptr) + return std::optional<std::int64_t>{}; + return immediateValue(*Value); +} + +[[nodiscard]] Expected<std::optional<std::int64_t>> +operandOrDefConstantValue(const MachineInstr &Instruction, const MachineFunction &Function, + std::size_t OperandIndex) { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + if (Operand == nullptr) + return std::optional<std::int64_t>{}; + if (const auto Value = immediateValue(*Operand)) + return Value; + return operandDefConstantValue(Instruction, Function, OperandIndex); +} + +[[nodiscard]] Expected<bool> matchesConstantRanges(const GIMatchTable &Table, std::size_t First, + std::size_t Count, + std::optional<std::int64_t> Value) { + if (Count == 0 || First > Table.ConstantRanges.size() || + Count > Table.ConstantRanges.size() - First) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid constant range"); + } + if (!Value) + return false; + for (std::size_t Index = First; Index < First + Count; ++Index) { + const GIMatchTableConstantRange &Range = Table.ConstantRanges[Index]; + if (Range.Minimum > Range.Maximum) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an inverted constant range"); + } + if (*Value >= Range.Minimum && *Value <= Range.Maximum) + return true; + } + return false; +} + +[[nodiscard]] Expected<bool> matchesOperandImmediateRanges(const GIMatchTable &Table, + const MachineInstr &Instruction, + const GIMatchTableInstruction &Step) { + const MachineOperand *Operand = Instruction.operandAt(Step.OperandIndex); + const std::optional<std::int64_t> Value = + Operand == nullptr ? std::optional<std::int64_t>{} : immediateValue(*Operand); + return matchesConstantRanges(Table, Step.Value, Step.Auxiliary, Value); +} + +[[nodiscard]] Expected<bool> matchesOperandDefConstantRanges(const GIMatchTable &Table, + const MachineInstr &Instruction, + const MachineFunction &Function, + const GIMatchTableInstruction &Step) { + auto Value = operandDefConstantValue(Instruction, Function, Step.OperandIndex); + if (!Value) + return std::unexpected(Value.error()); + return matchesConstantRanges(Table, Step.Value, Step.Auxiliary, *Value); +} + +[[nodiscard]] const MachineInstr *operandDefinition(const MachineInstr &Instruction, + const MachineFunction &Function, + std::size_t OperandIndex) { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + return Reg != nullptr && Reg->isVirtual() ? Function.registerInfo().getVRegDef(*Reg) : nullptr; +} + +[[nodiscard]] Expected<bool> matchesOperandDefOpcode(const MachineInstr &Instruction, + const MachineFunction &Function, + const GIMatchTableInstruction &Step) { + if (Step.Value == 0) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid def-use opcode"); + const MachineInstr *Definition = operandDefinition(Instruction, Function, Step.OperandIndex); + return Definition != nullptr && Definition->opcode().value() == Step.Value; +} + +[[nodiscard]] Expected<bool> +matchesOperandDefIntegerPredicate(const MachineInstr &Instruction, const MachineFunction &Function, + const GIMatchTableInstruction &Step) { + const MachineInstr *Definition = operandDefinition(Instruction, Function, Step.OperandIndex); + if (Definition == nullptr) + return false; + return matchesIntegerPredicate(*Definition, Step.Auxiliary, Step.Value); +} + +[[nodiscard]] Expected<bool> +matchesOperandDefOperandConstantRanges(const GIMatchTable &Table, const MachineInstr &Instruction, + const MachineFunction &Function, + const GIMatchTableInstruction &Step) { + if (Step.Immediate <= 0 || + static_cast<std::uint64_t>(Step.Immediate) > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid def-use constant range count"); + } + const std::size_t Count = static_cast<std::size_t>(Step.Immediate); + const MachineInstr *Definition = operandDefinition(Instruction, Function, Step.OperandIndex); + if (Definition == nullptr) + return false; + auto Value = operandOrDefConstantValue(*Definition, Function, Step.Auxiliary); + if (!Value) + return std::unexpected(Value.error()); + return matchesConstantRanges(Table, Step.Value, Count, *Value); +} + +[[nodiscard]] Expected<RegisterOperandFlags> registerFlagsFromRaw(std::uint32_t Raw) { + constexpr std::uint32_t ValidBits = + (std::uint32_t{1} << static_cast<std::uint16_t>(RegisterOperandFlag::Count)) - 1U; + if ((Raw & ~ValidBits) != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has invalid register flags"); + } + RegisterOperandFlags Flags; + for (std::uint16_t Index = 0; Index < static_cast<std::uint16_t>(RegisterOperandFlag::Count); + ++Index) { + if ((Raw & (std::uint32_t{1} << Index)) != 0) + Flags.set(static_cast<RegisterOperandFlag>(Index)); + } + return Flags; +} + +struct MatchedRule final { + struct CapturedOperand final { + std::uint32_t ID = 0; + MachineOperand Operand; + }; + + struct Temporary final { + std::uint32_t ID = 0; + LowLevelType Type; + }; + + struct RenderedOperand final { + enum class Kind : std::uint8_t { + RootOperand, + ComplexOperand, + CapturedOperand, + PhysicalRegister, + Temporary, + CapturedImmediate, + CapturedTypeBits, + Immediate, + }; + + Kind SourceKind = Kind::RootOperand; + std::uint32_t Value = 0; + std::uint32_t Flags = 0; + std::string_view ComplexRenderer; + GIMatchImmediateTransform Transform = GIMatchImmediateTransform::Identity; + std::int64_t Immediate = 0; + }; + + struct Instruction final { + Opcode ResultOpcode; + bool ReplacesRoot = false; + std::vector<RenderedOperand> Operands; + }; + + std::vector<Temporary> Temporaries; + std::vector<CapturedOperand> CapturedOperands; + std::vector<Instruction> Instructions; + std::vector<Register> EraseOperandDefinitions; + std::optional<std::uint32_t> ReplaceRootCapture; + std::uint32_t ClearInstructionFlags = 0; +}; + +[[nodiscard]] bool hasTemporary(const MatchedRule &Rule, std::uint32_t ID) { + return std::ranges::any_of(Rule.Temporaries, [ID](const MatchedRule::Temporary &Temporary) { + return Temporary.ID == ID; + }); +} + +[[nodiscard]] bool hasCapturedOperand(const MatchedRule &Rule, std::uint32_t ID) { + return std::ranges::any_of( + Rule.CapturedOperands, + [ID](const MatchedRule::CapturedOperand &Capture) { return Capture.ID == ID; }); +} + +[[nodiscard]] bool hasRootInstruction(const MatchedRule &Rule) { + return std::ranges::any_of(Rule.Instructions, &MatchedRule::Instruction::ReplacesRoot); +} + +[[nodiscard]] const MatchedRule::CapturedOperand * +findCapturedOperand(const std::vector<MatchedRule::CapturedOperand> &Captures, std::uint32_t ID) { + const auto Position = std::ranges::find(Captures, ID, &MatchedRule::CapturedOperand::ID); + return Position == Captures.end() ? nullptr : &*Position; +} + +template <typename TargetPredicateMatcher> +[[nodiscard]] Expected<std::optional<MatchedRule>> +matchRule(const GIMatchTable &Table, const GIMatchTableRule &Rule, const MachineInstr &Instruction, + const MachineFunction &Function, const TargetDescription &Target, + TargetPredicateMatcher &&MatchesTargetPredicate) { + const std::size_t First = Rule.FirstInstruction; + const std::size_t Count = Rule.InstructionCount; + if (First > Table.Instructions.size() || Count > Table.Instructions.size() - First) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table rule has an invalid instruction range"); + } + + MatchedRule Result; + bool Applying = false; + bool Complete = false; + for (std::size_t Index = First; Index < First + Count; ++Index) { + const GIMatchTableInstruction &Step = Table.Instructions[Index]; + switch (Step.Opcode) { + case GIMatchTableOpcode::CheckOpcode: + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + if (Instruction.opcode().value() != Step.Value) + return std::optional<MatchedRule>{}; + break; + case GIMatchTableOpcode::CheckOperandType: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesType(Instruction, Function, Step); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckInstructionFlag: + if (Applying || Step.Value == 0) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid instruction flag predicate"); + if (!matchesInstructionFlags(Instruction, Step)) + return std::optional<MatchedRule>{}; + break; + case GIMatchTableOpcode::ClearInstructionFlags: + if (!Applying || Complete || Step.Value == 0) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid flag clear action"); + if (Result.ClearInstructionFlags != 0) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table clears instruction flags twice"); + Result.ClearInstructionFlags = Step.Value; + break; + case GIMatchTableOpcode::CheckOperandBank: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesBank(Instruction, Function, Step); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckOperandClass: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesClass(Instruction, Function, Step); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckIntegerPredicate: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesIntegerPredicate(Instruction, Step.OperandIndex, Step.Value); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckFloatingPredicate: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesFloatingPredicate(Instruction, Step.OperandIndex, Step.Value); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckMemorySize: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesMemorySize(Instruction, Step); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckTargetPredicate: { + if (Applying || Step.Symbol.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid target predicate"); + } + auto Matched = MatchesTargetPredicate(Step.Symbol, Instruction, Function, Target); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckOperandImmediateRanges: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesOperandImmediateRanges(Table, Instruction, Step); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckOperandDefConstantRanges: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesOperandDefConstantRanges(Table, Instruction, Function, Step); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckOperandDefOpcode: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesOperandDefOpcode(Instruction, Function, Step); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckOperandDefIntegerPredicate: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesOperandDefIntegerPredicate(Instruction, Function, Step); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CheckOperandDefOperandConstantRanges: { + if (Applying) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table performs a predicate after an action"); + } + auto Matched = matchesOperandDefOperandConstantRanges(Table, Instruction, Function, Step); + if (!Matched) + return std::unexpected(Matched.error()); + if (!*Matched) + return std::optional<MatchedRule>{}; + break; + } + case GIMatchTableOpcode::CaptureDefOperand: { + if (Applying || Complete || Step.Value == 0 || hasCapturedOperand(Result, Step.Value)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid def-use capture"); + } + const MachineInstr *Definition = operandDefinition(Instruction, Function, Step.OperandIndex); + const MachineOperand *Operand = + Definition == nullptr ? nullptr : Definition->operandAt(Step.Auxiliary); + if (Operand == nullptr || Operand->isImplicit()) + return std::optional<MatchedRule>{}; + Result.CapturedOperands.push_back({ + .ID = Step.Value, + .Operand = Operand->cloneDetached(), + }); + break; + } + case GIMatchTableOpcode::CreateTemporary: { + if (Complete || Step.Value == 0 || hasTemporary(Result, Step.Value)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid temporary"); + } + auto Type = operandType(Instruction, Function, Step.OperandIndex); + if (!Type) + return std::unexpected(Type.error()); + if (!*Type) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table temporary has no type source"); + } + Applying = true; + Result.Temporaries.push_back({ + .ID = Step.Value, + .Type = **Type, + }); + break; + } + case GIMatchTableOpcode::CreateTemporaryType: { + if (Complete || Step.Value == 0 || hasTemporary(Result, Step.Value) || + Step.Auxiliary != static_cast<std::uint32_t>(GIMatchTypeKind::Scalar) || + Step.Immediate <= 0 || + static_cast<std::uint64_t>(Step.Immediate) > LowLevelType::MaxBitWidth) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid explicit temporary type"); + } + auto Type = + LowLevelType::makeCategoryNeutralScalar(static_cast<std::uint64_t>(Step.Immediate)); + if (!Type) + return std::unexpected(Type.error()); + Applying = true; + Result.Temporaries.push_back({.ID = Step.Value, .Type = *Type}); + break; + } + case GIMatchTableOpcode::MutateOpcode: + if (Complete || Step.Value == 0 || hasRootInstruction(Result)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid root mutation"); + } + Applying = true; + Result.Instructions.push_back({ + .ResultOpcode = Opcode::fromValue(Step.Value), + .ReplacesRoot = true, + .Operands = {}, + }); + break; + case GIMatchTableOpcode::BuildInstruction: + if (Complete || Step.Value == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid instruction action"); + } + Applying = true; + Result.Instructions.push_back({ + .ResultOpcode = Opcode::fromValue(Step.Value), + .ReplacesRoot = false, + .Operands = {}, + }); + break; + case GIMatchTableOpcode::RenderOperand: + if (Complete || Result.Instructions.empty() || + Instruction.operandAt(Step.OperandIndex) == nullptr || + Instruction.operandAt(Step.OperandIndex)->isImplicit()) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid operand renderer"); + } + Result.Instructions.back().Operands.push_back({ + .SourceKind = MatchedRule::RenderedOperand::Kind::RootOperand, + .Value = Step.OperandIndex, + .ComplexRenderer = {}, + }); + break; + case GIMatchTableOpcode::RenderComplexOperand: + if (Complete || Result.Instructions.empty() || Step.Symbol.empty() || + Instruction.operandAt(Step.OperandIndex) == nullptr || + Instruction.operandAt(Step.OperandIndex)->isImplicit()) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid complex operand renderer"); + } + Result.Instructions.back().Operands.push_back({ + .SourceKind = MatchedRule::RenderedOperand::Kind::ComplexOperand, + .Value = Step.OperandIndex, + .ComplexRenderer = Step.Symbol, + }); + break; + case GIMatchTableOpcode::RenderCapturedOperand: + if (Complete || Result.Instructions.empty() || Step.Value == 0 || + !hasCapturedOperand(Result, Step.Value)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid captured operand renderer"); + } + Result.Instructions.back().Operands.push_back({ + .SourceKind = MatchedRule::RenderedOperand::Kind::CapturedOperand, + .Value = Step.Value, + .ComplexRenderer = {}, + }); + break; + case GIMatchTableOpcode::RenderCapturedTypeBits: + if (Complete || Result.Instructions.empty() || Step.Value == 0 || + !hasCapturedOperand(Result, Step.Value)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid captured type renderer"); + } + Result.Instructions.back().Operands.push_back({ + .SourceKind = MatchedRule::RenderedOperand::Kind::CapturedTypeBits, + .Value = Step.Value, + .ComplexRenderer = {}, + }); + break; + case GIMatchTableOpcode::RenderPhysicalRegister: + if (Complete || Result.Instructions.empty() || Step.Value == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid physical register renderer"); + } + Result.Instructions.back().Operands.push_back({ + .SourceKind = MatchedRule::RenderedOperand::Kind::PhysicalRegister, + .Value = Step.Value, + .Flags = Step.Auxiliary, + .ComplexRenderer = {}, + }); + break; + case GIMatchTableOpcode::RenderTemporary: + if (Complete || Result.Instructions.empty() || !hasTemporary(Result, Step.Value)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid temporary renderer"); + } + Result.Instructions.back().Operands.push_back({ + .SourceKind = MatchedRule::RenderedOperand::Kind::Temporary, + .Value = Step.Value, + .Flags = Step.Auxiliary, + .ComplexRenderer = {}, + }); + break; + case GIMatchTableOpcode::RenderCapturedImmediate: { + if (Complete || Result.Instructions.empty() || + Step.Value > static_cast<std::uint32_t>(GIMatchImmediateTransform::PlusOne)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid captured immediate renderer"); + } + auto Value = operandDefConstantValue(Instruction, Function, Step.OperandIndex); + if (!Value) + return std::unexpected(Value.error()); + if (!*Value) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table lost a captured constant"); + } + Result.Instructions.back().Operands.push_back({ + .SourceKind = MatchedRule::RenderedOperand::Kind::CapturedImmediate, + .Value = Step.OperandIndex, + .ComplexRenderer = {}, + .Transform = static_cast<GIMatchImmediateTransform>(Step.Value), + }); + break; + } + case GIMatchTableOpcode::RenderImmediate: + if (Complete || Result.Instructions.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid immediate renderer"); + } + Result.Instructions.back().Operands.push_back({ + .SourceKind = MatchedRule::RenderedOperand::Kind::Immediate, + .ComplexRenderer = {}, + .Immediate = Step.Immediate, + }); + break; + case GIMatchTableOpcode::ReplaceRootWithCapturedOperand: + if (Table.Kind != GIMatchTableKind::Combiner || Complete || Applying || + !Result.Instructions.empty() || Step.Value == 0 || + !hasCapturedOperand(Result, Step.Value) || Result.ReplaceRootCapture) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid root replacement action"); + } + Applying = true; + Result.ReplaceRootCapture = Step.Value; + break; + case GIMatchTableOpcode::EraseOperandDefinition: { + if (Complete || (Result.Instructions.empty() && !Result.ReplaceRootCapture)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid erased definition"); + } + const MachineOperand *Operand = Instruction.operandAt(Step.OperandIndex); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table erases a non-virtual operand definition"); + } + Result.EraseOperandDefinitions.push_back(*Reg); + break; + } + case GIMatchTableOpcode::Complete: + if (Complete || (Result.Instructions.empty() && !Result.ReplaceRootCapture) || + Index + 1 != First + Count) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid completion action"); + } + Complete = true; + break; + } + } + + if (!Complete || + (Table.Kind == GIMatchTableKind::Selection && + (Result.Instructions.empty() || !hasRootInstruction(Result))) || + (Table.Kind == GIMatchTableKind::Combiner && Result.Instructions.empty() && + !Result.ReplaceRootCapture) || + (Table.Kind == GIMatchTableKind::Combiner && !Result.Instructions.empty() && + Result.ReplaceRootCapture)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table rule does not complete rendering"); + } + return std::optional<MatchedRule>(std::move(Result)); +} + +template <typename ComplexRenderer> +[[nodiscard]] Expected<bool> +complexRenderersMatch(const MatchedRule &Rule, const MachineInstr &Instruction, + const MachineFunction &Function, const TargetDescription &Target, + ComplexRenderer &&RenderComplex) { + for (const MatchedRule::Instruction &Action : Rule.Instructions) { + for (const MatchedRule::RenderedOperand &Rendered : Action.Operands) { + if (Rendered.SourceKind != MatchedRule::RenderedOperand::Kind::ComplexOperand) + continue; + auto Complex = + RenderComplex(Rendered.ComplexRenderer, Instruction, Function, Target, Rendered.Value); + if (!Complex) + return std::unexpected(Complex.error()); + if (!*Complex) + return false; + } + } + return true; +} + +[[nodiscard]] std::size_t explicitOperandCount(const MachineInstr &Instruction) noexcept { + std::size_t Count = 0; + while (Count < Instruction.operandCount() && !Instruction.operandAt(Count)->isImplicit()) + ++Count; + return Count; +} + +[[nodiscard]] Expected<void> +restoreInstruction(MachineInstr &Instruction, Opcode OriginalOpcode, + const std::vector<MachineOperand> &OriginalOperands, + const std::vector<std::pair<std::size_t, std::size_t>> &Ties) { + while (Instruction.operandCount() != 0) { + auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto RestoredOpcode = Instruction.setOpcode(OriginalOpcode); + if (!RestoredOpcode) + return std::unexpected(RestoredOpcode.error()); + for (const MachineOperand &Operand : OriginalOperands) { + auto Added = Instruction.addOperand(Operand.cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + } + for (const auto [DefIndex, UseIndex] : Ties) { + auto Tied = Instruction.tieOperands(DefIndex, UseIndex); + if (!Tied) + return std::unexpected(Tied.error()); + } + return {}; +} + +struct MaterializedTemporary final { + std::uint32_t ID = 0; + Register Reg; +}; + +struct RegisterConstraintSnapshot final { + Register Reg; + std::optional<RegisterBankID> Bank; + std::optional<RegisterClassID> RegClass; +}; + +[[nodiscard]] const MaterializedTemporary * +findTemporary(const std::vector<MaterializedTemporary> &Temporaries, std::uint32_t ID) { + const auto Position = std::ranges::find(Temporaries, ID, &MaterializedTemporary::ID); + return Position == Temporaries.end() ? nullptr : &*Position; +} + +[[nodiscard]] Expected<void> +eraseTemporaries(MachineRegisterInfo &Registers, + const std::vector<MaterializedTemporary> &Temporaries) { + for (auto Iterator = Temporaries.rbegin(); Iterator != Temporaries.rend(); ++Iterator) { + if (!Registers.isKnownVirtualRegister(Iterator->Reg)) + continue; + auto Erased = Registers.eraseVirtualRegister(Iterator->Reg); + if (!Erased) + return std::unexpected(Erased.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> eraseRegisters(MachineRegisterInfo &Registers, + const std::vector<Register> &RegistersToErase) { + for (auto Iterator = RegistersToErase.rbegin(); Iterator != RegistersToErase.rend(); ++Iterator) { + if (!Registers.isKnownVirtualRegister(*Iterator)) + continue; + auto Erased = Registers.eraseVirtualRegister(*Iterator); + if (!Erased) + return std::unexpected(Erased.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> +restoreRegisterConstraints(MachineRegisterInfo &Registers, + const std::vector<RegisterConstraintSnapshot> &Snapshots) { + for (auto Iterator = Snapshots.rbegin(); Iterator != Snapshots.rend(); ++Iterator) { + if (!Registers.isKnownVirtualRegister(Iterator->Reg)) + continue; + if (Iterator->Bank) { + auto Restored = Registers.setRegisterBank(Iterator->Reg, *Iterator->Bank); + if (!Restored) + return std::unexpected(Restored.error()); + } else { + auto Cleared = Registers.clearRegisterBank(Iterator->Reg); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + if (Iterator->RegClass) { + auto Restored = Registers.setRegisterClass(Iterator->Reg, *Iterator->RegClass); + if (!Restored) + return std::unexpected(Restored.error()); + } else { + auto Cleared = Registers.clearRegisterClass(Iterator->Reg); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + } + return {}; +} + +[[nodiscard]] Expected<void> +constrainRenderedInstruction(const InstructionDescriptor &Descriptor, + std::span<const MachineOperand> Operands, MachineFunction &Function, + const TargetDescription &Target, + std::vector<RegisterConstraintSnapshot> &Snapshots) { + if (Descriptor.Operands.empty() && !Operands.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table result has no operand descriptors"); + } + + MachineRegisterInfo &Registers = Function.registerInfo(); + const auto Remember = [&Snapshots](Register Reg, const VirtualRegisterInfo &Info) { + if (std::ranges::find(Snapshots, Reg, &RegisterConstraintSnapshot::Reg) == Snapshots.end()) + Snapshots.push_back({.Reg = Reg, .Bank = Info.Bank, .RegClass = Info.RegClass}); + }; + + for (std::size_t Index = 0; Index < Operands.size(); ++Index) { + const Register *Reg = Operands[Index].getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + + const OperandDescriptor &OperandDescriptor = Index < Descriptor.Operands.size() + ? Descriptor.Operands[Index] + : Descriptor.Operands.back(); + if (!OperandDescriptor.RegBank && !OperandDescriptor.RegClass) + continue; + + auto Info = Registers.virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (OperandDescriptor.RegBank) { + if (Info->get().Bank && *Info->get().Bank != *OperandDescriptor.RegBank) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table result has an incompatible register bank"); + } + if (!Info->get().Bank) { + Remember(*Reg, Info->get()); + auto SetBank = Registers.setRegisterBank(*Reg, *OperandDescriptor.RegBank); + if (!SetBank) + return std::unexpected(SetBank.error()); + } + } + if (OperandDescriptor.RegClass) { + if (Info->get().RegClass && + !Target.registers().isSubclass(*Info->get().RegClass, *OperandDescriptor.RegClass)) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table result has an incompatible register class"); + } + if (!Info->get().RegClass) { + Remember(*Reg, Info->get()); + auto SetClass = Registers.setRegisterClass(*Reg, *OperandDescriptor.RegClass); + if (!SetClass) + return std::unexpected(SetClass.error()); + } + } + } + return {}; +} + +[[nodiscard]] Expected<MachineOperand> +renderOperand(const MatchedRule::RenderedOperand &Rendered, const MachineInstr &Instruction, + const MachineFunction &Function, const TargetDescription &Target, + const std::vector<MaterializedTemporary> &Temporaries, + const std::vector<MatchedRule::CapturedOperand> &CapturedOperands, + std::size_t OriginalExplicitCount) { + switch (Rendered.SourceKind) { + case MatchedRule::RenderedOperand::Kind::RootOperand: + if (Rendered.Value >= OriginalExplicitCount) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table renders an implicit or missing operand"); + } + return Instruction.operandAt(Rendered.Value)->cloneDetached(); + case MatchedRule::RenderedOperand::Kind::ComplexOperand: + return makeUnexpected(ErrorCode::VerificationError, + "generated match table did not resolve a complex operand renderer"); + case MatchedRule::RenderedOperand::Kind::CapturedOperand: { + const MatchedRule::CapturedOperand *Captured = + findCapturedOperand(CapturedOperands, Rendered.Value); + if (Captured == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table renders an unknown captured operand"); + } + return Captured->Operand.cloneDetached(); + } + case MatchedRule::RenderedOperand::Kind::CapturedTypeBits: { + const MatchedRule::CapturedOperand *Captured = + findCapturedOperand(CapturedOperands, Rendered.Value); + if (Captured == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table renders an unknown captured type"); + const Register *Reg = Captured->Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table captured type is not a virtual register"); + auto Type = Function.registerInfo().getType(*Reg); + if (!Type) + return std::unexpected(Type.error()); + if (!Type->isCategoryNeutralScalar()) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table captured type is not scalar"); + return MachineOperand::immediate(static_cast<std::int64_t>(Type->scalarBitWidth())); + } + case MatchedRule::RenderedOperand::Kind::PhysicalRegister: { + auto Physical = PhysicalRegister::create(Rendered.Value); + if (!Physical || Target.registers().physicalRegister(*Physical) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table renders an unknown physical register"); + } + auto Flags = registerFlagsFromRaw(Rendered.Flags); + if (!Flags) + return std::unexpected(Flags.error()); + return MachineOperand::registerOperand(Register::fromPhysical(*Physical), *Flags); + } + case MatchedRule::RenderedOperand::Kind::Temporary: { + const MaterializedTemporary *Temporary = findTemporary(Temporaries, Rendered.Value); + if (Temporary == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table renders an unknown temporary"); + } + auto Flags = registerFlagsFromRaw(Rendered.Flags); + if (!Flags) + return std::unexpected(Flags.error()); + return MachineOperand::registerOperand(Temporary->Reg, *Flags); + } + case MatchedRule::RenderedOperand::Kind::CapturedImmediate: { + auto Captured = operandDefConstantValue(Instruction, Function, Rendered.Value); + if (!Captured) + return std::unexpected(Captured.error()); + if (!*Captured) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table lost a captured constant"); + } + std::int64_t Value = **Captured; + switch (Rendered.Transform) { + case GIMatchImmediateTransform::Identity: + break; + case GIMatchImmediateTransform::Negate: + if (Value == std::numeric_limits<std::int64_t>::min()) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table immediate negation overflows"); + } + Value = -Value; + break; + case GIMatchImmediateTransform::PlusOne: + if (Value == std::numeric_limits<std::int64_t>::max()) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table immediate increment overflows"); + } + ++Value; + break; + } + return MachineOperand::immediate(Value); + } + case MatchedRule::RenderedOperand::Kind::Immediate: + return MachineOperand::immediate(Rendered.Immediate); + } + return makeUnexpected(ErrorCode::VerificationError, + "generated match table has an invalid operand renderer"); +} + +[[nodiscard]] bool compatibleReplacement(Register Destination, Register Source, + const MachineFunction &Function, + const TargetDescription &Target) { + if (!Destination.isVirtual() || Source.isNoRegister()) + return false; + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(Destination); + if (!DestinationInfo) + return false; + const VirtualRegisterInfo &DestinationValue = DestinationInfo->get(); + if (Source.isPhysical()) { + const std::optional<PhysicalRegister> Physical = Source.physicalRegister(); + if (!Physical) + return false; + if (DestinationValue.RegClass && + !Target.registers().registerClassContains(*DestinationValue.RegClass, *Physical)) + return false; + return !DestinationValue.Bank || + Target.registers().bankContainsRegister(*DestinationValue.Bank, *Physical); + } + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo) + return false; + const VirtualRegisterInfo &SourceValue = SourceInfo->get(); + if (DestinationValue.Type != SourceValue.Type) + return false; + if (DestinationValue.Bank && + (!SourceValue.Bank || + !Target.registers().registerBanksOverlap(*DestinationValue.Bank, *SourceValue.Bank))) + return false; + if (DestinationValue.RegClass && + (!SourceValue.RegClass || !Target.registers().registerClassesOverlap( + *DestinationValue.RegClass, *SourceValue.RegClass))) + return false; + return true; +} + +[[nodiscard]] Expected<bool> replaceRootWithCapturedOperand(const MatchedRule &Rule, + MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + if (!Rule.ReplaceRootCapture) + return false; + const MatchedRule::CapturedOperand *Captured = + findCapturedOperand(Rule.CapturedOperands, *Rule.ReplaceRootCapture); + const MachineOperand *Definition = Instruction.operandAt(0); + const Register *Destination = Definition == nullptr ? nullptr : Definition->getIf<Register>(); + const Register *Source = Captured == nullptr ? nullptr : Captured->Operand.getIf<Register>(); + if (Destination == nullptr || Source == nullptr || !Definition->isDef() || Definition->isUse() || + (!Source->isVirtual() && !Source->isPhysical()) || Destination->isNoRegister() || + Source->isNoRegister() || !Destination->isVirtual() || + Function.registerInfo().getUniqueVRegDef(*Destination) != Definition || + !compatibleReplacement(*Destination, *Source, Function, Target)) { + return false; + } + if (Instruction.parent() == nullptr) + return makeUnexpected(ErrorCode::InternalError, "generated combiner root is detached"); + std::vector<MachineOperand *> Uses; + for (MachineOperand &Use : Function.registerInfo().useOperands(*Destination, true)) + Uses.push_back(&Use); + for (MachineOperand *Use : Uses) { + auto Replaced = Use->setRegister(*Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + auto Erased = eraseInstr(Instruction, Function.registerInfo(), Target); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +template <typename Finalizer, typename ComplexRenderer> +[[nodiscard]] Expected<void> applyRule(const MatchedRule &Rule, MachineInstr &Instruction, + MachineFunction &Function, const TargetDescription &Target, + Finalizer &&Finalize, ComplexRenderer &&RenderComplex) { + const InstructionDescriptor *InputDescriptor = Target.instructionDescriptor(Instruction.opcode()); + if (InputDescriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table input has no instruction descriptor"); + } + if (Rule.ReplaceRootCapture) { + auto Replaced = replaceRootWithCapturedOperand(Rule, Instruction, Function, Target); + if (!Replaced) + return std::unexpected(Replaced.error()); + if (!*Replaced) + return makeUnexpected(ErrorCode::VerificationError, + "generated combiner root replacement is incompatible"); + for (const Register Reg : Rule.EraseOperandDefinitions) { + MachineInstr *Definition = Function.registerInfo().getVRegDef(Reg); + if (Definition == nullptr || !isTriviallyDead(*Definition, Function.registerInfo(), Target)) + continue; + auto Erased = eraseInstr(*Definition, Function.registerInfo(), Target); + if (!Erased) + return std::unexpected(Erased.error()); + } + if (Rule.ClearInstructionFlags != 0) { + MachineInstrFlags Flags = Instruction.flags(); + for (std::uint32_t Index = 0; Index < static_cast<std::uint32_t>(MachineInstrFlag::Count); + ++Index) { + if ((Rule.ClearInstructionFlags & (std::uint32_t{1} << Index)) != 0) + Flags.clear(static_cast<MachineInstrFlag>(Index)); + } + Instruction.setFlags(Flags); + } + return {}; + } + const auto RootPosition = + std::ranges::find(Rule.Instructions, true, &MatchedRule::Instruction::ReplacesRoot); + if (RootPosition == Rule.Instructions.end()) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table selection has no root instruction"); + const std::size_t RootIndex = + static_cast<std::size_t>(std::distance(Rule.Instructions.begin(), RootPosition)); + const MachineInstrFlags InputFlags = Instruction.flags(); + const bool InputMayRaiseFPException = + InputDescriptor->Flags.has(InstructionFlag::MayRaiseFPException); + const std::size_t OriginalExplicitCount = explicitOperandCount(Instruction); + const std::size_t OriginalOperandCount = Instruction.operandCount(); + const Opcode OriginalOpcode = Instruction.opcode(); + + std::vector<MachineOperand> OriginalOperands; + OriginalOperands.reserve(OriginalOperandCount); + for (const MachineOperand &Operand : Instruction.operands()) + OriginalOperands.push_back(Operand.cloneDetached()); + + std::vector<std::pair<std::size_t, std::size_t>> Ties; + for (std::size_t Index = 0; Index < OriginalOperandCount; ++Index) { + const std::optional<std::size_t> Other = Instruction.operandAt(Index)->tiedOperandIndex(); + if (Other && Index < *Other) + Ties.emplace_back(Index, *Other); + } + + std::vector<MaterializedTemporary> Temporaries; + Temporaries.reserve(Rule.Temporaries.size()); + for (const MatchedRule::Temporary &Temporary : Rule.Temporaries) { + auto Reg = Function.registerInfo().createVirtualRegister(Temporary.Type); + if (!Reg) { + auto Cleanup = eraseTemporaries(Function.registerInfo(), Temporaries); + if (!Cleanup) + return std::unexpected(Cleanup.error()); + return std::unexpected(Reg.error()); + } + Temporaries.push_back({.ID = Temporary.ID, .Reg = *Reg}); + } + + struct PendingComplexInstruction final { + Opcode ResultOpcode; + std::vector<MachineOperand> Operands; + }; + + std::vector<Register> ComplexRegisters; + std::vector<PendingComplexInstruction> PendingComplexInstructions; + std::vector<MachineInstr *> AddedInstructions; + const auto CleanupGeneratedRegisters = [&]() -> Expected<void> { + auto ErasedTemporaries = eraseTemporaries(Function.registerInfo(), Temporaries); + if (!ErasedTemporaries) + return std::unexpected(ErasedTemporaries.error()); + return eraseRegisters(Function.registerInfo(), ComplexRegisters); + }; + + std::vector<std::vector<MachineOperand>> RenderedInstructions; + RenderedInstructions.reserve(Rule.Instructions.size()); + std::vector<std::optional<std::size_t>> SourceToOutput(OriginalOperandCount); + for (std::size_t InstructionIndex = 0; InstructionIndex < Rule.Instructions.size(); + ++InstructionIndex) { + const MatchedRule::Instruction &Planned = Rule.Instructions[InstructionIndex]; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Planned.ResultOpcode); + if (Descriptor == nullptr || + (!Descriptor->Variadic && Descriptor->Operands.size() != Planned.Operands.size())) { + auto Cleanup = CleanupGeneratedRegisters(); + if (!Cleanup) + return std::unexpected(Cleanup.error()); + return makeUnexpected(ErrorCode::VerificationError, + "generated match table result has an incompatible descriptor"); + } + std::vector<MachineOperand> Operands; + Operands.reserve(Planned.Operands.size()); + for (const MatchedRule::RenderedOperand &Rendered : Planned.Operands) { + Expected<MachineOperand> Operand = [&]() -> Expected<MachineOperand> { + if (Rendered.SourceKind != MatchedRule::RenderedOperand::Kind::ComplexOperand) + return renderOperand(Rendered, Instruction, Function, Target, Temporaries, + Rule.CapturedOperands, OriginalExplicitCount); + + auto Complex = + RenderComplex(Rendered.ComplexRenderer, Instruction, Function, Target, Rendered.Value); + if (!Complex) + return std::unexpected(Complex.error()); + if (!*Complex) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table complex operand unexpectedly did not match"); + if ((*Complex)->Operand && (*Complex)->Instruction) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table complex operand has two renderings"); + if ((*Complex)->Operand) + return std::move(*(*Complex)->Operand); + if (!(*Complex)->Instruction) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table complex operand has no rendering"); + + GIMatchComplexInstruction &Plan = *(*Complex)->Instruction; + if (!Plan.ResultType.isValid()) + return makeUnexpected(ErrorCode::VerificationError, + "generated match table complex instruction has no result type"); + auto Result = Function.registerInfo().createVirtualRegister(Plan.ResultType); + if (!Result) + return std::unexpected(Result.error()); + ComplexRegisters.push_back(*Result); + auto Definition = MachineOperand::registerOperand(*Result, {RegisterOperandFlag::Define}); + if (!Definition) + return std::unexpected(Definition.error()); + std::vector<MachineOperand> PlannedOperands; + PlannedOperands.reserve(Plan.Operands.size() + 1U); + PlannedOperands.push_back(std::move(*Definition)); + for (MachineOperand &PlanOperand : Plan.Operands) + PlannedOperands.push_back(std::move(PlanOperand)); + PendingComplexInstructions.push_back({ + .ResultOpcode = Plan.ResultOpcode, + .Operands = std::move(PlannedOperands), + }); + return MachineOperand::registerOperand(*Result, {RegisterOperandFlag::Use}); + }(); + if (!Operand) { + auto Cleanup = CleanupGeneratedRegisters(); + if (!Cleanup) + return std::unexpected(Cleanup.error()); + return std::unexpected(Operand.error()); + } + if (InstructionIndex == RootIndex && + Rendered.SourceKind == MatchedRule::RenderedOperand::Kind::RootOperand) { + if (SourceToOutput[Rendered.Value] && + Instruction.operandAt(Rendered.Value)->tiedOperandIndex()) { + auto Cleanup = CleanupGeneratedRegisters(); + if (!Cleanup) + return std::unexpected(Cleanup.error()); + return makeUnexpected(ErrorCode::VerificationError, + "generated match table duplicates a tied operand"); + } + SourceToOutput[Rendered.Value] = Operands.size(); + } + Operands.push_back(std::move(*Operand)); + } + RenderedInstructions.push_back(std::move(Operands)); + } + + std::vector<RegisterConstraintSnapshot> ConstraintSnapshots; + bool RootModified = false; + const auto Rollback = [&](Error Failure) -> Expected<void> { + for (auto Iterator = AddedInstructions.rbegin(); Iterator != AddedInstructions.rend(); + ++Iterator) { + if ((*Iterator)->parent() == nullptr) + continue; + auto Erased = eraseInstr(**Iterator, Function.registerInfo(), Target); + if (!Erased) + return std::unexpected(Erased.error()); + } + if (RootModified) { + auto Restored = restoreInstruction(Instruction, OriginalOpcode, OriginalOperands, Ties); + if (!Restored) + return std::unexpected(Restored.error()); + } + auto RestoredConstraints = + restoreRegisterConstraints(Function.registerInfo(), ConstraintSnapshots); + if (!RestoredConstraints) + return std::unexpected(RestoredConstraints.error()); + auto ErasedGenerated = CleanupGeneratedRegisters(); + if (!ErasedGenerated) + return std::unexpected(ErasedGenerated.error()); + return std::unexpected(std::move(Failure)); + }; + + for (const PendingComplexInstruction &Pending : PendingComplexInstructions) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Pending.ResultOpcode); + if (Descriptor == nullptr || + (!Descriptor->Variadic && Descriptor->Operands.size() != Pending.Operands.size())) { + return Rollback(Error(ErrorCode::VerificationError, + "generated match table complex renderer has an incompatible result")); + } + auto Constraint = constrainRenderedInstruction(*Descriptor, Pending.Operands, Function, Target, + ConstraintSnapshots); + if (!Constraint) + return Rollback(Constraint.error()); + } + + for (std::size_t Index = 0; Index < Rule.Instructions.size(); ++Index) { + const InstructionDescriptor *Descriptor = + Target.instructionDescriptor(Rule.Instructions[Index].ResultOpcode); + if (Descriptor == nullptr) + return Rollback(Error(ErrorCode::VerificationError, + "generated match table selects an unknown instruction opcode")); + auto Constraint = constrainRenderedInstruction(*Descriptor, RenderedInstructions[Index], + Function, Target, ConstraintSnapshots); + if (!Constraint) + return Rollback(Constraint.error()); + } + + MachineIRBuilder Builder(Function, Target); + MachineInstr *ComplexAnchor = nullptr; + for (PendingComplexInstruction &Pending : PendingComplexInstructions) { + auto Point = ComplexAnchor == nullptr ? Builder.setInsertPointBefore(Instruction) + : Builder.setInsertPointAfter(*ComplexAnchor); + if (!Point) + return Rollback(Point.error()); + auto Built = Builder.buildInstr(Pending.ResultOpcode); + if (!Built) + return Rollback(Built.error()); + for (MachineOperand &Operand : Pending.Operands) { + auto Added = Built->addOperand(std::move(Operand)); + if (!Added) + return Rollback(Added.error()); + } + auto Complete = Built->build(); + if (!Complete) + return Rollback(Complete.error()); + AddedInstructions.push_back(*Complete); + auto Finalized = Finalize(**Complete, InputFlags, InputMayRaiseFPException); + if (!Finalized) + return Rollback(Finalized.error()); + ComplexAnchor = *Complete; + } + + const auto BuildAction = [&](std::size_t Index, MachineInstr *&Anchor) -> Expected<void> { + auto Point = Anchor == nullptr ? Builder.setInsertPointBefore(Instruction) + : Builder.setInsertPointAfter(*Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(Rule.Instructions[Index].ResultOpcode); + if (!Pending) + return std::unexpected(Pending.error()); + for (MachineOperand &Operand : RenderedInstructions[Index]) { + auto Added = Pending->addOperand(std::move(Operand)); + if (!Added) + return std::unexpected(Added.error()); + } + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + AddedInstructions.push_back(*Built); + auto Finalized = Finalize(**Built, InputFlags, InputMayRaiseFPException); + if (!Finalized) + return std::unexpected(Finalized.error()); + Anchor = *Built; + return {}; + }; + + MachineInstr *BeforeAnchor = ComplexAnchor; + for (std::size_t Index = 0; Index < RootIndex; ++Index) { + auto Built = BuildAction(Index, BeforeAnchor); + if (!Built) + return Rollback(Built.error()); + } + + const MatchedRule::Instruction &RootAction = Rule.Instructions[RootIndex]; + RootModified = true; + auto SetOpcode = Instruction.setOpcode(RootAction.ResultOpcode); + if (!SetOpcode) + return Rollback(SetOpcode.error()); + while (Instruction.operandCount() != 0) { + auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); + if (!Erased) + return Rollback(Erased.error()); + } + for (MachineOperand &Operand : RenderedInstructions[RootIndex]) { + auto Added = Instruction.addOperand(std::move(Operand)); + if (!Added) + return Rollback(Added.error()); + } + for (std::size_t Index = OriginalExplicitCount; Index < OriginalOperandCount; ++Index) { + SourceToOutput[Index] = Instruction.operandCount(); + auto Added = Instruction.addOperand(OriginalOperands[Index].cloneDetached()); + if (!Added) + return Rollback(Added.error()); + } + for (const auto [OldDef, OldUse] : Ties) { + const std::optional<std::size_t> &NewDef = SourceToOutput[OldDef]; + const std::optional<std::size_t> &NewUse = SourceToOutput[OldUse]; + if (!NewDef || !NewUse) + continue; + auto Tied = Instruction.tieOperands(*NewDef, *NewUse); + if (!Tied) + return Rollback(Tied.error()); + } + + MachineInstr *AfterAnchor = &Instruction; + for (std::size_t Index = RootIndex + 1U; Index < Rule.Instructions.size(); ++Index) { + auto Built = BuildAction(Index, AfterAnchor); + if (!Built) + return Rollback(Built.error()); + } + + for (const Register Reg : Rule.EraseOperandDefinitions) { + MachineInstr *Definition = Function.registerInfo().getVRegDef(Reg); + if (Definition == nullptr || Definition == &Instruction || + !isTriviallyDead(*Definition, Function.registerInfo(), Target)) { + continue; + } + auto Erased = eraseInstr(*Definition, Function.registerInfo(), Target); + if (!Erased) + return Rollback(Erased.error()); + } + if (Rule.ClearInstructionFlags != 0) { + MachineInstrFlags Flags = Instruction.flags(); + for (std::uint32_t Index = 0; Index < static_cast<std::uint32_t>(MachineInstrFlag::Count); + ++Index) { + if ((Rule.ClearInstructionFlags & (std::uint32_t{1} << Index)) != 0) + Flags.clear(static_cast<MachineInstrFlag>(Index)); + } + Instruction.setFlags(Flags); + } + return {}; +} + +} // namespace + +Expected<void> GIMatchTableExecutor::finalizeGeneratedInstruction(MachineInstr &, MachineFunction &, + const TargetDescription &, + MachineInstrFlags, bool) const { + return {}; +} + +Expected<bool> GIMatchTableExecutor::matchesTargetPredicate(std::string_view Name, + const MachineInstr &, + const MachineFunction &, + const TargetDescription &) const { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table uses unknown target predicate '" + + std::string(Name) + "'"); +} + +Expected<std::optional<GIMatchComplexOperand>> +GIMatchTableExecutor::renderComplexOperand(std::string_view, const MachineInstr &, + const MachineFunction &, const TargetDescription &, + std::uint32_t) const { + return std::optional<GIMatchComplexOperand>{}; +} + +Expected<bool> GIMatchTableExecutor::match(const GIMatchTable &Table, + const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) const { + for (const GIMatchTableRule &Rule : Table.Rules) { + auto Match = matchRule(Table, Rule, Instruction, Function, Target, + [this](std::string_view Name, const MachineInstr &Input, + const MachineFunction &InputFunction, + const TargetDescription &InputTarget) { + return matchesTargetPredicate(Name, Input, InputFunction, InputTarget); + }); + if (!Match) + return std::unexpected(Match.error()); + if (!*Match) + continue; + auto Complex = complexRenderersMatch( + **Match, Instruction, Function, Target, + [this](std::string_view Name, const MachineInstr &Input, + const MachineFunction &InputFunction, const TargetDescription &InputTarget, + std::uint32_t OperandIndex) { + return renderComplexOperand(Name, Input, InputFunction, InputTarget, OperandIndex); + }); + if (!Complex) + return std::unexpected(Complex.error()); + if (!*Complex) + continue; + for (const MatchedRule::Instruction &Action : (*Match)->Instructions) { + if (Target.instructionDescriptor(Action.ResultOpcode) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "generated match table selects an unknown instruction opcode"); + } + } + return true; + } + return false; +} + +Expected<bool> GIMatchTableExecutor::select(const GIMatchTable &Table, MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const { + for (const GIMatchTableRule &Rule : Table.Rules) { + auto Match = matchRule(Table, Rule, Instruction, Function, Target, + [this](std::string_view Name, const MachineInstr &Input, + const MachineFunction &InputFunction, + const TargetDescription &InputTarget) { + return matchesTargetPredicate(Name, Input, InputFunction, InputTarget); + }); + if (!Match) + return std::unexpected(Match.error()); + if (!*Match) + continue; + auto Complex = complexRenderersMatch( + **Match, Instruction, Function, Target, + [this](std::string_view Name, const MachineInstr &Input, + const MachineFunction &InputFunction, const TargetDescription &InputTarget, + std::uint32_t OperandIndex) { + return renderComplexOperand(Name, Input, InputFunction, InputTarget, OperandIndex); + }); + if (!Complex) + return std::unexpected(Complex.error()); + if (!*Complex) + continue; + auto Applied = applyRule( + **Match, Instruction, Function, Target, + [this, &Function, &Target](MachineInstr &Output, MachineInstrFlags InputFlags, + bool InputMayRaiseFPException) { + return finalizeGeneratedInstruction(Output, Function, Target, InputFlags, + InputMayRaiseFPException); + }, + [this](std::string_view Name, const MachineInstr &Input, + const MachineFunction &InputFunction, const TargetDescription &InputTarget, + std::uint32_t OperandIndex) { + return renderComplexOperand(Name, Input, InputFunction, InputTarget, OperandIndex); + }); + if (!Applied) + return std::unexpected(Applied.error()); + return true; + } + return false; +} + +Expected<bool> GIMatchTableExecutor::combine(const GIMatchTable &Table, MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const { + if (Table.Kind != GIMatchTableKind::Combiner) + return makeUnexpected(ErrorCode::InvalidArgument, + "GlobalISel combiner received a selection match table"); + for (const GIMatchTableRule &Rule : Table.Rules) { + auto Match = matchRule(Table, Rule, Instruction, Function, Target, + [this](std::string_view Name, const MachineInstr &Input, + const MachineFunction &InputFunction, + const TargetDescription &InputTarget) { + return matchesTargetPredicate(Name, Input, InputFunction, InputTarget); + }); + if (!Match) + return std::unexpected(Match.error()); + if (!*Match) + continue; + if (!(*Match)->ReplaceRootCapture) { + for (const MatchedRule::Instruction &Action : (*Match)->Instructions) { + if (Target.instructionDescriptor(Action.ResultOpcode) == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "generated combiner selects an unknown instruction opcode"); + } + } + auto Applied = applyRule( + **Match, Instruction, Function, Target, + [this, &Function, &Target](MachineInstr &Output, MachineInstrFlags InputFlags, + bool InputMayRaiseFPException) { + return finalizeGeneratedInstruction(Output, Function, Target, InputFlags, + InputMayRaiseFPException); + }, + [this](std::string_view Name, const MachineInstr &Input, + const MachineFunction &InputFunction, const TargetDescription &InputTarget, + std::uint32_t OperandIndex) { + return renderComplexOperand(Name, Input, InputFunction, InputTarget, OperandIndex); + }); + if (!Applied) + return std::unexpected(Applied.error()); + return true; + } + return false; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp b/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp new file mode 100644 index 0000000..4ce1473 --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h" + +#include <algorithm> + +namespace rcc { + +void GISelObserverWrapper::addObserver(GISelChangeObserver &Observer) { + Observers.push_back(&Observer); +} + +void GISelObserverWrapper::removeObserver(GISelChangeObserver &Observer) { + const auto Position = std::ranges::find(Observers, &Observer); + if (Position != Observers.end()) + Observers.erase(Position); +} + +void GISelObserverWrapper::clearObservers() noexcept { + Observers.clear(); +} + +void GISelObserverWrapper::erasingInstr(MachineInstr &Instruction) { + for (GISelChangeObserver *Observer : Observers) + Observer->erasingInstr(Instruction); +} + +void GISelObserverWrapper::createdInstr(MachineInstr &Instruction) { + for (GISelChangeObserver *Observer : Observers) + Observer->createdInstr(Instruction); +} + +void GISelObserverWrapper::changingInstr(MachineInstr &Instruction) { + for (GISelChangeObserver *Observer : Observers) + Observer->changingInstr(Instruction); +} + +void GISelObserverWrapper::changedInstr(MachineInstr &Instruction) { + for (GISelChangeObserver *Observer : Observers) + Observer->changedInstr(Instruction); +} + +void GISelObserverWrapper::handleInsertion(MachineInstr &Instruction) { + createdInstr(Instruction); +} + +void GISelObserverWrapper::handleRemoval(MachineInstr &Instruction) { + erasingInstr(Instruction); +} + +MachineFunctionDelegateInstaller::MachineFunctionDelegateInstaller( + MachineFunction &Function, MachineFunction::Delegate &Delegate) + : Function(&Function), Delegate(&Delegate) { + Function.setDelegate(Delegate); +} + +MachineFunctionDelegateInstaller::~MachineFunctionDelegateInstaller() { + Function->resetDelegate(*Delegate); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/backend/lib/CodeGen/GlobalISel/GISelValueTracking.cpp new file mode 100644 index 0000000..85e552c --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/GISelValueTracking.cpp @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/GISelValueTracking.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/Utils.h" +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] std::optional<Register> registerOperand(const MachineInstr &Instruction, + std::size_t Index) noexcept { + const MachineOperand *Operand = Instruction.operandAt(Index); + return Operand == nullptr ? std::nullopt : Operand->registerValue(); +} + +[[nodiscard]] Expected<APInt> zero(std::uint32_t BitWidth) { + return APInt::fromUnsigned(BitWidth, 0); +} + +[[nodiscard]] Expected<APInt> resizeMask(const APInt &Value, std::uint32_t BitWidth) { + if (Value.bitWidth() == BitWidth) + return Value; + if (Value.bitWidth() == 0) + return zero(BitWidth); + if (Value.bitWidth() < BitWidth) + return Value.zext(BitWidth); + return Value.trunc(BitWidth); +} + +[[nodiscard]] Expected<APInt> highBitsMask(std::uint32_t BitWidth, std::uint32_t LowBitCount) { + auto LowBits = APInt::getLowBitsSet(BitWidth, LowBitCount); + if (!LowBits) + return std::unexpected(LowBits.error()); + return LowBits->bitwiseNot(); +} + +[[nodiscard]] Expected<APInt> mergeMasks(const APInt &Left, const APInt &Right) { + return Left.bitwiseOr(Right); +} + +[[nodiscard]] Expected<APInt> intersectMasks(const APInt &Left, const APInt &Right) { + return Left.bitwiseAnd(Right); +} + +[[nodiscard]] Expected<APInt> addMasks(const APInt &Left, const APInt &Right) { + if (Left.bitWidth() == 0 || Left.bitWidth() != Right.bitWidth()) + return makeUnexpected(ErrorCode::InvalidArgument, + "known-bits add operands must have the same nonzero width"); + std::vector<std::uint64_t> Words(Left.words().size(), 0); + unsigned __int128 Carry = 0; + for (std::size_t Index = 0; Index != Words.size(); ++Index) { + const unsigned __int128 Sum = static_cast<unsigned __int128>(Left.word(Index)) + + static_cast<unsigned __int128>(Right.word(Index)) + Carry; + Words[Index] = static_cast<std::uint64_t>(Sum); + Carry = Sum >> 64U; + } + return APInt::create(Left.bitWidth(), Words); +} + +struct MaskProduct final { + APInt Value; + bool Overflow = false; +}; + +[[nodiscard]] Expected<MaskProduct> multiplyMasks(const APInt &Left, const APInt &Right) { + if (Left.bitWidth() == 0 || Left.bitWidth() != Right.bitWidth()) + return makeUnexpected(ErrorCode::InvalidArgument, + "known-bits multiply operands must have the same nonzero width"); + + const std::size_t WordCount = Left.words().size(); + std::vector<std::uint64_t> Product(WordCount * 2U, 0); + for (std::size_t LeftIndex = 0; LeftIndex != WordCount; ++LeftIndex) { + for (std::size_t RightIndex = 0; RightIndex != WordCount; ++RightIndex) { + const std::size_t ProductIndex = LeftIndex + RightIndex; + const unsigned __int128 Partial = static_cast<unsigned __int128>(Left.word(LeftIndex)) * + static_cast<unsigned __int128>(Right.word(RightIndex)); + unsigned __int128 Sum = static_cast<unsigned __int128>(Product[ProductIndex]) + + static_cast<std::uint64_t>(Partial); + Product[ProductIndex] = static_cast<std::uint64_t>(Sum); + unsigned __int128 Carry = (Partial >> 64U) + (Sum >> 64U); + for (std::size_t Index = ProductIndex + 1U; Carry != 0 && Index != Product.size(); ++Index) { + Sum = static_cast<unsigned __int128>(Product[Index]) + static_cast<std::uint64_t>(Carry); + Product[Index] = static_cast<std::uint64_t>(Sum); + Carry = (Carry >> 64U) + (Sum >> 64U); + } + } + } + + bool Overflow = false; + for (std::size_t Index = WordCount; Index != Product.size(); ++Index) + Overflow |= Product[Index] != 0; + const std::uint32_t HighBits = Left.bitWidth() % 64U; + if (HighBits != 0) { + const std::uint64_t HighMask = (std::uint64_t{1} << HighBits) - 1U; + Overflow |= (Product[WordCount - 1U] & ~HighMask) != 0; + Product[WordCount - 1U] &= HighMask; + } + Product.resize(WordCount); + auto Value = APInt::create(Left.bitWidth(), Product); + if (!Value) + return std::unexpected(Value.error()); + return MaskProduct{.Value = std::move(*Value), .Overflow = Overflow}; +} + +[[nodiscard]] std::uint32_t countTrailingOnes(const APInt &Value) noexcept { + std::uint32_t Count = 0; + while (Count != Value.bitWidth() && Value.testBit(Count)) + ++Count; + return Count; +} + +[[nodiscard]] Expected<APInt> constantAtWidth(const APInt &Value, std::uint32_t BitWidth) { + return resizeMask(Value, BitWidth); +} + +[[nodiscard]] std::optional<std::uint32_t> +constantShiftAmount(Register Reg, const MachineRegisterInfo &MRI, std::uint32_t BitWidth) { + const auto Constant = getIConstantVRegValWithLookThrough(Reg, MRI); + if (!Constant || Constant->Value.activeBitWidth() > 32) + return std::nullopt; + const std::uint64_t Amount = Constant->Value.word(0); + if (Amount >= BitWidth) + return std::nullopt; + return static_cast<std::uint32_t>(Amount); +} + +[[nodiscard]] Expected<APInt> shiftMaskLeft(const APInt &Value, std::uint32_t Amount) { + std::vector<std::uint64_t> Words(Value.words().size(), 0); + for (std::uint32_t Bit = 0; Bit + Amount < Value.bitWidth(); ++Bit) { + if (Value.testBit(Bit)) { + const std::uint32_t DestinationBit = Bit + Amount; + Words[DestinationBit / 64U] |= std::uint64_t{1} << (DestinationBit % 64U); + } + } + return APInt::create(Value.bitWidth(), Words); +} + +[[nodiscard]] Expected<APInt> shiftMaskRight(const APInt &Value, std::uint32_t Amount) { + std::vector<std::uint64_t> Words(Value.words().size(), 0); + for (std::uint32_t Bit = Amount; Bit < Value.bitWidth(); ++Bit) { + if (Value.testBit(Bit)) { + const std::uint32_t DestinationBit = Bit - Amount; + Words[DestinationBit / 64U] |= std::uint64_t{1} << (DestinationBit % 64U); + } + } + return APInt::create(Value.bitWidth(), Words); +} + +[[nodiscard]] std::uint32_t constantNumSignBits(const APInt &Value) noexcept { + if (Value.bitWidth() == 0) + return 1; + if (Value.isNegative()) + return static_cast<std::uint32_t>(Value.countlOne()); + auto Inverted = Value.bitwiseNot(); + return Inverted ? static_cast<std::uint32_t>(Inverted->countlOne()) : 1U; +} + +} // namespace + +GISelValueTracking::GISelValueTracking(const MachineFunction &Function, + const TargetDescription &Target, unsigned MaxDepth) noexcept + : Function(Function), MRI(Function.registerInfo()), Target(Target), MaxDepth(MaxDepth) {} + +Expected<GISelValueTracking::KnownBits> +GISelValueTracking::makeUnknown(std::uint32_t BitWidth) const { + auto Zero = zero(BitWidth); + if (!Zero) + return std::unexpected(Zero.error()); + auto One = zero(BitWidth); + if (!One) + return std::unexpected(One.error()); + return KnownBits{.Zero = std::move(*Zero), .One = std::move(*One)}; +} + +Expected<GISelValueTracking::KnownBits> GISelValueTracking::getKnownBits(Register Reg, + unsigned Depth) const { + auto Type = MRI.getType(Reg); + if (!Type) + return std::unexpected(Type.error()); + const std::uint32_t BitWidth = Type->scalarBitWidth(); + auto Unknown = makeUnknown(BitWidth); + if (!Unknown) + return std::unexpected(Unknown.error()); + if (Depth >= MaxDepth) + return Unknown; + + const MachineInstr *Definition = MRI.getVRegDef(Reg); + if (Definition == nullptr) + return Unknown; + + const auto Opcode = static_cast<GenericOpcode>(Definition->opcode().value()); + if (Opcode == GenericOpcode::G_CONSTANT) { + const MachineOperand *ValueOperand = Definition->operandAt(1); + const auto *Immediate = + ValueOperand == nullptr ? nullptr : ValueOperand->getIf<IntegerImmediateOperand>(); + if (Immediate == nullptr) + return Unknown; + auto Value = constantAtWidth(Immediate->Value, BitWidth); + if (!Value) + return std::unexpected(Value.error()); + auto KnownZero = Value->bitwiseNot(); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*Value)}; + } + + if (Opcode == GenericOpcode::COPY) { + const auto Source = registerOperand(*Definition, 1); + if (!Source || !Source->isVirtual()) + return Unknown; + auto SourceBits = getKnownBits(*Source, Depth); + if (!SourceBits) + return std::unexpected(SourceBits.error()); + auto KnownZero = resizeMask(SourceBits->Zero, BitWidth); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto KnownOne = resizeMask(SourceBits->One, BitWidth); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)}; + } + + if (Opcode == GenericOpcode::G_ICMP || Opcode == GenericOpcode::G_FCMP) { + if (!Type->isVector() && BitWidth > 1 && + Target.getBooleanContents(false, Opcode == GenericOpcode::G_FCMP) == + BooleanContent::ZeroOrOne) { + auto KnownZero = highBitsMask(BitWidth, 1); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + Unknown->Zero = std::move(*KnownZero); + } + return Unknown; + } + + switch (Opcode) { + case GenericOpcode::G_UADDO: + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_SSUBE: + case GenericOpcode::G_UMULO: + case GenericOpcode::G_SMULO: { + const auto BooleanResult = registerOperand(*Definition, 1); + if (BooleanResult && *BooleanResult == Reg && !Type->isVector() && BitWidth > 1 && + Target.getBooleanContents(false, false) == BooleanContent::ZeroOrOne) { + auto KnownZero = highBitsMask(BitWidth, 1); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + Unknown->Zero = std::move(*KnownZero); + } + return Unknown; + } + default: + break; + } + + const auto Source = registerOperand(*Definition, 1); + if (!Source) + return Unknown; + + if (Opcode == GenericOpcode::G_TRUNC || Opcode == GenericOpcode::G_ANYEXT || + Opcode == GenericOpcode::G_ZEXT || Opcode == GenericOpcode::G_SEXT || + Opcode == GenericOpcode::G_ASSERT_ZEXT || Opcode == GenericOpcode::G_ASSERT_SEXT || + Opcode == GenericOpcode::G_SEXT_INREG) { + auto SourceBits = getKnownBits(*Source, Depth + 1U); + if (!SourceBits) + return std::unexpected(SourceBits.error()); + auto SourceType = MRI.getType(*Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + const std::uint32_t SourceWidth = SourceType->scalarBitWidth(); + auto KnownZero = resizeMask(SourceBits->Zero, BitWidth); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto KnownOne = resizeMask(SourceBits->One, BitWidth); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + + if (Opcode == GenericOpcode::G_ZEXT && SourceWidth < BitWidth) { + auto HighZero = highBitsMask(BitWidth, SourceWidth); + if (!HighZero) + return std::unexpected(HighZero.error()); + KnownZero = mergeMasks(*KnownZero, *HighZero); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + } + + std::uint32_t ExtendedFrom = SourceWidth; + if (Opcode == GenericOpcode::G_ASSERT_ZEXT || Opcode == GenericOpcode::G_ASSERT_SEXT || + Opcode == GenericOpcode::G_SEXT_INREG) { + const MachineOperand *WidthOperand = Definition->operandAt(2); + const auto *Width = WidthOperand == nullptr ? nullptr : WidthOperand->getIf<std::int64_t>(); + if (Width == nullptr || *Width <= 0 || static_cast<std::uint64_t>(*Width) > BitWidth) + return Unknown; + ExtendedFrom = static_cast<std::uint32_t>(*Width); + } + + if (Opcode == GenericOpcode::G_ASSERT_ZEXT) { + auto HighZero = highBitsMask(BitWidth, ExtendedFrom); + if (!HighZero) + return std::unexpected(HighZero.error()); + KnownZero = mergeMasks(*KnownZero, *HighZero); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + } + + if ((Opcode == GenericOpcode::G_SEXT || Opcode == GenericOpcode::G_ASSERT_SEXT || + Opcode == GenericOpcode::G_SEXT_INREG) && + ExtendedFrom < BitWidth) { + auto HighBits = highBitsMask(BitWidth, ExtendedFrom); + if (!HighBits) + return std::unexpected(HighBits.error()); + if (KnownZero->testBit(ExtendedFrom - 1U)) { + KnownZero = mergeMasks(*KnownZero, *HighBits); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + } else if (KnownOne->testBit(ExtendedFrom - 1U)) { + KnownOne = mergeMasks(*KnownOne, *HighBits); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + } + } + + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)}; + } + + if (Opcode == GenericOpcode::G_AND || Opcode == GenericOpcode::G_OR || + Opcode == GenericOpcode::G_XOR) { + const auto Right = registerOperand(*Definition, 2); + if (!Right) + return Unknown; + auto LeftBits = getKnownBits(*Source, Depth + 1U); + if (!LeftBits) + return std::unexpected(LeftBits.error()); + auto RightBits = getKnownBits(*Right, Depth + 1U); + if (!RightBits) + return std::unexpected(RightBits.error()); + + if (Opcode == GenericOpcode::G_AND) { + auto KnownZero = mergeMasks(LeftBits->Zero, RightBits->Zero); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto KnownOne = intersectMasks(LeftBits->One, RightBits->One); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)}; + } + if (Opcode == GenericOpcode::G_OR) { + auto KnownZero = intersectMasks(LeftBits->Zero, RightBits->Zero); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto KnownOne = mergeMasks(LeftBits->One, RightBits->One); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)}; + } + + auto BothZero = intersectMasks(LeftBits->Zero, RightBits->Zero); + if (!BothZero) + return std::unexpected(BothZero.error()); + auto BothOne = intersectMasks(LeftBits->One, RightBits->One); + if (!BothOne) + return std::unexpected(BothOne.error()); + auto KnownZero = mergeMasks(*BothZero, *BothOne); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto LeftZeroRightOne = intersectMasks(LeftBits->Zero, RightBits->One); + if (!LeftZeroRightOne) + return std::unexpected(LeftZeroRightOne.error()); + auto LeftOneRightZero = intersectMasks(LeftBits->One, RightBits->Zero); + if (!LeftOneRightZero) + return std::unexpected(LeftOneRightZero.error()); + auto KnownOne = mergeMasks(*LeftZeroRightOne, *LeftOneRightZero); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)}; + } + + if (Opcode == GenericOpcode::G_ADD) { + const auto Right = registerOperand(*Definition, 2); + if (!Right) + return Unknown; + auto LeftBits = getKnownBits(*Source, Depth + 1U); + if (!LeftBits) + return std::unexpected(LeftBits.error()); + auto RightBits = getKnownBits(*Right, Depth + 1U); + if (!RightBits) + return std::unexpected(RightBits.error()); + + auto LeftMaximum = LeftBits->Zero.bitwiseNot(); + if (!LeftMaximum) + return std::unexpected(LeftMaximum.error()); + auto RightMaximum = RightBits->Zero.bitwiseNot(); + if (!RightMaximum) + return std::unexpected(RightMaximum.error()); + auto PossibleSumZero = addMasks(*LeftMaximum, *RightMaximum); + if (!PossibleSumZero) + return std::unexpected(PossibleSumZero.error()); + auto PossibleSumOne = addMasks(LeftBits->One, RightBits->One); + if (!PossibleSumOne) + return std::unexpected(PossibleSumOne.error()); + + auto CarryKnownZero = PossibleSumZero->bitwiseXor(LeftBits->Zero); + if (!CarryKnownZero) + return std::unexpected(CarryKnownZero.error()); + CarryKnownZero = CarryKnownZero->bitwiseXor(RightBits->Zero); + if (!CarryKnownZero) + return std::unexpected(CarryKnownZero.error()); + CarryKnownZero = CarryKnownZero->bitwiseNot(); + if (!CarryKnownZero) + return std::unexpected(CarryKnownZero.error()); + auto CarryKnownOne = PossibleSumOne->bitwiseXor(LeftBits->One); + if (!CarryKnownOne) + return std::unexpected(CarryKnownOne.error()); + CarryKnownOne = CarryKnownOne->bitwiseXor(RightBits->One); + if (!CarryKnownOne) + return std::unexpected(CarryKnownOne.error()); + + auto LeftKnown = mergeMasks(LeftBits->Zero, LeftBits->One); + if (!LeftKnown) + return std::unexpected(LeftKnown.error()); + auto RightKnown = mergeMasks(RightBits->Zero, RightBits->One); + if (!RightKnown) + return std::unexpected(RightKnown.error()); + auto CarryKnown = mergeMasks(*CarryKnownZero, *CarryKnownOne); + if (!CarryKnown) + return std::unexpected(CarryKnown.error()); + auto Known = intersectMasks(*LeftKnown, *RightKnown); + if (!Known) + return std::unexpected(Known.error()); + Known = intersectMasks(*Known, *CarryKnown); + if (!Known) + return std::unexpected(Known.error()); + + auto NotPossibleSumZero = PossibleSumZero->bitwiseNot(); + if (!NotPossibleSumZero) + return std::unexpected(NotPossibleSumZero.error()); + auto KnownZero = intersectMasks(*NotPossibleSumZero, *Known); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto KnownOne = intersectMasks(*PossibleSumOne, *Known); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)}; + } + + if (Opcode == GenericOpcode::G_MUL) { + const auto Right = registerOperand(*Definition, 2); + if (!Right) + return Unknown; + auto LeftBits = getKnownBits(*Source, Depth + 1U); + if (!LeftBits) + return std::unexpected(LeftBits.error()); + auto RightBits = getKnownBits(*Right, Depth + 1U); + if (!RightBits) + return std::unexpected(RightBits.error()); + + auto LeftMaximum = LeftBits->Zero.bitwiseNot(); + if (!LeftMaximum) + return std::unexpected(LeftMaximum.error()); + auto RightMaximum = RightBits->Zero.bitwiseNot(); + if (!RightMaximum) + return std::unexpected(RightMaximum.error()); + auto MaximumProduct = multiplyMasks(*LeftMaximum, *RightMaximum); + if (!MaximumProduct) + return std::unexpected(MaximumProduct.error()); + const std::uint32_t LeadingZeroes = + MaximumProduct->Overflow + ? 0 + : BitWidth - static_cast<std::uint32_t>(MaximumProduct->Value.activeBitWidth()); + + auto LeftKnown = mergeMasks(LeftBits->Zero, LeftBits->One); + if (!LeftKnown) + return std::unexpected(LeftKnown.error()); + auto RightKnown = mergeMasks(RightBits->Zero, RightBits->One); + if (!RightKnown) + return std::unexpected(RightKnown.error()); + const std::uint32_t LeftTrailingKnown = countTrailingOnes(*LeftKnown); + const std::uint32_t RightTrailingKnown = countTrailingOnes(*RightKnown); + const std::uint32_t LeftTrailingZeroes = countTrailingOnes(LeftBits->Zero); + const std::uint32_t RightTrailingZeroes = countTrailingOnes(RightBits->Zero); + const std::uint32_t TrailingZeroes = LeftTrailingZeroes + RightTrailingZeroes; + const std::uint32_t SmallestOperand = + std::min(LeftTrailingKnown - LeftTrailingZeroes, RightTrailingKnown - RightTrailingZeroes); + const std::uint32_t ResultBitsKnown = std::min(SmallestOperand + TrailingZeroes, BitWidth); + + auto BottomKnown = multiplyMasks(LeftBits->One, RightBits->One); + if (!BottomKnown) + return std::unexpected(BottomKnown.error()); + auto LowMask = APInt::getLowBitsSet(BitWidth, ResultBitsKnown); + if (!LowMask) + return std::unexpected(LowMask.error()); + auto NotBottomKnown = BottomKnown->Value.bitwiseNot(); + if (!NotBottomKnown) + return std::unexpected(NotBottomKnown.error()); + auto KnownZero = intersectMasks(*NotBottomKnown, *LowMask); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto HighZero = highBitsMask(BitWidth, BitWidth - LeadingZeroes); + if (!HighZero) + return std::unexpected(HighZero.error()); + KnownZero = mergeMasks(*KnownZero, *HighZero); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto KnownOne = intersectMasks(BottomKnown->Value, *LowMask); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)}; + } + + if (Opcode == GenericOpcode::G_SHL || Opcode == GenericOpcode::G_LSHR || + Opcode == GenericOpcode::G_ASHR) { + const auto AmountReg = registerOperand(*Definition, 2); + if (!AmountReg) + return Unknown; + const auto Amount = constantShiftAmount(*AmountReg, MRI, BitWidth); + if (!Amount) + return Unknown; + auto SourceBits = getKnownBits(*Source, Depth + 1U); + if (!SourceBits) + return std::unexpected(SourceBits.error()); + + if (Opcode == GenericOpcode::G_SHL) { + auto KnownZero = shiftMaskLeft(SourceBits->Zero, *Amount); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto LowZero = APInt::getLowBitsSet(BitWidth, *Amount); + if (!LowZero) + return std::unexpected(LowZero.error()); + KnownZero = mergeMasks(*KnownZero, *LowZero); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto KnownOne = shiftMaskLeft(SourceBits->One, *Amount); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)}; + } + + auto KnownZero = shiftMaskRight(SourceBits->Zero, *Amount); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + auto KnownOne = shiftMaskRight(SourceBits->One, *Amount); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + auto HighBits = highBitsMask(BitWidth, BitWidth - *Amount); + if (!HighBits) + return std::unexpected(HighBits.error()); + if (Opcode == GenericOpcode::G_LSHR || SourceBits->Zero.testBit(BitWidth - 1U)) { + KnownZero = mergeMasks(*KnownZero, *HighBits); + if (!KnownZero) + return std::unexpected(KnownZero.error()); + } else if (SourceBits->One.testBit(BitWidth - 1U)) { + KnownOne = mergeMasks(*KnownOne, *HighBits); + if (!KnownOne) + return std::unexpected(KnownOne.error()); + } + return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)}; + } + + return Unknown; +} + +Expected<APInt> GISelValueTracking::getKnownZeroes(Register Reg) const { + auto Known = getKnownBits(Reg); + if (!Known) + return std::unexpected(Known.error()); + return std::move(Known->Zero); +} + +Expected<std::uint32_t> GISelValueTracking::computeNumSignBits(Register Reg, unsigned Depth) const { + auto Type = MRI.getType(Reg); + if (!Type) + return std::unexpected(Type.error()); + const std::uint32_t BitWidth = Type->scalarBitWidth(); + if (Depth >= MaxDepth) + return 1U; + + const MachineInstr *Definition = MRI.getVRegDef(Reg); + if (Definition == nullptr) + return 1U; + const auto Opcode = static_cast<GenericOpcode>(Definition->opcode().value()); + std::uint32_t FirstAnswer = 1U; + + if (Opcode == GenericOpcode::G_CONSTANT) { + const MachineOperand *ValueOperand = Definition->operandAt(1); + const auto *Immediate = + ValueOperand == nullptr ? nullptr : ValueOperand->getIf<IntegerImmediateOperand>(); + if (Immediate == nullptr) + return 1U; + auto Value = constantAtWidth(Immediate->Value, BitWidth); + if (!Value) + return std::unexpected(Value.error()); + return constantNumSignBits(*Value); + } + + const auto Source = registerOperand(*Definition, 1); + if (Opcode == GenericOpcode::COPY && Source && Source->isVirtual()) + return computeNumSignBits(*Source, Depth); + + if (Opcode == GenericOpcode::G_SEXT && Source) { + auto SourceType = MRI.getType(*Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + auto SourceSignBits = computeNumSignBits(*Source, Depth + 1U); + if (!SourceSignBits) + return std::unexpected(SourceSignBits.error()); + return std::min(BitWidth, *SourceSignBits + BitWidth - SourceType->scalarBitWidth()); + } + + if ((Opcode == GenericOpcode::G_ASSERT_SEXT || Opcode == GenericOpcode::G_SEXT_INREG) && Source) { + const MachineOperand *WidthOperand = Definition->operandAt(2); + const auto *Width = WidthOperand == nullptr ? nullptr : WidthOperand->getIf<std::int64_t>(); + if (Width != nullptr && *Width > 0 && static_cast<std::uint64_t>(*Width) <= BitWidth) { + auto SourceSignBits = computeNumSignBits(*Source, Depth + 1U); + if (!SourceSignBits) + return std::unexpected(SourceSignBits.error()); + return std::max(*SourceSignBits, BitWidth - static_cast<std::uint32_t>(*Width) + 1U); + } + } + + if ((Opcode == GenericOpcode::G_AND || Opcode == GenericOpcode::G_OR || + Opcode == GenericOpcode::G_XOR) && + Source) { + const auto Right = registerOperand(*Definition, 2); + if (Right) { + auto LeftSignBits = computeNumSignBits(*Source, Depth + 1U); + if (!LeftSignBits) + return std::unexpected(LeftSignBits.error()); + if (*LeftSignBits != 1) { + auto RightSignBits = computeNumSignBits(*Right, Depth + 1U); + if (!RightSignBits) + return std::unexpected(RightSignBits.error()); + if (*RightSignBits != 1) + FirstAnswer = std::min(*LeftSignBits, *RightSignBits); + } + } + } + + if (Opcode == GenericOpcode::G_ASHR && Source) { + const auto Right = registerOperand(*Definition, 2); + if (Right) { + auto SourceSignBits = computeNumSignBits(*Source, Depth + 1U); + if (!SourceSignBits) + return std::unexpected(SourceSignBits.error()); + FirstAnswer = *SourceSignBits; + if (const auto Amount = constantShiftAmount(*Right, MRI, BitWidth)) + FirstAnswer = std::min(BitWidth, FirstAnswer + *Amount); + } + } + + if (Opcode == GenericOpcode::G_TRUNC && Source) { + auto SourceType = MRI.getType(*Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + const std::uint32_t SourceWidth = SourceType->scalarBitWidth(); + auto SourceSignBits = computeNumSignBits(*Source, Depth + 1U); + if (!SourceSignBits) + return std::unexpected(SourceSignBits.error()); + if (SourceWidth > BitWidth && *SourceSignBits > SourceWidth - BitWidth) + return *SourceSignBits - (SourceWidth - BitWidth); + } + + if (Opcode == GenericOpcode::G_ICMP || Opcode == GenericOpcode::G_FCMP) { + if (BitWidth > 1) { + const BooleanContent Contents = + Target.getBooleanContents(Type->isVector(), Opcode == GenericOpcode::G_FCMP); + if (Contents == BooleanContent::ZeroOrNegativeOne) + return BitWidth; + if (Contents == BooleanContent::ZeroOrOne) + return BitWidth - 1U; + } + } + + switch (Opcode) { + case GenericOpcode::G_UADDO: + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_SSUBE: + case GenericOpcode::G_UMULO: + case GenericOpcode::G_SMULO: { + const auto BooleanResult = registerOperand(*Definition, 1); + if (BooleanResult && *BooleanResult == Reg && + Target.getBooleanContents(Type->isVector(), false) == BooleanContent::ZeroOrNegativeOne) + return BitWidth; + break; + } + default: + break; + } + + auto Known = getKnownBits(Reg, Depth); + if (!Known) + return std::unexpected(Known.error()); + std::uint32_t KnownSignBits = 1; + if (Known->Zero.testBit(BitWidth - 1U)) + KnownSignBits = static_cast<std::uint32_t>(Known->Zero.countlOne()); + else if (Known->One.testBit(BitWidth - 1U)) + KnownSignBits = static_cast<std::uint32_t>(Known->One.countlOne()); + return std::max(FirstAnswer, std::max(1U, KnownSignBits)); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp new file mode 100644 index 0000000..0814641 --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/InstructionSelect.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/Utils.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstdint> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &instructionSelectContract() { + static const MachinePassContract Contract{ + .Name = "instruction-select", + .Requires = {MachineFunctionProperty::Legalized, MachineFunctionProperty::RegBankSelected}, + .Sets = {MachineFunctionProperty::Selected}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + return Contract; +} + +struct OpcodeState final { + MachineInstrStableID Instruction; + Opcode OpcodeValue; + std::uint64_t OperandFingerprint = 0; + + friend bool operator==(const OpcodeState &Left, const OpcodeState &Right) noexcept = default; +}; + +struct OpcodeStateHash final { + [[nodiscard]] std::size_t operator()(const OpcodeState &State) const noexcept { + return std::hash<std::uint64_t>{}(State.Instruction.value()) ^ + (std::hash<std::uint32_t>{}(State.OpcodeValue.value()) << 1U) ^ + (std::hash<std::uint64_t>{}(State.OperandFingerprint) << 2U); + } +}; + +void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept { + State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U); +} + +[[nodiscard]] std::uint64_t operandFingerprint(const MachineInstr &Instruction) noexcept { + std::uint64_t Result = 0xcbf29ce484222325ULL; + combineFingerprint(Result, Instruction.flags().rawValue()); + combineFingerprint(Result, Instruction.operandCount()); + for (const MachineOperand &Operand : Instruction.operands()) { + combineFingerprint(Result, static_cast<std::uint64_t>(Operand.kind())); + combineFingerprint(Result, Operand.targetFlags()); + combineFingerprint(Result, Operand.registerFlags().rawValue()); + combineFingerprint(Result, Operand.subReg().value()); + if (const Register *Reg = Operand.getIf<Register>()) + combineFingerprint(Result, Reg->opaqueValue()); + else if (const std::int64_t *Immediate = Operand.getIf<std::int64_t>()) + combineFingerprint(Result, static_cast<std::uint64_t>(*Immediate)); + else if (const auto *Integer = Operand.getIf<IntegerImmediateOperand>()) + combineFingerprint(Result, Integer->Value.bitWidth()); + else if (const auto *Floating = Operand.getIf<FloatingImmediateOperand>()) + combineFingerprint(Result, static_cast<std::uint64_t>(Floating->Bits.semantics())); + else if (const auto *Block = Operand.getIf<BasicBlockOperand>()) + combineFingerprint(Result, Block->Block == nullptr ? 0 : Block->Block->stableID().value()); + } + return Result; +} + +[[nodiscard]] OpcodeState stateOf(const MachineInstr &Instruction) noexcept { + return {Instruction.stableID(), Instruction.opcode(), operandFingerprint(Instruction)}; +} + +[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) { + std::vector<MachineInstr *> Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) + Result.push_back(Instruction); + return Result; +} + +[[nodiscard]] std::vector<MachineBasicBlock *> stableCFGPostOrder(MachineFunction &Function) { + std::vector<MachineBasicBlock *> Roots; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + Roots.push_back(Block); + std::ranges::sort(Roots, {}, + [](const MachineBasicBlock *Block) { return Block->stableID().value(); }); + if (MachineBasicBlock *Entry = Function.firstBlock(); Entry != nullptr) { + const auto Position = std::ranges::find(Roots, Entry); + if (Position != Roots.end()) + std::rotate(Roots.begin(), Position, std::next(Position)); + } + + struct TraversalFrame final { + MachineBasicBlock *Block = nullptr; + std::vector<MachineBasicBlock *> Successors; + std::size_t NextSuccessor = 0; + }; + const auto MakeFrame = [](MachineBasicBlock &Block) { + TraversalFrame Frame; + Frame.Block = &Block; + for (const MachineSuccessor &Successor : Block.successors()) { + if (Successor.Block != nullptr) + Frame.Successors.push_back(Successor.Block); + } + std::ranges::sort(Frame.Successors, {}, [](const MachineBasicBlock *Successor) { + return Successor->stableID().value(); + }); + return Frame; + }; + + std::vector<MachineBasicBlock *> Result; + std::unordered_set<MachineBasicBlock *> Visited; + std::vector<TraversalFrame> Stack; + for (MachineBasicBlock *Root : Roots) { + if (!Visited.insert(Root).second) + continue; + Stack.push_back(MakeFrame(*Root)); + while (!Stack.empty()) { + TraversalFrame &Frame = Stack.back(); + if (Frame.NextSuccessor < Frame.Successors.size()) { + MachineBasicBlock *Successor = Frame.Successors[Frame.NextSuccessor++]; + if (Visited.insert(Successor).second) + Stack.push_back(MakeFrame(*Successor)); + continue; + } + Result.push_back(Frame.Block); + Stack.pop_back(); + } + } + return Result; +} + +[[nodiscard]] std::vector<MachineInstr *> bottomUpInstructions(MachineFunction &Function) { + std::vector<MachineInstr *> Result; + for (MachineBasicBlock *Block : stableCFGPostOrder(Function)) { + for (MachineInstr *Instruction = Block->lastInstruction(); Instruction != nullptr; + Instruction = Instruction->previous()) + Result.push_back(Instruction); + } + return Result; +} + +[[nodiscard]] Error passFailure(std::string Message, + ErrorCode Code = ErrorCode::VerificationError) { + return Error(Code, std::move(Message)); +} + +} // namespace + +Expected<void> InstructionSelector::propagateSelectionFlags(MachineInstr &Output, + MachineInstrFlags InputFlags, + bool InputMayRaiseFPException, + const TargetDescription &Target) const { + const InstructionDescriptor *OutputDescriptor = Target.instructionDescriptor(Output.opcode()); + if (OutputDescriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction selector produced an unknown instruction descriptor"); + } + if (!InputMayRaiseFPException && + OutputDescriptor->Flags.has(InstructionFlag::MayRaiseFPException)) { + InputFlags.set(MachineInstrFlag::NoFPExcept); + } + Output.setFlags(InputFlags); + return {}; +} + +Expected<void> InstructionSelector::propagateSelectionFlags(MachineInstr &Output, + const MachineInstr &Input, + const TargetDescription &Target) const { + const InstructionDescriptor *InputDescriptor = Target.instructionDescriptor(Input.opcode()); + if (InputDescriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction selector input has an unknown instruction descriptor"); + } + return propagateSelectionFlags(Output, Input.flags(), + InputDescriptor->Flags.has(InstructionFlag::MayRaiseFPException), + Target); +} + +Expected<void> InstructionSelector::finalizeGeneratedInstruction( + MachineInstr &Output, MachineFunction &Function, const TargetDescription &Target, + MachineInstrFlags InputFlags, bool InputMayRaiseFPException) const { + auto Flags = propagateSelectionFlags(Output, InputFlags, InputMayRaiseFPException, Target); + if (!Flags) + return std::unexpected(Flags.error()); + return constrainSelectedInstruction(Output, Function, Target); +} + +const MachinePassContract &InstructionSelect::contract() const noexcept { + return instructionSelectContract(); +} + +Expected<MachinePassResult> InstructionSelect::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + if (Function.properties().has(MachineFunctionProperty::FailedISel)) + return MachinePassResult{ + .Changed = false, + .Changes = {}, + .Preserved = PreservedAnalyses::all(), + }; + if (Info == nullptr) + return std::unexpected(passFailure("instruction select has no target behavior interface", + ErrorCode::InvalidArgument)); + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +InstructionSelect::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + + MachinePassResult Result; + std::vector<MachineInstr *> Worklist; + std::unordered_set<MachineInstrStableID> DiscoveredInstructions; + std::unordered_set<OpcodeState, OpcodeStateHash> SeenStates; + for (MachineInstr *Instruction : bottomUpInstructions(Function)) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return std::unexpected( + passFailure("instruction selector encountered an unknown instruction descriptor")); + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) + return std::unexpected( + withErrorContext(Validation.error(), "instruction selector input is invalid: ")); + Worklist.push_back(Instruction); + DiscoveredInstructions.insert(Instruction->stableID()); + } + const auto DiscoverNewInstructions = [&]() -> Expected<void> { + for (MachineInstr *Candidate : bottomUpInstructions(Function)) { + const InstructionDescriptor *CandidateDescriptor = + Target.instructionDescriptor(Candidate->opcode()); + if (CandidateDescriptor == nullptr) { + return std::unexpected( + passFailure("instruction selector produced an unknown instruction descriptor")); + } + if (DiscoveredInstructions.insert(Candidate->stableID()).second) + Worklist.push_back(Candidate); + } + return {}; + }; + + for (std::size_t Index = 0; Index < Worklist.size(); ++Index) { + MachineInstr *Instruction = Worklist[Index]; + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + if (isTriviallyDead(*Instruction, Function.registerInfo(), Target)) { + auto Removed = eraseInstr(*Instruction, Function.registerInfo(), Target); + if (!Removed) + return std::unexpected(Removed.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + continue; + } + const InstructionDescriptor *CurrentDescriptor = + Target.instructionDescriptor(Instruction->opcode()); + if (CurrentDescriptor == nullptr) + return std::unexpected( + passFailure("instruction selector encountered an unknown instruction descriptor")); + const bool IsGeneric = CurrentDescriptor->Flags.has(InstructionFlag::Generic); + const bool IsCopy = Instruction->opcode() == opcode(GenericOpcode::COPY); + if (!IsGeneric && !IsCopy) + continue; + const OpcodeState Before = stateOf(*Instruction); + if (!SeenStates.insert(Before).second) { + return std::unexpected(passFailure("instruction selector entered a repeated state for '" + + CurrentDescriptor->Name + "'", + ErrorCode::InternalError)); + } + std::string Rule = "select."; + Rule.append(CurrentDescriptor->Name); + auto Accepted = Analyses.requestAction(Rule, Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return std::unexpected(passFailure("required selection action was rejected for '" + + CurrentDescriptor->Name + "'", + ErrorCode::VerificationError)); + } + const MachineInstrFlags InputFlags = Instruction->flags(); + const bool InputMayRaiseFPException = + CurrentDescriptor->Flags.has(InstructionFlag::MayRaiseFPException); + auto Selection = Info->select(*Instruction, Function, Target); + if (!Selection) + return std::unexpected(Selection.error()); + if (Instruction->parent() == nullptr) { + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + auto Discovered = DiscoverNewInstructions(); + if (!Discovered) + return std::unexpected(Discovered.error()); + continue; + } + const InstructionDescriptor *Selected = Target.instructionDescriptor(Instruction->opcode()); + if (Selected == nullptr) + return std::unexpected( + passFailure("instruction selector produced an unknown instruction descriptor")); + if (IsGeneric) { + auto FlagPropagation = + Info->propagateSelectionFlags(*Instruction, InputFlags, InputMayRaiseFPException, Target); + if (!FlagPropagation) + return std::unexpected(FlagPropagation.error()); + auto Constraint = Info->constrainSelectedInstruction(*Instruction, Function, Target); + if (!Constraint) { + return std::unexpected(withErrorContext( + Constraint.error(), "instruction selector could not constrain selected registers: ")); + } + } + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) + return std::unexpected(withErrorContext(Validation.error(), + "instruction selector produced invalid MachineIR: ")); + const OpcodeState After = stateOf(*Instruction); + if (IsGeneric && After == Before) + return std::unexpected( + passFailure("instruction selector reported success without observable progress", + ErrorCode::InternalError)); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + if (Selected->Flags.has(InstructionFlag::Generic)) + Worklist.push_back(Instruction); + auto Discovered = DiscoverNewInstructions(); + if (!Discovered) + return std::unexpected(Discovered.error()); + } + for (MachineInstr *Instruction : layoutInstructions(Function)) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return std::unexpected( + passFailure("instruction selector produced an unknown instruction descriptor")); + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) + return std::unexpected(withErrorContext(Validation.error(), + "instruction selector produced invalid MachineIR: ")); + if (Descriptor->Flags.has(InstructionFlag::Generic)) + return std::unexpected( + passFailure("instruction selector left a generic opcode in the function")); + } + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->lastInstruction(); Instruction != nullptr;) { + MachineInstr *Previous = Instruction->previous(); + if (Instruction->opcode() != opcode(GenericOpcode::COPY)) { + Instruction = Previous; + continue; + } + const MachineOperand *SourceOperand = Instruction->operandAt(1); + const MachineOperand *DestinationOperand = Instruction->operandAt(0); + const Register *Source = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Source == nullptr || Destination == nullptr || !Source->isVirtual() || + !Destination->isVirtual() || SourceOperand->subReg().value() != 0) { + Instruction = Previous; + continue; + } + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); + if (!SourceInfo) + return std::unexpected(SourceInfo.error()); + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); + if (!DestinationInfo) + return std::unexpected(DestinationInfo.error()); + if (!SourceInfo->get().RegClass || + SourceInfo->get().RegClass != DestinationInfo->get().RegClass) { + Instruction = Previous; + continue; + } + auto Replaced = Function.registerInfo().replaceRegWith(*Destination, *Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + auto Removed = Instruction->eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + Instruction = Previous; + } + } + auto RegisterValidation = Target.validateRegisterInfo(Function.registerInfo()); + if (!RegisterValidation) + return std::unexpected(withErrorContext(RegisterValidation.error(), + "instruction selection produced invalid registers: ")); + Function.registerInfo().clearAllVirtualRegisterTypes(); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Registers); + Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + return Result; +} +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp new file mode 100644 index 0000000..0601040 --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp @@ -0,0 +1,493 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/Legalizer.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/CSEInfo.h" +#include "rcc/CodeGen/GlobalISel/CSEMIRBuilder.h" +#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "rcc/CodeGen/GlobalISel/GISelValueTracking.h" +#include "rcc/CodeGen/GlobalISel/GISelWorkList.h" +#include "rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h" +#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h" +#include "rcc/CodeGen/GlobalISel/Utils.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <array> +#include <cstddef> +#include <memory> +#include <ranges> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &legalizerContract() { + static const MachinePassContract Contract{ + .Name = "legalizer", + // LLVM's GlobalISel legalizer operates on SSA form and does not require + // the optional register-liveness tracking property. + .Requires = {MachineFunctionProperty::IsSSA}, + .Sets = {MachineFunctionProperty::Legalized}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Legalized, + }; + return Contract; +} + +[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) { + std::vector<MachineInstr *> Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) + Result.push_back(Instruction); + return Result; +} + +[[nodiscard]] bool isArtifact(const MachineInstr &Instruction) noexcept { + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + default: + return false; + case GenericOpcode::G_TRUNC: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_MERGE_VALUES: + case GenericOpcode::G_UNMERGE_VALUES: + case GenericOpcode::G_CONCAT_VECTORS: + case GenericOpcode::G_BUILD_VECTOR: + case GenericOpcode::G_EXTRACT: + case GenericOpcode::G_INSERT: + return true; + } +} + +[[nodiscard]] bool isIntrinsicOpcode(Opcode InstructionOpcode) noexcept { + switch (static_cast<GenericOpcode>(InstructionOpcode.value())) { + default: + return false; + case GenericOpcode::G_INTRINSIC: + case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS: + case GenericOpcode::G_INTRINSIC_CONVERGENT: + case GenericOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: + return true; + } +} + +using InstList = GISelWorkList<256>; +using ArtifactList = GISelWorkList<128>; + +class LegalizerWorkListManager final : public GISelChangeObserver { +public: + LegalizerWorkListManager(InstList &Instructions, ArtifactList &Artifacts) + : Instructions(Instructions), Artifacts(Artifacts) {} + + void erasingInstr(MachineInstr &Instruction) override { + Instructions.remove(&Instruction); + Artifacts.remove(&Instruction); + } + + void createdInstr(MachineInstr &Instruction) override { + createdOrChangedInstr(Instruction); + } + + void changingInstr(MachineInstr &) override {} + + void changedInstr(MachineInstr &Instruction) override { + createdOrChangedInstr(Instruction); + } + +private: + void createdOrChangedInstr(MachineInstr &Instruction) { + if (!isGenericOpcode(Instruction.opcode())) + return; + if (isArtifact(Instruction)) + Artifacts.insert(&Instruction); + else + Instructions.insert(&Instruction); + } + + InstList &Instructions; + ArtifactList &Artifacts; +}; + +[[nodiscard]] std::vector<MachineBasicBlock *> reversePostOrderBlocks(MachineFunction &Function) { + struct TraversalFrame final { + MachineBasicBlock *Block = nullptr; + std::vector<MachineBasicBlock *> Successors; + std::size_t NextSuccessor = 0; + }; + + const auto MakeFrame = [](MachineBasicBlock &Block) { + TraversalFrame Frame; + Frame.Block = &Block; + for (const MachineSuccessor &Successor : Block.successors()) { + if (Successor.Block != nullptr) + Frame.Successors.push_back(Successor.Block); + } + std::ranges::sort(Frame.Successors, {}, [](const MachineBasicBlock *Successor) { + return Successor->stableID().value(); + }); + return Frame; + }; + + std::vector<MachineBasicBlock *> Roots; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + Roots.push_back(Block); + std::ranges::sort(Roots, {}, + [](const MachineBasicBlock *Block) { return Block->stableID().value(); }); + if (MachineBasicBlock *Entry = Function.firstBlock(); Entry != nullptr) { + const auto Position = std::ranges::find(Roots, Entry); + if (Position != Roots.end()) + std::rotate(Roots.begin(), Position, std::next(Position)); + } + + std::vector<MachineBasicBlock *> PostOrder; + std::unordered_set<MachineBasicBlock *> Visited; + std::vector<TraversalFrame> Stack; + for (MachineBasicBlock *Root : Roots) { + if (!Visited.insert(Root).second) + continue; + Stack.push_back(MakeFrame(*Root)); + while (!Stack.empty()) { + TraversalFrame &Frame = Stack.back(); + if (Frame.NextSuccessor < Frame.Successors.size()) { + MachineBasicBlock *Successor = Frame.Successors[Frame.NextSuccessor++]; + if (Visited.insert(Successor).second) + Stack.push_back(MakeFrame(*Successor)); + continue; + } + PostOrder.push_back(Frame.Block); + Stack.pop_back(); + } + } + std::ranges::reverse(PostOrder); + return PostOrder; +} + +[[nodiscard]] Error passFailure(std::string Message, + ErrorCode Code = ErrorCode::VerificationError) { + return Error(Code, std::move(Message)); +} + +[[nodiscard]] std::string_view legalizationActionName(LegalizationAction Action) noexcept { + switch (Action) { + case LegalizationAction::Legal: + return "legal"; + case LegalizationAction::Unsupported: + return "unsupported"; + case LegalizationAction::Custom: + return "custom"; + case LegalizationAction::Lower: + return "lower"; + case LegalizationAction::Libcall: + return "libcall"; + case LegalizationAction::WidenScalar: + return "widen-scalar"; + case LegalizationAction::NarrowScalar: + return "narrow-scalar"; + case LegalizationAction::MoreElements: + return "more-elements"; + case LegalizationAction::FewerElements: + return "fewer-elements"; + case LegalizationAction::Bitcast: + return "bitcast"; + case LegalizationAction::ChangeElementType: + return "change-element-type"; + case LegalizationAction::ChangeAddressSpace: + return "change-address-space"; + } + return "unknown"; +} + +[[nodiscard]] Expected<void> clearTrackedLivenessFlags(MachineFunction &Function, + MachinePassResult &Result) { + if (!Result.Changed || !Function.properties().has(MachineFunctionProperty::TracksLiveness)) + return {}; + bool Changed = false; + for (MachineInstr *Instruction : layoutInstructions(Function)) { + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand &Operand = *Instruction->operandAt(Index); + if (!Operand.isRegister()) + continue; + RegisterOperandFlags Flags = Operand.registerFlags(); + if (!Flags.has(RegisterOperandFlag::Kill) && !Flags.has(RegisterOperandFlag::Dead)) + continue; + Flags.clear(RegisterOperandFlag::Kill); + Flags.clear(RegisterOperandFlag::Dead); + auto Updated = Operand.setRegisterFlags(Flags); + if (!Updated) + return std::unexpected(Updated.error()); + Changed = true; + } + } + if (Changed) + Result.Changes.add(MachinePassChange::Operands); + return {}; +} + +} // namespace + +const MachinePassContract &Legalizer::contract() const noexcept { + return legalizerContract(); +} + +Expected<MachinePassResult> Legalizer::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + if (Function.properties().has(MachineFunctionProperty::FailedISel)) + return MachinePassResult{ + .Changed = false, + .Changes = {}, + .Preserved = PreservedAnalyses::all(), + }; + if (Info == nullptr) + return std::unexpected( + passFailure("legalizer has no target behavior interface", ErrorCode::InvalidArgument)); + if (AbortMode != GlobalISelAbortMode::Enable) { + if (AbortMode != GlobalISelAbortMode::Disable && + AbortMode != GlobalISelAbortMode::DisableWithDiagnostic) { + return std::unexpected(passFailure("legalizer has an unknown GlobalISel abort mode", + ErrorCode::InvalidArgument)); + } + auto Result = runImplementation(Function, Target, Analyses); + if (Result || Result.error().code() != ErrorCode::Unsupported) + return Result; + Function.properties().set(MachineFunctionProperty::FailedISel); + return MachinePassResult{ + .Changed = true, + .Changes = {MachinePassChange::Instructions}, + .Preserved = PreservedAnalyses::none(), + }; + } + + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> Legalizer::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + MachinePassResult Result; + InstList InstructionWorklist; + ArtifactList ArtifactWorklist; + std::vector<MachineInstr *> RetryList; + + for (MachineBasicBlock *Block : reversePostOrderBlocks(Function)) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (!isGenericOpcode(Instruction->opcode())) + continue; + if (isArtifact(*Instruction)) + ArtifactWorklist.deferredInsert(Instruction); + else + InstructionWorklist.deferredInsert(Instruction); + } + } + ArtifactWorklist.finalize(); + InstructionWorklist.finalize(); + + LegalizerWorkListManager WorkListObserver(InstructionWorklist, ArtifactWorklist); + GISelCSEInfo CSEInfo; + CSEInfo.setCSEConfig(getStandardCSEConfigForOpt(Optimization)); + auto CSEAnalysis = CSEInfo.analyze(Function); + if (!CSEAnalysis) + return std::unexpected( + withErrorContext(CSEAnalysis.error(), "GlobalISel CSE analysis failed: ")); + std::array<GISelChangeObserver *, 2> AuxiliaryObservers{&WorkListObserver, &CSEInfo}; + GISelObserverWrapper Observer{std::span<GISelChangeObserver *const>(AuxiliaryObservers)}; + MachineFunctionDelegateInstaller ObserverInstaller(Function, Observer); + CSEMIRBuilder Builder(Function, Target); + Builder.setCSEInfo(&CSEInfo); + const LibcallLoweringInfo *ActiveLibcalls = + Libcalls.CallLowering == nullptr ? nullptr : &Libcalls; + LegalizerHelper Helper(Function, *Info, Observer, Builder, ActiveLibcalls); + GISelValueTracking ValueTracking(Function, Target); + LegalizationArtifactCombiner ArtifactCombiner(Builder, Function.registerInfo(), *Info, + &ValueTracking); + + do { + while (!InstructionWorklist.empty()) { + MachineInstr *Instruction = InstructionWorklist.popBack(); + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + + if (isTriviallyDead(*Instruction, Function.registerInfo(), Target)) { + auto Removed = eraseInstr(*Instruction, Function.registerInfo(), Target); + if (!Removed) + return std::unexpected(Removed.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + continue; + } + + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return std::unexpected( + passFailure("legalizer encountered an unknown instruction descriptor")); + if (!Descriptor->Flags.has(InstructionFlag::Generic)) + continue; + + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) { + return std::unexpected( + withErrorContext(Validation.error(), "legalizer input is invalid: ")); + } + + if (isIntrinsicOpcode(Instruction->opcode())) { + auto Accepted = + Analyses.requestAction("legalize.intrinsic", Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + return std::unexpected(passFailure("required intrinsic legalization action was rejected", + ErrorCode::VerificationError)); + auto IntrinsicResult = Info->legalizeIntrinsic(Helper, *Instruction); + if (!IntrinsicResult) + return std::unexpected(IntrinsicResult.error()); + if (!*IntrinsicResult) { + const std::string_view Name = Descriptor->Name; + return std::unexpected(passFailure("target legalizer rejected intrinsic instruction '" + + std::string(Name) + "'", + ErrorCode::Unsupported)); + } + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + continue; + } + + auto Decision = Info->getDecision(*Instruction, Function, Target); + if (!Decision) + return std::unexpected(Decision.error()); + if (Decision->Action == LegalizationAction::Legal) + continue; + if (Decision->Action == LegalizationAction::Unsupported) { + if (isArtifact(*Instruction)) { + RetryList.push_back(Instruction); + continue; + } + return std::unexpected( + passFailure("target legalizer rejected generic instruction '" + Descriptor->Name + "'", + ErrorCode::Unsupported)); + } + + auto LoweringValidation = Info->validateLowering(*Instruction, Function, Target, *Decision); + if (!LoweringValidation) { + return std::unexpected(withErrorContext(LoweringValidation.error(), + "legalizer lowering precondition failed: ")); + } + std::string Rule = "legalize."; + Rule.append(legalizationActionName(Decision->Action)); + Rule.push_back('.'); + Rule.append(Descriptor->Name); + auto Accepted = Analyses.requestAction(Rule, Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return std::unexpected( + passFailure("required legalizer action was rejected for '" + Descriptor->Name + "'", + ErrorCode::VerificationError)); + } + Expected<void> Lowered = Helper.legalize(*Instruction, *Decision); + if (!Lowered) { + if (Lowered.error().code() == ErrorCode::Unsupported && Instruction->parent() != nullptr && + isArtifact(*Instruction)) { + RetryList.push_back(Instruction); + continue; + } + return std::unexpected(Lowered.error()); + } + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + } + + if (!RetryList.empty()) { + if (!ArtifactWorklist.empty()) { + for (MachineInstr *Instruction : RetryList) + ArtifactWorklist.insert(Instruction); + RetryList.clear(); + } else { + MachineInstr *Instruction = RetryList.front(); + const InstructionDescriptor *Descriptor = + Instruction == nullptr ? nullptr : Target.instructionDescriptor(Instruction->opcode()); + return std::unexpected(passFailure( + "target legalizer rejected generic artifact '" + + std::string(Descriptor == nullptr ? "<unknown>" : Descriptor->Name) + "'", + ErrorCode::Unsupported)); + } + } + + while (!ArtifactWorklist.empty()) { + MachineInstr *Instruction = ArtifactWorklist.popBack(); + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + if (isTriviallyDead(*Instruction, Function.registerInfo(), Target)) { + auto Removed = eraseInstr(*Instruction, Function.registerInfo(), Target); + if (!Removed) + return std::unexpected(Removed.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + continue; + } + std::vector<MachineInstr *> DeadInstructions; + auto Combined = + ArtifactCombiner.tryCombineInstruction(*Instruction, DeadInstructions, Observer); + if (!Combined) + return std::unexpected(Combined.error()); + if (*Combined) { + auto Removed = eraseInstrs(DeadInstructions, Function.registerInfo(), Target); + if (!Removed) + return std::unexpected(Removed.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + continue; + } + InstructionWorklist.insert(Instruction); + } + } while (!InstructionWorklist.empty()); + + auto CSEValidation = CSEInfo.verify(); + if (!CSEValidation) { + return std::unexpected( + withErrorContext(CSEValidation.error(), "GlobalISel CSE verification failed: ")); + } + + for (MachineInstr *Instruction : layoutInstructions(Function)) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return std::unexpected(passFailure("legalizer produced an unknown instruction descriptor")); + if (!Descriptor->Flags.has(InstructionFlag::Generic)) + continue; + auto Decision = Info->getDecision(*Instruction, Function, Target); + if (!Decision) + return std::unexpected(Decision.error()); + if (Decision->Action != LegalizationAction::Legal && + Decision->Action != LegalizationAction::Custom) + return std::unexpected(passFailure( + "legalizer reached a fixed point with an illegal generic instruction '" + + Descriptor->Name + "'", + Decision->Action == LegalizationAction::Unsupported ? ErrorCode::Unsupported + : ErrorCode::InternalError)); + auto FinalValidation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!FinalValidation) + return std::unexpected( + withErrorContext(FinalValidation.error(), "legalizer produced invalid MachineIR: ")); + } + auto Liveness = clearTrackedLivenessFlags(Function, Result); + if (!Liveness) + return std::unexpected( + withErrorContext(Liveness.error(), "legalizer liveness-flag invalidation failed: ")); + Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp new file mode 100644 index 0000000..8bf6e80 --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -0,0 +1,7120 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h" + +#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "rcc/CodeGen/GlobalISel/Utils.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetCallLowering.h" + +#include <algorithm> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <initializer_list> +#include <limits> +#include <numeric> +#include <optional> +#include <span> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct ScalarLayout final { + std::uint64_t StoreSize = 0; + std::uint64_t AllocationSize = 0; + std::uint64_t ABIAlignment = 0; +}; + +[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) { + if (const std::int64_t *Value = Operand.getIf<std::int64_t>()) + return *Value; + const auto *Integer = Operand.getIf<IntegerImmediateOperand>(); + if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) { + return makeUnexpected(ErrorCode::VerificationError, + "legalizer requires an immediate no wider than 64 bits"); + } + std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front(); + const std::uint32_t Width = Integer->Value.bitWidth(); + if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0) + Raw |= ~std::uint64_t{0} << Width; + return std::bit_cast<std::int64_t>(Raw); +} + +[[nodiscard]] std::optional<ScalarLayout> scalarLayout(LowLevelType Type) noexcept { + if (!Type.isScalarValue() || Type.scalarBitWidth() == 0) + return std::nullopt; + const std::uint64_t StoreSize = (Type.scalarBitWidth() + 7U) / 8U; + const std::uint64_t AllocationSize = std::bit_ceil(StoreSize); + return ScalarLayout{StoreSize, AllocationSize, std::min<std::uint64_t>(AllocationSize, 8)}; +} + +[[nodiscard]] Expected<const MachineMemOperand *> createMemoryAccess(MachineFunction &Function, + MemoryAccessFlag Access, + LowLevelType Type, + const ScalarLayout &Layout) { + auto Size = MemorySize::fixed(Layout.StoreSize); + auto Align = Alignment::create(Layout.ABIAlignment); + if (!Size || !Align) + return makeUnexpected(ErrorCode::InternalError, + "legalizer cannot represent scalar memory layout"); + MachineMemOperandDesc Description; + Description.Accesses = {Access}; + Description.Size = *Size; + Description.MemoryType = Type; + Description.Align = *Align; + return Function.createMemOperand(std::move(Description)); +} + +[[nodiscard]] Expected<Register> resultRegister(MachineInstr &Instruction) { + const MachineOperand *Definition = Instruction.operandAt(0); + const Register *Result = Definition == nullptr ? nullptr : Definition->getIf<Register>(); + if (Result == nullptr || !Definition->isDef()) { + return makeUnexpected(ErrorCode::InternalError, + "machine IR builder returned an instruction without a register result"); + } + return *Result; +} + +[[nodiscard]] Expected<void> setInsertPointAtFirstTerminator(MachineIRBuilder &Builder, + MachineBasicBlock &Block) { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = + Builder.target().instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "PHI predecessor contains an unknown instruction"); + if (Descriptor->Flags.has(InstructionFlag::Terminator)) + return Builder.setInsertPointBefore(*Instruction); + } + return Builder.setInsertPoint(Block); +} + +[[nodiscard]] Expected<void> setInsertPointAfterPHIs(MachineIRBuilder &Builder, + MachineBasicBlock &Block) { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = + Builder.target().instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "PHI block contains an unknown instruction"); + if (!Descriptor->Flags.has(InstructionFlag::Phi) && + !Descriptor->Flags.has(InstructionFlag::Meta)) { + return Builder.setInsertPointBefore(*Instruction); + } + } + return Builder.setInsertPoint(Block); +} + +[[nodiscard]] Expected<Register> buildConstantRegister(MachineIRBuilder &Builder, LowLevelType Type, + APInt Value) { + auto Built = Builder.buildConstant(Type, std::move(Value)); + if (!Built) + return std::unexpected(Built.error()); + return resultRegister(**Built); +} + +[[nodiscard]] Expected<Register> buildConstantRegister(MachineIRBuilder &Builder, LowLevelType Type, + std::int64_t Value) { + auto Immediate = APInt::fromSigned(Type.scalarBitWidth(), Value); + if (!Immediate) + return std::unexpected(Immediate.error()); + return buildConstantRegister(Builder, Type, std::move(*Immediate)); +} + +[[nodiscard]] Expected<Register> buildResultInstruction(MachineIRBuilder &Builder, + GenericOpcode Generic, LowLevelType Type, + std::initializer_list<Register> Sources) { + auto Built = Builder.buildInstr(opcode(Generic), {Type}, Sources); + if (!Built) + return std::unexpected(Built.error()); + return resultRegister(**Built); +} + +[[nodiscard]] Expected<Register> +buildFlaggedBinaryInstruction(MachineIRBuilder &Builder, GenericOpcode Generic, LowLevelType Type, + Register Left, Register Right, MachineInstrFlags Flags) { + auto Destination = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Destination) + return std::unexpected(Destination.error()); + auto Pending = Builder.buildInstr(opcode(Generic)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Left); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Right); !Added) + return std::unexpected(Added.error()); + if (auto Set = Pending->setFlags(Flags); !Set) + return std::unexpected(Set.error()); + if (auto Built = Pending->build(); !Built) + return std::unexpected(Built.error()); + return *Destination; +} + +[[nodiscard]] Expected<std::int64_t> addMemoryOffset(std::int64_t Offset, + std::uint64_t ByteOffset) { + if (ByteOffset > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) { + return makeUnexpected(ErrorCode::Unsupported, + "memory split offset exceeds the representable machine memory range"); + } + const auto SignedByteOffset = static_cast<std::int64_t>(ByteOffset); + if (Offset > std::numeric_limits<std::int64_t>::max() - SignedByteOffset) { + return makeUnexpected(ErrorCode::Unsupported, + "memory split offset exceeds the representable machine memory range"); + } + return Offset + SignedByteOffset; +} + +[[nodiscard]] Expected<const MachineMemOperand *> +deriveMemoryOperand(MachineFunction &Function, const MachineMemOperand &Original, + LowLevelType MemoryType, std::uint64_t ByteOffset) { + if (!MemoryType.isScalarValue() || MemoryType.minimumBitWidth() == 0 || + MemoryType.minimumBitWidth() % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "memory splitting requires a byte-sized scalar memory type"); + } + + const std::uint64_t ByteSize = MemoryType.minimumBitWidth() / 8U; + auto Size = MemorySize::fixed(ByteSize); + if (!Size) + return std::unexpected(Size.error()); + + MachineMemOperandDesc Description = Original.description(); + auto Offset = addMemoryOffset(Description.Offset, ByteOffset); + if (!Offset) + return std::unexpected(Offset.error()); + Description.Offset = *Offset; + if (Description.OffsetRange) { + auto Lower = addMemoryOffset(Description.OffsetRange->Lower, ByteOffset); + auto Upper = addMemoryOffset(Description.OffsetRange->Upper, ByteOffset); + if (!Lower) + return std::unexpected(Lower.error()); + if (!Upper) + return std::unexpected(Upper.error()); + Description.OffsetRange = MemoryOffsetRange{*Lower, *Upper}; + } + + const std::optional<Alignment> BaseAlignment = Description.BaseAlign; + std::uint64_t EffectiveAlignment = BaseAlignment.value_or(Description.Align).bytes(); + if (Description.Offset != 0) { + const auto OffsetBits = static_cast<std::uint64_t>(Description.Offset); + const std::uint64_t OffsetAlignment = std::uint64_t{1} << std::countr_zero(OffsetBits); + EffectiveAlignment = std::min(EffectiveAlignment, OffsetAlignment); + } + auto AlignmentValue = Alignment::create(EffectiveAlignment); + if (!AlignmentValue) + return std::unexpected(AlignmentValue.error()); + + Description.Size = *Size; + Description.MemoryType = MemoryType; + Description.Align = *AlignmentValue; + Description.BaseAlign = BaseAlignment; + return Function.createMemOperand(std::move(Description)); +} + +[[nodiscard]] Expected<Register> +buildObjectPointerOffset(MachineIRBuilder &Builder, Register Pointer, std::uint64_t ByteOffset) { + if (ByteOffset == 0) + return Pointer; + + auto PointerType = Builder.function().registerInfo().getType(Pointer); + if (!PointerType) + return std::unexpected(PointerType.error()); + if (!PointerType->isPointer()) { + return makeUnexpected(ErrorCode::VerificationError, + "memory splitting requires a pointer address operand"); + } + const std::uint32_t PointerBits = PointerType->scalarBitWidth(); + if (PointerBits < 64U && (ByteOffset >> PointerBits) != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "memory split offset does not fit the pointer index type"); + } + auto OffsetType = LowLevelType::makeCategoryNeutralScalar(PointerBits); + if (!OffsetType) + return std::unexpected(OffsetType.error()); + auto OffsetValue = APInt::fromUnsigned(PointerBits, ByteOffset); + if (!OffsetValue) + return std::unexpected(OffsetValue.error()); + auto Offset = buildConstantRegister(Builder, *OffsetType, std::move(*OffsetValue)); + if (!Offset) + return std::unexpected(Offset.error()); + return buildFlaggedBinaryInstruction( + Builder, GenericOpcode::G_PTR_ADD, *PointerType, Pointer, *Offset, + MachineInstrFlags{MachineInstrFlag::NoUnsignedWrap, MachineInstrFlag::InBounds}); +} + +[[nodiscard]] Expected<APInt> repeatedByte(std::uint32_t BitWidth, std::uint8_t Byte) { + if (BitWidth == 0 || BitWidth > APInt::MaxBitWidth || BitWidth % 8U != 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "repeated-byte integer has an invalid bit width"); + constexpr std::uint64_t ByteSplat = 0x0101010101010101ULL; + std::vector<std::uint64_t> Words((static_cast<std::size_t>(BitWidth) + 63U) / 64U, + ByteSplat * Byte); + const std::uint32_t HighBits = BitWidth % 64U; + if (HighBits != 0) + Words.back() &= (std::uint64_t{1} << HighBits) - 1U; + return APInt::create(BitWidth, Words); +} + +[[nodiscard]] Expected<APInt> oneBitSet(std::uint32_t BitWidth, std::uint32_t Bit) { + if (BitWidth == 0 || BitWidth > APInt::MaxBitWidth || Bit >= BitWidth) + return makeUnexpected(ErrorCode::InvalidArgument, + "single-bit integer has an invalid bit position"); + std::vector<std::uint64_t> Words((static_cast<std::size_t>(BitWidth) + 63U) / 64U, 0); + Words[Bit / 64U] = std::uint64_t{1} << (Bit % 64U); + return APInt::create(BitWidth, Words); +} + +[[nodiscard]] Expected<Register> buildICmpRegister(MachineIRBuilder &Builder, + IntegerPredicate Predicate, + LowLevelType ResultType, Register Left, + Register Right) { + auto Built = Builder.buildICmp(Predicate, ResultType, Left, Right); + if (!Built) + return std::unexpected(Built.error()); + return resultRegister(**Built); +} + +[[nodiscard]] IntegerPredicate minMaxToCompare(GenericOpcode Generic) { + switch (Generic) { + case GenericOpcode::G_SMIN: + return IntegerPredicate::SignedLessThan; + case GenericOpcode::G_SMAX: + return IntegerPredicate::SignedGreaterThan; + case GenericOpcode::G_UMIN: + return IntegerPredicate::UnsignedLessThan; + case GenericOpcode::G_UMAX: + return IntegerPredicate::UnsignedGreaterThan; + default: + std::unreachable(); + } +} + +[[nodiscard]] bool isEqualityPredicate(IntegerPredicate Predicate) noexcept { + return Predicate == IntegerPredicate::Equal || Predicate == IntegerPredicate::NotEqual; +} + +[[nodiscard]] Expected<IntegerPredicate> unsignedPredicate(IntegerPredicate Predicate) { + switch (Predicate) { + case IntegerPredicate::Equal: + case IntegerPredicate::NotEqual: + case IntegerPredicate::UnsignedGreaterThan: + case IntegerPredicate::UnsignedGreaterEqual: + case IntegerPredicate::UnsignedLessThan: + case IntegerPredicate::UnsignedLessEqual: + return Predicate; + case IntegerPredicate::SignedGreaterThan: + return IntegerPredicate::UnsignedGreaterThan; + case IntegerPredicate::SignedGreaterEqual: + return IntegerPredicate::UnsignedGreaterEqual; + case IntegerPredicate::SignedLessThan: + return IntegerPredicate::UnsignedLessThan; + case IntegerPredicate::SignedLessEqual: + return IntegerPredicate::UnsignedLessEqual; + case IntegerPredicate::Count: + return makeUnexpected(ErrorCode::VerificationError, + "integer comparison has an invalid predicate"); + } + return makeUnexpected(ErrorCode::VerificationError, + "integer comparison has an invalid predicate"); +} + +[[nodiscard]] Expected<LowLevelType> booleanTypeFor(LowLevelType Type) { + auto BooleanElementType = LowLevelType::makeCategoryNeutralScalar(1); + if (!BooleanElementType) + return std::unexpected(BooleanElementType.error()); + if (!Type.isVector()) + return *BooleanElementType; + if (Type.isScalableVector()) + return LowLevelType::makeScalableVector(Type.minimumElementCount(), *BooleanElementType); + return LowLevelType::makeFixedVector(Type.minimumElementCount(), *BooleanElementType); +} + +[[nodiscard]] Expected<Register> buildConstantBefore(MachineIRBuilder &Builder, + MachineInstr &Anchor, LowLevelType Type, + std::uint64_t Value) { + auto Immediate = APInt::fromUnsigned(Type.scalarBitWidth(), Value); + if (!Immediate) + return std::unexpected(Immediate.error()); + if (auto Point = Builder.setInsertPointBefore(Anchor); !Point) + return std::unexpected(Point.error()); + return buildConstantRegister(Builder, Type, std::move(*Immediate)); +} + +[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LowLevelType Type, + Register Left, Register Right) { + auto Result = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + if (auto Point = Builder.setInsertPointBefore(Anchor); !Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(Generic)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Left); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Right); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected<std::string> integerLibcallName(GenericOpcode Generic, + std::uint32_t BitWidth) { + std::string_view Prefix; + switch (Generic) { + case GenericOpcode::G_MUL: + Prefix = "__mul"; + break; + case GenericOpcode::G_SDIV: + Prefix = "__div"; + break; + case GenericOpcode::G_UDIV: + Prefix = "__udiv"; + break; + case GenericOpcode::G_SREM: + Prefix = "__mod"; + break; + case GenericOpcode::G_UREM: + Prefix = "__umod"; + break; + default: + return makeUnexpected(ErrorCode::Unsupported, + "integer libcall has no runtime function mapping"); + } + + std::string_view Suffix; + switch (BitWidth) { + case 32: + Suffix = "si3"; + break; + case 64: + Suffix = "di3"; + break; + case 128: + Suffix = "ti3"; + break; + default: + return makeUnexpected(ErrorCode::Unsupported, + "integer libcall requires a 32, 64, or 128-bit scalar"); + } + return std::string(Prefix) + std::string(Suffix); +} + +[[nodiscard]] ABIExtensionKind integerLibcallExtension(GenericOpcode Generic, + std::uint32_t BitWidth) noexcept { + if (BitWidth >= 64) + return ABIExtensionKind::None; + if (Generic == GenericOpcode::G_SDIV || Generic == GenericOpcode::G_SREM) + return ABIExtensionKind::SignExtend; + if (Generic == GenericOpcode::G_UDIV || Generic == GenericOpcode::G_UREM) + return ABIExtensionKind::ZeroExtend; + return ABIExtensionKind::AnyExtend; +} + +[[nodiscard]] Expected<std::string_view> integerLibcallTypeCode(std::uint32_t BitWidth) { + switch (BitWidth) { + case 32: + return "si"; + case 64: + return "di"; + case 128: + return "ti"; + default: + return makeUnexpected(ErrorCode::Unsupported, + "conversion libcall requires a 32, 64, or 128-bit integer"); + } +} + +[[nodiscard]] Expected<std::string_view> floatingLibcallTypeCode(std::uint32_t BitWidth) { + switch (BitWidth) { + case 32: + return "sf"; + case 64: + return "df"; + case 128: + return "tf"; + default: + return makeUnexpected(ErrorCode::Unsupported, + "conversion libcall requires a 32, 64, or 128-bit float"); + } +} + +[[nodiscard]] Expected<std::string> floatingBinaryLibcallName(GenericOpcode Generic, + std::uint32_t BitWidth) { + if (Generic != GenericOpcode::G_FREM) { + return makeUnexpected(ErrorCode::Unsupported, + "floating libcall has no runtime function mapping"); + } + switch (BitWidth) { + case 32: + return "fmodf"; + case 64: + return "fmod"; + case 128: + return "fmodl"; + default: + return makeUnexpected(ErrorCode::Unsupported, + "floating libcall requires a 32, 64, or 128-bit scalar"); + } +} + +[[nodiscard]] Expected<std::string> conversionLibcallName(GenericOpcode Generic, + std::uint32_t DestinationWidth, + std::uint32_t SourceWidth) { + const bool DestinationIsFloat = + Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP || + Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC; + const bool SourceIsFloat = + Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI || + Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC; + auto DestinationCode = DestinationIsFloat ? floatingLibcallTypeCode(DestinationWidth) + : integerLibcallTypeCode(DestinationWidth); + auto SourceCode = + SourceIsFloat ? floatingLibcallTypeCode(SourceWidth) : integerLibcallTypeCode(SourceWidth); + if (!DestinationCode) + return std::unexpected(DestinationCode.error()); + if (!SourceCode) + return std::unexpected(SourceCode.error()); + + std::string Name; + switch (Generic) { + case GenericOpcode::G_FPEXT: + Name = "__extend"; + break; + case GenericOpcode::G_FPTRUNC: + Name = "__trunc"; + break; + case GenericOpcode::G_FPTOSI: + Name = "__fix"; + break; + case GenericOpcode::G_FPTOUI: + Name = "__fixuns"; + break; + case GenericOpcode::G_SITOFP: + Name = "__float"; + break; + case GenericOpcode::G_UITOFP: + Name = "__floatun"; + break; + default: + return makeUnexpected(ErrorCode::Unsupported, "opcode has no conversion libcall mapping"); + } + Name += *SourceCode; + Name += *DestinationCode; + if (Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC) + Name += '2'; + return Name; +} + +[[nodiscard]] Expected<ABIValue> +libcallABIValue(Register Value, LowLevelType Type, ABIValueCategory Category, + std::uint32_t OriginalValue, std::uint16_t FragmentIndex, + std::uint16_t FragmentCount, std::uint32_t FragmentOffsetBytes, + Alignment OriginalAlignment, ABIExtensionKind Extension) { + auto OneByte = Alignment::create(1); + if (!OneByte) + return std::unexpected(OneByte.error()); + return ABIValue{ + .Value = Value, + .Type = Type, + .Category = Category, + .OriginalValue = OriginalValue, + .FragmentIndex = FragmentIndex, + .FragmentCount = FragmentCount, + .FragmentOffsetBytes = FragmentOffsetBytes, + .OriginalAlignment = OriginalAlignment, + .Extension = Extension, + .Flags = {}, + .ByValSizeBytes = 0, + .ByValAlignment = *OneByte, + }; +} + +} // namespace + +LegalizerHelper::LegalizerHelper(MachineFunction &Function, const LegalizerInfo &Info, + GISelChangeObserver &Observer, MachineIRBuilder &MIRBuilder, + const LibcallLoweringInfo *Libcalls) + : MIRBuilder(MIRBuilder), Observer(Observer), Function(Function), MRI(Function.registerInfo()), + Info(Info), Libcalls(Libcalls) {} + +Expected<void> LegalizerHelper::widenScalarMergeValues(MachineInstr &Instruction, + std::size_t TypeIndex, + LowLevelType WideType) { + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::Unsupported, + "G_MERGE_VALUES widening requires type index one"); + if (Instruction.operandCount() < 3) + return makeUnexpected(ErrorCode::VerificationError, + "G_MERGE_VALUES requires a destination and sources"); + + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + const Register *FirstSource = Instruction.operandAt(1)->getIf<Register>(); + if (Destination == nullptr || FirstSource == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_MERGE_VALUES requires register operands"); + auto DestinationType = MRI.getType(*Destination); + auto SourceType = MRI.getType(*FirstSource); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (DestinationType->isVector()) + return makeUnexpected(ErrorCode::Unsupported, + "vector G_MERGE_VALUES widening is not supported"); + + const std::uint64_t DestinationSize = DestinationType->minimumBitWidth(); + const std::uint64_t SourceSize = SourceType->minimumBitWidth(); + const std::uint64_t WideSize = WideType.minimumBitWidth(); + const std::uint64_t MergeCount = (DestinationSize + WideSize - 1U) / WideSize; + const std::size_t SourceCount = Instruction.operandCount() - 1U; + const std::uint64_t PartSize = DestinationSize / SourceCount; + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + if (WideSize >= DestinationSize) { + auto BuiltResult = MIRBuilder.buildZExt(WideType, *FirstSource); + if (!BuiltResult) + return std::unexpected(BuiltResult.error()); + auto Result = resultRegister(**BuiltResult); + if (!Result) + return std::unexpected(Result.error()); + Register ResultRegister = *Result; + + for (std::size_t Index = 2; Index != Instruction.operandCount(); ++Index) { + const Register *Source = Instruction.operandAt(Index)->getIf<Register>(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_MERGE_VALUES requires register sources"); + auto ActualSourceType = MRI.getType(*Source); + if (!ActualSourceType) + return std::unexpected(ActualSourceType.error()); + if (*ActualSourceType != *SourceType || ActualSourceType->minimumBitWidth() != PartSize) + return makeUnexpected(ErrorCode::VerificationError, + "G_MERGE_VALUES source types are inconsistent"); + + auto BuiltExtendedInput = MIRBuilder.buildZExt(WideType, *Source); + if (!BuiltExtendedInput) + return std::unexpected(BuiltExtendedInput.error()); + auto ExtendedInput = resultRegister(**BuiltExtendedInput); + if (!ExtendedInput) + return std::unexpected(ExtendedInput.error()); + auto ShiftAmount = buildConstantRegister(MIRBuilder, WideType, + static_cast<std::int64_t>((Index - 1U) * PartSize)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto BuiltShiftedInput = MIRBuilder.buildShl(WideType, *ExtendedInput, *ShiftAmount); + if (!BuiltShiftedInput) + return std::unexpected(BuiltShiftedInput.error()); + auto ShiftedInput = resultRegister(**BuiltShiftedInput); + if (!ShiftedInput) + return std::unexpected(ShiftedInput.error()); + + Register NextResult = *Destination; + if (Index + 1U != Instruction.operandCount() || WideType != *DestinationType) { + auto NewResult = MRI.createVirtualRegister(WideType); + if (!NewResult) + return std::unexpected(NewResult.error()); + NextResult = *NewResult; + } + if (auto Built = MIRBuilder.buildOr(NextResult, ResultRegister, *ShiftedInput); !Built) + return std::unexpected(Built.error()); + ResultRegister = NextResult; + } + + if (WideSize > DestinationSize) { + if (auto Built = MIRBuilder.buildTrunc(*Destination, ResultRegister); !Built) + return std::unexpected(Built.error()); + } else if (DestinationType->isPointer()) { + if (auto Built = MIRBuilder.buildIntToPtr(*Destination, ResultRegister); !Built) + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); + } + + auto GCDType = LowLevelType::makeCategoryNeutralScalar(std::gcd(SourceSize, WideSize)); + auto WideDestinationType = LowLevelType::makeCategoryNeutralScalar(MergeCount * WideSize); + if (!GCDType) + return std::unexpected(GCDType.error()); + if (!WideDestinationType) + return std::unexpected(WideDestinationType.error()); + + std::vector<Register> UnmergedRegisters; + for (std::size_t Index = 1; Index != Instruction.operandCount(); ++Index) { + const Register *Source = Instruction.operandAt(Index)->getIf<Register>(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_MERGE_VALUES requires register sources"); + if (GCDType->minimumBitWidth() == SourceSize) { + UnmergedRegisters.push_back(*Source); + continue; + } + auto Extracted = extractGCDType(UnmergedRegisters, *GCDType, *Source); + if (!Extracted) + return std::unexpected(Extracted.error()); + } + + const std::uint64_t RequiredPartCount = MergeCount * WideSize; + if (UnmergedRegisters.size() != RequiredPartCount) { + auto BuiltUndef = MIRBuilder.buildUndef(*GCDType); + if (!BuiltUndef) + return std::unexpected(BuiltUndef.error()); + auto UndefRegister = resultRegister(**BuiltUndef); + if (!UndefRegister) + return std::unexpected(UndefRegister.error()); + while (UnmergedRegisters.size() != RequiredPartCount) + UnmergedRegisters.push_back(*UndefRegister); + } + + const std::size_t PartsPerGCD = static_cast<std::size_t>(WideSize / GCDType->minimumBitWidth()); + std::vector<Register> NewMergeRegisters; + NewMergeRegisters.reserve(static_cast<std::size_t>(MergeCount)); + for (std::size_t Index = 0; Index != MergeCount; ++Index) { + std::vector<Register> Parts( + UnmergedRegisters.begin() + static_cast<std::ptrdiff_t>(Index * PartsPerGCD), + UnmergedRegisters.begin() + static_cast<std::ptrdiff_t>((Index + 1U) * PartsPerGCD)); + auto BuiltMerge = MIRBuilder.buildMergeLikeInstr(WideType, Parts); + if (!BuiltMerge) + return std::unexpected(BuiltMerge.error()); + auto MergeRegister = resultRegister(**BuiltMerge); + if (!MergeRegister) + return std::unexpected(MergeRegister.error()); + NewMergeRegisters.push_back(*MergeRegister); + } + + if (DestinationSize == WideDestinationType->minimumBitWidth()) { + if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, NewMergeRegisters); !Built) + return std::unexpected(Built.error()); + } else { + auto BuiltFinalMerge = MIRBuilder.buildMergeLikeInstr(*WideDestinationType, NewMergeRegisters); + if (!BuiltFinalMerge) + return std::unexpected(BuiltFinalMerge.error()); + auto FinalMerge = resultRegister(**BuiltFinalMerge); + if (!FinalMerge) + return std::unexpected(FinalMerge.error()); + if (auto Built = MIRBuilder.buildTrunc(*Destination, *FinalMerge); !Built) + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::widenScalarUnmergeValues(MachineInstr &Instruction, + std::size_t TypeIndex, + LowLevelType WideType) { + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::Unsupported, + "G_UNMERGE_VALUES widening requires type index zero"); + if (Instruction.operandCount() < 2) + return makeUnexpected(ErrorCode::VerificationError, + "G_UNMERGE_VALUES requires destinations and a source"); + + const std::size_t DestinationCount = Instruction.operandCount() - 1U; + const Register *Source = Instruction.operandAt(DestinationCount)->getIf<Register>(); + const Register *FirstDestination = Instruction.operandAt(0)->getIf<Register>(); + if (Source == nullptr || FirstDestination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_UNMERGE_VALUES requires register operands"); + auto SourceType = MRI.getType(*Source); + auto DestinationType = MRI.getType(*FirstDestination); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (SourceType->isVector() || !DestinationType->isCategoryNeutralScalar()) + return makeUnexpected(ErrorCode::Unsupported, + "G_UNMERGE_VALUES widening requires scalar types"); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + Register SourceRegister = *Source; + if (WideType.minimumBitWidth() >= SourceType->minimumBitWidth()) { + if (SourceType->isPointer()) { + auto IntegerSourceType = + LowLevelType::makeCategoryNeutralScalar(SourceType->minimumBitWidth()); + if (!IntegerSourceType) + return std::unexpected(IntegerSourceType.error()); + auto IntegerSource = MRI.createVirtualRegister(*IntegerSourceType); + if (!IntegerSource) + return std::unexpected(IntegerSource.error()); + if (auto Built = MIRBuilder.buildPtrToInt(*IntegerSource, SourceRegister); !Built) + return std::unexpected(Built.error()); + *SourceType = *IntegerSourceType; + SourceRegister = *IntegerSource; + } + + if (WideType.minimumBitWidth() > SourceType->minimumBitWidth()) { + auto WideSource = MRI.createVirtualRegister(WideType); + if (!WideSource) + return std::unexpected(WideSource.error()); + if (auto Built = MIRBuilder.buildAnyExt(*WideSource, SourceRegister); !Built) + return std::unexpected(Built.error()); + *SourceType = WideType; + SourceRegister = *WideSource; + } + + const std::uint64_t DestinationSize = DestinationType->minimumBitWidth(); + if (auto Built = MIRBuilder.buildTrunc(*FirstDestination, SourceRegister); !Built) + return std::unexpected(Built.error()); + for (std::size_t Index = 1; Index != DestinationCount; ++Index) { + const Register *Destination = Instruction.operandAt(Index)->getIf<Register>(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_UNMERGE_VALUES requires register destinations"); + auto ShiftedSource = MRI.createVirtualRegister(*SourceType); + if (!ShiftedSource) + return std::unexpected(ShiftedSource.error()); + auto ShiftAmount = buildConstantRegister(MIRBuilder, *SourceType, + static_cast<std::int64_t>(DestinationSize * Index)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + if (auto Built = MIRBuilder.buildLShr(*ShiftedSource, SourceRegister, *ShiftAmount); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildTrunc(*Destination, *ShiftedSource); !Built) + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); + } + + auto LCMType = getLCMType(*SourceType, WideType); + if (!LCMType) + return std::unexpected(LCMType.error()); + Register WideSource = SourceRegister; + if (LCMType->minimumBitWidth() != SourceType->minimumBitWidth()) { + if (SourceType->isPointer()) + return makeUnexpected(ErrorCode::Unsupported, + "widening a pointer G_UNMERGE_VALUES source is not implemented"); + auto ExtendedSource = MRI.createVirtualRegister(*LCMType); + if (!ExtendedSource) + return std::unexpected(ExtendedSource.error()); + if (auto Built = MIRBuilder.buildAnyExt(*ExtendedSource, WideSource); !Built) + return std::unexpected(Built.error()); + WideSource = *ExtendedSource; + } + + const std::size_t UnmergeCount = + static_cast<std::size_t>(LCMType->minimumBitWidth() / WideType.minimumBitWidth()); + std::vector<Register> UnmergeRegisters; + UnmergeRegisters.reserve(UnmergeCount); + for (std::size_t Index = 0; Index != UnmergeCount; ++Index) { + auto Part = MRI.createVirtualRegister(WideType); + if (!Part) + return std::unexpected(Part.error()); + UnmergeRegisters.push_back(*Part); + } + if (auto Built = MIRBuilder.buildUnmergeValues(UnmergeRegisters, WideSource); !Built) + return std::unexpected(Built.error()); + + auto GCDType = getGCDType(WideType, *DestinationType); + if (!GCDType) + return std::unexpected(GCDType.error()); + const std::size_t PartsPerRemerge = + static_cast<std::size_t>(DestinationType->minimumBitWidth() / GCDType->minimumBitWidth()); + + if (PartsPerRemerge == 1) { + const std::size_t PartsPerUnmerge = + static_cast<std::size_t>(WideType.minimumBitWidth() / DestinationType->minimumBitWidth()); + for (std::size_t Index = 0; Index != UnmergeRegisters.size(); ++Index) { + std::vector<Register> Destinations; + Destinations.reserve(PartsPerUnmerge); + for (std::size_t PartIndex = 0; PartIndex != PartsPerUnmerge; ++PartIndex) { + const std::size_t DestinationIndex = Index * PartsPerUnmerge + PartIndex; + if (DestinationIndex < DestinationCount) { + const Register *Destination = Instruction.operandAt(DestinationIndex)->getIf<Register>(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_UNMERGE_VALUES requires register destinations"); + Destinations.push_back(*Destination); + } else { + auto DeadDestination = MRI.createVirtualRegister(*DestinationType); + if (!DeadDestination) + return std::unexpected(DeadDestination.error()); + Destinations.push_back(*DeadDestination); + } + } + if (auto Built = MIRBuilder.buildUnmergeValues(Destinations, UnmergeRegisters[Index]); !Built) + return std::unexpected(Built.error()); + } + } else { + std::vector<Register> Parts; + for (Register UnmergeRegister : UnmergeRegisters) { + auto Extracted = extractGCDType(Parts, *GCDType, UnmergeRegister); + if (!Extracted) + return std::unexpected(Extracted.error()); + } + for (std::size_t Index = 0; Index != DestinationCount; ++Index) { + const Register *Destination = Instruction.operandAt(Index)->getIf<Register>(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_UNMERGE_VALUES requires register destinations"); + std::vector<Register> RemergeParts( + Parts.begin() + static_cast<std::ptrdiff_t>(Index * PartsPerRemerge), + Parts.begin() + static_cast<std::ptrdiff_t>((Index + 1U) * PartsPerRemerge)); + if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, RemergeParts); !Built) + return std::unexpected(Built.error()); + } + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::insertParts(Register Destination, LowLevelType ResultType, + LowLevelType PartType, + const std::vector<Register> &PartRegisters, + LowLevelType LeftoverType, + const std::vector<Register> &LeftoverRegisters) { + if (!LeftoverType.isValid()) { + if (!LeftoverRegisters.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, + "leftover registers require a leftover type"); + return MIRBuilder.buildMergeLikeInstr(Destination, PartRegisters).transform([](MachineInstr *) { + }); + } + + if (ResultType.isVector()) { + if (LeftoverRegisters.size() != 1) + return makeUnexpected(ErrorCode::InvalidArgument, + "mixed subvector merge requires one leftover register"); + std::vector<Register> AllRegisters = PartRegisters; + AllRegisters.insert(AllRegisters.end(), LeftoverRegisters.begin(), LeftoverRegisters.end()); + return MIRBuilder.buildMergeLikeInstr(Destination, AllRegisters).transform([](MachineInstr *) { + }); + } + + std::vector<Register> GCDRegisters; + auto FirstGCD = getGCDType(ResultType, LeftoverType); + if (!FirstGCD) + return std::unexpected(FirstGCD.error()); + auto GCDType = getGCDType(*FirstGCD, PartType); + if (!GCDType) + return std::unexpected(GCDType.error()); + for (Register Part : PartRegisters) { + auto Extracted = extractGCDType(GCDRegisters, *GCDType, Part); + if (!Extracted) + return std::unexpected(Extracted.error()); + } + for (Register Part : LeftoverRegisters) { + auto Extracted = extractGCDType(GCDRegisters, *GCDType, Part); + if (!Extracted) + return std::unexpected(Extracted.error()); + } + auto LCMType = buildLCMMergePieces(ResultType, LeftoverType, *GCDType, GCDRegisters); + if (!LCMType) + return std::unexpected(LCMType.error()); + return buildWidenedRemergeToDst(Destination, *LCMType, GCDRegisters); +} + +Expected<void> LegalizerHelper::extractGCDType(std::vector<Register> &Parts, LowLevelType GCDType, + Register Source) { + auto SourceType = MRI.getType(Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (*SourceType == GCDType) { + Parts.push_back(Source); + return {}; + } + const std::uint64_t SourceSize = SourceType->minimumBitWidth(); + const std::uint64_t GCDSize = GCDType.minimumBitWidth(); + if (GCDSize == 0 || SourceSize % GCDSize != 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "GCD type does not evenly divide the source type"); + return extractParts(Source, GCDType, static_cast<int>(SourceSize / GCDSize), Parts, MIRBuilder, + MRI); +} + +Expected<LowLevelType> LegalizerHelper::extractGCDType(std::vector<Register> &Parts, + LowLevelType DestinationType, + LowLevelType NarrowType, Register Source) { + auto SourceType = MRI.getType(Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + auto FirstGCD = getGCDType(*SourceType, NarrowType); + if (!FirstGCD) + return std::unexpected(FirstGCD.error()); + auto GCDType = getGCDType(*FirstGCD, DestinationType); + if (!GCDType) + return std::unexpected(GCDType.error()); + auto Extracted = extractGCDType(Parts, *GCDType, Source); + if (!Extracted) + return std::unexpected(Extracted.error()); + return *GCDType; +} + +Expected<LowLevelType> LegalizerHelper::buildLCMMergePieces(LowLevelType DestinationType, + LowLevelType NarrowType, + LowLevelType GCDType, + std::vector<Register> &Registers, + GenericOpcode PadStrategy) { + auto LCMType = getLCMType(DestinationType, NarrowType); + if (!LCMType) + return std::unexpected(LCMType.error()); + const int PartCount = static_cast<int>(LCMType->minimumBitWidth() / NarrowType.minimumBitWidth()); + const int SubPartCount = + static_cast<int>(NarrowType.minimumBitWidth() / GCDType.minimumBitWidth()); + const int OriginalSourceCount = static_cast<int>(Registers.size()); + + Register PadRegister; + if (OriginalSourceCount < PartCount * SubPartCount) { + if (PadStrategy == GenericOpcode::G_ZEXT) { + auto Pad = buildConstantRegister(MIRBuilder, GCDType, 0); + if (!Pad) + return std::unexpected(Pad.error()); + PadRegister = *Pad; + } else if (PadStrategy == GenericOpcode::G_ANYEXT) { + auto Built = MIRBuilder.buildUndef(GCDType); + if (!Built) + return std::unexpected(Built.error()); + auto Pad = resultRegister(**Built); + if (!Pad) + return std::unexpected(Pad.error()); + PadRegister = *Pad; + } else if (PadStrategy == GenericOpcode::G_SEXT) { + if (Registers.empty()) + return makeUnexpected(ErrorCode::InternalError, + "sign-extension padding requires a source part"); + auto ShiftType = LowLevelType::makeCategoryNeutralScalar(64); + if (!ShiftType) + return std::unexpected(ShiftType.error()); + auto ShiftAmount = buildConstantRegister( + MIRBuilder, *ShiftType, static_cast<std::int64_t>(GCDType.minimumBitWidth() - 1U)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Pad = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, GCDType, + {Registers.back(), *ShiftAmount}); + if (!Pad) + return std::unexpected(Pad.error()); + PadRegister = *Pad; + } else { + return makeUnexpected(ErrorCode::InternalError, + "LCM merge received an invalid extension strategy"); + } + } + + std::vector<Register> Remerge(static_cast<std::size_t>(PartCount)); + std::vector<Register> SubMerge(static_cast<std::size_t>(SubPartCount)); + Register AllPadRegister; + for (int PartIndex = 0; PartIndex != PartCount; ++PartIndex) { + bool AllMergePartsArePadding = true; + for (int SubPartIndex = 0; SubPartIndex != SubPartCount; ++SubPartIndex) { + const int Index = PartIndex * SubPartCount + SubPartIndex; + if (Index >= OriginalSourceCount) { + SubMerge[static_cast<std::size_t>(SubPartIndex)] = PadRegister; + continue; + } + SubMerge[static_cast<std::size_t>(SubPartIndex)] = Registers[static_cast<std::size_t>(Index)]; + AllMergePartsArePadding = false; + } + + if (AllMergePartsArePadding && AllPadRegister.isNoRegister()) { + if (PadStrategy == GenericOpcode::G_ZEXT) { + auto Pad = buildConstantRegister(MIRBuilder, NarrowType, 0); + if (!Pad) + return std::unexpected(Pad.error()); + AllPadRegister = *Pad; + } else if (PadStrategy == GenericOpcode::G_ANYEXT) { + auto Built = MIRBuilder.buildUndef(NarrowType); + if (!Built) + return std::unexpected(Built.error()); + auto Pad = resultRegister(**Built); + if (!Pad) + return std::unexpected(Pad.error()); + AllPadRegister = *Pad; + } + } + if (!AllPadRegister.isNoRegister()) { + Remerge[static_cast<std::size_t>(PartIndex)] = AllPadRegister; + continue; + } + if (SubPartCount == 1) { + Remerge[static_cast<std::size_t>(PartIndex)] = SubMerge.front(); + } else { + auto Built = MIRBuilder.buildMergeLikeInstr(NarrowType, SubMerge); + if (!Built) + return std::unexpected(Built.error()); + auto RemergeRegister = resultRegister(**Built); + if (!RemergeRegister) + return std::unexpected(RemergeRegister.error()); + Remerge[static_cast<std::size_t>(PartIndex)] = *RemergeRegister; + } + if (AllMergePartsArePadding && AllPadRegister.isNoRegister()) + AllPadRegister = Remerge[static_cast<std::size_t>(PartIndex)]; + } + Registers = std::move(Remerge); + return *LCMType; +} + +Expected<void> +LegalizerHelper::buildWidenedRemergeToDst(Register Destination, LowLevelType LCMType, + const std::vector<Register> &RemergeRegisters) { + auto DestinationType = MRI.getType(Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (*DestinationType == LCMType) + return MIRBuilder.buildMergeLikeInstr(Destination, RemergeRegisters) + .transform([](MachineInstr *) {}); + + auto Remerge = MRI.createVirtualRegister(LCMType); + if (!Remerge) + return std::unexpected(Remerge.error()); + auto BuiltMerge = MIRBuilder.buildMergeLikeInstr(*Remerge, RemergeRegisters); + if (!BuiltMerge) + return std::unexpected(BuiltMerge.error()); + if (!DestinationType->isVector() && !LCMType.isVector()) + return MIRBuilder.buildTrunc(Destination, *Remerge).transform([](MachineInstr *) {}); + + if (LCMType.isVector()) { + const std::size_t DefinitionCount = + static_cast<std::size_t>(LCMType.minimumBitWidth() / DestinationType->minimumBitWidth()); + std::vector<Register> Definitions; + Definitions.reserve(DefinitionCount); + Definitions.push_back(Destination); + for (std::size_t Index = 1; Index != DefinitionCount; ++Index) { + auto Definition = MRI.createVirtualRegister(*DestinationType); + if (!Definition) + return std::unexpected(Definition.error()); + Definitions.push_back(*Definition); + } + return MIRBuilder.buildUnmergeValues(Definitions, *Remerge).transform([](MachineInstr *) {}); + } + return makeUnexpected(ErrorCode::Unsupported, + "widened remerge has an unsupported type combination"); +} + +Expected<void> LegalizerHelper::multiplyRegisters(std::vector<Register> &Destinations, + const std::vector<Register> &FirstSources, + const std::vector<Register> &SecondSources, + LowLevelType NarrowType) { + if (FirstSources.empty() || FirstSources.size() != SecondSources.size() || + (Destinations.size() != FirstSources.size() && + Destinations.size() != FirstSources.size() * 2U)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "multi-precision multiplication has incompatible register parts"); + } + auto BooleanType = LowLevelType::makeCategoryNeutralScalar(1); + if (!BooleanType) + return std::unexpected(BooleanType.error()); + + const auto BuildUAddo = [&](Register Left, + Register Right) -> Expected<std::pair<Register, Register>> { + auto Sum = MRI.createVirtualRegister(NarrowType); + auto Carry = MRI.createVirtualRegister(*BooleanType); + if (!Sum) + return std::unexpected(Sum.error()); + if (!Carry) + return std::unexpected(Carry.error()); + auto Built = + MIRBuilder.buildInstr(opcode(GenericOpcode::G_UADDO), {*Sum, *Carry}, {Left, Right}); + if (!Built) + return std::unexpected(Built.error()); + return std::pair{*Sum, *Carry}; + }; + + const std::size_t SourcePartCount = FirstSources.size(); + const std::size_t DestinationPartCount = Destinations.size(); + auto Product = buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, NarrowType, + {FirstSources.front(), SecondSources.front()}); + if (!Product) + return std::unexpected(Product.error()); + Destinations.front() = *Product; + + Register PreviousCarrySum; + std::vector<Register> Factors; + for (std::size_t DestinationIndex = 1; DestinationIndex != DestinationPartCount; + ++DestinationIndex) { + // Collect high halves from the preceding product diagonal. + const std::size_t HighStart = + DestinationIndex < SourcePartCount ? 0 : DestinationIndex - SourcePartCount; + const std::size_t HighEnd = std::min(DestinationIndex - 1U, SourcePartCount - 1U); + for (std::size_t Index = HighStart; Index <= HighEnd; ++Index) { + auto HighProduct = buildResultInstruction( + MIRBuilder, GenericOpcode::G_UMULH, NarrowType, + {FirstSources[DestinationIndex - 1U - Index], SecondSources[Index]}); + if (!HighProduct) + return std::unexpected(HighProduct.error()); + Factors.push_back(*HighProduct); + } + + // Visit the current low-half diagonal from its low first-source part. + const std::size_t LowStart = + DestinationIndex + 1U < SourcePartCount ? 0 : DestinationIndex - SourcePartCount + 1U; + const std::size_t LowEnd = std::min(DestinationIndex, SourcePartCount - 1U); + for (std::size_t ReverseIndex = LowEnd + 1U; ReverseIndex != LowStart; --ReverseIndex) { + const std::size_t Index = ReverseIndex - 1U; + auto LowProduct = + buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, NarrowType, + {FirstSources[DestinationIndex - Index], SecondSources[Index]}); + if (!LowProduct) + return std::unexpected(LowProduct.error()); + Factors.push_back(*LowProduct); + } + if (DestinationIndex != 1U) + Factors.push_back(PreviousCarrySum); + if (Factors.size() < 2) + return makeUnexpected(ErrorCode::InternalError, + "multi-precision multiplication has too few factors"); + + Register FactorSum; + Register CarrySum; + if (DestinationIndex + 1U != DestinationPartCount) { + auto SumAndCarry = BuildUAddo(Factors[0], Factors[1]); + if (!SumAndCarry) + return std::unexpected(SumAndCarry.error()); + FactorSum = SumAndCarry->first; + auto ExtendedCarry = MRI.createVirtualRegister(NarrowType); + if (!ExtendedCarry) + return std::unexpected(ExtendedCarry.error()); + if (auto Built = MIRBuilder.buildZExt(*ExtendedCarry, SumAndCarry->second); !Built) + return std::unexpected(Built.error()); + CarrySum = *ExtendedCarry; + + for (std::size_t Index = 2; Index != Factors.size(); ++Index) { + SumAndCarry = BuildUAddo(FactorSum, Factors[Index]); + if (!SumAndCarry) + return std::unexpected(SumAndCarry.error()); + FactorSum = SumAndCarry->first; + auto ExtendedCarryPart = MRI.createVirtualRegister(NarrowType); + if (!ExtendedCarryPart) + return std::unexpected(ExtendedCarryPart.error()); + if (auto Built = MIRBuilder.buildZExt(*ExtendedCarryPart, SumAndCarry->second); !Built) + return std::unexpected(Built.error()); + auto NextCarrySum = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, NarrowType, + {CarrySum, *ExtendedCarryPart}); + if (!NextCarrySum) + return std::unexpected(NextCarrySum.error()); + CarrySum = *NextCarrySum; + } + } else { + auto Sum = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, NarrowType, + {Factors[0], Factors[1]}); + if (!Sum) + return std::unexpected(Sum.error()); + FactorSum = *Sum; + for (std::size_t Index = 2; Index != Factors.size(); ++Index) { + Sum = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, NarrowType, + {FactorSum, Factors[Index]}); + if (!Sum) + return std::unexpected(Sum.error()); + FactorSum = *Sum; + } + } + + PreviousCarrySum = CarrySum; + Destinations[DestinationIndex] = FactorSum; + Factors.clear(); + } + return {}; +} + +Expected<void> LegalizerHelper::widenScalarSrc(MachineInstr &Instruction, LowLevelType WideType, + std::size_t OperandIndex, + GenericOpcode ExtensionOpcode) { + MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Source == nullptr || !Source->isVirtual() || !Operand->isUse()) + return makeUnexpected(ErrorCode::VerificationError, + "scalar widening source is not a virtual-register use"); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto Extension = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {WideType}, {*Source}); + if (!Extension) + return std::unexpected(Extension.error()); + auto WideSource = resultRegister(**Extension); + if (!WideSource) + return std::unexpected(WideSource.error()); + return Operand->setRegister(*WideSource); +} + +Expected<void> LegalizerHelper::narrowScalarSrc(MachineInstr &Instruction, LowLevelType NarrowType, + std::size_t OperandIndex) { + MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Source == nullptr || !Source->isVirtual() || !Operand->isUse()) + return makeUnexpected(ErrorCode::VerificationError, + "scalar narrowing source is not a virtual-register use"); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto Truncation = MIRBuilder.buildTrunc(NarrowType, *Source); + if (!Truncation) + return std::unexpected(Truncation.error()); + auto NarrowSource = resultRegister(**Truncation); + if (!NarrowSource) + return std::unexpected(NarrowSource.error()); + return Operand->setRegister(*NarrowSource); +} + +Expected<void> LegalizerHelper::narrowScalarDst(MachineInstr &Instruction, LowLevelType NarrowType, + std::size_t OperandIndex, + GenericOpcode ExtensionOpcode) { + MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *Destination = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Destination == nullptr || !Destination->isVirtual() || !Operand->isDef()) + return makeUnexpected(ErrorCode::VerificationError, + "scalar narrowing destination is not a virtual-register definition"); + + const Register OriginalDestination = *Destination; + auto NarrowDestination = MRI.createVirtualRegister(NarrowType); + if (!NarrowDestination) + return std::unexpected(NarrowDestination.error()); + if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Extension = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {OriginalDestination}, + {*NarrowDestination}); + !Extension) { + return std::unexpected(Extension.error()); + } + return Operand->setRegister(*NarrowDestination); +} + +Expected<void> LegalizerHelper::widenScalarDst(MachineInstr &Instruction, LowLevelType WideType, + std::size_t OperandIndex, + GenericOpcode TruncOpcode) { + MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *Destination = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Destination == nullptr || !Destination->isVirtual() || !Operand->isDef()) + return makeUnexpected(ErrorCode::VerificationError, + "scalar widening destination is not a virtual-register definition"); + + auto WideDestination = MRI.createVirtualRegister(WideType); + if (!WideDestination) + return std::unexpected(WideDestination.error()); + if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point) + return std::unexpected(Point.error()); + auto Truncation = MIRBuilder.buildInstr(opcode(TruncOpcode)); + if (!Truncation) + return std::unexpected(Truncation.error()); + if (auto Added = Truncation->addDef(*Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Truncation->addUse(*WideDestination); !Added) + return std::unexpected(Added.error()); + if (auto Built = Truncation->build(); !Built) + return std::unexpected(Built.error()); + return Operand->setRegister(*WideDestination); +} + +Expected<void> LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &Instruction, + std::size_t TypeIndex, + LowLevelType WideType) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + GenericOpcode ArithmeticOpcode; + GenericOpcode ExtensionOpcode; + bool HasCarryIn = false; + switch (Generic) { + case GenericOpcode::G_SADDO: + ArithmeticOpcode = GenericOpcode::G_ADD; + ExtensionOpcode = GenericOpcode::G_SEXT; + break; + case GenericOpcode::G_SSUBO: + ArithmeticOpcode = GenericOpcode::G_SUB; + ExtensionOpcode = GenericOpcode::G_SEXT; + break; + case GenericOpcode::G_UADDO: + ArithmeticOpcode = GenericOpcode::G_ADD; + ExtensionOpcode = GenericOpcode::G_ZEXT; + break; + case GenericOpcode::G_USUBO: + ArithmeticOpcode = GenericOpcode::G_SUB; + ExtensionOpcode = GenericOpcode::G_ZEXT; + break; + case GenericOpcode::G_SADDE: + ArithmeticOpcode = GenericOpcode::G_UADDE; + ExtensionOpcode = GenericOpcode::G_SEXT; + HasCarryIn = true; + break; + case GenericOpcode::G_SSUBE: + ArithmeticOpcode = GenericOpcode::G_USUBE; + ExtensionOpcode = GenericOpcode::G_SEXT; + HasCarryIn = true; + break; + case GenericOpcode::G_UADDE: + ArithmeticOpcode = GenericOpcode::G_UADDE; + ExtensionOpcode = GenericOpcode::G_ZEXT; + HasCarryIn = true; + break; + case GenericOpcode::G_USUBE: + ArithmeticOpcode = GenericOpcode::G_USUBE; + ExtensionOpcode = GenericOpcode::G_ZEXT; + HasCarryIn = true; + break; + default: + return makeUnexpected(ErrorCode::InternalError, + "unexpected opcode during add/sub overflow widening"); + } + + if (TypeIndex == 1) { + if (HasCarryIn) { + if (auto Result = widenScalarSrc(Instruction, WideType, 4, + MIRBuilder.getBoolExtOp(WideType.isVector(), false)); + !Result) { + return Result; + } + } + return widenScalarDst(Instruction, WideType, 1); + } + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "add/sub overflow widening requires type index zero or one"); + if (Instruction.operandCount() != (HasCarryIn ? 5U : 4U)) + return makeUnexpected(ErrorCode::VerificationError, + "add/sub overflow widening has an invalid operand count"); + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *OverflowOperand = Instruction.operandAt(1); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const MachineOperand *CarryInOperand = HasCarryIn ? Instruction.operandAt(4) : nullptr; + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Overflow = + OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf<Register>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + const Register *CarryIn = CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf<Register>(); + if (Destination == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr || + (HasCarryIn && CarryIn == nullptr)) { + return makeUnexpected(ErrorCode::VerificationError, + "add/sub overflow widening requires register operands"); + } + + auto OriginalType = MRI.getType(*Destination); + auto OverflowType = MRI.getType(*Overflow); + if (!OriginalType) + return std::unexpected(OriginalType.error()); + if (!OverflowType) + return std::unexpected(OverflowType.error()); + + auto WideLeft = MRI.createVirtualRegister(WideType); + auto WideRight = MRI.createVirtualRegister(WideType); + auto WideResult = MRI.createVirtualRegister(WideType); + auto TruncatedResult = MRI.createVirtualRegister(*OriginalType); + auto ExtendedResult = MRI.createVirtualRegister(WideType); + if (!WideLeft) + return std::unexpected(WideLeft.error()); + if (!WideRight) + return std::unexpected(WideRight.error()); + if (!WideResult) + return std::unexpected(WideResult.error()); + if (!TruncatedResult) + return std::unexpected(TruncatedResult.error()); + if (!ExtendedResult) + return std::unexpected(ExtendedResult.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*WideLeft}, {*Left}); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*WideRight}, {*Right}); !Built) + return std::unexpected(Built.error()); + if (HasCarryIn) { + if (auto Built = MIRBuilder.buildInstr(opcode(ArithmeticOpcode), {*WideResult, *OverflowType}, + {*WideLeft, *WideRight, *CarryIn}); + !Built) { + return std::unexpected(Built.error()); + } + } else if (auto Built = MIRBuilder.buildInstr(opcode(ArithmeticOpcode), {*WideResult}, + {*WideLeft, *WideRight}); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildTrunc(*TruncatedResult, *WideResult); !Built) + return std::unexpected(Built.error()); + if (auto Built = + MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*ExtendedResult}, {*TruncatedResult}); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::NotEqual, *Overflow, *WideResult, *ExtendedResult); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildTrunc(*Destination, *WideResult); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::widenScalarMulo(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType WideType) { + if (TypeIndex == 1) + return widenScalarDst(Instruction, WideType, 1); + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "multiply-overflow widening requires type index zero or one"); + if (Instruction.operandCount() != 4) + return makeUnexpected(ErrorCode::VerificationError, + "multiply-overflow widening requires four register operands"); + + const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SMULO); + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + const Register *OriginalOverflow = Instruction.operandAt(1)->getIf<Register>(); + const Register *Left = Instruction.operandAt(2)->getIf<Register>(); + const Register *Right = Instruction.operandAt(3)->getIf<Register>(); + if (Destination == nullptr || OriginalOverflow == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "multiply-overflow widening requires register operands"); + + auto SourceType = MRI.getType(*Left); + auto OverflowType = MRI.getType(*OriginalOverflow); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (!OverflowType) + return std::unexpected(OverflowType.error()); + const std::uint32_t SourceBitWidth = SourceType->scalarBitWidth(); + if (SourceBitWidth == 0 || !WideType.isCategoryNeutralScalar()) + return makeUnexpected(ErrorCode::VerificationError, + "multiply-overflow widening requires scalar types"); + + const GenericOpcode ExtensionOpcode = IsSigned ? GenericOpcode::G_SEXT : GenericOpcode::G_ZEXT; + auto WideLeft = buildResultInstruction(MIRBuilder, ExtensionOpcode, WideType, {*Left}); + auto WideRight = buildResultInstruction(MIRBuilder, ExtensionOpcode, WideType, {*Right}); + if (!WideLeft) + return std::unexpected(WideLeft.error()); + if (!WideRight) + return std::unexpected(WideRight.error()); + + const bool WideMulCanOverflow = static_cast<std::uint64_t>(WideType.scalarBitWidth()) < + 2U * static_cast<std::uint64_t>(SourceBitWidth); + Register Mul; + Register WideOverflow; + if (WideMulCanOverflow) { + auto Built = + MIRBuilder.buildInstr(opcode(IsSigned ? GenericOpcode::G_SMULO : GenericOpcode::G_UMULO), + {WideType, *OverflowType}, {*WideLeft, *WideRight}); + if (!Built) + return std::unexpected(Built.error()); + const MachineOperand *ResultOperand = (*Built)->operandAt(0); + const MachineOperand *OverflowOperand = (*Built)->operandAt(1); + const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>(); + const Register *Overflow = + OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf<Register>(); + if (Result == nullptr || Overflow == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "multiply-overflow widening produced invalid results"); + Mul = *Result; + WideOverflow = *Overflow; + } else { + auto Built = + buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, WideType, {*WideLeft, *WideRight}); + if (!Built) + return std::unexpected(Built.error()); + Mul = *Built; + } + + if (auto Built = MIRBuilder.buildTrunc(*Destination, Mul); !Built) + return std::unexpected(Built.error()); + + Register ExtendedResult; + if (IsSigned) { + auto Built = MIRBuilder.buildSExtInReg(WideType, Mul, SourceBitWidth); + if (!Built) + return std::unexpected(Built.error()); + auto Result = resultRegister(**Built); + if (!Result) + return std::unexpected(Result.error()); + ExtendedResult = *Result; + } else { + auto Mask = APInt::getLowBitsSet(WideType.scalarBitWidth(), SourceBitWidth); + if (!Mask) + return std::unexpected(Mask.error()); + auto MaskRegister = buildConstantRegister(MIRBuilder, WideType, std::move(*Mask)); + if (!MaskRegister) + return std::unexpected(MaskRegister.error()); + auto Built = + buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, WideType, {Mul, *MaskRegister}); + if (!Built) + return std::unexpected(Built.error()); + ExtendedResult = *Built; + } + + if (WideMulCanOverflow) { + auto Overflow = MRI.createVirtualRegister(*OverflowType); + if (!Overflow) + return std::unexpected(Overflow.error()); + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::NotEqual, *Overflow, Mul, ExtendedResult); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildOr(*OriginalOverflow, WideOverflow, *Overflow); !Built) + return std::unexpected(Built.error()); + } else if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::NotEqual, *OriginalOverflow, Mul, + ExtendedResult); + !Built) { + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::widenScalar(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType WideType) { + auto Query = LegalizerInfo::buildQuery(Instruction, Function, MIRBuilder.target()); + if (!Query) + return std::unexpected(Query.error()); + if (!WideType.isCategoryNeutralScalar() || TypeIndex >= Query->Types.size() || + !Query->Types[TypeIndex].isCategoryNeutralScalar() || + WideType.scalarBitWidth() <= Query->Types[TypeIndex].scalarBitWidth()) { + return makeUnexpected(ErrorCode::VerificationError, + "scalar widening has an invalid type index or target type"); + } + + const auto WidenSource = [&](std::size_t OperandIndex, GenericOpcode ExtensionOpcode) { + return widenScalarSrc(Instruction, WideType, OperandIndex, ExtensionOpcode); + }; + const auto WidenDestination = [&](std::size_t OperandIndex = 0, + GenericOpcode TruncOpcode = GenericOpcode::G_TRUNC) { + return widenScalarDst(Instruction, WideType, OperandIndex, TruncOpcode); + }; + const auto Apply = [](Expected<void> Result) -> Expected<void> { + if (!Result) + return std::unexpected(Result.error()); + return {}; + }; + + Observer.changingInstr(Instruction); + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + case GenericOpcode::G_MERGE_VALUES: + return widenScalarMergeValues(Instruction, TypeIndex, WideType); + case GenericOpcode::G_UNMERGE_VALUES: + return widenScalarUnmergeValues(Instruction, TypeIndex, WideType); + case GenericOpcode::G_ATOMICRMW_XCHG: + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + case GenericOpcode::G_ATOMICRMW_AND: + case GenericOpcode::G_ATOMICRMW_OR: + case GenericOpcode::G_ATOMICRMW_XOR: + case GenericOpcode::G_ATOMICRMW_MIN: + case GenericOpcode::G_ATOMICRMW_MAX: + case GenericOpcode::G_ATOMICRMW_UMIN: + case GenericOpcode::G_ATOMICRMW_UMAX: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "atomic RMW value widening has an invalid type index"); + if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_ATOMIC_CMPXCHG: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "compare-exchange widening has an invalid type index"); + if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenSource(3, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: + if (TypeIndex == 0) { + if (auto Result = Apply(WidenSource(3, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenSource(4, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "compare-exchange success widening has an invalid type index"); + if (auto Result = Apply(WidenDestination(1)); !Result) + return Result; + break; + + case GenericOpcode::G_CONSTANT: { + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "constant widening has an invalid type index"); + MachineOperand *SourceOperand = Instruction.operandAt(1); + const auto *Immediate = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf<IntegerImmediateOperand>(); + if (Immediate == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_CONSTANT widening requires an integer immediate"); + const GenericOpcode ExtensionOpcode = + Info.getExtOpcodeForWideningConstant(Query->Types[TypeIndex]); + if (ExtensionOpcode != GenericOpcode::G_SEXT && ExtensionOpcode != GenericOpcode::G_ZEXT && + ExtensionOpcode != GenericOpcode::G_ANYEXT) { + return makeUnexpected(ErrorCode::InternalError, + "constant widening selected an invalid extension opcode"); + } + auto Extended = ExtensionOpcode == GenericOpcode::G_SEXT + ? Immediate->Value.sext(WideType.scalarBitWidth()) + : Immediate->Value.zext(WideType.scalarBitWidth()); + if (!Extended) + return std::unexpected(Extended.error()); + if (auto Replaced = SourceOperand->setIntegerImmediate(std::move(*Extended)); !Replaced) + return std::unexpected(Replaced.error()); + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + } + + case GenericOpcode::G_IMPLICIT_DEF: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "implicit definition widening has an invalid type index"); + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_BRCOND: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "G_BRCOND widening requires type index zero"); + if (auto Result = Apply(WidenSource(0, MIRBuilder.getBoolExtOp(false, false))); !Result) + return Result; + break; + + case GenericOpcode::G_CTTZ: + case GenericOpcode::G_CTTZ_ZERO_POISON: + case GenericOpcode::G_CTLZ: + case GenericOpcode::G_CTLZ_ZERO_POISON: + case GenericOpcode::G_CTLS: + case GenericOpcode::G_CTPOP: { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (TypeIndex == 0) { + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "bit-count widening has an invalid type index"); + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "bit-count widening requires register operands"); + auto SourceType = MRI.getType(*Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + + GenericOpcode ExtensionOpcode = GenericOpcode::G_ZEXT; + switch (Generic) { + case GenericOpcode::G_CTTZ: + case GenericOpcode::G_CTTZ_ZERO_POISON: + case GenericOpcode::G_CTLZ_ZERO_POISON: + ExtensionOpcode = GenericOpcode::G_ANYEXT; + break; + case GenericOpcode::G_CTLS: + ExtensionOpcode = GenericOpcode::G_SEXT; + break; + default: + break; + } + + auto WideSource = buildResultInstruction(MIRBuilder, ExtensionOpcode, WideType, {*Source}); + if (!WideSource) + return std::unexpected(WideSource.error()); + GenericOpcode WideOpcode = Generic; + if (Generic == GenericOpcode::G_CTTZ) { + auto TopBitValue = oneBitSet(WideType.scalarBitWidth(), SourceType->scalarBitWidth()); + if (!TopBitValue) + return std::unexpected(TopBitValue.error()); + auto TopBit = buildConstantRegister(MIRBuilder, WideType, std::move(*TopBitValue)); + if (!TopBit) + return std::unexpected(TopBit.error()); + auto NonZeroSource = + buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, WideType, {*WideSource, *TopBit}); + if (!NonZeroSource) + return std::unexpected(NonZeroSource.error()); + WideSource = *NonZeroSource; + WideOpcode = GenericOpcode::G_CTTZ_ZERO_POISON; + } + + const std::uint32_t SizeDifference = WideType.scalarBitWidth() - SourceType->scalarBitWidth(); + if (Generic == GenericOpcode::G_CTLZ_ZERO_POISON) { + auto ShiftAmount = + buildConstantRegister(MIRBuilder, WideType, static_cast<std::int64_t>(SizeDifference)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, WideType, + {*WideSource, *ShiftAmount}); + if (!Shifted) + return std::unexpected(Shifted.error()); + WideSource = *Shifted; + } + + auto Result = buildResultInstruction(MIRBuilder, WideOpcode, WideType, {*WideSource}); + if (!Result) + return std::unexpected(Result.error()); + if (Generic == GenericOpcode::G_CTLZ || Generic == GenericOpcode::G_CTLS) { + auto Difference = + buildConstantRegister(MIRBuilder, WideType, static_cast<std::int64_t>(SizeDifference)); + if (!Difference) + return std::unexpected(Difference.error()); + if (Generic == GenericOpcode::G_CTLZ) { + Result = buildFlaggedBinaryInstruction(MIRBuilder, GenericOpcode::G_SUB, WideType, *Result, + *Difference, + MachineInstrFlags{MachineInstrFlag::NoUnsignedWrap}); + } else { + Result = buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, WideType, + {*Result, *Difference}); + } + if (!Result) + return std::unexpected(Result.error()); + } + + if (auto Built = MIRBuilder.buildZExtOrTrunc(*Destination, *Result); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); + } + + case GenericOpcode::G_BSWAP: { + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "byte-swap widening has an invalid type index"); + MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "byte-swap widening requires a register destination"); + const Register OriginalDestination = *Destination; + auto OriginalType = MRI.getType(OriginalDestination); + if (!OriginalType) + return std::unexpected(OriginalType.error()); + auto WideDestination = MRI.createVirtualRegister(WideType); + auto Shifted = MRI.createVirtualRegister(WideType); + if (!WideDestination) + return std::unexpected(WideDestination.error()); + if (!Shifted) + return std::unexpected(Shifted.error()); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Replaced = DestinationOperand->setRegister(*WideDestination); !Replaced) + return std::unexpected(Replaced.error()); + + if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point) + return std::unexpected(Point.error()); + const std::uint32_t Difference = WideType.scalarBitWidth() - OriginalType->scalarBitWidth(); + auto ShiftAmount = buildConstantRegister(MIRBuilder, WideType, Difference); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + if (auto Built = MIRBuilder.buildLShr(*Shifted, *WideDestination, *ShiftAmount); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildTrunc(OriginalDestination, *Shifted); !Built) + return std::unexpected(Built.error()); + break; + } + + case GenericOpcode::G_LOAD: + case GenericOpcode::G_SEXTLOAD: + case GenericOpcode::G_ZEXTLOAD: + case GenericOpcode::G_FPEXTLOAD: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "load widening has an invalid type index"); + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_STORE: { + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "store widening has an invalid type index"); + const Register *Source = Instruction.operandAt(0)->getIf<Register>(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "store widening requires a register source"); + auto SourceType = MRI.getType(*Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (!SourceType->isCategoryNeutralScalar()) + return makeUnexpected(ErrorCode::Unsupported, "non-scalar store widening is not implemented"); + const GenericOpcode ExtensionOpcode = + SourceType->scalarBitWidth() == 1 ? GenericOpcode::G_ZEXT : GenericOpcode::G_ANYEXT; + if (auto Result = Apply(WidenSource(0, ExtensionOpcode)); !Result) + return Result; + break; + } + + case GenericOpcode::G_FPTRUNCSTORE: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "floating truncating store widening has an invalid type index"); + if (auto Result = Apply(WidenSource(0, GenericOpcode::G_FPEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_UADDO: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBE: + return widenScalarAddSubOverflow(Instruction, TypeIndex, WideType); + + case GenericOpcode::G_SMULO: + case GenericOpcode::G_UMULO: + if (TypeIndex != 1) + return widenScalarMulo(Instruction, TypeIndex, WideType); + if (auto Result = widenScalarMulo(Instruction, TypeIndex, WideType); !Result) + return Result; + break; + + case GenericOpcode::G_ADD: + case GenericOpcode::G_SUB: + case GenericOpcode::G_MUL: + case GenericOpcode::G_AND: + case GenericOpcode::G_OR: + case GenericOpcode::G_XOR: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "binary operation widening has an invalid type index"); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_SDIV: + case GenericOpcode::G_SREM: + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_ABDS: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "signed operation widening has an invalid type index"); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_SEXT)); !Result) + return Result; + if (auto Result = Apply(WidenSource(2, GenericOpcode::G_SEXT)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_UDIV: + case GenericOpcode::G_UREM: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: + case GenericOpcode::G_ABDU: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "unsigned operation widening has an invalid type index"); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ZEXT)); !Result) + return Result; + if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ZEXT)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_SHL: + case GenericOpcode::G_LSHR: + case GenericOpcode::G_ASHR: + if (TypeIndex == 0) { + GenericOpcode ExtensionOpcode = GenericOpcode::G_ANYEXT; + if (Instruction.opcode() == opcode(GenericOpcode::G_LSHR)) + ExtensionOpcode = GenericOpcode::G_ZEXT; + else if (Instruction.opcode() == opcode(GenericOpcode::G_ASHR)) + ExtensionOpcode = GenericOpcode::G_SEXT; + if (auto Result = Apply(WidenSource(1, ExtensionOpcode)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "shift widening has an invalid type index"); + if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ZEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_ROTL: + case GenericOpcode::G_ROTR: + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::Unsupported, "rotate value widening is not defined"); + if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ZEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_FREEZE: + case GenericOpcode::G_CONSTANT_FOLD_BARRIER: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "unary operation widening has an invalid type index"); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_ABS: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "absolute-value widening has an invalid type index"); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_SEXT)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_ICMP: + if (TypeIndex == 0) { + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "integer comparison widening has an invalid type index"); + { + const MachineOperand *PredicateMachineOperand = Instruction.operandAt(1); + const auto *Predicate = PredicateMachineOperand == nullptr + ? nullptr + : PredicateMachineOperand->getIf<PredicateOperand>(); + const IntegerPredicate *Integer = + Predicate == nullptr ? nullptr : Predicate->integerPredicate(); + if (Integer == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_ICMP widening requires an integer predicate"); + + bool IsSigned = false; + switch (*Integer) { + case IntegerPredicate::SignedGreaterThan: + case IntegerPredicate::SignedGreaterEqual: + case IntegerPredicate::SignedLessThan: + case IntegerPredicate::SignedLessEqual: + IsSigned = true; + break; + case IntegerPredicate::Equal: + case IntegerPredicate::NotEqual: + case IntegerPredicate::UnsignedGreaterThan: + case IntegerPredicate::UnsignedGreaterEqual: + case IntegerPredicate::UnsignedLessThan: + case IntegerPredicate::UnsignedLessEqual: + break; + case IntegerPredicate::Count: + return makeUnexpected(ErrorCode::VerificationError, + "G_ICMP widening received an invalid integer predicate"); + } + const GenericOpcode ExtensionOpcode = + IsSigned || Info.isSExtCheaperThanZExt(Query->Types[TypeIndex], WideType) + ? GenericOpcode::G_SEXT + : GenericOpcode::G_ZEXT; + if (auto Result = Apply(WidenSource(2, ExtensionOpcode)); !Result) + return Result; + if (auto Result = Apply(WidenSource(3, ExtensionOpcode)); !Result) + return Result; + } + break; + + case GenericOpcode::G_FCMP: + if (TypeIndex == 0) { + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "floating comparison widening has an invalid type index"); + if (auto Result = Apply(WidenSource(2, GenericOpcode::G_FPEXT)); !Result) + return Result; + if (auto Result = Apply(WidenSource(3, GenericOpcode::G_FPEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_SELECT: + if (TypeIndex == 0) { + if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenSource(3, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "select widening has an invalid type index"); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ZEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_PHI: { + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "G_PHI widening requires type index zero"); + if (Instruction.operandCount() < 3 || Instruction.operandCount() % 2U == 0) + return makeUnexpected(ErrorCode::VerificationError, + "G_PHI widening requires incoming value/block pairs"); + + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { + MachineOperand *ValueOperand = Instruction.operandAt(Index); + const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const auto *Incoming = + BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>(); + if (Value == nullptr || Incoming == nullptr || Incoming->Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_PHI widening has an invalid incoming pair"); + if (auto Point = setInsertPointAtFirstTerminator(MIRBuilder, *Incoming->Block); !Point) + return std::unexpected(Point.error()); + auto Extension = MIRBuilder.buildInstr(opcode(GenericOpcode::G_ANYEXT), {WideType}, {*Value}); + if (!Extension) + return std::unexpected(Extension.error()); + auto WideValue = resultRegister(**Extension); + if (!WideValue) + return std::unexpected(WideValue.error()); + if (auto Changed = ValueOperand->setRegister(*WideValue); !Changed) + return std::unexpected(Changed.error()); + } + + MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + MachineBasicBlock *Block = Instruction.parent(); + if (Destination == nullptr || Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_PHI widening has an invalid destination"); + const Register OriginalDestination = *Destination; + auto WideDestination = MRI.createVirtualRegister(WideType); + if (!WideDestination) + return std::unexpected(WideDestination.error()); + if (auto Changed = DestinationOperand->setRegister(*WideDestination); !Changed) + return std::unexpected(Changed.error()); + if (auto Point = setInsertPointAfterPHIs(MIRBuilder, *Block); !Point) + return std::unexpected(Point.error()); + if (auto Truncation = MIRBuilder.buildTrunc(OriginalDestination, *WideDestination); + !Truncation) { + return std::unexpected(Truncation.error()); + } + break; + } + + case GenericOpcode::G_FADD: + case GenericOpcode::G_FMUL: + case GenericOpcode::G_FSUB: + case GenericOpcode::G_FMA: + case GenericOpcode::G_FMAD: + case GenericOpcode::G_FNEG: + case GenericOpcode::G_FABS: + case GenericOpcode::G_FCANONICALIZE: + case GenericOpcode::G_FMINNUM: + case GenericOpcode::G_FMAXNUM: + case GenericOpcode::G_FMINNUM_IEEE: + case GenericOpcode::G_FMAXNUM_IEEE: + case GenericOpcode::G_FMINIMUM: + case GenericOpcode::G_FMAXIMUM: + case GenericOpcode::G_FMINIMUMNUM: + case GenericOpcode::G_FMAXIMUMNUM: + case GenericOpcode::G_FDIV: + case GenericOpcode::G_FREM: + case GenericOpcode::G_FCEIL: + case GenericOpcode::G_FFLOOR: + case GenericOpcode::G_FCOS: + case GenericOpcode::G_FSIN: + case GenericOpcode::G_FTAN: + case GenericOpcode::G_FACOS: + case GenericOpcode::G_FASIN: + case GenericOpcode::G_FATAN: + case GenericOpcode::G_FATAN2: + case GenericOpcode::G_FCOSH: + case GenericOpcode::G_FSINH: + case GenericOpcode::G_FTANH: + case GenericOpcode::G_FLOG10: + case GenericOpcode::G_FLOG: + case GenericOpcode::G_FLOG2: + case GenericOpcode::G_FRINT: + case GenericOpcode::G_FNEARBYINT: + case GenericOpcode::G_FSQRT: + case GenericOpcode::G_FEXP: + case GenericOpcode::G_FEXP2: + case GenericOpcode::G_FEXP10: + case GenericOpcode::G_FPOW: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "floating operation widening has an invalid type index"); + for (std::size_t OperandIndex = 1; OperandIndex < Instruction.operandCount(); ++OperandIndex) { + if (auto Result = Apply(WidenSource(OperandIndex, GenericOpcode::G_FPEXT)); !Result) + return Result; + } + if (auto Result = Apply(WidenDestination(0, GenericOpcode::G_FPTRUNC)); !Result) + return Result; + break; + + case GenericOpcode::G_FPEXT: + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::Unsupported, + "floating-point extension destination widening is not defined"); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_FPEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_FPTOSI: + case GenericOpcode::G_FPTOUI: + case GenericOpcode::G_INTRINSIC_LRINT: + case GenericOpcode::G_INTRINSIC_LLRINT: + case GenericOpcode::G_IS_FPCLASS: + if (TypeIndex == 0) { + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "floating-to-integer widening has an invalid type index"); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_FPEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_SITOFP: + case GenericOpcode::G_UITOFP: + if (TypeIndex == 0) { + if (auto Result = Apply(WidenDestination(0, GenericOpcode::G_FPTRUNC)); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "integer-to-floating widening has an invalid type index"); + if (auto Result = Apply(WidenSource(1, Instruction.opcode() == opcode(GenericOpcode::G_SITOFP) + ? GenericOpcode::G_SEXT + : GenericOpcode::G_ZEXT)); + !Result) + return Result; + break; + + case GenericOpcode::G_INTTOPTR: + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::Unsupported, + "G_INTTOPTR destination widening is not defined"); + if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ZEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_PTRTOINT: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::Unsupported, "G_PTRTOINT source widening is not defined"); + if (auto Result = Apply(WidenDestination()); !Result) + return Result; + break; + + default: + return makeUnexpected(ErrorCode::Unsupported, + "scalar widening is not implemented for this opcode"); + } + + Observer.changedInstr(Instruction); + return {}; +} + +Expected<void> LegalizerHelper::narrowScalarAddSub(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType) { + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::Unsupported, "add/sub narrowing requires type index zero"); + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "add/sub narrowing requires a register destination"); + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (DestinationType->isVector()) + return makeUnexpected(ErrorCode::Unsupported, "vector add/sub narrowing is not implemented"); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + GenericOpcode FirstOpcode; + GenericOpcode MiddleOpcode; + GenericOpcode FinalOpcode; + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_UADDO: + case GenericOpcode::G_UADDE: + case GenericOpcode::G_ADD: + FirstOpcode = GenericOpcode::G_UADDO; + MiddleOpcode = GenericOpcode::G_UADDE; + FinalOpcode = Instruction.opcode() == opcode(GenericOpcode::G_SADDO) || + Instruction.opcode() == opcode(GenericOpcode::G_SADDE) + ? GenericOpcode::G_SADDE + : GenericOpcode::G_UADDE; + break; + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_SSUBE: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SUB: + FirstOpcode = GenericOpcode::G_USUBO; + MiddleOpcode = GenericOpcode::G_USUBE; + FinalOpcode = Instruction.opcode() == opcode(GenericOpcode::G_SSUBO) || + Instruction.opcode() == opcode(GenericOpcode::G_SSUBE) + ? GenericOpcode::G_SSUBE + : GenericOpcode::G_USUBE; + break; + default: + return makeUnexpected(ErrorCode::InternalError, + "unexpected add/sub opcode during scalar narrowing"); + } + + const InstructionDescriptor *Descriptor = + MIRBuilder.target().instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "add/sub narrowing has no instruction descriptor"); + const std::size_t DefinitionCount = Descriptor->NumExplicitDefs; + const MachineOperand *FirstSourceOperand = Instruction.operandAt(DefinitionCount); + const MachineOperand *SecondSourceOperand = Instruction.operandAt(DefinitionCount + 1); + const Register *FirstSource = + FirstSourceOperand == nullptr ? nullptr : FirstSourceOperand->getIf<Register>(); + const Register *SecondSource = + SecondSourceOperand == nullptr ? nullptr : SecondSourceOperand->getIf<Register>(); + if (FirstSource == nullptr || SecondSource == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "add/sub narrowing requires two register sources"); + + Register CarryDestination; + Register CarryIn; + if (DefinitionCount == 2) { + const MachineOperand *CarryOperand = Instruction.operandAt(1); + const Register *Carry = CarryOperand == nullptr ? nullptr : CarryOperand->getIf<Register>(); + if (Carry == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "add/sub narrowing has an invalid carry destination"); + CarryDestination = *Carry; + } + if (Instruction.operandCount() == DefinitionCount + 3) { + const MachineOperand *CarryOperand = Instruction.operandAt(DefinitionCount + 2); + const Register *Carry = CarryOperand == nullptr ? nullptr : CarryOperand->getIf<Register>(); + if (Carry == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "add/sub narrowing has an invalid carry input"); + CarryIn = *Carry; + } + + LowLevelType LeftoverType; + LowLevelType DummyType; + std::vector<Register> FirstParts; + std::vector<Register> SecondParts; + std::vector<Register> FirstLeftovers; + std::vector<Register> SecondLeftovers; + std::vector<Register> DestinationParts; + auto FirstExtracted = extractParts(*FirstSource, *DestinationType, NarrowType, LeftoverType, + FirstParts, FirstLeftovers, MIRBuilder, MRI); + if (!FirstExtracted) + return std::unexpected(FirstExtracted.error()); + auto SecondExtracted = extractParts(*SecondSource, *DestinationType, NarrowType, DummyType, + SecondParts, SecondLeftovers, MIRBuilder, MRI); + if (!SecondExtracted) + return std::unexpected(SecondExtracted.error()); + + const std::size_t NarrowPartCount = FirstParts.size(); + FirstParts.insert(FirstParts.end(), FirstLeftovers.begin(), FirstLeftovers.end()); + SecondParts.insert(SecondParts.end(), SecondLeftovers.begin(), SecondLeftovers.end()); + if (FirstParts.size() != SecondParts.size()) + return makeUnexpected(ErrorCode::InternalError, + "add/sub source splits have different part counts"); + DestinationParts.reserve(FirstParts.size()); + + auto BooleanType = LowLevelType::makeCategoryNeutralScalar(1); + if (!BooleanType) + return std::unexpected(BooleanType.error()); + for (std::size_t Index = 0; Index != FirstParts.size(); ++Index) { + auto PartType = MRI.getType(FirstParts[Index]); + if (!PartType) + return std::unexpected(PartType.error()); + auto PartDestination = MRI.createVirtualRegister(*PartType); + if (!PartDestination) + return std::unexpected(PartDestination.error()); + + Register CarryOut; + if (Index + 1 == FirstParts.size() && !CarryDestination.isNoRegister()) { + CarryOut = CarryDestination; + } else { + auto NewCarry = MRI.createVirtualRegister(*BooleanType); + if (!NewCarry) + return std::unexpected(NewCarry.error()); + CarryOut = *NewCarry; + } + + Expected<MachineInstr *> Built; + if (CarryIn.isNoRegister()) { + Built = MIRBuilder.buildInstr(opcode(FirstOpcode), {*PartDestination, CarryOut}, + {FirstParts[Index], SecondParts[Index]}); + } else if (Index + 1 == FirstParts.size()) { + Built = MIRBuilder.buildInstr(opcode(FinalOpcode), {*PartDestination, CarryOut}, + {FirstParts[Index], SecondParts[Index], CarryIn}); + } else { + Built = MIRBuilder.buildInstr(opcode(MiddleOpcode), {*PartDestination, CarryOut}, + {FirstParts[Index], SecondParts[Index], CarryIn}); + } + if (!Built) + return std::unexpected(Built.error()); + DestinationParts.push_back(*PartDestination); + CarryIn = CarryOut; + } + + std::vector<Register> MainDestinations(DestinationParts.begin(), + DestinationParts.begin() + + static_cast<std::ptrdiff_t>(NarrowPartCount)); + std::vector<Register> LeftoverDestinations(DestinationParts.begin() + + static_cast<std::ptrdiff_t>(NarrowPartCount), + DestinationParts.end()); + auto Inserted = insertParts(*Destination, *DestinationType, NarrowType, MainDestinations, + LeftoverType, LeftoverDestinations); + if (!Inserted) + return std::unexpected(Inserted.error()); + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::narrowScalarMul(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType) { + if (TypeIndex != 0 || Instruction.operandCount() != 3) + return makeUnexpected(ErrorCode::Unsupported, + "multiply narrowing requires type index zero and two sources"); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *FirstSourceOperand = Instruction.operandAt(1); + const MachineOperand *SecondSourceOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *FirstSource = + FirstSourceOperand == nullptr ? nullptr : FirstSourceOperand->getIf<Register>(); + const Register *SecondSource = + SecondSourceOperand == nullptr ? nullptr : SecondSourceOperand->getIf<Register>(); + if (Destination == nullptr || FirstSource == nullptr || SecondSource == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "multiply narrowing requires register operands"); + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (DestinationType->isVector() || !NarrowType.isCategoryNeutralScalar()) + return makeUnexpected(ErrorCode::Unsupported, + "multiply narrowing requires scalar integer types"); + + const std::uint64_t DestinationSize = DestinationType->minimumBitWidth(); + const std::uint64_t NarrowSize = NarrowType.minimumBitWidth(); + if (NarrowSize == 0 || NarrowSize >= DestinationSize || DestinationSize % NarrowSize != 0) + return makeUnexpected(ErrorCode::Unsupported, + "multiply narrowing requires an even smaller scalar type"); + const std::size_t PartCount = static_cast<std::size_t>(DestinationSize / NarrowSize); + const bool IsMultiplyHigh = Instruction.opcode() == opcode(GenericOpcode::G_UMULH); + const std::size_t TemporaryPartCount = PartCount * (IsMultiplyHigh ? 2U : 1U); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + std::vector<Register> FirstParts; + std::vector<Register> SecondParts; + std::vector<Register> TemporaryDestinations(TemporaryPartCount); + if (auto Extracted = extractParts(*FirstSource, NarrowType, static_cast<int>(PartCount), + FirstParts, MIRBuilder, MRI); + !Extracted) { + return std::unexpected(Extracted.error()); + } + if (auto Extracted = extractParts(*SecondSource, NarrowType, static_cast<int>(PartCount), + SecondParts, MIRBuilder, MRI); + !Extracted) { + return std::unexpected(Extracted.error()); + } + if (auto Multiplied = + multiplyRegisters(TemporaryDestinations, FirstParts, SecondParts, NarrowType); + !Multiplied) { + return std::unexpected(Multiplied.error()); + } + + std::vector<Register> ResultParts(TemporaryDestinations.end() - + static_cast<std::ptrdiff_t>(PartCount), + TemporaryDestinations.end()); + if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, ResultParts); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::narrowScalarExt(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType) { + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::Unsupported, "extension narrowing requires type index zero"); + + const auto Destination = Instruction.operandAt(0)->registerValue(); + const auto Source = Instruction.operandAt(1)->registerValue(); + if (!Destination || !Source) + return makeUnexpected(ErrorCode::VerificationError, + "extension narrowing requires register operands"); + + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (DestinationType->isVector()) + return makeUnexpected(ErrorCode::Unsupported, "vector extension narrowing is not implemented"); + + std::vector<Register> Parts; + auto GCDType = extractGCDType(Parts, *DestinationType, NarrowType, *Source); + if (!GCDType) + return std::unexpected(GCDType.error()); + auto LCMType = buildLCMMergePieces(*DestinationType, NarrowType, *GCDType, Parts, + static_cast<GenericOpcode>(Instruction.opcode().value())); + if (!LCMType) + return std::unexpected(LCMType.error()); + auto Remerged = buildWidenedRemergeToDst(*Destination, *LCMType, Parts); + if (!Remerged) + return std::unexpected(Remerged.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::narrowScalarBasic(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType) { + if (TypeIndex != 0 || Instruction.operandCount() != 3) + return makeUnexpected(ErrorCode::Unsupported, + "basic scalar narrowing requires type index zero and two sources"); + + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + const Register *FirstSource = Instruction.operandAt(1)->getIf<Register>(); + const Register *SecondSource = Instruction.operandAt(2)->getIf<Register>(); + if (Destination == nullptr || FirstSource == nullptr || SecondSource == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "basic scalar narrowing requires register operands"); + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + + LowLevelType LeftoverType; + LowLevelType UnusedType; + std::vector<Register> FirstParts; + std::vector<Register> FirstLeftovers; + std::vector<Register> SecondParts; + std::vector<Register> SecondLeftovers; + auto FirstExtracted = extractParts(*FirstSource, *DestinationType, NarrowType, LeftoverType, + FirstParts, FirstLeftovers, MIRBuilder, MRI); + if (!FirstExtracted) + return std::unexpected(FirstExtracted.error()); + auto SecondExtracted = extractParts(*SecondSource, *DestinationType, NarrowType, UnusedType, + SecondParts, SecondLeftovers, MIRBuilder, MRI); + if (!SecondExtracted) + return std::unexpected(SecondExtracted.error()); + if (!*FirstExtracted || !*SecondExtracted || FirstParts.size() != SecondParts.size() || + FirstLeftovers.size() != SecondLeftovers.size()) { + return makeUnexpected(ErrorCode::Unsupported, + "basic scalar operands cannot be split consistently"); + } + + std::vector<Register> DestinationParts; + DestinationParts.reserve(FirstParts.size()); + for (std::size_t Index = 0; Index != FirstParts.size(); ++Index) { + auto Built = MIRBuilder.buildInstr(Instruction.opcode(), {NarrowType}, + {FirstParts[Index], SecondParts[Index]}); + if (!Built) + return std::unexpected(Built.error()); + auto Result = resultRegister(**Built); + if (!Result) + return std::unexpected(Result.error()); + DestinationParts.push_back(*Result); + } + + std::vector<Register> DestinationLeftovers; + DestinationLeftovers.reserve(FirstLeftovers.size()); + for (std::size_t Index = 0; Index != FirstLeftovers.size(); ++Index) { + auto Built = MIRBuilder.buildInstr(Instruction.opcode(), {LeftoverType}, + {FirstLeftovers[Index], SecondLeftovers[Index]}); + if (!Built) + return std::unexpected(Built.error()); + auto Result = resultRegister(**Built); + if (!Result) + return std::unexpected(Result.error()); + DestinationLeftovers.push_back(*Result); + } + + if (auto Inserted = insertParts(*Destination, *DestinationType, NarrowType, DestinationParts, + LeftoverType, DestinationLeftovers); + !Inserted) { + return std::unexpected(Inserted.error()); + } + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::narrowScalarSelect(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType) { + if (TypeIndex != 0 || Instruction.operandCount() != 4) + return makeUnexpected(ErrorCode::Unsupported, + "select narrowing requires type index zero and three sources"); + + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + const Register *Condition = Instruction.operandAt(1)->getIf<Register>(); + const Register *TrueSource = Instruction.operandAt(2)->getIf<Register>(); + const Register *FalseSource = Instruction.operandAt(3)->getIf<Register>(); + if (Destination == nullptr || Condition == nullptr || TrueSource == nullptr || + FalseSource == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "select narrowing requires register operands"); + } + auto DestinationType = MRI.getType(*Destination); + auto ConditionType = MRI.getType(*Condition); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!ConditionType) + return std::unexpected(ConditionType.error()); + if (ConditionType->isVector()) + return makeUnexpected(ErrorCode::Unsupported, "vector select narrowing is not implemented"); + + LowLevelType LeftoverType; + LowLevelType UnusedType; + std::vector<Register> TrueParts; + std::vector<Register> TrueLeftovers; + std::vector<Register> FalseParts; + std::vector<Register> FalseLeftovers; + auto TrueExtracted = extractParts(*TrueSource, *DestinationType, NarrowType, LeftoverType, + TrueParts, TrueLeftovers, MIRBuilder, MRI); + if (!TrueExtracted) + return std::unexpected(TrueExtracted.error()); + auto FalseExtracted = extractParts(*FalseSource, *DestinationType, NarrowType, UnusedType, + FalseParts, FalseLeftovers, MIRBuilder, MRI); + if (!FalseExtracted) + return std::unexpected(FalseExtracted.error()); + if (!*TrueExtracted || !*FalseExtracted || TrueParts.size() != FalseParts.size() || + TrueLeftovers.size() != FalseLeftovers.size()) { + return makeUnexpected(ErrorCode::Unsupported, "select operands cannot be split consistently"); + } + + std::vector<Register> DestinationParts; + DestinationParts.reserve(TrueParts.size()); + for (std::size_t Index = 0; Index != TrueParts.size(); ++Index) { + auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {NarrowType}, + {*Condition, TrueParts[Index], FalseParts[Index]}); + if (!Built) + return std::unexpected(Built.error()); + auto Result = resultRegister(**Built); + if (!Result) + return std::unexpected(Result.error()); + DestinationParts.push_back(*Result); + } + + std::vector<Register> DestinationLeftovers; + DestinationLeftovers.reserve(TrueLeftovers.size()); + for (std::size_t Index = 0; Index != TrueLeftovers.size(); ++Index) { + auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {LeftoverType}, + {*Condition, TrueLeftovers[Index], FalseLeftovers[Index]}); + if (!Built) + return std::unexpected(Built.error()); + auto Result = resultRegister(**Built); + if (!Result) + return std::unexpected(Result.error()); + DestinationLeftovers.push_back(*Result); + } + + if (auto Inserted = insertParts(*Destination, *DestinationType, NarrowType, DestinationParts, + LeftoverType, DestinationLeftovers); + !Inserted) { + return std::unexpected(Inserted.error()); + } + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &Instruction, + const APInt &Amount, + LowLevelType HalfType, + LowLevelType AmountType) { + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + const Register *Source = Instruction.operandAt(1)->getIf<Register>(); + if (Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "constant shift narrowing requires register operands"); + + auto LowPart = MRI.createVirtualRegister(HalfType); + auto HighPart = MRI.createVirtualRegister(HalfType); + if (!LowPart) + return std::unexpected(LowPart.error()); + if (!HighPart) + return std::unexpected(HighPart.error()); + if (auto Built = MIRBuilder.buildUnmergeValues({*LowPart, *HighPart}, *Source); !Built) + return std::unexpected(Built.error()); + + if (Amount.isZero()) { + if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, {*LowPart, *HighPart}); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); + } + + const std::uint64_t HalfBits = HalfType.minimumBitWidth(); + const std::uint64_t TotalBits = HalfBits * 2U; + const bool AmountExceeds64Bits = Amount.activeBitWidth() > 64; + const std::uint64_t AmountValue = Amount.word(0); + const bool ExceedsTotal = AmountExceeds64Bits || AmountValue > TotalBits; + const bool ExceedsHalf = AmountExceeds64Bits || AmountValue > HalfBits; + + const auto BuildConstant = [&](LowLevelType Type, std::uint64_t Value) { + return buildConstantRegister(MIRBuilder, Type, static_cast<std::int64_t>(Value)); + }; + const auto BuildBinary = [&](GenericOpcode Generic, Register Left, Register Right) { + return buildResultInstruction(MIRBuilder, Generic, HalfType, {Left, Right}); + }; + + Register LowResult; + Register HighResult; + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic == GenericOpcode::G_SHL) { + if (ExceedsTotal) { + auto Zero = BuildConstant(HalfType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + LowResult = *Zero; + HighResult = *Zero; + } else if (ExceedsHalf) { + auto Zero = BuildConstant(HalfType, 0); + auto Shift = BuildConstant(AmountType, AmountValue - HalfBits); + if (!Zero) + return std::unexpected(Zero.error()); + if (!Shift) + return std::unexpected(Shift.error()); + auto Shifted = BuildBinary(GenericOpcode::G_SHL, *LowPart, *Shift); + if (!Shifted) + return std::unexpected(Shifted.error()); + LowResult = *Zero; + HighResult = *Shifted; + } else if (AmountValue == HalfBits) { + auto Zero = BuildConstant(HalfType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + LowResult = *Zero; + HighResult = *LowPart; + } else { + auto Shift = BuildConstant(AmountType, AmountValue); + auto InverseShift = BuildConstant(AmountType, HalfBits - AmountValue); + if (!Shift) + return std::unexpected(Shift.error()); + if (!InverseShift) + return std::unexpected(InverseShift.error()); + auto LowShift = BuildBinary(GenericOpcode::G_SHL, *LowPart, *Shift); + auto HighShift = BuildBinary(GenericOpcode::G_SHL, *HighPart, *Shift); + auto Carry = BuildBinary(GenericOpcode::G_LSHR, *LowPart, *InverseShift); + if (!LowShift) + return std::unexpected(LowShift.error()); + if (!HighShift) + return std::unexpected(HighShift.error()); + if (!Carry) + return std::unexpected(Carry.error()); + auto Combined = BuildBinary(GenericOpcode::G_OR, *HighShift, *Carry); + if (!Combined) + return std::unexpected(Combined.error()); + LowResult = *LowShift; + HighResult = *Combined; + } + } else if (Generic == GenericOpcode::G_LSHR) { + if (ExceedsTotal) { + auto Zero = BuildConstant(HalfType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + LowResult = *Zero; + HighResult = *Zero; + } else if (ExceedsHalf) { + auto Shift = BuildConstant(AmountType, AmountValue - HalfBits); + auto Zero = BuildConstant(HalfType, 0); + if (!Shift) + return std::unexpected(Shift.error()); + if (!Zero) + return std::unexpected(Zero.error()); + auto Shifted = BuildBinary(GenericOpcode::G_LSHR, *HighPart, *Shift); + if (!Shifted) + return std::unexpected(Shifted.error()); + LowResult = *Shifted; + HighResult = *Zero; + } else if (AmountValue == HalfBits) { + auto Zero = BuildConstant(HalfType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + LowResult = *HighPart; + HighResult = *Zero; + } else { + auto Shift = BuildConstant(AmountType, AmountValue); + auto InverseShift = BuildConstant(AmountType, HalfBits - AmountValue); + if (!Shift) + return std::unexpected(Shift.error()); + if (!InverseShift) + return std::unexpected(InverseShift.error()); + auto LowShift = BuildBinary(GenericOpcode::G_LSHR, *LowPart, *Shift); + auto Carry = BuildBinary(GenericOpcode::G_SHL, *HighPart, *InverseShift); + auto HighShift = BuildBinary(GenericOpcode::G_LSHR, *HighPart, *Shift); + if (!LowShift) + return std::unexpected(LowShift.error()); + if (!Carry) + return std::unexpected(Carry.error()); + if (!HighShift) + return std::unexpected(HighShift.error()); + auto Combined = BuildBinary(GenericOpcode::G_OR, *LowShift, *Carry); + if (!Combined) + return std::unexpected(Combined.error()); + LowResult = *Combined; + HighResult = *HighShift; + } + } else if (Generic == GenericOpcode::G_ASHR) { + auto SignShift = BuildConstant(AmountType, HalfBits - 1U); + if (!SignShift) + return std::unexpected(SignShift.error()); + auto Sign = BuildBinary(GenericOpcode::G_ASHR, *HighPart, *SignShift); + if (!Sign) + return std::unexpected(Sign.error()); + if (ExceedsTotal) { + LowResult = *Sign; + HighResult = *Sign; + } else if (ExceedsHalf) { + auto Shift = BuildConstant(AmountType, AmountValue - HalfBits); + if (!Shift) + return std::unexpected(Shift.error()); + auto Shifted = BuildBinary(GenericOpcode::G_ASHR, *HighPart, *Shift); + if (!Shifted) + return std::unexpected(Shifted.error()); + LowResult = *Shifted; + HighResult = *Sign; + } else if (AmountValue == HalfBits) { + LowResult = *HighPart; + HighResult = *Sign; + } else { + auto Shift = BuildConstant(AmountType, AmountValue); + auto InverseShift = BuildConstant(AmountType, HalfBits - AmountValue); + if (!Shift) + return std::unexpected(Shift.error()); + if (!InverseShift) + return std::unexpected(InverseShift.error()); + auto LowShift = BuildBinary(GenericOpcode::G_LSHR, *LowPart, *Shift); + auto Carry = BuildBinary(GenericOpcode::G_SHL, *HighPart, *InverseShift); + auto HighShift = BuildBinary(GenericOpcode::G_ASHR, *HighPart, *Shift); + if (!LowShift) + return std::unexpected(LowShift.error()); + if (!Carry) + return std::unexpected(Carry.error()); + if (!HighShift) + return std::unexpected(HighShift.error()); + auto Combined = BuildBinary(GenericOpcode::G_OR, *LowShift, *Carry); + if (!Combined) + return std::unexpected(Combined.error()); + LowResult = *Combined; + HighResult = *HighShift; + } + } else { + return makeUnexpected(ErrorCode::InternalError, + "constant shift narrowing received a non-shift opcode"); + } + + if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, {LowResult, HighResult}); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::narrowScalarShift(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType) { + if (Instruction.operandCount() != 3) + return makeUnexpected(ErrorCode::VerificationError, + "shift narrowing requires one destination and two sources"); + if (TypeIndex == 1) { + Observer.changingInstr(Instruction); + auto Narrowed = narrowScalarSrc(Instruction, NarrowType, 2); + if (!Narrowed) + return std::unexpected(Narrowed.error()); + Observer.changedInstr(Instruction); + return {}; + } + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::Unsupported, "shift narrowing has an invalid type index"); + + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + const Register *Source = Instruction.operandAt(1)->getIf<Register>(); + const Register *Amount = Instruction.operandAt(2)->getIf<Register>(); + if (Destination == nullptr || Source == nullptr || Amount == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "shift narrowing requires register operands"); + auto DestinationType = MRI.getType(*Destination); + auto AmountType = MRI.getType(*Amount); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!AmountType) + return std::unexpected(AmountType.error()); + if (!DestinationType->isCategoryNeutralScalar()) + return makeUnexpected(ErrorCode::Unsupported, "vector shift narrowing is not implemented"); + + const std::uint64_t DestinationBits = DestinationType->scalarBitWidth(); + if (DestinationBits == 0 || DestinationBits % 2U != 0) + return makeUnexpected(ErrorCode::Unsupported, "shift narrowing requires an even scalar width"); + + if (NarrowType.isCategoryNeutralScalar() && NarrowType.scalarBitWidth() != 0 && + DestinationBits % NarrowType.scalarBitWidth() == 0 && + DestinationBits / NarrowType.scalarBitWidth() >= 8U) { + return narrowScalarShiftMultiway(Instruction, NarrowType); + } + + auto HalfType = LowLevelType::makeCategoryNeutralScalar(DestinationBits / 2U); + if (!HalfType) + return std::unexpected(HalfType.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Constant = getIConstantVRegValWithLookThrough(*Amount, MRI)) + return narrowScalarShiftByConstant(Instruction, Constant->Value, *HalfType, *AmountType); + + auto HalfBits = buildConstantRegister(MIRBuilder, *AmountType, + static_cast<std::int64_t>(DestinationBits / 2U)); + if (!HalfBits) + return std::unexpected(HalfBits.error()); + auto LowPart = MRI.createVirtualRegister(*HalfType); + auto HighPart = MRI.createVirtualRegister(*HalfType); + if (!LowPart) + return std::unexpected(LowPart.error()); + if (!HighPart) + return std::unexpected(HighPart.error()); + if (auto Built = MIRBuilder.buildUnmergeValues({*LowPart, *HighPart}, *Source); !Built) + return std::unexpected(Built.error()); + + auto AmountExcess = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, {*Amount, *HalfBits}); + auto AmountLack = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, {*HalfBits, *Amount}); + auto ZeroAmount = buildConstantRegister(MIRBuilder, *AmountType, 0); + auto ConditionType = LowLevelType::makeCategoryNeutralScalar(1); + if (!AmountExcess) + return std::unexpected(AmountExcess.error()); + if (!AmountLack) + return std::unexpected(AmountLack.error()); + if (!ZeroAmount) + return std::unexpected(ZeroAmount.error()); + if (!ConditionType) + return std::unexpected(ConditionType.error()); + auto IsShort = buildICmpRegister(MIRBuilder, IntegerPredicate::UnsignedLessThan, *ConditionType, + *Amount, *HalfBits); + auto IsZero = + buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *ConditionType, *Amount, *ZeroAmount); + if (!IsShort) + return std::unexpected(IsShort.error()); + if (!IsZero) + return std::unexpected(IsZero.error()); + + const auto Build = [&](GenericOpcode Generic, Register Left, Register Right) { + return buildResultInstruction(MIRBuilder, Generic, *HalfType, {Left, Right}); + }; + const auto Select = [&](Register Condition, Register TrueValue, Register FalseValue) { + return buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, *HalfType, + {Condition, TrueValue, FalseValue}); + }; + + Register LowResult; + Register HighResult; + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic == GenericOpcode::G_SHL) { + auto LowShort = Build(GenericOpcode::G_SHL, *LowPart, *Amount); + auto LowCarry = Build(GenericOpcode::G_LSHR, *LowPart, *AmountLack); + auto HighShift = Build(GenericOpcode::G_SHL, *HighPart, *Amount); + if (!LowShort) + return std::unexpected(LowShort.error()); + if (!LowCarry) + return std::unexpected(LowCarry.error()); + if (!HighShift) + return std::unexpected(HighShift.error()); + auto HighShort = Build(GenericOpcode::G_OR, *LowCarry, *HighShift); + auto LowLong = buildConstantRegister(MIRBuilder, *HalfType, 0); + auto HighLong = Build(GenericOpcode::G_SHL, *LowPart, *AmountExcess); + if (!HighShort) + return std::unexpected(HighShort.error()); + if (!LowLong) + return std::unexpected(LowLong.error()); + if (!HighLong) + return std::unexpected(HighLong.error()); + auto Low = Select(*IsShort, *LowShort, *LowLong); + auto HighNonZero = Select(*IsShort, *HighShort, *HighLong); + if (!Low) + return std::unexpected(Low.error()); + if (!HighNonZero) + return std::unexpected(HighNonZero.error()); + auto High = Select(*IsZero, *HighPart, *HighNonZero); + if (!High) + return std::unexpected(High.error()); + LowResult = *Low; + HighResult = *High; + } else if (Generic == GenericOpcode::G_LSHR || Generic == GenericOpcode::G_ASHR) { + auto HighShort = Build(Generic, *HighPart, *Amount); + auto LowShift = Build(GenericOpcode::G_LSHR, *LowPart, *Amount); + auto HighCarry = Build(GenericOpcode::G_SHL, *HighPart, *AmountLack); + if (!HighShort) + return std::unexpected(HighShort.error()); + if (!LowShift) + return std::unexpected(LowShift.error()); + if (!HighCarry) + return std::unexpected(HighCarry.error()); + auto LowShort = Build(GenericOpcode::G_OR, *LowShift, *HighCarry); + if (!LowShort) + return std::unexpected(LowShort.error()); + + Expected<Register> HighLong = + Generic == GenericOpcode::G_LSHR + ? buildConstantRegister(MIRBuilder, *HalfType, 0) + : buildConstantRegister(MIRBuilder, *AmountType, + static_cast<std::int64_t>(DestinationBits / 2U - 1U)) + .and_then([&](Register Shift) { + return Build(GenericOpcode::G_ASHR, *HighPart, Shift); + }); + auto LowLong = Build(Generic, *HighPart, *AmountExcess); + if (!HighLong) + return std::unexpected(HighLong.error()); + if (!LowLong) + return std::unexpected(LowLong.error()); + auto LowNonZero = Select(*IsShort, *LowShort, *LowLong); + if (!LowNonZero) + return std::unexpected(LowNonZero.error()); + auto Low = Select(*IsZero, *LowPart, *LowNonZero); + auto High = Select(*IsShort, *HighShort, *HighLong); + if (!Low) + return std::unexpected(Low.error()); + if (!High) + return std::unexpected(High.error()); + LowResult = *Low; + HighResult = *High; + } else { + return makeUnexpected(ErrorCode::InternalError, "shift narrowing received a non-shift opcode"); + } + + if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, {LowResult, HighResult}); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); +} + +Expected<Register> LegalizerHelper::buildConstantShiftPart(GenericOpcode Generic, + std::size_t PartIndex, + std::size_t PartCount, + const std::vector<Register> &SourceParts, + const ShiftParameters &Parameters, + LowLevelType TargetType) { + const auto WordShift = getIConstantVRegValWithLookThrough(Parameters.WordShift, MRI, false); + const auto BitShift = getIConstantVRegValWithLookThrough(Parameters.BitShift, MRI, false); + if (!WordShift || !BitShift || WordShift->Value.activeBitWidth() > 64 || + BitShift->Value.activeBitWidth() > 64) { + return makeUnexpected(ErrorCode::InternalError, + "multiway shift parameters are not representable constants"); + } + const std::uint64_t ShiftWords = WordShift->Value.word(0); + const std::uint64_t ShiftBits = BitShift->Value.word(0); + const bool NeedsCarry = ShiftBits != 0; + const auto Build = [&](GenericOpcode Opcode, Register Left, Register Right) { + return buildResultInstruction(MIRBuilder, Opcode, TargetType, {Left, Right}); + }; + + if (Generic == GenericOpcode::G_SHL) { + if (PartIndex < ShiftWords) + return Parameters.Zero; + const std::size_t SourceIndex = PartIndex - static_cast<std::size_t>(ShiftWords); + if (!NeedsCarry) + return SourceParts[SourceIndex]; + auto High = Build(GenericOpcode::G_SHL, SourceParts[SourceIndex], Parameters.BitShift); + if (!High) + return std::unexpected(High.error()); + if (SourceIndex == 0) + return *High; + auto Low = + Build(GenericOpcode::G_LSHR, SourceParts[SourceIndex - 1U], Parameters.InverseBitShift); + if (!Low) + return std::unexpected(Low.error()); + return Build(GenericOpcode::G_OR, *High, *Low); + } + + if (Generic == GenericOpcode::G_LSHR) { + const std::uint64_t SourceIndex = PartIndex + ShiftWords; + if (SourceIndex >= PartCount) + return Parameters.Zero; + if (!NeedsCarry) + return SourceParts[static_cast<std::size_t>(SourceIndex)]; + auto Low = Build(GenericOpcode::G_LSHR, SourceParts[static_cast<std::size_t>(SourceIndex)], + Parameters.BitShift); + if (!Low) + return std::unexpected(Low.error()); + if (SourceIndex + 1U >= PartCount) + return *Low; + auto High = Build(GenericOpcode::G_SHL, SourceParts[static_cast<std::size_t>(SourceIndex + 1U)], + Parameters.InverseBitShift); + if (!High) + return std::unexpected(High.error()); + return Build(GenericOpcode::G_OR, *Low, *High); + } + + if (Generic == GenericOpcode::G_ASHR) { + const std::uint64_t SourceIndex = PartIndex + ShiftWords; + if (SourceIndex >= PartCount) + return Parameters.SignBit; + if (!NeedsCarry) + return SourceParts[static_cast<std::size_t>(SourceIndex)]; + const GenericOpcode MainOpcode = + SourceIndex + 1U == PartCount ? GenericOpcode::G_ASHR : GenericOpcode::G_LSHR; + auto Low = + Build(MainOpcode, SourceParts[static_cast<std::size_t>(SourceIndex)], Parameters.BitShift); + if (!Low) + return std::unexpected(Low.error()); + const Register HighSource = SourceIndex + 1U < PartCount + ? SourceParts[static_cast<std::size_t>(SourceIndex + 1U)] + : Parameters.SignBit; + auto High = Build(GenericOpcode::G_SHL, HighSource, Parameters.InverseBitShift); + if (!High) + return std::unexpected(High.error()); + return Build(GenericOpcode::G_OR, *Low, *High); + } + + return makeUnexpected(ErrorCode::InternalError, + "constant shift part received a non-shift opcode"); +} + +Expected<Register> LegalizerHelper::buildVariableShiftPart(GenericOpcode Generic, + Register MainOperand, + Register ShiftAmount, + LowLevelType TargetType, + Register CarryOperand) { + const GenericOpcode MainOpcode = + Generic == GenericOpcode::G_ASHR ? GenericOpcode::G_LSHR : Generic; + auto MainShifted = + buildResultInstruction(MIRBuilder, MainOpcode, TargetType, {MainOperand, ShiftAmount}); + if (!MainShifted) + return std::unexpected(MainShifted.error()); + if (CarryOperand.isNoRegister()) + return *MainShifted; + + auto AmountType = MRI.getType(ShiftAmount); + if (!AmountType) + return std::unexpected(AmountType.error()); + auto ZeroAmount = buildConstantRegister(MIRBuilder, *AmountType, 0); + auto BoolType = LowLevelType::makeCategoryNeutralScalar(1); + if (!ZeroAmount) + return std::unexpected(ZeroAmount.error()); + if (!BoolType) + return std::unexpected(BoolType.error()); + auto IsZero = + buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *BoolType, ShiftAmount, *ZeroAmount); + auto TargetBits = buildConstantRegister(MIRBuilder, *AmountType, + static_cast<std::int64_t>(TargetType.scalarBitWidth())); + if (!IsZero) + return std::unexpected(IsZero.error()); + if (!TargetBits) + return std::unexpected(TargetBits.error()); + auto InverseAmount = buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, + {*TargetBits, ShiftAmount}); + if (!InverseAmount) + return std::unexpected(InverseAmount.error()); + + const GenericOpcode CarryOpcode = + Generic == GenericOpcode::G_SHL ? GenericOpcode::G_LSHR : GenericOpcode::G_SHL; + auto CarryBits = + buildResultInstruction(MIRBuilder, CarryOpcode, TargetType, {CarryOperand, *InverseAmount}); + auto Zero = buildConstantRegister(MIRBuilder, TargetType, 0); + if (!CarryBits) + return std::unexpected(CarryBits.error()); + if (!Zero) + return std::unexpected(Zero.error()); + auto SafeCarry = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, TargetType, + {*IsZero, *Zero, *CarryBits}); + if (!SafeCarry) + return std::unexpected(SafeCarry.error()); + return buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, TargetType, + {*MainShifted, *SafeCarry}); +} + +Expected<void> LegalizerHelper::narrowScalarShiftByConstantMultiway(MachineInstr &Instruction, + const APInt &Amount, + LowLevelType TargetType, + LowLevelType AmountType) { + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + const Register *Source = Instruction.operandAt(1)->getIf<Register>(); + if (Destination == nullptr || Source == nullptr || Amount.activeBitWidth() > 64) + return makeUnexpected(ErrorCode::Unsupported, + "constant multiway shift requires representable register operands"); + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + const std::uint64_t DestinationBits = DestinationType->scalarBitWidth(); + const std::uint64_t TargetBits = TargetType.scalarBitWidth(); + if (TargetBits == 0 || DestinationBits % TargetBits != 0) + return makeUnexpected(ErrorCode::VerificationError, + "multiway shift target type does not divide the destination"); + const std::size_t PartCount = static_cast<std::size_t>(DestinationBits / TargetBits); + + std::vector<Register> SourceParts; + auto Extracted = + extractParts(*Source, TargetType, static_cast<int>(PartCount), SourceParts, MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + if (Amount.isZero()) { + if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, SourceParts); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); + } + + const std::uint64_t AmountValue = Amount.word(0); + const std::uint64_t ShiftWords = AmountValue / TargetBits; + const std::uint64_t ShiftBits = AmountValue % TargetBits; + ShiftParameters Parameters; + auto WordShift = + buildConstantRegister(MIRBuilder, AmountType, static_cast<std::int64_t>(ShiftWords)); + auto BitShift = + buildConstantRegister(MIRBuilder, AmountType, static_cast<std::int64_t>(ShiftBits)); + auto InverseBitShift = buildConstantRegister(MIRBuilder, AmountType, + static_cast<std::int64_t>(TargetBits - ShiftBits)); + auto Zero = buildConstantRegister(MIRBuilder, TargetType, 0); + if (!WordShift) + return std::unexpected(WordShift.error()); + if (!BitShift) + return std::unexpected(BitShift.error()); + if (!InverseBitShift) + return std::unexpected(InverseBitShift.error()); + if (!Zero) + return std::unexpected(Zero.error()); + Parameters.WordShift = *WordShift; + Parameters.BitShift = *BitShift; + Parameters.InverseBitShift = *InverseBitShift; + Parameters.Zero = *Zero; + + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic == GenericOpcode::G_ASHR) { + auto SignAmount = + buildConstantRegister(MIRBuilder, AmountType, static_cast<std::int64_t>(TargetBits - 1U)); + if (!SignAmount) + return std::unexpected(SignAmount.error()); + auto SignBit = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, TargetType, + {SourceParts.back(), *SignAmount}); + if (!SignBit) + return std::unexpected(SignBit.error()); + Parameters.SignBit = *SignBit; + } + + std::vector<Register> DestinationParts; + DestinationParts.reserve(PartCount); + for (std::size_t Index = 0; Index != PartCount; ++Index) { + auto Part = + buildConstantShiftPart(Generic, Index, PartCount, SourceParts, Parameters, TargetType); + if (!Part) + return std::unexpected(Part.error()); + DestinationParts.push_back(*Part); + } + if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, DestinationParts); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::narrowScalarShiftMultiway(MachineInstr &Instruction, + LowLevelType TargetType) { + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + const Register *Source = Instruction.operandAt(1)->getIf<Register>(); + const Register *Amount = Instruction.operandAt(2)->getIf<Register>(); + if (Destination == nullptr || Source == nullptr || Amount == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "multiway shift narrowing requires register operands"); + auto DestinationType = MRI.getType(*Destination); + auto AmountType = MRI.getType(*Amount); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!AmountType) + return std::unexpected(AmountType.error()); + const std::uint64_t DestinationBits = DestinationType->scalarBitWidth(); + const std::uint64_t TargetBits = TargetType.scalarBitWidth(); + if (TargetBits == 0 || DestinationBits % TargetBits != 0 || !std::has_single_bit(TargetBits)) { + return makeUnexpected(ErrorCode::VerificationError, + "multiway shift requires a power-of-two divisor type"); + } + const std::size_t PartCount = static_cast<std::size_t>(DestinationBits / TargetBits); + if (auto Constant = getIConstantVRegValWithLookThrough(*Amount, MRI)) { + return narrowScalarShiftByConstantMultiway(Instruction, Constant->Value, TargetType, + *AmountType); + } + + std::vector<Register> SourceParts; + auto Extracted = + extractParts(*Source, TargetType, static_cast<int>(PartCount), SourceParts, MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + + auto ZeroAmount = buildConstantRegister(MIRBuilder, *AmountType, 0); + auto BoolType = LowLevelType::makeCategoryNeutralScalar(1); + if (!ZeroAmount) + return std::unexpected(ZeroAmount.error()); + if (!BoolType) + return std::unexpected(BoolType.error()); + auto IsZero = + buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *BoolType, *Amount, *ZeroAmount); + if (!IsZero) + return std::unexpected(IsZero.error()); + + const auto TargetLog2 = static_cast<std::int64_t>(std::countr_zero(TargetBits)); + auto Log2 = buildConstantRegister(MIRBuilder, *AmountType, TargetLog2); + auto Mask = + buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(TargetBits - 1U)); + if (!Log2) + return std::unexpected(Log2.error()); + if (!Mask) + return std::unexpected(Mask.error()); + auto WordShift = + buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *AmountType, {*Amount, *Log2}); + auto BitShift = + buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType, {*Amount, *Mask}); + auto Zero = buildConstantRegister(MIRBuilder, TargetType, 0); + if (!WordShift) + return std::unexpected(WordShift.error()); + if (!BitShift) + return std::unexpected(BitShift.error()); + if (!Zero) + return std::unexpected(Zero.error()); + + Register FillValue = *Zero; + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic == GenericOpcode::G_ASHR) { + auto SignAmount = + buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(TargetBits - 1U)); + if (!SignAmount) + return std::unexpected(SignAmount.error()); + auto Sign = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, TargetType, + {SourceParts.back(), *SignAmount}); + if (!Sign) + return std::unexpected(Sign.error()); + FillValue = *Sign; + } + + std::vector<Register> DestinationParts(PartCount); + for (std::size_t PartIndex = 0; PartIndex != PartCount; ++PartIndex) { + Register InBoundsResult = FillValue; + for (std::size_t WordIndex = 0; WordIndex != PartCount; ++WordIndex) { + auto WordConstant = + buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(WordIndex)); + if (!WordConstant) + return std::unexpected(WordConstant.error()); + auto IsWord = buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *BoolType, *WordShift, + *WordConstant); + if (!IsWord) + return std::unexpected(IsWord.error()); + + std::ptrdiff_t MainIndex; + std::ptrdiff_t CarryIndex; + if (Generic == GenericOpcode::G_SHL) { + MainIndex = static_cast<std::ptrdiff_t>(PartIndex) - static_cast<std::ptrdiff_t>(WordIndex); + CarryIndex = MainIndex - 1; + } else if (Generic == GenericOpcode::G_LSHR || Generic == GenericOpcode::G_ASHR) { + MainIndex = static_cast<std::ptrdiff_t>(PartIndex) + static_cast<std::ptrdiff_t>(WordIndex); + CarryIndex = MainIndex + 1; + } else { + return makeUnexpected(ErrorCode::InternalError, + "multiway shift received a non-shift opcode"); + } + + Register Result = FillValue; + if (MainIndex >= 0 && MainIndex < static_cast<std::ptrdiff_t>(PartCount)) { + Register Carry; + if (CarryIndex >= 0 && CarryIndex < static_cast<std::ptrdiff_t>(PartCount)) + Carry = SourceParts[static_cast<std::size_t>(CarryIndex)]; + else if (Generic == GenericOpcode::G_ASHR && + CarryIndex >= static_cast<std::ptrdiff_t>(PartCount)) + Carry = FillValue; + auto Shifted = + buildVariableShiftPart(Generic, SourceParts[static_cast<std::size_t>(MainIndex)], + *BitShift, TargetType, Carry); + if (!Shifted) + return std::unexpected(Shifted.error()); + Result = *Shifted; + } + + auto Selected = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, TargetType, + {*IsWord, Result, InBoundsResult}); + if (!Selected) + return std::unexpected(Selected.error()); + InBoundsResult = *Selected; + } + auto Selected = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, TargetType, + {*IsZero, SourceParts[PartIndex], InBoundsResult}); + if (!Selected) + return std::unexpected(Selected.error()); + DestinationParts[PartIndex] = *Selected; + } + + if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, DestinationParts); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::narrowScalarLoadStore(MachineInstr &Instruction, + std::size_t TypeIndex, + LowLevelType NarrowType) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic != GenericOpcode::G_LOAD && Generic != GenericOpcode::G_STORE) { + return makeUnexpected(ErrorCode::InvalidArgument, + "memory scalar narrowing received an unsupported opcode"); + } + if (TypeIndex != 0 || !NarrowType.isCategoryNeutralScalar() || + NarrowType.minimumBitWidth() == 0 || NarrowType.minimumBitWidth() % 8U != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "memory scalar narrowing requires a byte-sized scalar value type"); + } + if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 || + Instruction.memOperands().front() == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory scalar narrowing requires one address and one memory operand"); + } + + const MachineOperand *ValueOperand = Instruction.operandAt(0); + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>(); + if (Value == nullptr || Address == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory scalar narrowing requires register value and address operands"); + } + auto ValueType = MRI.getType(*Value); + if (!ValueType) + return std::unexpected(ValueType.error()); + if (!ValueType->isCategoryNeutralScalar() || + NarrowType.minimumBitWidth() >= ValueType->minimumBitWidth()) { + return makeUnexpected(ErrorCode::Unsupported, + "memory scalar narrowing requires a smaller integer scalar type"); + } + + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType) { + return makeUnexpected(ErrorCode::Unsupported, + "memory scalar narrowing requires a fixed typed memory operand"); + } + const std::uint64_t MemorySizeInBits = Description.Size.minimumBytes() * 8U; + if (MemorySizeInBits != ValueType->minimumBitWidth()) { + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + if (Generic == GenericOpcode::G_LOAD) { + auto Temporary = MRI.createVirtualRegister(NarrowType); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto Load = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!Load) + return std::unexpected(Load.error()); + if (auto Added = Load->addDef(*Temporary); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addUse(*Address); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addMemOperand(Memory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Load->build(); !Built) + return std::unexpected(Built.error()); + if (auto Extended = MIRBuilder.buildAnyExt(*Value, *Temporary); !Extended) + return std::unexpected(Extended.error()); + } else { + auto Temporary = MRI.createVirtualRegister(NarrowType); + if (!Temporary) + return std::unexpected(Temporary.error()); + if (auto Truncated = MIRBuilder.buildTrunc(*Temporary, *Value); !Truncated) + return std::unexpected(Truncated.error()); + auto Store = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!Store) + return std::unexpected(Store.error()); + if (auto Added = Store->addUse(*Temporary); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addUse(*Address); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addMemOperand(Memory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Store->build(); !Built) + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); + } + + return reduceLoadStoreWidth(Instruction, TypeIndex, NarrowType); +} + +Expected<void> LegalizerHelper::reduceLoadStoreWidth(MachineInstr &Instruction, + std::size_t TypeIndex, + LowLevelType NarrowType) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic != GenericOpcode::G_LOAD && Generic != GenericOpcode::G_STORE) { + return makeUnexpected(ErrorCode::InvalidArgument, + "memory width reduction received an unsupported opcode"); + } + if (TypeIndex != 0 || !NarrowType.isCategoryNeutralScalar() || + NarrowType.minimumBitWidth() == 0 || NarrowType.minimumBitWidth() % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction requires a byte-sized scalar target type"); + } + if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 || + Instruction.memOperands().front() == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory width reduction requires one address and one memory operand"); + } + + const bool IsLoad = Generic == GenericOpcode::G_LOAD; + const MachineOperand *ValueOperand = Instruction.operandAt(0); + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>(); + if (Value == nullptr || Address == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory width reduction requires register value and address operands"); + } + auto ValueType = MRI.getType(*Value); + if (!ValueType) + return std::unexpected(ValueType.error()); + if (!ValueType->isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction does not support pointer or vector values"); + } + + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + if (Memory.isAtomic()) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction does not support atomic operations"); + } + if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType || + Description.MemoryType->isVector()) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction requires a fixed scalar memory operand"); + } + const std::uint64_t ValueBits = ValueType->minimumBitWidth(); + if (ValueBits != Description.Size.minimumBytes() * 8U || + NarrowType.minimumBitWidth() >= ValueBits) { + return makeUnexpected( + ErrorCode::Unsupported, + "memory width reduction cannot split the requested value and memory widths"); + } + + const std::size_t PartCount = static_cast<std::size_t>(ValueBits / NarrowType.minimumBitWidth()); + const std::uint64_t LeftoverBits = ValueBits - PartCount * NarrowType.minimumBitWidth(); + LowLevelType ExpectedLeftoverType; + if (LeftoverBits != 0) { + auto Type = LowLevelType::makeCategoryNeutralScalar(LeftoverBits); + if (!Type) + return std::unexpected(Type.error()); + if (LeftoverBits % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction leaves a non-byte-sized scalar fragment"); + } + ExpectedLeftoverType = *Type; + } + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + std::vector<Register> Parts; + std::vector<Register> Leftovers; + LowLevelType LeftoverType; + if (IsLoad) { + Parts.reserve(PartCount); + for (std::size_t Index = 0; Index != PartCount; ++Index) { + auto Part = MRI.createVirtualRegister(NarrowType); + if (!Part) + return std::unexpected(Part.error()); + Parts.push_back(*Part); + } + LeftoverType = ExpectedLeftoverType; + if (ExpectedLeftoverType.isValid()) { + auto Leftover = MRI.createVirtualRegister(ExpectedLeftoverType); + if (!Leftover) + return std::unexpected(Leftover.error()); + Leftovers.push_back(*Leftover); + } + } else { + auto Extracted = extractParts(*Value, *ValueType, NarrowType, LeftoverType, Parts, Leftovers, + MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + if (!*Extracted || Parts.size() != PartCount || LeftoverType != ExpectedLeftoverType || + (ExpectedLeftoverType.isValid() && Leftovers.size() != 1U)) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction cannot split the store value consistently"); + } + } + + const auto SplitPieces = [&](LowLevelType PartType, const std::vector<Register> &Values, + std::uint64_t &Offset) -> Expected<void> { + const std::uint64_t PartBytes = PartType.minimumBitWidth() / 8U; + for (Register Part : Values) { + auto AddressPart = buildObjectPointerOffset(MIRBuilder, *Address, Offset); + if (!AddressPart) + return std::unexpected(AddressPart.error()); + auto MemoryPart = deriveMemoryOperand(Function, Memory, PartType, Offset); + if (!MemoryPart) + return std::unexpected(MemoryPart.error()); + + auto Access = + MIRBuilder.buildInstr(opcode(IsLoad ? GenericOpcode::G_LOAD : GenericOpcode::G_STORE)); + if (!Access) + return std::unexpected(Access.error()); + if (IsLoad) { + if (auto Added = Access->addDef(Part); !Added) + return std::unexpected(Added.error()); + } else if (auto Added = Access->addUse(Part); !Added) { + return std::unexpected(Added.error()); + } + if (auto Added = Access->addUse(*AddressPart); !Added) + return std::unexpected(Added.error()); + if (auto Added = Access->addMemOperand(**MemoryPart); !Added) + return std::unexpected(Added.error()); + if (auto Built = Access->build(); !Built) + return std::unexpected(Built.error()); + Offset += PartBytes; + } + return {}; + }; + + std::uint64_t Offset = 0; + if (auto Split = SplitPieces(NarrowType, Parts, Offset); !Split) + return std::unexpected(Split.error()); + if (LeftoverType.isValid()) { + if (auto Split = SplitPieces(LeftoverType, Leftovers, Offset); !Split) + return std::unexpected(Split.error()); + } + + if (IsLoad) { + if (auto Inserted = insertParts(*Value, *ValueType, NarrowType, Parts, LeftoverType, Leftovers); + !Inserted) { + return std::unexpected(Inserted.error()); + } + } + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::narrowScalar(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType) { + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + case GenericOpcode::G_IMPLICIT_DEF: { + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_IMPLICIT_DEF narrowing requires a register destination"); + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + + const std::uint64_t DestinationSize = DestinationType->minimumBitWidth(); + const std::uint64_t NarrowSize = NarrowType.minimumBitWidth(); + if (NarrowSize == 0 || NarrowSize >= DestinationSize) + return makeUnexpected(ErrorCode::VerificationError, + "G_IMPLICIT_DEF narrowing requires a smaller nonzero type"); + + if (DestinationSize % NarrowSize != 0) { + LowLevelType ImplicitType = NarrowType; + if (DestinationType->isVector()) { + auto VectorType = + DestinationType->isScalableVector() + ? LowLevelType::makeScalableVector(DestinationType->minimumElementCount(), + NarrowType) + : LowLevelType::makeFixedVector(DestinationType->minimumElementCount(), NarrowType); + if (!VectorType) + return std::unexpected(VectorType.error()); + ImplicitType = *VectorType; + } + auto Implicit = MIRBuilder.buildUndef(ImplicitType); + if (!Implicit) + return std::unexpected(Implicit.error()); + auto ImplicitReg = resultRegister(**Implicit); + if (!ImplicitReg) + return std::unexpected(ImplicitReg.error()); + if (auto Extended = MIRBuilder.buildAnyExt(*Destination, *ImplicitReg); !Extended) + return std::unexpected(Extended.error()); + } else { + const std::size_t PartCount = static_cast<std::size_t>(DestinationSize / NarrowSize); + std::vector<Register> Parts; + Parts.reserve(PartCount); + for (std::size_t Index = 0; Index != PartCount; ++Index) { + auto Part = MIRBuilder.buildUndef(NarrowType); + if (!Part) + return std::unexpected(Part.error()); + auto PartReg = resultRegister(**Part); + if (!PartReg) + return std::unexpected(PartReg.error()); + Parts.push_back(*PartReg); + } + if (auto Merged = MIRBuilder.buildMergeLikeInstr(*Destination, Parts); !Merged) + return std::unexpected(Merged.error()); + } + + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_CONSTANT: { + const Register *Destination = Instruction.operandAt(0)->getIf<Register>(); + const auto *Immediate = Instruction.operandAt(1)->getIf<IntegerImmediateOperand>(); + if (Destination == nullptr || Immediate == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_CONSTANT narrowing requires register and integer operands"); + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + + const std::uint64_t TotalSize = DestinationType->minimumBitWidth(); + const std::uint64_t NarrowSize = NarrowType.minimumBitWidth(); + if (NarrowSize == 0 || NarrowSize >= TotalSize || NarrowSize > APInt::MaxBitWidth || + TotalSize > APInt::MaxBitWidth) { + return makeUnexpected(ErrorCode::VerificationError, + "G_CONSTANT narrowing requires a smaller representable type"); + } + + const std::size_t PartCount = static_cast<std::size_t>(TotalSize / NarrowSize); + std::vector<Register> Parts; + Parts.reserve(PartCount); + for (std::size_t Index = 0; Index != PartCount; ++Index) { + auto Shifted = Immediate->Value.logicalShiftRight(Index * NarrowSize); + if (!Shifted) + return std::unexpected(Shifted.error()); + APInt PartValue = std::move(*Shifted); + if (PartValue.bitWidth() > NarrowSize) { + auto Truncated = PartValue.trunc(static_cast<std::uint32_t>(NarrowSize)); + if (!Truncated) + return std::unexpected(Truncated.error()); + PartValue = std::move(*Truncated); + } + auto Part = buildConstantRegister(MIRBuilder, NarrowType, std::move(PartValue)); + if (!Part) + return std::unexpected(Part.error()); + Parts.push_back(*Part); + } + + LowLevelType LeftoverType; + std::vector<Register> Leftovers; + const std::uint64_t LeftoverBits = TotalSize - PartCount * NarrowSize; + if (LeftoverBits != 0) { + auto Type = LowLevelType::makeCategoryNeutralScalar(LeftoverBits); + if (!Type) + return std::unexpected(Type.error()); + LeftoverType = *Type; + auto Shifted = Immediate->Value.logicalShiftRight(PartCount * NarrowSize); + if (!Shifted) + return std::unexpected(Shifted.error()); + APInt LeftoverValue = std::move(*Shifted); + if (LeftoverValue.bitWidth() > LeftoverBits) { + auto Truncated = LeftoverValue.trunc(static_cast<std::uint32_t>(LeftoverBits)); + if (!Truncated) + return std::unexpected(Truncated.error()); + LeftoverValue = std::move(*Truncated); + } + auto Leftover = buildConstantRegister(MIRBuilder, LeftoverType, std::move(LeftoverValue)); + if (!Leftover) + return std::unexpected(Leftover.error()); + Leftovers.push_back(*Leftover); + } + + if (auto Inserted = + insertParts(*Destination, *DestinationType, NarrowType, Parts, LeftoverType, Leftovers); + !Inserted) { + return std::unexpected(Inserted.error()); + } + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_ADD: + case GenericOpcode::G_SUB: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: + case GenericOpcode::G_UADDO: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBE: + return narrowScalarAddSub(Instruction, TypeIndex, NarrowType); + case GenericOpcode::G_MUL: + case GenericOpcode::G_UMULH: + return narrowScalarMul(Instruction, TypeIndex, NarrowType); + case GenericOpcode::G_LOAD: + case GenericOpcode::G_STORE: + return narrowScalarLoadStore(Instruction, TypeIndex, NarrowType); + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_ANYEXT: + return narrowScalarExt(Instruction, TypeIndex, NarrowType); + case GenericOpcode::G_INTTOPTR: + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::Unsupported, + "G_INTTOPTR destination narrowing is not defined"); + Observer.changingInstr(Instruction); + if (auto Narrowed = narrowScalarSrc(Instruction, NarrowType, 1); !Narrowed) + return std::unexpected(Narrowed.error()); + Observer.changedInstr(Instruction); + return {}; + case GenericOpcode::G_PTRTOINT: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::Unsupported, "G_PTRTOINT source narrowing is not defined"); + Observer.changingInstr(Instruction); + if (auto Narrowed = narrowScalarDst(Instruction, NarrowType, 0, GenericOpcode::G_ZEXT); + !Narrowed) { + return std::unexpected(Narrowed.error()); + } + Observer.changedInstr(Instruction); + return {}; + case GenericOpcode::G_PHI: { + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::Unsupported, "G_PHI narrowing requires type index zero"); + if (Instruction.operandCount() < 3 || Instruction.operandCount() % 2U == 0) + return makeUnexpected(ErrorCode::VerificationError, + "G_PHI narrowing requires incoming value/block pairs"); + MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_PHI narrowing requires a register destination"); + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + const std::uint64_t DestinationSize = DestinationType->minimumBitWidth(); + const std::uint64_t NarrowSize = NarrowType.minimumBitWidth(); + if (NarrowSize == 0 || NarrowSize >= DestinationSize || DestinationSize % NarrowSize != 0) + return makeUnexpected(ErrorCode::Unsupported, + "G_PHI narrowing requires an exact scalar split"); + + const std::size_t PartCount = static_cast<std::size_t>(DestinationSize / NarrowSize); + const std::size_t IncomingCount = (Instruction.operandCount() - 1U) / 2U; + std::vector<std::vector<Register>> IncomingParts(IncomingCount); + std::vector<MachineBasicBlock *> IncomingBlocks(IncomingCount); + Observer.changingInstr(Instruction); + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { + const MachineOperand *ValueOperand = Instruction.operandAt(Index); + const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const auto *Incoming = + BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>(); + if (Value == nullptr || Incoming == nullptr || Incoming->Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_PHI narrowing has an invalid incoming pair"); + if (auto Point = setInsertPointAtFirstTerminator(MIRBuilder, *Incoming->Block); !Point) + return std::unexpected(Point.error()); + const std::size_t IncomingIndex = Index / 2U; + IncomingBlocks[IncomingIndex] = Incoming->Block; + auto &Parts = IncomingParts[IncomingIndex]; + Parts.reserve(PartCount); + for (std::size_t PartIndex = 0; PartIndex != PartCount; ++PartIndex) { + auto Part = MRI.createVirtualRegister(NarrowType); + if (!Part) + return std::unexpected(Part.error()); + Parts.push_back(*Part); + } + if (auto Unmerged = MIRBuilder.buildUnmergeValues(Parts, *Value); !Unmerged) + return std::unexpected(Unmerged.error()); + } + + MachineBasicBlock &Block = *Instruction.parent(); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + std::vector<Register> Destinations; + Destinations.reserve(PartCount); + for (std::size_t PartIndex = 0; PartIndex != PartCount; ++PartIndex) { + auto DestinationPart = MRI.createVirtualRegister(NarrowType); + if (!DestinationPart) + return std::unexpected(DestinationPart.error()); + auto Phi = MIRBuilder.buildInstr(opcode(GenericOpcode::G_PHI)); + if (!Phi) + return std::unexpected(Phi.error()); + if (auto Added = Phi->addDef(*DestinationPart); !Added) + return std::unexpected(Added.error()); + for (std::size_t IncomingIndex = 0; IncomingIndex != IncomingCount; ++IncomingIndex) { + if (auto Added = Phi->addUse(IncomingParts[IncomingIndex][PartIndex]); !Added) + return std::unexpected(Added.error()); + if (auto Added = Phi->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added) + return std::unexpected(Added.error()); + } + if (auto Built = Phi->build(); !Built) + return std::unexpected(Built.error()); + Destinations.push_back(*DestinationPart); + } + + if (auto Point = setInsertPointAfterPHIs(MIRBuilder, Block); !Point) + return std::unexpected(Point.error()); + if (auto Merged = MIRBuilder.buildMergeLikeInstr(*Destination, Destinations); !Merged) + return std::unexpected(Merged.error()); + Observer.changedInstr(Instruction); + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_FCMP: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::Unsupported, + "floating comparison source narrowing is not defined"); + Observer.changingInstr(Instruction); + if (auto Narrowed = narrowScalarDst(Instruction, NarrowType, 0, GenericOpcode::G_ZEXT); + !Narrowed) { + return std::unexpected(Narrowed.error()); + } + Observer.changedInstr(Instruction); + return {}; + case GenericOpcode::G_SEXT_INREG: { + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::Unsupported, + "G_SEXT_INREG narrowing requires type index zero"); + + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *SourceOperand = Instruction.operandAt(1); + MachineOperand *WidthOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Destination == nullptr || Source == nullptr || WidthOperand == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_SEXT_INREG narrowing requires register and width operands"); + auto SizeInBits = signedImmediate(*WidthOperand); + if (!SizeInBits) + return std::unexpected(SizeInBits.error()); + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + + const std::uint64_t TotalSize = DestinationType->minimumBitWidth(); + const std::uint64_t NarrowSize = NarrowType.minimumBitWidth(); + if (*SizeInBits <= 0 || static_cast<std::uint64_t>(*SizeInBits) > TotalSize || + NarrowSize == 0 || NarrowSize >= TotalSize) { + return makeUnexpected(ErrorCode::VerificationError, + "G_SEXT_INREG narrowing has invalid scalar widths"); + } + + if (NarrowSize > static_cast<std::uint64_t>(*SizeInBits)) { + const Register OriginalDestination = *Destination; + Observer.changingInstr(Instruction); + auto NarrowedSource = narrowScalarSrc(Instruction, NarrowType, 1); + if (!NarrowedSource) + return std::unexpected(NarrowedSource.error()); + auto NarrowDestination = MRI.createVirtualRegister(NarrowType); + if (!NarrowDestination) + return std::unexpected(NarrowDestination.error()); + if (auto Changed = DestinationOperand->setRegister(*NarrowDestination); !Changed) + return std::unexpected(Changed.error()); + if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Extended = MIRBuilder.buildSExt(OriginalDestination, *NarrowDestination); !Extended) + return std::unexpected(Extended.error()); + Observer.changedInstr(Instruction); + return {}; + } + + if (TotalSize % NarrowSize != 0) + return makeUnexpected(ErrorCode::Unsupported, + "G_SEXT_INREG cannot be split evenly to the requested type"); + const std::size_t PartCount = static_cast<std::size_t>(TotalSize / NarrowSize); + std::vector<Register> SourceParts; + SourceParts.reserve(PartCount); + for (std::size_t Index = 0; Index != PartCount; ++Index) { + auto Part = MRI.createVirtualRegister(NarrowType); + if (!Part) + return std::unexpected(Part.error()); + SourceParts.push_back(*Part); + } + if (auto Unmerged = MIRBuilder.buildUnmergeValues(SourceParts, *Source); !Unmerged) + return std::unexpected(Unmerged.error()); + + auto ShiftAmount = + buildConstantRegister(MIRBuilder, NarrowType, static_cast<std::int64_t>(NarrowSize - 1U)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + + std::vector<Register> DestinationParts; + DestinationParts.reserve(PartCount); + Register FullExtension; + Register PartialExtension; + const std::uint64_t ExtensionBits = static_cast<std::uint64_t>(*SizeInBits); + for (std::size_t Index = 0; Index != PartCount; ++Index) { + if ((Index + 1U) * NarrowSize <= ExtensionBits) { + DestinationParts.push_back(SourceParts[Index]); + PartialExtension = SourceParts[Index]; + continue; + } + if (Index * NarrowSize >= ExtensionBits) { + if (PartialExtension.isNoRegister()) + return makeUnexpected(ErrorCode::InternalError, + "G_SEXT_INREG split has no sign-bearing part"); + if (FullExtension.isNoRegister()) { + auto Extended = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, NarrowType, + {PartialExtension, *ShiftAmount}); + if (!Extended) + return std::unexpected(Extended.error()); + FullExtension = *Extended; + } + DestinationParts.push_back(FullExtension); + continue; + } + + auto Extended = MIRBuilder.buildSExtInReg( + NarrowType, SourceParts[Index], static_cast<std::uint32_t>(ExtensionBits % NarrowSize)); + if (!Extended) + return std::unexpected(Extended.error()); + auto ExtendedRegister = resultRegister(**Extended); + if (!ExtendedRegister) + return std::unexpected(ExtendedRegister.error()); + DestinationParts.push_back(*ExtendedRegister); + PartialExtension = *ExtendedRegister; + } + + if (auto Merged = MIRBuilder.buildMergeLikeInstr(*Destination, DestinationParts); !Merged) + return std::unexpected(Merged.error()); + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_AND: + case GenericOpcode::G_OR: + case GenericOpcode::G_XOR: + return narrowScalarBasic(Instruction, TypeIndex, NarrowType); + case GenericOpcode::G_SELECT: + return narrowScalarSelect(Instruction, TypeIndex, NarrowType); + case GenericOpcode::G_SHL: + case GenericOpcode::G_LSHR: + case GenericOpcode::G_ASHR: + return narrowScalarShift(Instruction, TypeIndex, NarrowType); + case GenericOpcode::G_ICMP: { + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::Unsupported, + "G_ICMP narrowing requires the source type index"); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *PredicateMachineOperand = Instruction.operandAt(1); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const auto *PredicateValue = PredicateMachineOperand == nullptr + ? nullptr + : PredicateMachineOperand->getIf<PredicateOperand>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + const IntegerPredicate *Predicate = + PredicateValue == nullptr ? nullptr : PredicateValue->integerPredicate(); + if (Destination == nullptr || Left == nullptr || Right == nullptr || Predicate == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "G_ICMP narrowing requires an integer predicate and register operands"); + } + auto SourceType = MRI.getType(*Left); + auto ResultType = MRI.getType(*Destination); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (!ResultType) + return std::unexpected(ResultType.error()); + + LowLevelType LeftoverType; + LowLevelType UnusedType; + std::vector<Register> LeftParts; + std::vector<Register> LeftLeftovers; + std::vector<Register> RightParts; + std::vector<Register> RightLeftovers; + auto LeftExtracted = extractParts(*Left, *SourceType, NarrowType, LeftoverType, LeftParts, + LeftLeftovers, MIRBuilder, MRI); + if (!LeftExtracted) + return std::unexpected(LeftExtracted.error()); + auto RightExtracted = extractParts(*Right, *SourceType, NarrowType, UnusedType, RightParts, + RightLeftovers, MIRBuilder, MRI); + if (!RightExtracted) + return std::unexpected(RightExtracted.error()); + if (!*LeftExtracted || !*RightExtracted || LeftParts.size() != RightParts.size() || + LeftLeftovers.size() != RightLeftovers.size()) { + return makeUnexpected(ErrorCode::Unsupported, "G_ICMP operands cannot be split consistently"); + } + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (isEqualityPredicate(*Predicate)) { + auto Zero = buildConstantRegister(MIRBuilder, NarrowType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + + std::vector<Register> Xors; + Xors.reserve(LeftParts.size() + LeftLeftovers.size()); + for (std::size_t Index = 0; Index != LeftParts.size(); ++Index) { + auto Xor = buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, NarrowType, + {LeftParts[Index], RightParts[Index]}); + if (!Xor) + return std::unexpected(Xor.error()); + Xors.push_back(*Xor); + } + + std::vector<Register> WidenedXors; + for (std::size_t Index = 0; Index != LeftLeftovers.size(); ++Index) { + auto Xor = buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, LeftoverType, + {LeftLeftovers[Index], RightLeftovers[Index]}); + if (!Xor) + return std::unexpected(Xor.error()); + auto GCDType = extractGCDType(WidenedXors, NarrowType, LeftoverType, *Xor); + if (!GCDType) + return std::unexpected(GCDType.error()); + auto LCMType = buildLCMMergePieces(LeftoverType, NarrowType, *GCDType, WidenedXors, + GenericOpcode::G_ZEXT); + if (!LCMType) + return std::unexpected(LCMType.error()); + Xors.insert(Xors.end(), WidenedXors.begin(), WidenedXors.end()); + } + + if (Xors.size() < 2) { + return makeUnexpected(ErrorCode::InternalError, + "G_ICMP narrowing produced fewer than two comparison parts"); + } + auto Combined = + buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, NarrowType, {Xors[0], Xors[1]}); + if (!Combined) + return std::unexpected(Combined.error()); + for (std::size_t Index = 2; Index != Xors.size(); ++Index) { + auto Next = buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, NarrowType, + {*Combined, Xors[Index]}); + if (!Next) + return std::unexpected(Next.error()); + Combined = std::move(Next); + } + if (auto Built = MIRBuilder.buildICmp(*Predicate, *Destination, *Combined, *Zero); !Built) + return std::unexpected(Built.error()); + } else { + Register ComparisonInput; + const auto buildComparisonPart = [&](Register LeftPart, Register RightPart, + bool IsFinal) -> Expected<Register> { + IntegerPredicate PartPredicate = *Predicate; + if (!IsFinal) { + auto Unsigned = unsignedPredicate(*Predicate); + if (!Unsigned) + return std::unexpected(Unsigned.error()); + PartPredicate = *Unsigned; + } + Register ComparisonOutput = *Destination; + if (!IsFinal) { + auto Temporary = MRI.createVirtualRegister(*ResultType); + if (!Temporary) + return std::unexpected(Temporary.error()); + ComparisonOutput = *Temporary; + } + if (ComparisonInput.isNoRegister()) { + auto Built = MIRBuilder.buildICmp(PartPredicate, ComparisonOutput, LeftPart, RightPart); + if (!Built) + return std::unexpected(Built.error()); + } else { + auto Comparison = MRI.createVirtualRegister(*ResultType); + auto Equal = MRI.createVirtualRegister(*ResultType); + if (!Comparison) + return std::unexpected(Comparison.error()); + if (!Equal) + return std::unexpected(Equal.error()); + if (auto Built = MIRBuilder.buildICmp(PartPredicate, *Comparison, LeftPart, RightPart); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::Equal, *Equal, LeftPart, RightPart); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = + MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {ComparisonOutput}, + {*Equal, ComparisonInput, *Comparison}); + !Built) { + return std::unexpected(Built.error()); + } + } + return ComparisonOutput; + }; + + for (std::size_t Index = 0; Index != LeftParts.size(); ++Index) { + const bool IsFinal = Index + 1U == LeftParts.size() && LeftLeftovers.empty(); + auto Comparison = buildComparisonPart(LeftParts[Index], RightParts[Index], IsFinal); + if (!Comparison) + return std::unexpected(Comparison.error()); + ComparisonInput = *Comparison; + } + for (std::size_t Index = 0; Index != LeftLeftovers.size(); ++Index) { + const bool IsFinal = Index + 1U == LeftLeftovers.size(); + auto Comparison = buildComparisonPart(LeftLeftovers[Index], RightLeftovers[Index], IsFinal); + if (!Comparison) + return std::unexpected(Comparison.error()); + ComparisonInput = *Comparison; + } + } + + return Instruction.eraseFromParent(); + } + default: + return makeUnexpected(ErrorCode::Unsupported, + "scalar narrowing is not implemented for this opcode"); + } +} + +Expected<void> LegalizerHelper::fewerElements(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType NarrowType) { + auto Query = LegalizerInfo::buildQuery(Instruction, Function, MIRBuilder.target()); + if (!Query) + return std::unexpected(Query.error()); + if (TypeIndex >= Query->Types.size() || TypeIndex >= Query->TypeOperands.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "vector legalization has an invalid type index"); + } + + const LowLevelType VectorType = Query->Types[TypeIndex]; + const std::optional<LowLevelType> ElementType = VectorType.scalarElementType(); + if (!VectorType.isVector() || VectorType.isScalableVector() || !ElementType || + *ElementType != NarrowType || !NarrowType.isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "vector legalization requires fixed vectors split into scalar elements"); + } + + std::size_t ExplicitOperandCount = 0; + while (ExplicitOperandCount < Instruction.operandCount() && + !Instruction.operandAt(ExplicitOperandCount)->isImplicit()) { + ++ExplicitOperandCount; + } + const std::vector<std::size_t> &TypedOperands = Query->TypeOperands[TypeIndex]; + if (TypedOperands.size() != ExplicitOperandCount || TypedOperands.empty()) { + return makeUnexpected( + ErrorCode::Unsupported, + "vector legalization requires all explicit operands to share one vector type"); + } + + std::vector<std::size_t> DefinitionIndices; + std::vector<std::size_t> SourceIndices; + DefinitionIndices.reserve(TypedOperands.size()); + SourceIndices.reserve(TypedOperands.size()); + for (const std::size_t OperandIndex : TypedOperands) { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *RegisterOperand = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (RegisterOperand == nullptr || !RegisterOperand->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "vector legalization requires virtual-register operands"); + } + auto OperandType = MRI.getType(*RegisterOperand); + if (!OperandType) + return std::unexpected(OperandType.error()); + if (*OperandType != VectorType) { + return makeUnexpected( + ErrorCode::VerificationError, + "vector legalization type operands do not match the requested vector type"); + } + if (Operand->isDef() && !Operand->isUse()) { + DefinitionIndices.push_back(OperandIndex); + } else if (Operand->isUse() && !Operand->isDef()) { + SourceIndices.push_back(OperandIndex); + } else { + return makeUnexpected(ErrorCode::Unsupported, + "vector legalization does not support tied or non-register operands"); + } + } + if (DefinitionIndices.empty() || SourceIndices.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "vector legalization requires register definitions and sources"); + } + + const std::size_t ElementCount = VectorType.minimumElementCount(); + std::vector<std::vector<Register>> SourceParts; + SourceParts.reserve(SourceIndices.size()); + for (const std::size_t SourceIndex : SourceIndices) { + const Register Source = *Instruction.operandAt(SourceIndex)->getIf<Register>(); + std::vector<Register> Parts; + auto Extracted = + extractParts(Source, NarrowType, static_cast<int>(ElementCount), Parts, MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + SourceParts.push_back(std::move(Parts)); + } + + std::vector<std::vector<Register>> DestinationParts(DefinitionIndices.size()); + for (std::vector<Register> &Parts : DestinationParts) + Parts.reserve(ElementCount); + for (std::size_t ElementIndex = 0; ElementIndex < ElementCount; ++ElementIndex) { + auto ScalarInstruction = MIRBuilder.buildInstr(Instruction.opcode()); + if (!ScalarInstruction) + return std::unexpected(ScalarInstruction.error()); + if (auto SetFlags = ScalarInstruction->setFlags(Instruction.flags()); !SetFlags) + return std::unexpected(SetFlags.error()); + for (std::size_t DefinitionNumber = 0; DefinitionNumber < DefinitionIndices.size(); + ++DefinitionNumber) { + auto Destination = MRI.createVirtualRegister(NarrowType); + if (!Destination) + return std::unexpected(Destination.error()); + if (auto Added = ScalarInstruction->addDef(*Destination); !Added) + return std::unexpected(Added.error()); + DestinationParts[DefinitionNumber].push_back(*Destination); + } + for (const std::vector<Register> &Parts : SourceParts) { + if (auto Added = ScalarInstruction->addUse(Parts[ElementIndex]); !Added) + return std::unexpected(Added.error()); + } + if (auto Built = ScalarInstruction->build(); !Built) + return std::unexpected(Built.error()); + } + + for (std::size_t DefinitionNumber = 0; DefinitionNumber < DefinitionIndices.size(); + ++DefinitionNumber) { + const Register Destination = + *Instruction.operandAt(DefinitionIndices[DefinitionNumber])->getIf<Register>(); + if (auto Merged = + MIRBuilder.buildMergeLikeInstr(Destination, DestinationParts[DefinitionNumber]); + !Merged) { + return std::unexpected(Merged.error()); + } + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerVAArg(MachineInstr &Instruction) { + if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 || + !Instruction.memOperands().empty()) { + return makeUnexpected(ErrorCode::VerificationError, "G_VAARG has invalid operands"); + } + + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *ListOperand = Instruction.operandAt(1); + MachineOperand *AlignmentOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>(); + if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() || + !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() || + AlignmentOperand == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "G_VAARG requires virtual-register operands"); + } + + auto DestinationInfo = MRI.virtualRegisterInfo(*Destination); + auto PointerInfo = MRI.virtualRegisterInfo(*ListPointer); + auto RequestedAlignment = signedImmediate(*AlignmentOperand); + if (!DestinationInfo || !PointerInfo) + return makeUnexpected(ErrorCode::VerificationError, "G_VAARG has invalid register information"); + const std::optional<LowLevelType> &DestinationTypeInfo = DestinationInfo->get().Type; + const std::optional<LowLevelType> &PointerTypeInfo = PointerInfo->get().Type; + if (!DestinationTypeInfo.has_value() || !PointerTypeInfo.has_value() || !RequestedAlignment || + *RequestedAlignment <= 0 || + !std::has_single_bit(static_cast<std::uint64_t>(*RequestedAlignment))) { + return makeUnexpected(ErrorCode::VerificationError, + "G_VAARG has invalid type or alignment information"); + } + + const LowLevelType DestinationType = DestinationTypeInfo.value(); + const LowLevelType PointerType = PointerTypeInfo.value(); + const auto DestinationLayout = scalarLayout(DestinationType); + const auto PointerLayout = scalarLayout(PointerType); + if (!DestinationLayout || !PointerLayout || !PointerType.isPointer()) { + return makeUnexpected(ErrorCode::Unsupported, + "G_VAARG requires scalar result and pointer list types"); + } + + auto ListLoadMemory = + createMemoryAccess(Function, MemoryAccessFlag::Load, PointerType, *PointerLayout); + auto ListStoreMemory = + createMemoryAccess(Function, MemoryAccessFlag::Store, PointerType, *PointerLayout); + if (!ListLoadMemory) + return std::unexpected(ListLoadMemory.error()); + if (!ListStoreMemory) + return std::unexpected(ListStoreMemory.error()); + + auto CurrentList = MRI.createVirtualRegister(PointerType); + if (!CurrentList) + return std::unexpected(CurrentList.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto ListLoad = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!ListLoad) + return std::unexpected(ListLoad.error()); + if (auto Added = ListLoad->addDef(*CurrentList); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListLoad->addUse(*ListPointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ListLoad->build(); !Built) + return std::unexpected(Built.error()); + + Register ValueAddress = *CurrentList; + const std::uint64_t AlignmentValue = static_cast<std::uint64_t>(*RequestedAlignment); + auto PointerScalarType = LowLevelType::makeCategoryNeutralScalar(PointerType.scalarBitWidth()); + if (!PointerScalarType) + return std::unexpected(PointerScalarType.error()); + if (AlignmentValue > 1) { + auto AlignmentOffset = + buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, AlignmentValue - 1U); + if (!AlignmentOffset) + return std::unexpected(AlignmentOffset.error()); + auto BiasedAddress = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTR_ADD, + PointerType, ValueAddress, *AlignmentOffset); + if (!BiasedAddress) + return std::unexpected(BiasedAddress.error()); + auto AlignmentMask = + buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, ~(AlignmentValue - 1U)); + if (!AlignmentMask) + return std::unexpected(AlignmentMask.error()); + auto AlignedAddress = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTRMASK, + PointerType, *BiasedAddress, *AlignmentMask); + if (!AlignedAddress) + return std::unexpected(AlignedAddress.error()); + ValueAddress = *AlignedAddress; + } + + auto Increment = buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, + DestinationLayout->AllocationSize); + if (!Increment) + return std::unexpected(Increment.error()); + auto NextList = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTR_ADD, PointerType, + ValueAddress, *Increment); + if (!NextList) + return std::unexpected(NextList.error()); + + auto ListStore = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!ListStore) + return std::unexpected(ListStore.error()); + if (auto Added = ListStore->addUse(*NextList); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListStore->addUse(*ListPointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ListStore->build(); !Built) + return std::unexpected(Built.error()); + + if (!MRI.useEmpty(*Destination)) { + auto ValueLoadMemory = + createMemoryAccess(Function, MemoryAccessFlag::Load, DestinationType, *DestinationLayout); + if (!ValueLoadMemory) + return std::unexpected(ValueLoadMemory.error()); + auto ValueLoad = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!ValueLoad) + return std::unexpected(ValueLoad.error()); + if (auto Added = ValueLoad->addDef(*Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = ValueLoad->addUse(ValueAddress); !Added) + return std::unexpected(Added.error()); + if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ValueLoad->build(); !Built) + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerLoad(MachineInstr &Instruction) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic != GenericOpcode::G_LOAD && Generic != GenericOpcode::G_SEXTLOAD && + Generic != GenericOpcode::G_ZEXTLOAD) { + return makeUnexpected(ErrorCode::InvalidArgument, + "load lowering received an unsupported opcode"); + } + if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 || + Instruction.memOperands().front() == nullptr) { + return makeUnexpected( + ErrorCode::VerificationError, + "load lowering requires one destination, one address, and one memory operand"); + } + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>(); + if (Destination == nullptr || Address == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "load lowering requires register destination and address operands"); + } + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!DestinationType->isScalarValue()) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering does not support vector result types"); + } + + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + if (Memory.isAtomic()) + return makeUnexpected(ErrorCode::Unsupported, + "load lowering does not support atomic operations"); + if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType || + Description.MemoryType->isVector()) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering requires a fixed scalar memory operand"); + } + const LowLevelType MemoryType = *Description.MemoryType; + const std::uint64_t MemoryBits = MemoryType.minimumBitWidth(); + const std::uint64_t MemoryStoreBits = Description.Size.minimumBytes() * 8U; + if (MemoryBits == 0 || MemoryBits > MemoryStoreBits) { + return makeUnexpected(ErrorCode::VerificationError, + "load lowering has an inconsistent memory type and size"); + } + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + const auto BuildLoad = [&](GenericOpcode LoadOpcode, Register Result, Register Pointer, + const MachineMemOperand &LoadMemory) -> Expected<void> { + auto Load = MIRBuilder.buildInstr(opcode(LoadOpcode)); + if (!Load) + return std::unexpected(Load.error()); + if (auto Added = Load->addDef(Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addUse(Pointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addMemOperand(LoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Load->build(); !Built) + return std::unexpected(Built.error()); + return {}; + }; + + if (MemoryBits != MemoryStoreBits) { + if (!MemoryType.isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "non-byte-sized pointer loads are not supported"); + } + auto WideMemoryType = LowLevelType::makeCategoryNeutralScalar(MemoryStoreBits); + if (!WideMemoryType) + return std::unexpected(WideMemoryType.error()); + auto WideMemory = deriveMemoryOperand(Function, Memory, *WideMemoryType, 0); + if (!WideMemory) + return std::unexpected(WideMemory.error()); + + LowLevelType LoadType = *DestinationType; + Register LoadDestination = *Destination; + if (MemoryStoreBits > DestinationType->minimumBitWidth()) { + LoadType = *WideMemoryType; + auto Temporary = MRI.createVirtualRegister(LoadType); + if (!Temporary) + return std::unexpected(Temporary.error()); + LoadDestination = *Temporary; + } + + if (Generic == GenericOpcode::G_SEXTLOAD) { + auto Loaded = MRI.createVirtualRegister(LoadType); + if (!Loaded) + return std::unexpected(Loaded.error()); + if (auto Built = BuildLoad(GenericOpcode::G_LOAD, *Loaded, *Address, **WideMemory); !Built) + return std::unexpected(Built.error()); + if (auto Extended = MIRBuilder.buildSExtInReg(LoadDestination, *Loaded, + static_cast<std::uint32_t>(MemoryBits)); + !Extended) { + return std::unexpected(Extended.error()); + } + } else if (Generic == GenericOpcode::G_ZEXTLOAD || *WideMemoryType == LoadType) { + auto Loaded = MRI.createVirtualRegister(LoadType); + if (!Loaded) + return std::unexpected(Loaded.error()); + if (auto Built = BuildLoad(GenericOpcode::G_LOAD, *Loaded, *Address, **WideMemory); !Built) + return std::unexpected(Built.error()); + auto Assert = MIRBuilder.buildInstr(opcode(GenericOpcode::G_ASSERT_ZEXT)); + if (!Assert) + return std::unexpected(Assert.error()); + if (auto Added = Assert->addDef(LoadDestination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Assert->addUse(*Loaded); !Added) + return std::unexpected(Added.error()); + if (auto Added = Assert->addImmediate(static_cast<std::int64_t>(MemoryBits)); !Added) + return std::unexpected(Added.error()); + if (auto Built = Assert->build(); !Built) + return std::unexpected(Built.error()); + } else if (auto Built = + BuildLoad(GenericOpcode::G_LOAD, LoadDestination, *Address, **WideMemory); + !Built) { + return std::unexpected(Built.error()); + } + + if (*DestinationType != LoadType) { + if (auto Truncated = MIRBuilder.buildTrunc(*Destination, LoadDestination); !Truncated) + return std::unexpected(Truncated.error()); + } + return Instruction.eraseFromParent(); + } + + std::uint64_t LargeSplitBits = 0; + std::uint64_t SmallSplitBits = 0; + if (!std::has_single_bit(MemoryBits)) { + LargeSplitBits = std::bit_floor(MemoryBits); + SmallSplitBits = MemoryBits - LargeSplitBits; + } else { + if (MemoryBits < 16U) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering cannot split a byte-sized memory operation"); + } + LargeSplitBits = MemoryBits / 2U; + SmallSplitBits = LargeSplitBits; + } + if (LargeSplitBits == 0 || SmallSplitBits == 0 || LargeSplitBits % 8U != 0 || + SmallSplitBits % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering cannot split the requested memory width into bytes"); + } + + const std::uint64_t AnyExtBits = std::bit_ceil(DestinationType->minimumBitWidth()); + if (AnyExtBits == 0 || AnyExtBits > LowLevelType::MaxBitWidth) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering result width cannot be represented by the machine IR"); + } + auto AnyExtType = LowLevelType::makeCategoryNeutralScalar(AnyExtBits); + auto LargeMemoryType = LowLevelType::makeCategoryNeutralScalar(LargeSplitBits); + auto SmallMemoryType = LowLevelType::makeCategoryNeutralScalar(SmallSplitBits); + if (!AnyExtType) + return std::unexpected(AnyExtType.error()); + if (!LargeMemoryType) + return std::unexpected(LargeMemoryType.error()); + if (!SmallMemoryType) + return std::unexpected(SmallMemoryType.error()); + auto LargeMemory = deriveMemoryOperand(Function, Memory, *LargeMemoryType, 0); + auto SmallMemory = deriveMemoryOperand(Function, Memory, *SmallMemoryType, LargeSplitBits / 8U); + if (!LargeMemory) + return std::unexpected(LargeMemory.error()); + if (!SmallMemory) + return std::unexpected(SmallMemory.error()); + + auto LargeLoad = MRI.createVirtualRegister(*AnyExtType); + if (!LargeLoad) + return std::unexpected(LargeLoad.error()); + if (auto Built = BuildLoad(GenericOpcode::G_ZEXTLOAD, *LargeLoad, *Address, **LargeMemory); + !Built) + return std::unexpected(Built.error()); + auto SmallAddress = buildObjectPointerOffset(MIRBuilder, *Address, LargeSplitBits / 8U); + if (!SmallAddress) + return std::unexpected(SmallAddress.error()); + auto SmallLoad = MRI.createVirtualRegister(*AnyExtType); + if (!SmallLoad) + return std::unexpected(SmallLoad.error()); + if (auto Built = BuildLoad(Generic, *SmallLoad, *SmallAddress, **SmallMemory); !Built) + return std::unexpected(Built.error()); + auto ShiftAmount = + buildConstantRegister(MIRBuilder, *AnyExtType, static_cast<std::int64_t>(LargeSplitBits)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *AnyExtType, + {*SmallLoad, *ShiftAmount}); + if (!Shifted) + return std::unexpected(Shifted.error()); + + if (*AnyExtType == *DestinationType) { + if (auto Combined = MIRBuilder.buildOr(*Destination, *Shifted, *LargeLoad); !Combined) + return std::unexpected(Combined.error()); + } else { + auto Combined = buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *AnyExtType, + {*Shifted, *LargeLoad}); + if (!Combined) + return std::unexpected(Combined.error()); + if (AnyExtBits != DestinationType->minimumBitWidth()) { + if (auto Truncated = MIRBuilder.buildTrunc(*Destination, *Combined); !Truncated) + return std::unexpected(Truncated.error()); + } else if (DestinationType->isPointer()) { + if (auto Converted = MIRBuilder.buildIntToPtr(*Destination, *Combined); !Converted) + return std::unexpected(Converted.error()); + } else { + return makeUnexpected(ErrorCode::InternalError, + "load lowering produced an incompatible scalar result type"); + } + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerStore(MachineInstr &Instruction) { + if (Instruction.opcode() != opcode(GenericOpcode::G_STORE)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "store lowering received an unsupported opcode"); + } + if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 || + Instruction.memOperands().front() == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "store lowering requires one value, one address, and one memory operand"); + } + + const MachineOperand *ValueOperand = Instruction.operandAt(0); + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>(); + if (Value == nullptr || Address == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "store lowering requires register value and address operands"); + } + auto ValueType = MRI.getType(*Value); + if (!ValueType) + return std::unexpected(ValueType.error()); + if (!ValueType->isScalarValue()) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering does not support vector value types"); + } + + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + if (Memory.isAtomic()) + return makeUnexpected(ErrorCode::Unsupported, + "store lowering does not support atomic operations"); + if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType || + Description.MemoryType->isVector()) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering requires a fixed scalar memory operand"); + } + const LowLevelType MemoryType = *Description.MemoryType; + const std::uint64_t StoreBits = MemoryType.minimumBitWidth(); + const std::uint64_t StoreSizeBits = Description.Size.minimumBytes() * 8U; + if (StoreBits == 0 || StoreBits > StoreSizeBits) { + return makeUnexpected(ErrorCode::VerificationError, + "store lowering has an inconsistent memory type and size"); + } + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + const auto BuildStore = [&](Register StoredValue, Register Pointer, + const MachineMemOperand &StoreMemory) -> Expected<void> { + auto Store = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!Store) + return std::unexpected(Store.error()); + if (auto Added = Store->addUse(StoredValue); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addUse(Pointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addMemOperand(StoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Store->build(); !Built) + return std::unexpected(Built.error()); + return {}; + }; + + Register Source = *Value; + LowLevelType SourceType = *ValueType; + if (StoreBits != StoreSizeBits) { + if (!MemoryType.isCategoryNeutralScalar() || !SourceType.isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "non-byte-sized pointer stores are not supported"); + } + auto WideMemoryType = LowLevelType::makeCategoryNeutralScalar(StoreSizeBits); + if (!WideMemoryType) + return std::unexpected(WideMemoryType.error()); + if (StoreSizeBits > SourceType.minimumBitWidth()) { + auto Extended = + buildResultInstruction(MIRBuilder, GenericOpcode::G_ANYEXT, *WideMemoryType, {Source}); + if (!Extended) + return std::unexpected(Extended.error()); + Source = *Extended; + SourceType = *WideMemoryType; + } + auto MaskValue = + APInt::getLowBitsSet(SourceType.scalarBitWidth(), static_cast<std::uint32_t>(StoreBits)); + if (!MaskValue) + return std::unexpected(MaskValue.error()); + auto Mask = buildConstantRegister(MIRBuilder, SourceType, std::move(*MaskValue)); + if (!Mask) + return std::unexpected(Mask.error()); + auto ZeroExtended = + buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, SourceType, {Source, *Mask}); + if (!ZeroExtended) + return std::unexpected(ZeroExtended.error()); + auto WideMemory = deriveMemoryOperand(Function, Memory, *WideMemoryType, 0); + if (!WideMemory) + return std::unexpected(WideMemory.error()); + if (auto Built = BuildStore(*ZeroExtended, *Address, **WideMemory); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); + } + + std::uint64_t LargeSplitBits = 0; + std::uint64_t SmallSplitBits = 0; + if (!std::has_single_bit(StoreBits)) { + LargeSplitBits = std::bit_floor(StoreBits); + SmallSplitBits = StoreBits - LargeSplitBits; + } else { + if (StoreBits < 16U) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering cannot split a byte-sized memory operation"); + } + LargeSplitBits = StoreBits / 2U; + SmallSplitBits = LargeSplitBits; + } + if (LargeSplitBits == 0 || SmallSplitBits == 0 || LargeSplitBits % 8U != 0 || + SmallSplitBits % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering cannot split the requested memory width into bytes"); + } + + const std::uint64_t AnyExtBits = std::bit_ceil(StoreBits); + if (AnyExtBits == 0 || AnyExtBits > LowLevelType::MaxBitWidth) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering value width cannot be represented by the machine IR"); + } + auto NewSourceType = LowLevelType::makeCategoryNeutralScalar(AnyExtBits); + auto LargeMemoryType = LowLevelType::makeCategoryNeutralScalar(LargeSplitBits); + auto SmallMemoryType = LowLevelType::makeCategoryNeutralScalar(SmallSplitBits); + if (!NewSourceType) + return std::unexpected(NewSourceType.error()); + if (!LargeMemoryType) + return std::unexpected(LargeMemoryType.error()); + if (!SmallMemoryType) + return std::unexpected(SmallMemoryType.error()); + + if (SourceType.isPointer()) { + auto IntegerPointerType = LowLevelType::makeCategoryNeutralScalar(SourceType.scalarBitWidth()); + if (!IntegerPointerType) + return std::unexpected(IntegerPointerType.error()); + auto Converted = buildResultInstruction(MIRBuilder, GenericOpcode::G_PTRTOINT, + *IntegerPointerType, {Source}); + if (!Converted) + return std::unexpected(Converted.error()); + Source = *Converted; + SourceType = *IntegerPointerType; + } + auto Extended = MIRBuilder.buildAnyExtOrTrunc(*NewSourceType, Source); + if (!Extended) + return std::unexpected(Extended.error()); + auto ExtendedValue = resultRegister(**Extended); + if (!ExtendedValue) + return std::unexpected(ExtendedValue.error()); + auto ShiftAmount = + buildConstantRegister(MIRBuilder, *NewSourceType, static_cast<std::int64_t>(LargeSplitBits)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto SmallValue = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *NewSourceType, + {*ExtendedValue, *ShiftAmount}); + if (!SmallValue) + return std::unexpected(SmallValue.error()); + auto SmallAddress = buildObjectPointerOffset(MIRBuilder, *Address, LargeSplitBits / 8U); + if (!SmallAddress) + return std::unexpected(SmallAddress.error()); + auto LargeMemory = deriveMemoryOperand(Function, Memory, *LargeMemoryType, 0); + auto SmallMemory = deriveMemoryOperand(Function, Memory, *SmallMemoryType, LargeSplitBits / 8U); + if (!LargeMemory) + return std::unexpected(LargeMemory.error()); + if (!SmallMemory) + return std::unexpected(SmallMemory.error()); + if (auto Built = BuildStore(*ExtendedValue, *Address, **LargeMemory); !Built) + return std::unexpected(Built.error()); + if (auto Built = BuildStore(*SmallValue, *SmallAddress, **SmallMemory); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lower(MachineInstr &Instruction, + const LegalizationDecision &Decision) { + if (Decision.Action != LegalizationAction::Lower) + return makeUnexpected(ErrorCode::InvalidArgument, + "lower received a non-lower legalization action"); + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + case GenericOpcode::G_ABS: + return lowerAbsToAddXor(Instruction); + case GenericOpcode::G_LOAD: + case GenericOpcode::G_SEXTLOAD: + case GenericOpcode::G_ZEXTLOAD: + return lowerLoad(Instruction); + case GenericOpcode::G_STORE: + return lowerStore(Instruction); + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: + return lowerMinMax(Instruction); + case GenericOpcode::G_SCMP: + case GenericOpcode::G_UCMP: + return lowerThreewayCompare(Instruction); + case GenericOpcode::G_SDIVREM: + case GenericOpcode::G_UDIVREM: + return lowerDivRem(Instruction); + case GenericOpcode::G_SMULH: + case GenericOpcode::G_UMULH: + return lowerHighMultiply(Instruction); + case GenericOpcode::G_SMULO: + case GenericOpcode::G_UMULO: + return lowerMulo(Instruction); + case GenericOpcode::G_CTLZ_ZERO_POISON: + case GenericOpcode::G_CTTZ_ZERO_POISON: + case GenericOpcode::G_CTLZ: + case GenericOpcode::G_CTTZ: + case GenericOpcode::G_CTPOP: + case GenericOpcode::G_CTLS: + return lowerBitCount(Instruction); + case GenericOpcode::G_BSWAP: + return lowerBswap(Instruction); + case GenericOpcode::G_FSHL: + case GenericOpcode::G_FSHR: + return lowerFunnelShift(Instruction); + case GenericOpcode::G_ROTL: + case GenericOpcode::G_ROTR: + return lowerRotate(Instruction); + case GenericOpcode::G_DYN_STACKALLOC: + return lowerDynStackAlloc(Instruction); + case GenericOpcode::G_STACKSAVE: + return lowerStackSave(Instruction); + case GenericOpcode::G_STACKRESTORE: + return lowerStackRestore(Instruction); + case GenericOpcode::G_MEMCPY_INLINE: + return lowerMemcpyInline(Instruction); + case GenericOpcode::G_ABDS: + case GenericOpcode::G_ABDU: { + const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_ABDS); + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + auto Type = MRI.getType(Destination); + if (!Type) + return std::unexpected(Type.error()); + + const GenericOpcode MinOpcode = IsSigned ? GenericOpcode::G_SMIN : GenericOpcode::G_UMIN; + const GenericOpcode MaxOpcode = IsSigned ? GenericOpcode::G_SMAX : GenericOpcode::G_UMAX; + auto MinLegal = Info.isLegal(LegalizationQuery(opcode(MinOpcode), {*Type})); + if (!MinLegal) + return std::unexpected(MinLegal.error()); + auto MaxLegal = Info.isLegal(LegalizationQuery(opcode(MaxOpcode), {*Type})); + if (!MaxLegal) + return std::unexpected(MaxLegal.error()); + if (*MinLegal && *MaxLegal) + return lowerAbsDiffToMinMax(Instruction); + return lowerAbsDiffToSelect(Instruction); + } + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: + return lowerSignedAddSubOverflow(Instruction); + case GenericOpcode::G_UADDSAT: + case GenericOpcode::G_SADDSAT: + case GenericOpcode::G_USUBSAT: + case GenericOpcode::G_SSUBSAT: { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + const bool IsSigned = + Generic == GenericOpcode::G_SADDSAT || Generic == GenericOpcode::G_SSUBSAT; + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "saturating add/sub lowering requires a register destination"); + auto Type = MRI.getType(*Destination); + if (!Type) + return std::unexpected(Type.error()); + + const GenericOpcode MinOpcode = IsSigned ? GenericOpcode::G_SMIN : GenericOpcode::G_UMIN; + auto MinLegal = Info.isLegal(LegalizationQuery(opcode(MinOpcode), {*Type})); + if (!MinLegal) + return std::unexpected(MinLegal.error()); + if (!IsSigned && *MinLegal) + return lowerSaturatingAddSubToMinMax(Instruction); + if (IsSigned) { + auto MaxLegal = Info.isLegal(LegalizationQuery(opcode(GenericOpcode::G_SMAX), {*Type})); + if (!MaxLegal) + return std::unexpected(MaxLegal.error()); + if (*MinLegal && *MaxLegal) + return lowerSaturatingAddSubToMinMax(Instruction); + } + return lowerSaturatingAddSubToOverflow(Instruction); + } + case GenericOpcode::G_SSHLSAT: + case GenericOpcode::G_USHLSAT: + return lowerSaturatingShiftLeft(Instruction); + case GenericOpcode::G_SEXT_INREG: { + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Source = *Instruction.operandAt(1)->getIf<Register>(); + auto SizeInBits = signedImmediate(*Instruction.operandAt(2)); + if (!SizeInBits) + return std::unexpected(SizeInBits.error()); + auto Type = MRI.getType(Destination); + if (!Type) + return std::unexpected(Type.error()); + + auto Temporary = MRI.createVirtualRegister(*Type); + if (!Temporary) + return std::unexpected(Temporary.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + const std::int64_t Shift = static_cast<std::int64_t>(Type->scalarBitWidth()) - *SizeInBits; + auto ShiftAmount = buildConstantRegister(MIRBuilder, *Type, Shift); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + if (auto Built = MIRBuilder.buildShl(*Temporary, Source, *ShiftAmount); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildAShr(Destination, *Temporary, *ShiftAmount); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_VAARG: + return lowerVAArg(Instruction); + case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { + const Register OldValue = *Instruction.operandAt(0)->getIf<Register>(); + const Register Success = *Instruction.operandAt(1)->getIf<Register>(); + const Register Address = *Instruction.operandAt(2)->getIf<Register>(); + const Register Compare = *Instruction.operandAt(3)->getIf<Register>(); + const Register NewValue = *Instruction.operandAt(4)->getIf<Register>(); + auto NewOldValue = MRI.cloneVirtualRegister(OldValue); + if (!NewOldValue) + return std::unexpected(NewOldValue.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto Atomic = MIRBuilder.buildInstr(opcode(GenericOpcode::G_ATOMIC_CMPXCHG)); + if (!Atomic) + return std::unexpected(Atomic.error()); + if (auto Added = Atomic->addDef(*NewOldValue); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addUse(Address); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addUse(Compare); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addUse(NewValue); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addMemOperand(*Instruction.memOperands().front()); !Added) + return std::unexpected(Added.error()); + if (auto Built = Atomic->build(); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::Equal, Success, *NewOldValue, Compare); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildCopy(OldValue, *NewOldValue); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_ATOMICRMW_SUB: { + const Register Result = *Instruction.operandAt(0)->getIf<Register>(); + const Register Address = *Instruction.operandAt(1)->getIf<Register>(); + const Register Value = *Instruction.operandAt(2)->getIf<Register>(); + auto ValueType = MRI.getType(Value); + if (!ValueType) + return std::unexpected(ValueType.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto Zero = buildConstantRegister(MIRBuilder, *ValueType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Negated = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *ValueType, {*Zero, Value}); + if (!Negated) + return std::unexpected(Negated.error()); + + auto AtomicAdd = MIRBuilder.buildInstr(opcode(GenericOpcode::G_ATOMICRMW_ADD)); + if (!AtomicAdd) + return std::unexpected(AtomicAdd.error()); + if (auto Added = AtomicAdd->addDef(Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = AtomicAdd->addUse(Address); !Added) + return std::unexpected(Added.error()); + if (auto Added = AtomicAdd->addUse(*Negated); !Added) + return std::unexpected(Added.error()); + if (auto Added = AtomicAdd->addMemOperand(*Instruction.memOperands().front()); !Added) + return std::unexpected(Added.error()); + if (auto Built = AtomicAdd->build(); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_UADDO: { + const Register Result = *Instruction.operandAt(0)->getIf<Register>(); + const Register CarryOut = *Instruction.operandAt(1)->getIf<Register>(); + const Register Left = *Instruction.operandAt(2)->getIf<Register>(); + const Register Right = *Instruction.operandAt(3)->getIf<Register>(); + auto NewResult = MRI.cloneVirtualRegister(Result); + if (!NewResult) + return std::unexpected(NewResult.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildAdd(*NewResult, Left, Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::UnsignedLessThan, CarryOut, *NewResult, Right); + !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildCopy(Result, *NewResult); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_UADDE: { + const Register Result = *Instruction.operandAt(0)->getIf<Register>(); + const Register CarryOut = *Instruction.operandAt(1)->getIf<Register>(); + const Register Left = *Instruction.operandAt(2)->getIf<Register>(); + const Register Right = *Instruction.operandAt(3)->getIf<Register>(); + const Register CarryIn = *Instruction.operandAt(4)->getIf<Register>(); + auto ResultType = MRI.getType(Result); + auto ConditionType = MRI.getType(CarryOut); + if (!ResultType) + return std::unexpected(ResultType.error()); + if (!ConditionType) + return std::unexpected(ConditionType.error()); + auto NewResult = MRI.cloneVirtualRegister(Result); + auto TemporaryResult = MRI.createVirtualRegister(*ResultType); + auto Carry = MRI.createVirtualRegister(*ConditionType); + auto ExtendedCarryIn = MRI.createVirtualRegister(*ResultType); + auto ResultIsZero = MRI.createVirtualRegister(*ConditionType); + auto SecondCarry = MRI.createVirtualRegister(*ConditionType); + if (!NewResult) + return std::unexpected(NewResult.error()); + if (!TemporaryResult) + return std::unexpected(TemporaryResult.error()); + if (!Carry) + return std::unexpected(Carry.error()); + if (!ExtendedCarryIn) + return std::unexpected(ExtendedCarryIn.error()); + if (!ResultIsZero) + return std::unexpected(ResultIsZero.error()); + if (!SecondCarry) + return std::unexpected(SecondCarry.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildAdd(*TemporaryResult, Left, Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::UnsignedLessThan, *Carry, + *TemporaryResult, Left); + !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildZExt(*ExtendedCarryIn, CarryIn); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildAdd(*NewResult, *TemporaryResult, *ExtendedCarryIn); !Built) + return std::unexpected(Built.error()); + auto Zero = buildConstantRegister(MIRBuilder, *ResultType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::Equal, *ResultIsZero, *NewResult, *Zero); + !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildAnd(*SecondCarry, *ResultIsZero, CarryIn); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildOr(CarryOut, *Carry, *SecondCarry); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildCopy(Result, *NewResult); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_USUBO: { + const Register Result = *Instruction.operandAt(0)->getIf<Register>(); + const Register BorrowOut = *Instruction.operandAt(1)->getIf<Register>(); + const Register Left = *Instruction.operandAt(2)->getIf<Register>(); + const Register Right = *Instruction.operandAt(3)->getIf<Register>(); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildSub(Result, Left, Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::UnsignedLessThan, BorrowOut, Left, Right); + !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); + } + case GenericOpcode::G_USUBE: { + const Register Result = *Instruction.operandAt(0)->getIf<Register>(); + const Register BorrowOut = *Instruction.operandAt(1)->getIf<Register>(); + const Register Left = *Instruction.operandAt(2)->getIf<Register>(); + const Register Right = *Instruction.operandAt(3)->getIf<Register>(); + const Register BorrowIn = *Instruction.operandAt(4)->getIf<Register>(); + auto ResultType = MRI.getType(Result); + auto ConditionType = MRI.getType(BorrowOut); + if (!ResultType) + return std::unexpected(ResultType.error()); + if (!ConditionType) + return std::unexpected(ConditionType.error()); + auto TemporaryResult = MRI.createVirtualRegister(*ResultType); + auto Borrow = MRI.createVirtualRegister(*ConditionType); + auto ExtendedBorrowIn = MRI.createVirtualRegister(*ResultType); + auto TemporaryResultIsZero = MRI.createVirtualRegister(*ConditionType); + auto SecondBorrow = MRI.createVirtualRegister(*ConditionType); + if (!TemporaryResult) + return std::unexpected(TemporaryResult.error()); + if (!Borrow) + return std::unexpected(Borrow.error()); + if (!ExtendedBorrowIn) + return std::unexpected(ExtendedBorrowIn.error()); + if (!TemporaryResultIsZero) + return std::unexpected(TemporaryResultIsZero.error()); + if (!SecondBorrow) + return std::unexpected(SecondBorrow.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildSub(*TemporaryResult, Left, Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::UnsignedGreaterThan, *Borrow, + *TemporaryResult, Left); + !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildZExt(*ExtendedBorrowIn, BorrowIn); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildSub(Result, *TemporaryResult, *ExtendedBorrowIn); !Built) + return std::unexpected(Built.error()); + auto Zero = buildConstantRegister(MIRBuilder, *ResultType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::Equal, *TemporaryResultIsZero, + *TemporaryResult, *Zero); + !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildAnd(*SecondBorrow, *TemporaryResultIsZero, BorrowIn); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildOr(BorrowOut, *Borrow, *SecondBorrow); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); + } + default: { + const InstructionDescriptor *Descriptor = + MIRBuilder.target().instructionDescriptor(Instruction.opcode()); + const std::string_view Name = + Descriptor == nullptr ? std::string_view{"<unknown>"} : Descriptor->Name; + return makeUnexpected(ErrorCode::Unsupported, + "generic lowering is not implemented for opcode '" + std::string(Name) + + "'"); + } + } +} + +Expected<void> LegalizerHelper::lowerDynStackAlloc(MachineInstr &Instruction) { + if (Instruction.operandCount() != 3) + return makeUnexpected(ErrorCode::VerificationError, + "G_DYN_STACKALLOC requires destination, size, and alignment operands"); + if (MIRBuilder.target().stackGrowthDirection() != StackGrowthDirection::Down) + return makeUnexpected(ErrorCode::Unsupported, + "dynamic stack allocation requires a downward-growing stack"); + + const PhysicalRegister StackPointer = MIRBuilder.target().stackPointerRegisterToSaveRestore(); + if (!StackPointer.isValid()) + return makeUnexpected(ErrorCode::Unsupported, + "target does not define a stack pointer for dynamic allocation"); + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SizeOperand = Instruction.operandAt(1); + const MachineOperand *AlignmentOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *AllocationSize = + SizeOperand == nullptr ? nullptr : SizeOperand->getIf<Register>(); + if (Destination == nullptr || AllocationSize == nullptr || AlignmentOperand == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "G_DYN_STACKALLOC has invalid operands"); + + auto PointerType = MRI.getType(*Destination); + auto AllocationSizeType = MRI.getType(*AllocationSize); + auto AlignmentValue = signedImmediate(*AlignmentOperand); + if (!PointerType) + return std::unexpected(PointerType.error()); + if (!AllocationSizeType) + return std::unexpected(AllocationSizeType.error()); + if (!AlignmentValue) + return std::unexpected(AlignmentValue.error()); + if (!PointerType->isPointer() || !AllocationSizeType->isCategoryNeutralScalar() || + PointerType->scalarBitWidth() != AllocationSizeType->scalarBitWidth()) { + return makeUnexpected(ErrorCode::VerificationError, + "G_DYN_STACKALLOC requires pointer-width integer allocation size"); + } + if (*AlignmentValue <= 0 || !std::has_single_bit(static_cast<std::uint64_t>(*AlignmentValue))) { + return makeUnexpected(ErrorCode::VerificationError, + "G_DYN_STACKALLOC alignment must be a positive power of two"); + } + + auto StackPointerCopy = MRI.createVirtualRegister(*PointerType); + auto IntegerStackPointer = MRI.createVirtualRegister(*AllocationSizeType); + auto Allocation = MRI.createVirtualRegister(*AllocationSizeType); + if (!StackPointerCopy) + return std::unexpected(StackPointerCopy.error()); + if (!IntegerStackPointer) + return std::unexpected(IntegerStackPointer.error()); + if (!Allocation) + return std::unexpected(Allocation.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildCopy(*StackPointerCopy, Register::fromPhysical(StackPointer)); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildPtrToInt(*IntegerStackPointer, *StackPointerCopy); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildSub(*Allocation, *IntegerStackPointer, *AllocationSize); !Built) + return std::unexpected(Built.error()); + + Register AlignedAllocation = *Allocation; + if (*AlignmentValue > 1) { + auto LowBits = APInt::fromUnsigned(AllocationSizeType->scalarBitWidth(), + static_cast<std::uint64_t>(*AlignmentValue) - 1U); + if (!LowBits) + return std::unexpected(LowBits.error()); + auto AlignmentMask = LowBits->bitwiseNot(); + if (!AlignmentMask) + return std::unexpected(AlignmentMask.error()); + auto Mask = buildConstantRegister(MIRBuilder, *AllocationSizeType, std::move(*AlignmentMask)); + if (!Mask) + return std::unexpected(Mask.error()); + auto MaskedAllocation = MRI.createVirtualRegister(*AllocationSizeType); + if (!MaskedAllocation) + return std::unexpected(MaskedAllocation.error()); + if (auto Built = MIRBuilder.buildAnd(*MaskedAllocation, *Allocation, *Mask); !Built) + return std::unexpected(Built.error()); + AlignedAllocation = *MaskedAllocation; + } + + auto TargetPointer = MRI.createVirtualRegister(*PointerType); + if (!TargetPointer) + return std::unexpected(TargetPointer.error()); + if (auto Built = MIRBuilder.buildIntToPtr(*TargetPointer, AlignedAllocation); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildCopy(Register::fromPhysical(StackPointer), *TargetPointer); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildCopy(*Destination, *TargetPointer); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerStackSave(MachineInstr &Instruction) { + const PhysicalRegister StackPointer = MIRBuilder.target().stackPointerRegisterToSaveRestore(); + if (!StackPointer.isValid()) + return makeUnexpected(ErrorCode::Unsupported, + "target does not define a stack pointer for stack save"); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Instruction.operandCount() != 1 || Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_STACKSAVE requires one register destination"); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildCopy(*Destination, Register::fromPhysical(StackPointer)); + !Built) { + return std::unexpected(Built.error()); + } + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerStackRestore(MachineInstr &Instruction) { + const PhysicalRegister StackPointer = MIRBuilder.target().stackPointerRegisterToSaveRestore(); + if (!StackPointer.isValid()) + return makeUnexpected(ErrorCode::Unsupported, + "target does not define a stack pointer for stack restore"); + const MachineOperand *SourceOperand = Instruction.operandAt(0); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Instruction.operandCount() != 1 || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "G_STACKRESTORE requires one register source"); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildCopy(Register::fromPhysical(StackPointer), *Source); !Built) + return std::unexpected(Built.error()); + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerMemcpyInline(MachineInstr &Instruction) { + if (Instruction.operandCount() != 3 || Instruction.memOperands().size() != 2 || + Instruction.memOperands()[0] == nullptr || Instruction.memOperands()[1] == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "G_MEMCPY_INLINE requires destination, source, length, and two MMOs"); + } + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const MachineOperand *LengthOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + const Register *Length = LengthOperand == nullptr ? nullptr : LengthOperand->getIf<Register>(); + if (Destination == nullptr || Source == nullptr || Length == nullptr || + !Destination->isVirtual() || !Source->isVirtual() || !Length->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "G_MEMCPY_INLINE requires virtual-register operands"); + } + + const MachineMemOperand &StoreMemory = *Instruction.memOperands()[0]; + const MachineMemOperand &LoadMemory = *Instruction.memOperands()[1]; + if (!StoreMemory.isStore() || StoreMemory.isLoad() || !LoadMemory.isLoad() || + LoadMemory.isStore() || StoreMemory.isAtomic() || LoadMemory.isAtomic()) { + return makeUnexpected(ErrorCode::VerificationError, + "G_MEMCPY_INLINE requires non-atomic store and load memory operands"); + } + + const std::optional<ValueAndVReg> ConstantLength = + getIConstantVRegValWithLookThrough(*Length, MRI); + if (!ConstantLength || ConstantLength->Value.bitWidth() == 0 || + ConstantLength->Value.bitWidth() > 64) { + return makeUnexpected(ErrorCode::Unsupported, + "G_MEMCPY_INLINE requires a constant length no wider than 64 bits"); + } + const std::uint64_t KnownLength = ConstantLength->Value.words().front(); + if (KnownLength == 0) { + return Instruction.eraseFromParent(); + } + + auto ByteType = LowLevelType::makeCategoryNeutralScalar(8); + if (!ByteType) + return std::unexpected(ByteType.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + MachineFunction &Owner = MIRBuilder.function(); + for (std::uint64_t Offset = 0; Offset != KnownLength; ++Offset) { + auto LoadAddress = buildObjectPointerOffset(MIRBuilder, *Source, Offset); + if (!LoadAddress) + return std::unexpected(LoadAddress.error()); + auto ByteLoadMemory = deriveMemoryOperand(Owner, LoadMemory, *ByteType, Offset); + if (!ByteLoadMemory) + return std::unexpected(ByteLoadMemory.error()); + auto LoadedByte = MRI.createVirtualRegister(*ByteType); + if (!LoadedByte) + return std::unexpected(LoadedByte.error()); + auto Load = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!Load) + return std::unexpected(Load.error()); + if (auto Added = Load->addDef(*LoadedByte); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addUse(*LoadAddress); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addMemOperand(**ByteLoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Load->build(); !Built) + return std::unexpected(Built.error()); + + auto StoreAddress = buildObjectPointerOffset(MIRBuilder, *Destination, Offset); + if (!StoreAddress) + return std::unexpected(StoreAddress.error()); + auto ByteStoreMemory = deriveMemoryOperand(Owner, StoreMemory, *ByteType, Offset); + if (!ByteStoreMemory) + return std::unexpected(ByteStoreMemory.error()); + auto Store = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!Store) + return std::unexpected(Store.error()); + if (auto Added = Store->addUse(*LoadedByte); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addUse(*StoreAddress); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addMemOperand(**ByteStoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Store->build(); !Built) + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerAbsToAddXor(MachineInstr &Instruction) { + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Source = *Instruction.operandAt(1)->getIf<Register>(); + auto Type = MRI.getType(Destination); + if (!Type) + return std::unexpected(Type.error()); + + auto Shift = MRI.createVirtualRegister(*Type); + auto Add = MRI.createVirtualRegister(*Type); + if (!Shift) + return std::unexpected(Shift.error()); + if (!Add) + return std::unexpected(Add.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto ShiftAmount = buildConstantRegister(MIRBuilder, *Type, + static_cast<std::int64_t>(Type->scalarBitWidth() - 1U)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + if (auto Built = MIRBuilder.buildAShr(*Shift, Source, *ShiftAmount); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildAdd(*Add, Source, *Shift); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildXor(Destination, *Add, *Shift); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerAbsDiffToSelect(MachineInstr &Instruction) { + const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_ABDS); + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Left = *Instruction.operandAt(1)->getIf<Register>(); + const Register Right = *Instruction.operandAt(2)->getIf<Register>(); + auto Type = MRI.getType(Left); + if (!Type) + return std::unexpected(Type.error()); + auto ConditionType = LowLevelType::makeCategoryNeutralScalar(1); + if (!ConditionType) + return std::unexpected(ConditionType.error()); + + auto LeftSub = MRI.createVirtualRegister(*Type); + auto RightSub = MRI.createVirtualRegister(*Type); + auto Condition = MRI.createVirtualRegister(*ConditionType); + if (!LeftSub) + return std::unexpected(LeftSub.error()); + if (!RightSub) + return std::unexpected(RightSub.error()); + if (!Condition) + return std::unexpected(Condition.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildSub(*LeftSub, Left, Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildSub(*RightSub, Right, Left); !Built) + return std::unexpected(Built.error()); + const IntegerPredicate Predicate = + IsSigned ? IntegerPredicate::SignedGreaterThan : IntegerPredicate::UnsignedGreaterThan; + if (auto Built = MIRBuilder.buildICmp(Predicate, *Condition, Left, Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {Destination}, + {*Condition, *LeftSub, *RightSub}); + !Built) { + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerAbsDiffToMinMax(MachineInstr &Instruction) { + const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_ABDS); + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Left = *Instruction.operandAt(1)->getIf<Register>(); + const Register Right = *Instruction.operandAt(2)->getIf<Register>(); + auto Type = MRI.getType(Left); + if (!Type) + return std::unexpected(Type.error()); + + auto Maximum = MRI.createVirtualRegister(*Type); + auto Minimum = MRI.createVirtualRegister(*Type); + if (!Maximum) + return std::unexpected(Maximum.error()); + if (!Minimum) + return std::unexpected(Minimum.error()); + + const GenericOpcode MaxOpcode = IsSigned ? GenericOpcode::G_SMAX : GenericOpcode::G_UMAX; + const GenericOpcode MinOpcode = IsSigned ? GenericOpcode::G_SMIN : GenericOpcode::G_UMIN; + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(MaxOpcode), {*Maximum}, {Left, Right}); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(MinOpcode), {*Minimum}, {Left, Right}); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildSub(Destination, *Maximum, *Minimum); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerMinMax(MachineInstr &Instruction) { + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Left = *Instruction.operandAt(1)->getIf<Register>(); + const Register Right = *Instruction.operandAt(2)->getIf<Register>(); + + auto DestinationType = MRI.getType(Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto ConditionType = booleanTypeFor(*DestinationType); + if (!ConditionType) + return std::unexpected(ConditionType.error()); + + auto Condition = MRI.createVirtualRegister(*ConditionType); + if (!Condition) + return std::unexpected(Condition.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + const auto Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (auto Built = MIRBuilder.buildICmp(minMaxToCompare(Generic), *Condition, Left, Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {Destination}, + {*Condition, Left, Right}); + !Built) { + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerThreewayCompare(MachineInstr &Instruction) { + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Left = *Instruction.operandAt(1)->getIf<Register>(); + const Register Right = *Instruction.operandAt(2)->getIf<Register>(); + auto DestinationType = MRI.getType(Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto ConditionType = booleanTypeFor(*DestinationType); + if (!ConditionType) + return std::unexpected(ConditionType.error()); + + auto IsGreater = MRI.createVirtualRegister(*ConditionType); + auto IsLess = MRI.createVirtualRegister(*ConditionType); + if (!IsGreater) + return std::unexpected(IsGreater.error()); + if (!IsLess) + return std::unexpected(IsLess.error()); + + const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SCMP); + const IntegerPredicate GreaterPredicate = + IsSigned ? IntegerPredicate::SignedGreaterThan : IntegerPredicate::UnsignedGreaterThan; + const IntegerPredicate LessPredicate = + IsSigned ? IntegerPredicate::SignedLessThan : IntegerPredicate::UnsignedLessThan; + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = MIRBuilder.buildICmp(GreaterPredicate, *IsGreater, Left, Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildICmp(LessPredicate, *IsLess, Left, Right); !Built) + return std::unexpected(Built.error()); + + const BooleanContent Contents = + MIRBuilder.target().getBooleanContents(DestinationType->isVector(), false); + if (Contents == BooleanContent::Undefined) { + auto Zero = buildConstantRegister(MIRBuilder, *DestinationType, 0); + auto One = buildConstantRegister(MIRBuilder, *DestinationType, 1); + auto MinusOne = buildConstantRegister(MIRBuilder, *DestinationType, -1); + if (!Zero) + return std::unexpected(Zero.error()); + if (!One) + return std::unexpected(One.error()); + if (!MinusOne) + return std::unexpected(MinusOne.error()); + auto ZeroOrOne = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, *DestinationType, + {*IsGreater, *One, *Zero}); + if (!ZeroOrOne) + return std::unexpected(ZeroOrOne.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {Destination}, + {*IsLess, *MinusOne, *ZeroOrOne}); + !Built) { + return std::unexpected(Built.error()); + } + } else { + if (Contents == BooleanContent::ZeroOrNegativeOne) + std::swap(IsGreater, IsLess); + const GenericOpcode ExtensionOpcode = Contents == BooleanContent::ZeroOrNegativeOne + ? GenericOpcode::G_SEXT + : GenericOpcode::G_ZEXT; + auto ExtendedGreater = MRI.createVirtualRegister(*DestinationType); + auto ExtendedLess = MRI.createVirtualRegister(*DestinationType); + if (!ExtendedGreater) + return std::unexpected(ExtendedGreater.error()); + if (!ExtendedLess) + return std::unexpected(ExtendedLess.error()); + if (auto Built = + MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*ExtendedGreater}, {*IsGreater}); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*ExtendedLess}, {*IsLess}); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildSub(Destination, *ExtendedGreater, *ExtendedLess); !Built) + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerDivRem(MachineInstr &Instruction) { + const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SDIVREM); + const Register Quotient = *Instruction.operandAt(0)->getIf<Register>(); + const Register Remainder = *Instruction.operandAt(1)->getIf<Register>(); + const Register Dividend = *Instruction.operandAt(2)->getIf<Register>(); + const Register Divisor = *Instruction.operandAt(3)->getIf<Register>(); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + const GenericOpcode DivideOpcode = IsSigned ? GenericOpcode::G_SDIV : GenericOpcode::G_UDIV; + const GenericOpcode RemainderOpcode = IsSigned ? GenericOpcode::G_SREM : GenericOpcode::G_UREM; + if (auto Built = MIRBuilder.buildInstr(opcode(DivideOpcode), {Quotient}, {Dividend, Divisor}); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildInstr(opcode(RemainderOpcode), {Remainder}, {Dividend, Divisor}); + !Built) { + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerHighMultiply(MachineInstr &Instruction) { + if (Instruction.operandCount() != 3) + return makeUnexpected(ErrorCode::VerificationError, + "high multiply lowering requires a destination and two sources"); + const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SMULH); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *FirstSourceOperand = Instruction.operandAt(1); + const MachineOperand *SecondSourceOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *FirstSource = + FirstSourceOperand == nullptr ? nullptr : FirstSourceOperand->getIf<Register>(); + const Register *SecondSource = + SecondSourceOperand == nullptr ? nullptr : SecondSourceOperand->getIf<Register>(); + if (Destination == nullptr || FirstSource == nullptr || SecondSource == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "high multiply lowering requires register operands"); + auto OriginalType = MRI.getType(*Destination); + if (!OriginalType) + return std::unexpected(OriginalType.error()); + if (!OriginalType->isCategoryNeutralScalar() || OriginalType->scalarBitWidth() == 0 || + OriginalType->scalarBitWidth() > LowLevelType::MaxBitWidth / 2U) { + return makeUnexpected( + ErrorCode::Unsupported, + "high multiply lowering requires a scalar type with a representable double width"); + } + const std::uint32_t SizeInBits = OriginalType->scalarBitWidth(); + auto WideType = LowLevelType::makeCategoryNeutralScalar(SizeInBits * 2U); + if (!WideType) + return std::unexpected(WideType.error()); + + auto WideFirstSource = MRI.createVirtualRegister(*WideType); + auto WideSecondSource = MRI.createVirtualRegister(*WideType); + if (!WideFirstSource) + return std::unexpected(WideFirstSource.error()); + if (!WideSecondSource) + return std::unexpected(WideSecondSource.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = IsSigned ? MIRBuilder.buildSExt(*WideFirstSource, *FirstSource) + : MIRBuilder.buildZExt(*WideFirstSource, *FirstSource); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = IsSigned ? MIRBuilder.buildSExt(*WideSecondSource, *SecondSource) + : MIRBuilder.buildZExt(*WideSecondSource, *SecondSource); + !Built) { + return std::unexpected(Built.error()); + } + auto Product = buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, *WideType, + {*WideFirstSource, *WideSecondSource}); + if (!Product) + return std::unexpected(Product.error()); + auto ShiftAmount = buildConstantRegister(MIRBuilder, *WideType, SizeInBits); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Shifted = + buildResultInstruction(MIRBuilder, IsSigned ? GenericOpcode::G_ASHR : GenericOpcode::G_LSHR, + *WideType, {*Product, *ShiftAmount}); + if (!Shifted) + return std::unexpected(Shifted.error()); + if (auto Built = MIRBuilder.buildTrunc(*Destination, *Shifted); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerMulo(MachineInstr &Instruction) { + if (Instruction.operandCount() != 4) + return makeUnexpected(ErrorCode::VerificationError, + "multiply-overflow lowering requires four register operands"); + + const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SMULO); + const Register *ResultOperand = Instruction.operandAt(0)->getIf<Register>(); + const Register *OverflowOperand = Instruction.operandAt(1)->getIf<Register>(); + const Register *LeftOperand = Instruction.operandAt(2)->getIf<Register>(); + const Register *RightOperand = Instruction.operandAt(3)->getIf<Register>(); + if (ResultOperand == nullptr || OverflowOperand == nullptr || LeftOperand == nullptr || + RightOperand == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "multiply-overflow lowering requires register operands"); + } + const Register Result = *ResultOperand; + const Register Overflow = *OverflowOperand; + const Register Left = *LeftOperand; + const Register Right = *RightOperand; + auto ResultType = MRI.getType(Result); + if (!ResultType) + return std::unexpected(ResultType.error()); + if (!ResultType->isCategoryNeutralScalar() || ResultType->scalarBitWidth() == 0) + return makeUnexpected(ErrorCode::Unsupported, + "multiply-overflow lowering requires a scalar result type"); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + Observer.changingInstr(Instruction); + if (auto Changed = Instruction.setOpcode(opcode(GenericOpcode::G_MUL)); !Changed) + return std::unexpected(Changed.error()); + if (auto Changed = Instruction.eraseOperand(1); !Changed) + return std::unexpected(Changed.error()); + Observer.changedInstr(Instruction); + + auto HighPart = + buildResultInstruction(MIRBuilder, IsSigned ? GenericOpcode::G_SMULH : GenericOpcode::G_UMULH, + *ResultType, {Left, Right}); + if (!HighPart) + return std::unexpected(HighPart.error()); + auto Zero = buildConstantRegister(MIRBuilder, *ResultType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + + if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point) + return std::unexpected(Point.error()); + if (IsSigned) { + auto ShiftAmount = + buildConstantRegister(MIRBuilder, *ResultType, ResultType->scalarBitWidth() - 1U); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, *ResultType, + {Result, *ShiftAmount}); + if (!Shifted) + return std::unexpected(Shifted.error()); + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::NotEqual, Overflow, *HighPart, *Shifted); + !Built) { + return std::unexpected(Built.error()); + } + } else if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::NotEqual, Overflow, *HighPart, *Zero); + !Built) { + return std::unexpected(Built.error()); + } + return {}; +} + +Expected<void> LegalizerHelper::lowerBitCount(MachineInstr &Instruction) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "bit-count lowering requires register operands"); + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto SourceType = MRI.getType(*Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (!DestinationType->isCategoryNeutralScalar() || !SourceType->isCategoryNeutralScalar()) + return makeUnexpected(ErrorCode::Unsupported, + "non-scalar bit-count lowering is not implemented"); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + const auto IsSupported = [this](GenericOpcode Opcode, LowLevelType ResultType, + LowLevelType OperandType) -> Expected<bool> { + auto Decision = Info.getDecision(LegalizationQuery(opcode(Opcode), {ResultType, OperandType})); + if (!Decision) + return std::unexpected(Decision.error()); + return Decision->Action == LegalizationAction::Legal || + Decision->Action == LegalizationAction::Libcall || + Decision->Action == LegalizationAction::Custom; + }; + const auto EraseOriginal = [&Instruction]() -> Expected<void> { + return Instruction.eraseFromParent(); + }; + + switch (Generic) { + case GenericOpcode::G_CTLZ_ZERO_POISON: + case GenericOpcode::G_CTTZ_ZERO_POISON: { + const GenericOpcode Replacement = Generic == GenericOpcode::G_CTLZ_ZERO_POISON + ? GenericOpcode::G_CTLZ + : GenericOpcode::G_CTTZ; + Observer.changingInstr(Instruction); + if (auto Changed = Instruction.setOpcode(opcode(Replacement)); !Changed) + return std::unexpected(Changed.error()); + Observer.changedInstr(Instruction); + return {}; + } + + case GenericOpcode::G_CTLZ: { + const std::uint32_t Size = SourceType->scalarBitWidth(); + auto PoisonSupported = + IsSupported(GenericOpcode::G_CTLZ_ZERO_POISON, *DestinationType, *SourceType); + if (!PoisonSupported) + return std::unexpected(PoisonSupported.error()); + if (*PoisonSupported) { + auto PoisonCount = buildResultInstruction(MIRBuilder, GenericOpcode::G_CTLZ_ZERO_POISON, + *DestinationType, {*Source}); + if (!PoisonCount) + return std::unexpected(PoisonCount.error()); + auto Zero = buildConstantRegister(MIRBuilder, *SourceType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto ConditionType = booleanTypeFor(*SourceType); + if (!ConditionType) + return std::unexpected(ConditionType.error()); + auto IsZero = + buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *ConditionType, *Source, *Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + auto SizeConstant = + buildConstantRegister(MIRBuilder, *DestinationType, static_cast<std::int64_t>(Size)); + if (!SizeConstant) + return std::unexpected(SizeConstant.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {*Destination}, + {*IsZero, *SizeConstant, *PoisonCount}); + !Built) { + return std::unexpected(Built.error()); + } + return EraseOriginal(); + } + + Register Value = *Source; + const std::uint32_t ExpandedSize = std::bit_ceil(Size); + for (std::uint32_t Shift = 1; Shift <= ExpandedSize / 2U; Shift <<= 1U) { + auto ShiftAmount = + buildConstantRegister(MIRBuilder, *SourceType, static_cast<std::int64_t>(Shift)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType, + {Value, *ShiftAmount}); + if (!Shifted) + return std::unexpected(Shifted.error()); + auto Propagated = + buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *SourceType, {Value, *Shifted}); + if (!Propagated) + return std::unexpected(Propagated.error()); + Value = *Propagated; + } + auto Population = + buildResultInstruction(MIRBuilder, GenericOpcode::G_CTPOP, *DestinationType, {Value}); + if (!Population) + return std::unexpected(Population.error()); + auto SizeConstant = + buildConstantRegister(MIRBuilder, *DestinationType, static_cast<std::int64_t>(Size)); + if (!SizeConstant) + return std::unexpected(SizeConstant.error()); + if (auto Built = MIRBuilder.buildSub(*Destination, *SizeConstant, *Population); !Built) + return std::unexpected(Built.error()); + return EraseOriginal(); + } + + case GenericOpcode::G_CTTZ: { + const std::uint32_t Size = SourceType->scalarBitWidth(); + auto PoisonSupported = + IsSupported(GenericOpcode::G_CTTZ_ZERO_POISON, *DestinationType, *SourceType); + if (!PoisonSupported) + return std::unexpected(PoisonSupported.error()); + if (*PoisonSupported) { + auto PoisonCount = buildResultInstruction(MIRBuilder, GenericOpcode::G_CTTZ_ZERO_POISON, + *DestinationType, {*Source}); + if (!PoisonCount) + return std::unexpected(PoisonCount.error()); + auto Zero = buildConstantRegister(MIRBuilder, *SourceType, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto ConditionType = booleanTypeFor(*DestinationType); + if (!ConditionType) + return std::unexpected(ConditionType.error()); + auto IsZero = + buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *ConditionType, *Source, *Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + auto SizeConstant = + buildConstantRegister(MIRBuilder, *DestinationType, static_cast<std::int64_t>(Size)); + if (!SizeConstant) + return std::unexpected(SizeConstant.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {*Destination}, + {*IsZero, *SizeConstant, *PoisonCount}); + !Built) { + return std::unexpected(Built.error()); + } + return EraseOriginal(); + } + + auto MinusOne = buildConstantRegister(MIRBuilder, *SourceType, -1); + if (!MinusOne) + return std::unexpected(MinusOne.error()); + auto Inverted = + buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, *SourceType, {*Source, *MinusOne}); + if (!Inverted) + return std::unexpected(Inverted.error()); + auto Decremented = + buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType, {*Source, *MinusOne}); + if (!Decremented) + return std::unexpected(Decremented.error()); + auto Masked = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType, + {*Inverted, *Decremented}); + if (!Masked) + return std::unexpected(Masked.error()); + + auto PopulationSupported = IsSupported(GenericOpcode::G_CTPOP, *SourceType, *SourceType); + if (!PopulationSupported) + return std::unexpected(PopulationSupported.error()); + auto LeadingZeroSupported = IsSupported(GenericOpcode::G_CTLZ, *SourceType, *SourceType); + if (!LeadingZeroSupported) + return std::unexpected(LeadingZeroSupported.error()); + if (!*PopulationSupported && *LeadingZeroSupported) { + auto SizeConstant = + buildConstantRegister(MIRBuilder, *SourceType, static_cast<std::int64_t>(Size)); + if (!SizeConstant) + return std::unexpected(SizeConstant.error()); + auto LeadingZeros = + buildResultInstruction(MIRBuilder, GenericOpcode::G_CTLZ, *SourceType, {*Masked}); + if (!LeadingZeros) + return std::unexpected(LeadingZeros.error()); + if (auto Built = MIRBuilder.buildSub(*Destination, *SizeConstant, *LeadingZeros); !Built) + return std::unexpected(Built.error()); + return EraseOriginal(); + } + + Observer.changingInstr(Instruction); + if (auto Changed = Instruction.setOpcode(opcode(GenericOpcode::G_CTPOP)); !Changed) + return std::unexpected(Changed.error()); + if (auto Changed = SourceOperand->setRegister(*Masked); !Changed) + return std::unexpected(Changed.error()); + Observer.changedInstr(Instruction); + return {}; + } + + case GenericOpcode::G_CTPOP: { + const std::uint32_t Size = SourceType->scalarBitWidth(); + if (Size > 128 || Size % 8U != 0) + return makeUnexpected( + ErrorCode::Unsupported, + "population-count lowering requires a byte-sized scalar up to 128 bits"); + + auto One = buildConstantRegister(MIRBuilder, *SourceType, 1); + if (!One) + return std::unexpected(One.error()); + auto ShiftedByOne = + buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType, {*Source, *One}); + if (!ShiftedByOne) + return std::unexpected(ShiftedByOne.error()); + auto PairMaskValue = repeatedByte(Size, 0x55); + if (!PairMaskValue) + return std::unexpected(PairMaskValue.error()); + auto PairMask = buildConstantRegister(MIRBuilder, *SourceType, std::move(*PairMaskValue)); + if (!PairMask) + return std::unexpected(PairMask.error()); + auto HighPairBits = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType, + {*ShiftedByOne, *PairMask}); + if (!HighPairBits) + return std::unexpected(HighPairBits.error()); + auto PairCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *SourceType, + {*Source, *HighPairBits}); + if (!PairCounts) + return std::unexpected(PairCounts.error()); + + auto Two = buildConstantRegister(MIRBuilder, *SourceType, 2); + if (!Two) + return std::unexpected(Two.error()); + auto ShiftedPairs = + buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType, {*PairCounts, *Two}); + if (!ShiftedPairs) + return std::unexpected(ShiftedPairs.error()); + auto NibbleMaskValue = repeatedByte(Size, 0x33); + if (!NibbleMaskValue) + return std::unexpected(NibbleMaskValue.error()); + auto NibbleMask = buildConstantRegister(MIRBuilder, *SourceType, std::move(*NibbleMaskValue)); + if (!NibbleMask) + return std::unexpected(NibbleMask.error()); + auto HighNibbleCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType, + {*ShiftedPairs, *NibbleMask}); + if (!HighNibbleCounts) + return std::unexpected(HighNibbleCounts.error()); + auto LowNibbleCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType, + {*PairCounts, *NibbleMask}); + if (!LowNibbleCounts) + return std::unexpected(LowNibbleCounts.error()); + auto NibbleCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType, + {*HighNibbleCounts, *LowNibbleCounts}); + if (!NibbleCounts) + return std::unexpected(NibbleCounts.error()); + + auto Four = buildConstantRegister(MIRBuilder, *SourceType, 4); + if (!Four) + return std::unexpected(Four.error()); + auto ShiftedNibbles = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType, + {*NibbleCounts, *Four}); + if (!ShiftedNibbles) + return std::unexpected(ShiftedNibbles.error()); + auto DirtyByteCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType, + {*ShiftedNibbles, *NibbleCounts}); + if (!DirtyByteCounts) + return std::unexpected(DirtyByteCounts.error()); + auto ByteMaskValue = repeatedByte(Size, 0x0f); + if (!ByteMaskValue) + return std::unexpected(ByteMaskValue.error()); + auto ByteMask = buildConstantRegister(MIRBuilder, *SourceType, std::move(*ByteMaskValue)); + if (!ByteMask) + return std::unexpected(ByteMask.error()); + auto ByteCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType, + {*DirtyByteCounts, *ByteMask}); + if (!ByteCounts) + return std::unexpected(ByteCounts.error()); + + if (Size == 16) { + auto Eight = buildConstantRegister(MIRBuilder, *SourceType, 8); + if (!Eight) + return std::unexpected(Eight.error()); + auto HighByteCount = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType, + {*ByteCounts, *Eight}); + if (!HighByteCount) + return std::unexpected(HighByteCount.error()); + auto Sum = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType, + {*ByteCounts, *HighByteCount}); + if (!Sum) + return std::unexpected(Sum.error()); + auto ResultMaskValue = APInt::fromUnsigned(Size, 0xff); + if (!ResultMaskValue) + return std::unexpected(ResultMaskValue.error()); + auto ResultMask = buildConstantRegister(MIRBuilder, *SourceType, std::move(*ResultMaskValue)); + if (!ResultMask) + return std::unexpected(ResultMask.error()); + if (auto Built = MIRBuilder.buildAnd(*Destination, *Sum, *ResultMask); !Built) + return std::unexpected(Built.error()); + return EraseOriginal(); + } + + auto MultiplyMaskValue = repeatedByte(Size, 0x01); + if (!MultiplyMaskValue) + return std::unexpected(MultiplyMaskValue.error()); + auto MultiplyMask = + buildConstantRegister(MIRBuilder, *SourceType, std::move(*MultiplyMaskValue)); + if (!MultiplyMask) + return std::unexpected(MultiplyMask.error()); + auto FinalShift = + buildConstantRegister(MIRBuilder, *SourceType, static_cast<std::int64_t>(Size - 8U)); + if (!FinalShift) + return std::unexpected(FinalShift.error()); + + auto MultiplyDecision = + Info.getDecision(LegalizationQuery(opcode(GenericOpcode::G_MUL), {*SourceType})); + if (!MultiplyDecision) + return std::unexpected(MultiplyDecision.error()); + const bool IsMultiplySupported = MultiplyDecision->Action == LegalizationAction::Legal || + MultiplyDecision->Action == LegalizationAction::WidenScalar || + MultiplyDecision->Action == LegalizationAction::Custom; + if (IsMultiplySupported) { + auto Product = buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, *SourceType, + {*ByteCounts, *MultiplyMask}); + if (!Product) + return std::unexpected(Product.error()); + if (auto Built = MIRBuilder.buildLShr(*Destination, *Product, *FinalShift); !Built) + return std::unexpected(Built.error()); + } else { + Register Sum = *ByteCounts; + for (std::uint32_t Shift = 8; Shift < Size; Shift *= 2U) { + auto ShiftAmount = + buildConstantRegister(MIRBuilder, *SourceType, static_cast<std::int64_t>(Shift)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *SourceType, + {Sum, *ShiftAmount}); + if (!Shifted) + return std::unexpected(Shifted.error()); + auto NewSum = + buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType, {Sum, *Shifted}); + if (!NewSum) + return std::unexpected(NewSum.error()); + Sum = *NewSum; + } + if (auto Built = MIRBuilder.buildLShr(*Destination, Sum, *FinalShift); !Built) + return std::unexpected(Built.error()); + } + return EraseOriginal(); + } + + case GenericOpcode::G_CTLS: { + auto SignIndex = buildConstantRegister( + MIRBuilder, *SourceType, static_cast<std::int64_t>(SourceType->scalarBitWidth() - 1U)); + if (!SignIndex) + return std::unexpected(SignIndex.error()); + auto One = buildConstantRegister(MIRBuilder, *DestinationType, 1); + if (!One) + return std::unexpected(One.error()); + auto Sign = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, *SourceType, + {*Source, *SignIndex}); + if (!Sign) + return std::unexpected(Sign.error()); + auto DifferentBits = + buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, *SourceType, {*Source, *Sign}); + if (!DifferentBits) + return std::unexpected(DifferentBits.error()); + auto LeadingZeros = buildResultInstruction(MIRBuilder, GenericOpcode::G_CTLZ, *DestinationType, + {*DifferentBits}); + if (!LeadingZeros) + return std::unexpected(LeadingZeros.error()); + if (auto Built = MIRBuilder.buildSub(*Destination, *LeadingZeros, *One); !Built) + return std::unexpected(Built.error()); + return EraseOriginal(); + } + + default: + return makeUnexpected(ErrorCode::Unsupported, + "bit-count lowering received an unexpected opcode"); + } +} + +Expected<void> LegalizerHelper::lowerBswap(MachineInstr &Instruction) { + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Source = *Instruction.operandAt(1)->getIf<Register>(); + auto Type = MRI.getType(Source); + if (!Type) + return std::unexpected(Type.error()); + + const std::uint32_t SizeInBytes = (Type->scalarBitWidth() + 7U) / 8U; + const std::uint32_t BaseShiftAmount = (SizeInBytes - 1U) * 8U; + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto ShiftAmount = buildConstantRegister(MIRBuilder, *Type, BaseShiftAmount); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto LowByteShiftedLeft = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *Type, {Source, *ShiftAmount}); + auto HighByteShiftedRight = + buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *Type, {Source, *ShiftAmount}); + if (!LowByteShiftedLeft) + return std::unexpected(LowByteShiftedLeft.error()); + if (!HighByteShiftedRight) + return std::unexpected(HighByteShiftedRight.error()); + + Register Result; + if (SizeInBytes == 2U) { + if (auto Built = MIRBuilder.buildOr(Destination, *HighByteShiftedRight, *LowByteShiftedLeft); + !Built) { + return std::unexpected(Built.error()); + } + Result = Destination; + } else { + auto OuterBytes = buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *Type, + {*HighByteShiftedRight, *LowByteShiftedLeft}); + if (!OuterBytes) + return std::unexpected(OuterBytes.error()); + Result = *OuterBytes; + } + + for (std::uint32_t Index = 1; Index < SizeInBytes / 2U; ++Index) { + const std::uint32_t MaskBits = std::uint32_t{0xff} << (Index * 8U); + const std::int64_t MaskValue = static_cast<std::int32_t>(MaskBits); + auto Mask = buildConstantRegister(MIRBuilder, *Type, MaskValue); + auto InnerShiftAmount = buildConstantRegister(MIRBuilder, *Type, BaseShiftAmount - 16U * Index); + if (!Mask) + return std::unexpected(Mask.error()); + if (!InnerShiftAmount) + return std::unexpected(InnerShiftAmount.error()); + + auto LowByte = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *Type, {Source, *Mask}); + if (!LowByte) + return std::unexpected(LowByte.error()); + auto LowByteShifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *Type, + {*LowByte, *InnerShiftAmount}); + if (!LowByteShifted) + return std::unexpected(LowByteShifted.error()); + auto WithLowByte = + buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *Type, {Result, *LowByteShifted}); + if (!WithLowByte) + return std::unexpected(WithLowByte.error()); + + auto SourceShifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *Type, + {Source, *InnerShiftAmount}); + if (!SourceShifted) + return std::unexpected(SourceShifted.error()); + auto HighByte = + buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *Type, {*SourceShifted, *Mask}); + if (!HighByte) + return std::unexpected(HighByte.error()); + + if (Index + 1U == SizeInBytes / 2U) { + if (auto Built = MIRBuilder.buildOr(Destination, *WithLowByte, *HighByte); !Built) + return std::unexpected(Built.error()); + Result = Destination; + } else { + auto Combined = + buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *Type, {*WithLowByte, *HighByte}); + if (!Combined) + return std::unexpected(Combined.error()); + Result = *Combined; + } + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerFunnelShift(MachineInstr &Instruction) { + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Left = *Instruction.operandAt(1)->getIf<Register>(); + const Register Right = *Instruction.operandAt(2)->getIf<Register>(); + const Register Amount = *Instruction.operandAt(3)->getIf<Register>(); + auto DestinationType = MRI.getType(Destination); + auto AmountType = MRI.getType(Amount); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!AmountType) + return std::unexpected(AmountType.error()); + + const std::uint32_t ElementWidth = DestinationType->scalarBitWidth(); + const bool IsLeft = Instruction.opcode() == opcode(GenericOpcode::G_FSHL); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + auto Mask = + buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(ElementWidth - 1U)); + if (!Mask) + return std::unexpected(Mask.error()); + + Register ShiftAmount; + Register InverseShiftAmount; + if (std::has_single_bit(ElementWidth)) { + auto Shift = + buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType, {Amount, *Mask}); + auto AllOnes = buildConstantRegister(MIRBuilder, *AmountType, -1); + if (!Shift) + return std::unexpected(Shift.error()); + if (!AllOnes) + return std::unexpected(AllOnes.error()); + auto Inverted = + buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, *AmountType, {Amount, *AllOnes}); + if (!Inverted) + return std::unexpected(Inverted.error()); + auto Inverse = + buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType, {*Inverted, *Mask}); + if (!Inverse) + return std::unexpected(Inverse.error()); + ShiftAmount = *Shift; + InverseShiftAmount = *Inverse; + } else { + auto Width = + buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(ElementWidth)); + if (!Width) + return std::unexpected(Width.error()); + auto Shift = + buildResultInstruction(MIRBuilder, GenericOpcode::G_UREM, *AmountType, {Amount, *Width}); + if (!Shift) + return std::unexpected(Shift.error()); + auto Inverse = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, {*Mask, *Shift}); + if (!Inverse) + return std::unexpected(Inverse.error()); + ShiftAmount = *Shift; + InverseShiftAmount = *Inverse; + } + + auto One = buildConstantRegister(MIRBuilder, *AmountType, 1); + if (!One) + return std::unexpected(One.error()); + + Register ShiftedLeft; + Register ShiftedRight; + if (IsLeft) { + auto ShiftLeft = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *DestinationType, + {Left, ShiftAmount}); + if (!ShiftLeft) + return std::unexpected(ShiftLeft.error()); + auto FirstShiftRight = + buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *DestinationType, {Right, *One}); + if (!FirstShiftRight) + return std::unexpected(FirstShiftRight.error()); + auto ShiftRight = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *DestinationType, + {*FirstShiftRight, InverseShiftAmount}); + if (!ShiftRight) + return std::unexpected(ShiftRight.error()); + ShiftedLeft = *ShiftLeft; + ShiftedRight = *ShiftRight; + } else { + auto FirstShiftLeft = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *DestinationType, {Left, *One}); + if (!FirstShiftLeft) + return std::unexpected(FirstShiftLeft.error()); + auto ShiftLeft = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *DestinationType, + {*FirstShiftLeft, InverseShiftAmount}); + if (!ShiftLeft) + return std::unexpected(ShiftLeft.error()); + auto ShiftRight = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *DestinationType, + {Right, ShiftAmount}); + if (!ShiftRight) + return std::unexpected(ShiftRight.error()); + ShiftedLeft = *ShiftLeft; + ShiftedRight = *ShiftRight; + } + + auto Result = MIRBuilder.buildInstr(opcode(GenericOpcode::G_OR)); + if (!Result) + return std::unexpected(Result.error()); + if (auto Added = Result->addDef(Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Result->addUse(ShiftedLeft); !Added) + return std::unexpected(Added.error()); + if (auto Added = Result->addUse(ShiftedRight); !Added) + return std::unexpected(Added.error()); + if (auto Flags = Result->setFlags({MachineInstrFlag::Disjoint}); !Flags) + return std::unexpected(Flags.error()); + if (auto Built = Result->build(); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerRotate(MachineInstr &Instruction) { + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Source = *Instruction.operandAt(1)->getIf<Register>(); + const Register Amount = *Instruction.operandAt(2)->getIf<Register>(); + auto DestinationType = MRI.getType(Destination); + auto AmountType = MRI.getType(Amount); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!AmountType) + return std::unexpected(AmountType.error()); + + const std::uint32_t ElementWidth = DestinationType->scalarBitWidth(); + const bool IsLeft = Instruction.opcode() == opcode(GenericOpcode::G_ROTL); + const GenericOpcode ShiftOpcode = IsLeft ? GenericOpcode::G_SHL : GenericOpcode::G_LSHR; + const GenericOpcode ReverseShiftOpcode = IsLeft ? GenericOpcode::G_LSHR : GenericOpcode::G_SHL; + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto Zero = buildConstantRegister(MIRBuilder, *AmountType, 0); + auto WidthMinusOne = + buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(ElementWidth - 1U)); + if (!Zero) + return std::unexpected(Zero.error()); + if (!WidthMinusOne) + return std::unexpected(WidthMinusOne.error()); + + Register Shifted; + Register ReverseShifted; + if (std::has_single_bit(ElementWidth)) { + auto NegativeAmount = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, {*Zero, Amount}); + auto ShiftAmount = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType, + {Amount, *WidthMinusOne}); + if (!NegativeAmount) + return std::unexpected(NegativeAmount.error()); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Shift = + buildResultInstruction(MIRBuilder, ShiftOpcode, *DestinationType, {Source, *ShiftAmount}); + auto ReverseAmount = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType, + {*NegativeAmount, *WidthMinusOne}); + if (!Shift) + return std::unexpected(Shift.error()); + if (!ReverseAmount) + return std::unexpected(ReverseAmount.error()); + auto ReverseShift = buildResultInstruction(MIRBuilder, ReverseShiftOpcode, *DestinationType, + {Source, *ReverseAmount}); + if (!ReverseShift) + return std::unexpected(ReverseShift.error()); + Shifted = *Shift; + ReverseShifted = *ReverseShift; + } else { + auto Width = + buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(ElementWidth)); + if (!Width) + return std::unexpected(Width.error()); + auto ShiftAmount = + buildResultInstruction(MIRBuilder, GenericOpcode::G_UREM, *AmountType, {Amount, *Width}); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Shift = + buildResultInstruction(MIRBuilder, ShiftOpcode, *DestinationType, {Source, *ShiftAmount}); + auto ReverseAmount = buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, + {*WidthMinusOne, *ShiftAmount}); + auto One = buildConstantRegister(MIRBuilder, *AmountType, 1); + if (!Shift) + return std::unexpected(Shift.error()); + if (!ReverseAmount) + return std::unexpected(ReverseAmount.error()); + if (!One) + return std::unexpected(One.error()); + auto Inner = + buildResultInstruction(MIRBuilder, ReverseShiftOpcode, *DestinationType, {Source, *One}); + if (!Inner) + return std::unexpected(Inner.error()); + auto ReverseShift = buildResultInstruction(MIRBuilder, ReverseShiftOpcode, *DestinationType, + {*Inner, *ReverseAmount}); + if (!ReverseShift) + return std::unexpected(ReverseShift.error()); + Shifted = *Shift; + ReverseShifted = *ReverseShift; + } + + auto Result = MIRBuilder.buildInstr(opcode(GenericOpcode::G_OR)); + if (!Result) + return std::unexpected(Result.error()); + if (auto Added = Result->addDef(Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Result->addUse(Shifted); !Added) + return std::unexpected(Added.error()); + if (auto Added = Result->addUse(ReverseShifted); !Added) + return std::unexpected(Added.error()); + if (auto Flags = Result->setFlags({MachineInstrFlag::Disjoint}); !Flags) + return std::unexpected(Flags.error()); + if (auto Built = Result->build(); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::libcall(MachineInstr &Instruction) { + if (Libcalls == nullptr || Libcalls->CallLowering == nullptr) { + return makeUnexpected(ErrorCode::Unsupported, + "libcall legalization requires target call lowering information"); + } + + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + auto *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr || !DestinationOperand->isDef()) { + return makeUnexpected(ErrorCode::VerificationError, "libcall requires a register destination"); + } + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!DestinationType->isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "libcall requires a category-neutral scalar destination"); + } + + const bool IsConversion = + Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC || + Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI || + Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP; + const bool IsFloatingBinary = Generic == GenericOpcode::G_FREM; + if (Instruction.operandCount() < 2 || (IsConversion && Instruction.operandCount() != 2)) { + return makeUnexpected(ErrorCode::VerificationError, + "libcall has an invalid number of operands"); + } + + const MachineOperand *FirstSourceOperand = Instruction.operandAt(1); + const Register *FirstSource = + FirstSourceOperand == nullptr ? nullptr : FirstSourceOperand->getIf<Register>(); + if (FirstSource == nullptr || !FirstSourceOperand->isUse()) { + return makeUnexpected(ErrorCode::VerificationError, + "libcall requires register source operands"); + } + auto SourceType = MRI.getType(*FirstSource); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (!SourceType->isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "libcall requires category-neutral scalar sources"); + } + + Expected<std::string> Name = + IsConversion ? conversionLibcallName(Generic, DestinationType->scalarBitWidth(), + SourceType->scalarBitWidth()) + : IsFloatingBinary ? floatingBinaryLibcallName(Generic, DestinationType->scalarBitWidth()) + : integerLibcallName(Generic, DestinationType->scalarBitWidth()); + if (!Name) + return std::unexpected(Name.error()); + + const bool DestinationIsFloat = + Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC || + Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP || IsFloatingBinary; + const bool SourceIsFloat = + Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC || + Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI || IsFloatingBinary; + const ABIValueCategory DestinationCategory = + DestinationIsFloat ? ABIValueCategory::FloatingPoint : ABIValueCategory::Integer; + const ABIValueCategory SourceCategory = + SourceIsFloat ? ABIValueCategory::FloatingPoint : ABIValueCategory::Integer; + auto ExtensionFor = [&](ABIValueCategory Category, bool IsResult, + std::uint32_t BitWidth) noexcept { + if (Category != ABIValueCategory::Integer || BitWidth >= 64) + return ABIExtensionKind::None; + if (!IsConversion) + return integerLibcallExtension(Generic, BitWidth); + if ((IsResult && Generic == GenericOpcode::G_FPTOSI) || + (!IsResult && Generic == GenericOpcode::G_SITOFP)) { + return ABIExtensionKind::SignExtend; + } + return ABIExtensionKind::ZeroExtend; + }; + + auto FragmentType = LowLevelType::makeCategoryNeutralScalar(64); + if (!FragmentType) + return std::unexpected(FragmentType.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + auto AppendValue = [&](Register Value, LowLevelType Type, ABIValueCategory Category, + ABIExtensionKind Extension, std::uint32_t OriginalValue, bool IsResult, + std::vector<ABIValue> &Values, + std::vector<Register> *Fragments) -> Expected<void> { + const std::uint32_t BitWidth = Type.scalarBitWidth(); + auto OriginalAlignment = Alignment::create((BitWidth + 7U) / 8U); + if (!OriginalAlignment) + return std::unexpected(OriginalAlignment.error()); + if (BitWidth != 128) { + auto ABI = libcallABIValue(Value, Type, Category, OriginalValue, 0, 1, 0, *OriginalAlignment, + Extension); + if (!ABI) + return std::unexpected(ABI.error()); + Values.push_back(*ABI); + if (Fragments != nullptr) + Fragments->push_back(Value); + return {}; + } + + std::vector<Register> PartRegisters; + PartRegisters.reserve(2); + for (std::size_t Index = 0; Index != 2; ++Index) { + auto Part = MRI.createVirtualRegister(*FragmentType); + if (!Part) + return std::unexpected(Part.error()); + PartRegisters.push_back(*Part); + } + if (!IsResult) { + auto Unmerged = MIRBuilder.buildUnmergeValues(PartRegisters, Value); + if (!Unmerged) + return std::unexpected(Unmerged.error()); + } + for (std::uint16_t Index = 0; Index != 2; ++Index) { + auto ABI = libcallABIValue(PartRegisters[Index], *FragmentType, ABIValueCategory::Integer, + OriginalValue, Index, 2, static_cast<std::uint32_t>(Index) * 8U, + *OriginalAlignment, ABIExtensionKind::None); + if (!ABI) + return std::unexpected(ABI.error()); + Values.push_back(*ABI); + } + if (Fragments != nullptr) + *Fragments = std::move(PartRegisters); + return {}; + }; + + std::vector<ABIValue> Arguments; + Arguments.reserve((Instruction.operandCount() - 1U) * + (SourceType->scalarBitWidth() == 128 ? 2U : 1U)); + std::uint32_t OriginalArgument = 0; + for (std::size_t Index = 1; Index != Instruction.operandCount(); ++Index) { + const MachineOperand *Operand = Instruction.operandAt(Index); + const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Source == nullptr || !Operand->isUse()) { + return makeUnexpected(ErrorCode::VerificationError, + "libcall requires register source operands"); + } + auto ArgumentType = MRI.getType(*Source); + if (!ArgumentType) + return std::unexpected(ArgumentType.error()); + if (*ArgumentType != *SourceType) { + return makeUnexpected(ErrorCode::VerificationError, "libcall source types must match"); + } + if (!IsConversion && *ArgumentType != *DestinationType) { + return makeUnexpected(ErrorCode::VerificationError, + "binary libcall source and destination types must match"); + } + const ABIExtensionKind Extension = + ExtensionFor(SourceCategory, false, ArgumentType->scalarBitWidth()); + if (auto Appended = AppendValue(*Source, *ArgumentType, SourceCategory, Extension, + OriginalArgument++, false, Arguments, nullptr); + !Appended) { + return std::unexpected(Appended.error()); + } + } + + std::vector<ABIValue> Results; + std::vector<Register> ResultFragments; + const ABIExtensionKind ResultExtension = + ExtensionFor(DestinationCategory, true, DestinationType->scalarBitWidth()); + if (auto Appended = AppendValue(*Destination, *DestinationType, DestinationCategory, + ResultExtension, 0, true, Results, &ResultFragments); + !Appended) { + return std::unexpected(Appended.error()); + } + + auto Callee = MachineCallTarget::externalSymbol(std::move(*Name)); + if (!Callee) + return std::unexpected(Callee.error()); + CallLoweringRequest Request; + Request.Convention = Libcalls->Convention; + Request.Callee = std::move(*Callee); + Request.Arguments = std::move(Arguments); + Request.Results = std::move(Results); + Request.FixedArgumentCount = OriginalArgument; + if (auto Lowered = Libcalls->CallLowering->lowerCall(MIRBuilder, Request); !Lowered) + return std::unexpected(Lowered.error()); + + if (DestinationType->scalarBitWidth() == 128) { + auto Merged = MIRBuilder.buildMergeLikeInstr(*Destination, ResultFragments); + if (!Merged) + return std::unexpected(Merged.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerSignedAddSubOverflow(MachineInstr &Instruction) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic == GenericOpcode::G_SADDE || Generic == GenericOpcode::G_SSUBE) { + if (Instruction.operandCount() != 5) + return makeUnexpected(ErrorCode::VerificationError, + "signed extended add/sub lowering requires five register operands"); + + const bool IsAdd = Generic == GenericOpcode::G_SADDE; + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *OverflowOperand = Instruction.operandAt(1); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const MachineOperand *CarryInOperand = Instruction.operandAt(4); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Overflow = + OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf<Register>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + const Register *CarryIn = + CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf<Register>(); + if (Destination == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr || + CarryIn == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "signed extended add/sub lowering requires register operands"); + } + + auto Type = MRI.getType(*Destination); + if (!Type) + return std::unexpected(Type.error()); + auto Result = MRI.cloneVirtualRegister(*Destination); + auto ExtendedCarry = MRI.createVirtualRegister(*Type); + auto FirstXor = MRI.createVirtualRegister(*Type); + auto SecondXor = MRI.createVirtualRegister(*Type); + auto OverflowBits = MRI.createVirtualRegister(*Type); + if (!Result) + return std::unexpected(Result.error()); + if (!ExtendedCarry) + return std::unexpected(ExtendedCarry.error()); + if (!FirstXor) + return std::unexpected(FirstXor.error()); + if (!SecondXor) + return std::unexpected(SecondXor.error()); + if (!OverflowBits) + return std::unexpected(OverflowBits.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + if (IsAdd) { + auto Temporary = MRI.createVirtualRegister(*Type); + if (!Temporary) + return std::unexpected(Temporary.error()); + if (auto Built = MIRBuilder.buildAdd(*Temporary, *Left, *Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildZExt(*ExtendedCarry, *CarryIn); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildAdd(*Result, *Temporary, *ExtendedCarry); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildCopy(*Destination, *Result); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildXor(*FirstXor, *Result, *Left); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildXor(*SecondXor, *Result, *Right); !Built) + return std::unexpected(Built.error()); + } else { + auto RightWithCarry = MRI.createVirtualRegister(*Type); + if (!RightWithCarry) + return std::unexpected(RightWithCarry.error()); + if (auto Built = MIRBuilder.buildZExt(*ExtendedCarry, *CarryIn); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildAdd(*RightWithCarry, *Right, *ExtendedCarry); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildSub(*Result, *Left, *RightWithCarry); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildCopy(*Destination, *Result); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildXor(*FirstXor, *Left, *Right); !Built) + return std::unexpected(Built.error()); + if (auto Built = MIRBuilder.buildXor(*SecondXor, *Left, *Result); !Built) + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildAnd(*OverflowBits, *FirstXor, *SecondXor); !Built) + return std::unexpected(Built.error()); + auto Zero = buildConstantRegister(MIRBuilder, *Type, 0); + if (!Zero) + return std::unexpected(Zero.error()); + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *Overflow, *OverflowBits, *Zero); + !Built) { + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); + } + + const bool IsAdd = Instruction.opcode() == opcode(GenericOpcode::G_SADDO); + const Register Destination = *Instruction.operandAt(0)->getIf<Register>(); + const Register Overflow = *Instruction.operandAt(1)->getIf<Register>(); + const Register Left = *Instruction.operandAt(2)->getIf<Register>(); + const Register Right = *Instruction.operandAt(3)->getIf<Register>(); + auto Type = MRI.getType(Destination); + auto BooleanType = MRI.getType(Overflow); + if (!Type) + return std::unexpected(Type.error()); + if (!BooleanType) + return std::unexpected(BooleanType.error()); + + auto Result = MRI.cloneVirtualRegister(Destination); + auto FirstComparison = MRI.createVirtualRegister(*BooleanType); + auto SecondComparison = MRI.createVirtualRegister(*BooleanType); + if (!Result) + return std::unexpected(Result.error()); + if (!FirstComparison) + return std::unexpected(FirstComparison.error()); + if (!SecondComparison) + return std::unexpected(SecondComparison.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = IsAdd ? MIRBuilder.buildAdd(*Result, Left, Right) + : MIRBuilder.buildSub(*Result, Left, Right); + !Built) { + return std::unexpected(Built.error()); + } + auto Zero = buildConstantRegister(MIRBuilder, *Type, 0); + if (!Zero) + return std::unexpected(Zero.error()); + if (IsAdd) { + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *FirstComparison, *Result, Left); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *SecondComparison, Right, *Zero); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildXor(Overflow, *SecondComparison, *FirstComparison); !Built) + return std::unexpected(Built.error()); + } else { + if (auto Built = + MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *FirstComparison, Left, Right); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *SecondComparison, + *Result, *Zero); + !Built) { + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildXor(Overflow, *FirstComparison, *SecondComparison); !Built) + return std::unexpected(Built.error()); + } + if (auto Built = MIRBuilder.buildCopy(Destination, *Result); !Built) + return std::unexpected(Built.error()); + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerSaturatingAddSubToMinMax(MachineInstr &Instruction) { + if (Instruction.operandCount() != 3) + return makeUnexpected(ErrorCode::VerificationError, + "saturating add/sub min/max lowering requires three register operands"); + + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + const bool IsSigned = Generic == GenericOpcode::G_SADDSAT || Generic == GenericOpcode::G_SSUBSAT; + const bool IsAdd = Generic == GenericOpcode::G_UADDSAT || Generic == GenericOpcode::G_SADDSAT; + const GenericOpcode ArithmeticOpcode = IsAdd ? GenericOpcode::G_ADD : GenericOpcode::G_SUB; + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + if (Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "saturating add/sub min/max lowering requires register operands"); + + auto Type = MRI.getType(*Destination); + if (!Type) + return std::unexpected(Type.error()); + if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0) + return makeUnexpected(ErrorCode::Unsupported, + "saturating add/sub min/max lowering requires a scalar integer type"); + const std::uint32_t BitWidth = Type->scalarBitWidth(); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (IsSigned) { + auto MaximumValue = APInt::getLowBitsSet(BitWidth, BitWidth - 1U); + auto MinimumValue = oneBitSet(BitWidth, BitWidth - 1U); + if (!MaximumValue) + return std::unexpected(MaximumValue.error()); + if (!MinimumValue) + return std::unexpected(MinimumValue.error()); + auto Maximum = buildConstantRegister(MIRBuilder, *Type, std::move(*MaximumValue)); + auto Minimum = buildConstantRegister(MIRBuilder, *Type, std::move(*MinimumValue)); + if (!Maximum) + return std::unexpected(Maximum.error()); + if (!Minimum) + return std::unexpected(Minimum.error()); + + Register LowerBound; + Register UpperBound; + if (IsAdd) { + auto Zero = buildConstantRegister(MIRBuilder, *Type, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto LeftMaximum = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SMAX, *Type, {*Left, *Zero}); + if (!LeftMaximum) + return std::unexpected(LeftMaximum.error()); + auto High = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *Type, {*Maximum, *LeftMaximum}); + if (!High) + return std::unexpected(High.error()); + auto LeftMinimum = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SMIN, *Type, {*Left, *Zero}); + if (!LeftMinimum) + return std::unexpected(LeftMinimum.error()); + auto Low = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *Type, {*Minimum, *LeftMinimum}); + if (!Low) + return std::unexpected(Low.error()); + LowerBound = *Low; + UpperBound = *High; + } else { + auto NegativeOne = buildConstantRegister(MIRBuilder, *Type, -1); + if (!NegativeOne) + return std::unexpected(NegativeOne.error()); + auto LeftMaximum = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SMAX, *Type, {*Left, *NegativeOne}); + if (!LeftMaximum) + return std::unexpected(LeftMaximum.error()); + auto Low = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *Type, {*LeftMaximum, *Maximum}); + if (!Low) + return std::unexpected(Low.error()); + auto LeftMinimum = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SMIN, *Type, {*Left, *NegativeOne}); + if (!LeftMinimum) + return std::unexpected(LeftMinimum.error()); + auto High = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *Type, {*LeftMinimum, *Minimum}); + if (!High) + return std::unexpected(High.error()); + LowerBound = *Low; + UpperBound = *High; + } + + auto LowerClamped = + buildResultInstruction(MIRBuilder, GenericOpcode::G_SMAX, *Type, {LowerBound, *Right}); + if (!LowerClamped) + return std::unexpected(LowerClamped.error()); + auto ClampedRight = buildResultInstruction(MIRBuilder, GenericOpcode::G_SMIN, *Type, + {*LowerClamped, UpperBound}); + if (!ClampedRight) + return std::unexpected(ClampedRight.error()); + if (auto Built = + MIRBuilder.buildInstr(opcode(ArithmeticOpcode), {*Destination}, {*Left, *ClampedRight}); + !Built) { + return std::unexpected(Built.error()); + } + } else { + Register Available; + if (IsAdd) { + auto AllBits = APInt::getLowBitsSet(BitWidth, BitWidth); + if (!AllBits) + return std::unexpected(AllBits.error()); + auto AllOnes = buildConstantRegister(MIRBuilder, *Type, std::move(*AllBits)); + if (!AllOnes) + return std::unexpected(AllOnes.error()); + auto Inverted = + buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, *Type, {*Left, *AllOnes}); + if (!Inverted) + return std::unexpected(Inverted.error()); + Available = *Inverted; + } else { + Available = *Left; + } + auto Minimum = + buildResultInstruction(MIRBuilder, GenericOpcode::G_UMIN, *Type, {Available, *Right}); + if (!Minimum) + return std::unexpected(Minimum.error()); + if (auto Built = + MIRBuilder.buildInstr(opcode(ArithmeticOpcode), {*Destination}, {*Left, *Minimum}); + !Built) { + return std::unexpected(Built.error()); + } + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerSaturatingAddSubToOverflow(MachineInstr &Instruction) { + if (Instruction.operandCount() != 3) + return makeUnexpected(ErrorCode::VerificationError, + "saturating add/sub overflow lowering requires three register operands"); + + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + const bool IsSigned = Generic == GenericOpcode::G_SADDSAT || Generic == GenericOpcode::G_SSUBSAT; + const bool IsAdd = Generic == GenericOpcode::G_UADDSAT || Generic == GenericOpcode::G_SADDSAT; + const GenericOpcode OverflowOpcode = + IsAdd ? (IsSigned ? GenericOpcode::G_SADDO : GenericOpcode::G_UADDO) + : (IsSigned ? GenericOpcode::G_SSUBO : GenericOpcode::G_USUBO); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + if (Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "saturating add/sub overflow lowering requires register operands"); + + auto Type = MRI.getType(*Destination); + if (!Type) + return std::unexpected(Type.error()); + if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0) + return makeUnexpected(ErrorCode::Unsupported, + "saturating add/sub overflow lowering requires a scalar integer type"); + auto BooleanType = booleanTypeFor(*Type); + if (!BooleanType) + return std::unexpected(BooleanType.error()); + auto Temporary = MRI.createVirtualRegister(*Type); + auto Overflow = MRI.createVirtualRegister(*BooleanType); + if (!Temporary) + return std::unexpected(Temporary.error()); + if (!Overflow) + return std::unexpected(Overflow.error()); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (auto Built = + MIRBuilder.buildInstr(opcode(OverflowOpcode), {*Temporary, *Overflow}, {*Left, *Right}); + !Built) { + return std::unexpected(Built.error()); + } + + Register Clamp; + if (IsSigned) { + const std::uint32_t BitWidth = Type->scalarBitWidth(); + auto ShiftAmount = + buildConstantRegister(MIRBuilder, *Type, static_cast<std::int64_t>(BitWidth - 1U)); + auto MinimumValue = oneBitSet(BitWidth, BitWidth - 1U); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + if (!MinimumValue) + return std::unexpected(MinimumValue.error()); + auto Sign = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, *Type, + {*Temporary, *ShiftAmount}); + auto Minimum = buildConstantRegister(MIRBuilder, *Type, std::move(*MinimumValue)); + if (!Sign) + return std::unexpected(Sign.error()); + if (!Minimum) + return std::unexpected(Minimum.error()); + auto Saturated = + buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *Type, {*Sign, *Minimum}); + if (!Saturated) + return std::unexpected(Saturated.error()); + Clamp = *Saturated; + } else if (IsAdd) { + auto AllBits = APInt::getLowBitsSet(Type->scalarBitWidth(), Type->scalarBitWidth()); + if (!AllBits) + return std::unexpected(AllBits.error()); + auto Saturated = buildConstantRegister(MIRBuilder, *Type, std::move(*AllBits)); + if (!Saturated) + return std::unexpected(Saturated.error()); + Clamp = *Saturated; + } else { + auto Saturated = buildConstantRegister(MIRBuilder, *Type, 0); + if (!Saturated) + return std::unexpected(Saturated.error()); + Clamp = *Saturated; + } + + if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {*Destination}, + {*Overflow, Clamp, *Temporary}); + !Built) { + return std::unexpected(Built.error()); + } + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::lowerSaturatingShiftLeft(MachineInstr &Instruction) { + if (Instruction.operandCount() != 3) + return makeUnexpected(ErrorCode::VerificationError, + "saturating shift-left lowering requires three register operands"); + + const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SSHLSAT); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + if (Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "saturating shift-left lowering requires register operands"); + + auto Type = MRI.getType(*Destination); + if (!Type) + return std::unexpected(Type.error()); + if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0) + return makeUnexpected(ErrorCode::Unsupported, + "saturating shift-left lowering requires a scalar integer type"); + auto BooleanType = booleanTypeFor(*Type); + if (!BooleanType) + return std::unexpected(BooleanType.error()); + const std::uint32_t BitWidth = Type->scalarBitWidth(); + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *Type, {*Left, *Right}); + if (!Shifted) + return std::unexpected(Shifted.error()); + auto Restored = + buildResultInstruction(MIRBuilder, IsSigned ? GenericOpcode::G_ASHR : GenericOpcode::G_LSHR, + *Type, {*Shifted, *Right}); + if (!Restored) + return std::unexpected(Restored.error()); + + Register Saturated; + if (IsSigned) { + auto MinimumValue = oneBitSet(BitWidth, BitWidth - 1U); + auto MaximumValue = APInt::getLowBitsSet(BitWidth, BitWidth - 1U); + if (!MinimumValue) + return std::unexpected(MinimumValue.error()); + if (!MaximumValue) + return std::unexpected(MaximumValue.error()); + auto Minimum = buildConstantRegister(MIRBuilder, *Type, std::move(*MinimumValue)); + auto Maximum = buildConstantRegister(MIRBuilder, *Type, std::move(*MaximumValue)); + auto Zero = buildConstantRegister(MIRBuilder, *Type, 0); + if (!Minimum) + return std::unexpected(Minimum.error()); + if (!Maximum) + return std::unexpected(Maximum.error()); + if (!Zero) + return std::unexpected(Zero.error()); + auto Negative = + buildICmpRegister(MIRBuilder, IntegerPredicate::SignedLessThan, *BooleanType, *Left, *Zero); + if (!Negative) + return std::unexpected(Negative.error()); + auto SignSaturated = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, *Type, + {*Negative, *Minimum, *Maximum}); + if (!SignSaturated) + return std::unexpected(SignSaturated.error()); + Saturated = *SignSaturated; + } else { + auto AllBits = APInt::getLowBitsSet(BitWidth, BitWidth); + if (!AllBits) + return std::unexpected(AllBits.error()); + auto Maximum = buildConstantRegister(MIRBuilder, *Type, std::move(*AllBits)); + if (!Maximum) + return std::unexpected(Maximum.error()); + Saturated = *Maximum; + } + + auto Overflow = + buildICmpRegister(MIRBuilder, IntegerPredicate::NotEqual, *BooleanType, *Left, *Restored); + if (!Overflow) + return std::unexpected(Overflow.error()); + if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {*Destination}, + {*Overflow, Saturated, *Shifted}); + !Built) { + return std::unexpected(Built.error()); + } + + return Instruction.eraseFromParent(); +} + +Expected<void> LegalizerHelper::legalize(MachineInstr &Instruction, + const LegalizationDecision &Decision) { + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + switch (Decision.Action) { + case LegalizationAction::WidenScalar: + return widenScalar(Instruction, Decision.TypeIndex, Decision.NewType); + case LegalizationAction::NarrowScalar: + return narrowScalar(Instruction, Decision.TypeIndex, Decision.NewType); + case LegalizationAction::Custom: + return Info.legalizeCustom(*this, Instruction, Decision); + case LegalizationAction::Lower: + return lower(Instruction, Decision); + case LegalizationAction::Libcall: + return libcall(Instruction); + case LegalizationAction::FewerElements: + return fewerElements(Instruction, Decision.TypeIndex, Decision.NewType); + case LegalizationAction::Legal: + return {}; + case LegalizationAction::Unsupported: + case LegalizationAction::MoreElements: + case LegalizationAction::Bitcast: + case LegalizationAction::ChangeElementType: + case LegalizationAction::ChangeAddressSpace: + return makeUnexpected(ErrorCode::Unsupported, + "legalizer helper does not implement the requested action"); + } + return makeUnexpected(ErrorCode::InternalError, "unknown legalization action"); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp new file mode 100644 index 0000000..bdc783b --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <tuple> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] LegalityPredicate typeInSet(std::size_t TypeIndex, std::vector<LowLevelType> Types) { + return [TypeIndex, Types = std::move(Types)](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && + std::ranges::find(Types, Query.Types[TypeIndex]) != Types.end(); + }; +} + +[[nodiscard]] LegalityPredicate +typePairInSet(std::vector<std::pair<LowLevelType, LowLevelType>> Types) { + return [Types = std::move(Types)](const LegalizationQuery &Query) { + return Query.Types.size() >= 2 && + std::ranges::find(Types, std::pair{Query.Types[0], Query.Types[1]}) != Types.end(); + }; +} + +[[nodiscard]] LegalityPredicate +typeTripleInSet(std::vector<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) { + return [Types = std::move(Types)](const LegalizationQuery &Query) { + return Query.Types.size() >= 3 && + std::ranges::find(Types, std::tuple{Query.Types[0], Query.Types[1], Query.Types[2]}) != + Types.end(); + }; +} + +[[nodiscard]] LowLevelType scalarType(std::uint32_t BitWidth) { + auto Type = LowLevelType::makeCategoryNeutralScalar(BitWidth); + return Type ? *Type : LowLevelType{}; +} + +[[nodiscard]] std::uint32_t nextPowerOfTwo(std::uint32_t Width, std::uint32_t MinimumBitWidth) { + return std::max(std::bit_ceil(Width), MinimumBitWidth); +} + +[[nodiscard]] bool memoryRuleMatches(const LegalizationQuery &Query, + const LegalizeMemoryRule &Rule) { + if (Query.Types.size() < 2 || Query.MemoryOperands.size() != 1 || + Query.Types[0] != Rule.ValueType || Query.Types[1] != Rule.AddressType) + return false; + const LegalizationMemoryDesc &Memory = Query.MemoryOperands.front(); + if (Memory.AlignmentInBits < Rule.MinimumAlignmentInBits) + return false; + if (Memory.Type) + return Memory.Type->minimumBitWidth() == Rule.MemoryType.minimumBitWidth(); + return Memory.Size.isKnown() && !Memory.Size.isScalable() && + Memory.Size.minimumBytes() * 8U == Rule.MemoryType.scalarBitWidth(); +} + +} // namespace + +LegalizeRule::LegalizeRule(LegalityPredicate Predicate, LegalizationAction Action, + LegalizeMutation Mutation) + : Predicate(std::move(Predicate)), Action(Action), Mutation(std::move(Mutation)) {} + +bool LegalizeRule::matches(const LegalizationQuery &Query) const { + return Predicate(Query); +} + +LegalizationDecision LegalizeRule::apply(const LegalizationQuery &Query) const { + if (!Mutation) + return {.Action = Action, .TypeIndex = 0, .NewType = {}}; + auto [TypeIndex, NewType] = Mutation(Query); + return {.Action = Action, .TypeIndex = TypeIndex, .NewType = NewType}; +} + +LegalizeRuleSet &LegalizeRuleSet::actionIf(LegalizationAction Action, LegalityPredicate Predicate, + LegalizeMutation Mutation) { + Rules.emplace_back(std::move(Predicate), Action, std::move(Mutation)); + return *this; +} + +LegalizeRuleSet &LegalizeRuleSet::actionFor(LegalizationAction Action, + std::initializer_list<LowLevelType> Types) { + return actionIf(Action, typeInSet(0, std::vector<LowLevelType>(Types))); +} + +LegalizeRuleSet & +LegalizeRuleSet::actionFor(LegalizationAction Action, + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionIf(Action, typePairInSet(std::vector<std::pair<LowLevelType, LowLevelType>>(Types))); +} + +LegalizeRuleSet &LegalizeRuleSet::actionFor( + LegalizationAction Action, + std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) { + return actionIf( + Action, + typeTripleInSet(std::vector<std::tuple<LowLevelType, LowLevelType, LowLevelType>>(Types))); +} + +LegalizeRuleSet &LegalizeRuleSet::legalIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Legal, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::customIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Custom, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::lowerIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Lower, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::libcallIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Libcall, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::unsupportedIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Unsupported, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::widenScalarIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::WidenScalar, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::narrowScalarIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::NarrowScalar, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::moreElementsIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::MoreElements, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::fewerElementsIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::FewerElements, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::bitcastIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::Bitcast, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::legalFor(std::initializer_list<LowLevelType> Types) { + return actionFor(LegalizationAction::Legal, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::legalFor(bool Enabled, + std::initializer_list<LowLevelType> Types) { + return Enabled ? legalFor(Types) : *this; +} + +LegalizeRuleSet & +LegalizeRuleSet::legalFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Legal, Types); +} + +LegalizeRuleSet & +LegalizeRuleSet::legalFor(bool Enabled, + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return Enabled ? legalFor(Types) : *this; +} + +LegalizeRuleSet &LegalizeRuleSet::legalFor( + std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Legal, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::customFor(std::initializer_list<LowLevelType> Types) { + return actionFor(LegalizationAction::Custom, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::customFor(bool Enabled, + std::initializer_list<LowLevelType> Types) { + return Enabled ? customFor(Types) : *this; +} + +LegalizeRuleSet & +LegalizeRuleSet::customFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Custom, Types); +} + +LegalizeRuleSet & +LegalizeRuleSet::customFor(bool Enabled, + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return Enabled ? customFor(Types) : *this; +} + +LegalizeRuleSet &LegalizeRuleSet::lowerFor(std::initializer_list<LowLevelType> Types) { + return actionFor(LegalizationAction::Lower, Types); +} + +LegalizeRuleSet & +LegalizeRuleSet::lowerFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Lower, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::libcallFor(std::initializer_list<LowLevelType> Types) { + return actionFor(LegalizationAction::Libcall, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::libcallFor(bool Enabled, + std::initializer_list<LowLevelType> Types) { + return Enabled ? libcallFor(Types) : *this; +} + +LegalizeRuleSet & +LegalizeRuleSet::libcallFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Libcall, Types); +} + +LegalizeRuleSet & +LegalizeRuleSet::libcallFor(bool Enabled, + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return Enabled ? libcallFor(Types) : *this; +} + +LegalizeRuleSet & +LegalizeRuleSet::legalForTypesWithMemDesc(std::initializer_list<LegalizeMemoryRule> MemoryRules) { + const std::vector<LegalizeMemoryRule> MemoryRuleList(MemoryRules); + return legalIf([MemoryRuleList](const LegalizationQuery &Query) { + return std::ranges::any_of(MemoryRuleList, [&Query](const LegalizeMemoryRule &Rule) { + return memoryRuleMatches(Query, Rule); + }); + }); +} + +LegalizeRuleSet &LegalizeRuleSet::widenScalarToNextPow2(std::size_t TypeIndex, + std::uint32_t MinimumBitWidth) { + return widenScalarIf( + [TypeIndex, MinimumBitWidth](const LegalizationQuery &Query) { + if (TypeIndex >= Query.Types.size() || !Query.Types[TypeIndex].isCategoryNeutralScalar()) + return false; + const std::uint32_t Width = Query.Types[TypeIndex].scalarBitWidth(); + return nextPowerOfTwo(Width, MinimumBitWidth) > Width; + }, + [TypeIndex, MinimumBitWidth](const LegalizationQuery &Query) { + return std::pair{TypeIndex, scalarType(nextPowerOfTwo( + Query.Types[TypeIndex].scalarBitWidth(), MinimumBitWidth))}; + }); +} + +LegalizeRuleSet &LegalizeRuleSet::scalarize(std::size_t TypeIndex) { + return fewerElementsIf( + [TypeIndex](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isVector(); + }, + [TypeIndex](const LegalizationQuery &Query) { + return std::pair{TypeIndex, *Query.Types[TypeIndex].scalarElementType()}; + }); +} + +LegalizeRuleSet &LegalizeRuleSet::minScalar(std::size_t TypeIndex, LowLevelType MinimumType) { + return widenScalarIf( + [TypeIndex, MinimumType](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isCategoryNeutralScalar() && + Query.Types[TypeIndex].scalarBitWidth() < MinimumType.scalarBitWidth(); + }, + [TypeIndex, MinimumType](const LegalizationQuery &) { + return std::pair{TypeIndex, MinimumType}; + }); +} + +LegalizeRuleSet &LegalizeRuleSet::minScalar(bool Enabled, std::size_t TypeIndex, + LowLevelType MinimumType) { + return Enabled ? minScalar(TypeIndex, MinimumType) : *this; +} + +LegalizeRuleSet &LegalizeRuleSet::maxScalar(std::size_t TypeIndex, LowLevelType MaximumType) { + return narrowScalarIf( + [TypeIndex, MaximumType](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isCategoryNeutralScalar() && + Query.Types[TypeIndex].scalarBitWidth() > MaximumType.scalarBitWidth(); + }, + [TypeIndex, MaximumType](const LegalizationQuery &) { + return std::pair{TypeIndex, MaximumType}; + }); +} + +LegalizeRuleSet &LegalizeRuleSet::maxScalar(bool Enabled, std::size_t TypeIndex, + LowLevelType MaximumType) { + return Enabled ? maxScalar(TypeIndex, MaximumType) : *this; +} + +LegalizeRuleSet &LegalizeRuleSet::clampScalar(std::size_t TypeIndex, LowLevelType MinimumType, + LowLevelType MaximumType) { + return minScalar(TypeIndex, MinimumType).maxScalar(TypeIndex, MaximumType); +} + +LegalizeRuleSet &LegalizeRuleSet::scalarSameSizeAs(std::size_t TypeIndex, + std::size_t ReferenceTypeIndex) { + const auto ChangeToReference = [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) { + return std::pair{TypeIndex, scalarType(Query.Types[ReferenceTypeIndex].scalarBitWidth())}; + }; + widenScalarIf( + [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && ReferenceTypeIndex < Query.Types.size() && + Query.Types[TypeIndex].isCategoryNeutralScalar() && + Query.Types[ReferenceTypeIndex].isCategoryNeutralScalar() && + Query.Types[TypeIndex].scalarBitWidth() < + Query.Types[ReferenceTypeIndex].scalarBitWidth(); + }, + ChangeToReference); + return narrowScalarIf( + [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && ReferenceTypeIndex < Query.Types.size() && + Query.Types[TypeIndex].isCategoryNeutralScalar() && + Query.Types[ReferenceTypeIndex].isCategoryNeutralScalar() && + Query.Types[TypeIndex].scalarBitWidth() > + Query.Types[ReferenceTypeIndex].scalarBitWidth(); + }, + ChangeToReference); +} + +LegalizeRuleSet &LegalizeRuleSet::lowerIfMemSizeNotByteSizePow2(std::size_t MemoryIndex) { + return lowerIf([MemoryIndex](const LegalizationQuery &Query) { + if (MemoryIndex >= Query.MemoryOperands.size()) + return false; + const MemorySize Size = Query.MemoryOperands[MemoryIndex].Size; + return !Size.isKnown() || Size.isScalable() || !std::has_single_bit(Size.minimumBytes()); + }); +} + +LegalizeRuleSet &LegalizeRuleSet::custom() { + return customIf([](const LegalizationQuery &) { return true; }); +} + +LegalizeRuleSet &LegalizeRuleSet::alwaysLegal() { + return legalIf([](const LegalizationQuery &) { return true; }); +} + +LegalizeRuleSet &LegalizeRuleSet::lower() { + return lowerIf([](const LegalizationQuery &) { return true; }); +} + +LegalizeRuleSet &LegalizeRuleSet::libcall() { + return libcallIf([](const LegalizationQuery &) { return true; }); +} + +std::optional<LegalizationDecision> LegalizeRuleSet::apply(const LegalizationQuery &Query) const { + for (const LegalizeRule &Rule : Rules) { + if (Rule.matches(Query)) + return Rule.apply(Query); + } + return std::nullopt; +} + +LegalizeRuleSet &LegalizerInfo::getActionDefinitionsBuilder(Opcode InstructionOpcode) { + return RuleSets[InstructionOpcode]; +} + +LegalizeRuleSet & +LegalizerInfo::getActionDefinitionsBuilder(std::initializer_list<Opcode> InstructionOpcodes) { + const Opcode Representative = *InstructionOpcodes.begin(); + for (const Opcode InstructionOpcode : InstructionOpcodes) { + if (InstructionOpcode != Representative) + RuleAliases.emplace(InstructionOpcode, Representative); + } + return getActionDefinitionsBuilder(Representative); +} + +Expected<LegalizationQuery> LegalizerInfo::buildQuery(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "legalization query requires a registered instruction opcode"); + + std::size_t ExplicitCount = 0; + while (ExplicitCount < Instruction.operandCount() && + !Instruction.operandAt(ExplicitCount)->isImplicit()) { + ++ExplicitCount; + } + auto OperandDescriptors = resolveExplicitOperandDescriptors(*Descriptor, ExplicitCount); + if (!OperandDescriptors) + return std::unexpected(OperandDescriptors.error()); + + std::size_t TypeCount = 0; + for (const OperandDescriptor *Operand : *OperandDescriptors) { + if (Operand->TypeConstraint && Operand->TypeConstraint->EqualityGroup) { + TypeCount = std::max(TypeCount, + static_cast<std::size_t>(*Operand->TypeConstraint->EqualityGroup) + 1U); + } + } + + std::vector<std::optional<LowLevelType>> OptionalTypes(TypeCount); + std::vector<std::vector<std::size_t>> TypeOperands(TypeCount); + for (std::size_t Index = 0; Index < ExplicitCount; ++Index) { + const OperandDescriptor &OperandDescriptor = *(*OperandDescriptors)[Index]; + if (!OperandDescriptor.TypeConstraint || !OperandDescriptor.TypeConstraint->EqualityGroup) { + continue; + } + const std::size_t TypeIndex = *OperandDescriptor.TypeConstraint->EqualityGroup; + const Register *Reg = Instruction.operandAt(Index)->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return makeUnexpected(ErrorCode::VerificationError, + "legalization type operand must name a virtual register"); + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + const std::optional<LowLevelType> &RegisterType = Info->get().Type; + if (!RegisterType.has_value()) + return makeUnexpected(ErrorCode::VerificationError, + "legalization type operand has no LowLevelType"); + std::optional<LowLevelType> &ExistingType = OptionalTypes[TypeIndex]; + if (ExistingType.has_value() && ExistingType.value() != RegisterType.value()) + return makeUnexpected(ErrorCode::VerificationError, + "legalization type operands disagree within one type index"); + ExistingType = RegisterType.value(); + TypeOperands[TypeIndex].push_back(Index); + } + + std::vector<LowLevelType> Types; + Types.reserve(TypeCount); + for (const std::optional<LowLevelType> &Type : OptionalTypes) { + if (!Type) + return makeUnexpected(ErrorCode::VerificationError, + "legalization query contains a sparse type index"); + Types.push_back(*Type); + } + + std::vector<LegalizationMemoryDesc> MemoryOperands; + MemoryOperands.reserve(Instruction.memOperands().size()); + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + const MachineMemOperandDesc &Description = Memory->description(); + MemoryOperands.push_back({Description.MemoryType, Description.Size, + Description.Align.bytes() * 8U, Description.AddressSpace, + Description.AtomicKind, Description.SuccessOrdering, + Description.FailureOrdering, Description.SyncScope}); + } + + LegalizationQuery Query(Instruction.opcode(), std::move(Types), std::move(MemoryOperands)); + Query.TypeOperands = std::move(TypeOperands); + return Query; +} + +Expected<LegalizationDecision> LegalizerInfo::getDecision(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) const { + auto Query = buildQuery(Instruction, Function, Target); + if (!Query) + return std::unexpected(Query.error()); + return getDecision(*Query); +} + +Expected<LegalizationDecision> LegalizerInfo::getDecision(const LegalizationQuery &Query) const { + if (!isGenericOpcode(Query.InstructionOpcode)) + return LegalizationDecision{.Action = LegalizationAction::Legal, .TypeIndex = 0, .NewType = {}}; + + Opcode RuleOpcode = Query.InstructionOpcode; + if (const auto Alias = RuleAliases.find(RuleOpcode); Alias != RuleAliases.end()) + RuleOpcode = Alias->second; + const auto RuleSet = RuleSets.find(RuleOpcode); + if (RuleSet == RuleSets.end()) + return LegalizationDecision{ + .Action = LegalizationAction::Unsupported, .TypeIndex = 0, .NewType = {}}; + const std::optional<LegalizationDecision> Decision = RuleSet->second.apply(Query); + return Decision.value_or(LegalizationDecision{ + .Action = LegalizationAction::Unsupported, .TypeIndex = 0, .NewType = {}}); +} + +Expected<bool> LegalizerInfo::isLegal(const LegalizationQuery &Query) const { + auto Decision = getDecision(Query); + if (!Decision) + return std::unexpected(Decision.error()); + return Decision->Action == LegalizationAction::Legal; +} + +Expected<bool> LegalizerInfo::isUnsupported(const LegalizationQuery &Query) const { + auto Decision = getDecision(Query); + if (!Decision) + return std::unexpected(Decision.error()); + return Decision->Action == LegalizationAction::Unsupported; +} + +GenericOpcode LegalizerInfo::getExtOpcodeForWideningConstant(LowLevelType SmallType) const { + return SmallType.minimumBitWidth() % 8U == 0 ? GenericOpcode::G_SEXT : GenericOpcode::G_ZEXT; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp new file mode 100644 index 0000000..32f2f9f --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/RegBankSelect.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cstdint> +#include <optional> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &bankSelectContract() { + static const MachinePassContract Contract{ + .Name = "regbankselect", + .Requires = {MachineFunctionProperty::Legalized}, + .Sets = {MachineFunctionProperty::RegBankSelected}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegBankSelected, + }; + return Contract; +} + +void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept { + State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U); +} + +[[nodiscard]] std::uint64_t +registerConfigurationFingerprint(const MachineRegisterInfo &RegInfo) noexcept { + std::uint64_t Result = 0xcbf29ce484222325ULL; + for (const VirtualRegisterInfo &Info : RegInfo.virtualRegisters()) { + combineFingerprint(Result, Info.IsTombstone ? 1U : 0U); + combineFingerprint(Result, Info.Bank ? Info.Bank->value() : 0U); + combineFingerprint(Result, Info.RegClass ? Info.RegClass->value() : 0U); + } + return Result; +} + +[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) { + std::vector<MachineInstr *> Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) + Result.push_back(Instruction); + return Result; +} + +[[nodiscard]] Expected<void> restoreRegisterConfiguration( + MachineRegisterInfo &RegInfo, + const std::vector<std::pair<Register, std::pair<std::optional<RegisterBankID>, + std::optional<RegisterClassID>>>> &Snapshot) { + for (const auto &[Reg, Configuration] : Snapshot) { + auto ClearBank = RegInfo.clearRegisterBank(Reg); + if (!ClearBank) + return std::unexpected(ClearBank.error()); + if (Configuration.first) { + auto SetBank = RegInfo.setRegisterBank(Reg, *Configuration.first); + if (!SetBank) + return std::unexpected(SetBank.error()); + } + auto ClearClass = RegInfo.clearRegisterClass(Reg); + if (!ClearClass) + return std::unexpected(ClearClass.error()); + if (Configuration.second) { + auto SetClass = RegInfo.setRegisterClass(Reg, *Configuration.second); + if (!SetClass) + return std::unexpected(SetClass.error()); + } + } + return {}; +} + +[[nodiscard]] Error passFailure(std::string Message, + ErrorCode Code = ErrorCode::VerificationError) { + return Error(Code, std::move(Message)); +} + +struct PlannedRegisterBank final { + Register Reg; + RegisterBankID Bank; + bool IsConstrained = false; +}; + +struct MappingMutation final { + MachineOperand *Operand = nullptr; + Register Original; + Register Fresh; + MachineInstr *Copy = nullptr; +}; + +[[nodiscard]] Expected<MachineInstr *> buildBankCopy(MachineFunction &Function, + const TargetDescription &Target, + MachineInstr &Instruction, bool InsertBefore, + Register Destination, Register Source) { + MachineIRBuilder Builder(Function, Target); + auto Point = InsertBefore ? Builder.setInsertPointBefore(Instruction) + : Builder.setInsertPointAfter(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::COPY)); + if (!Pending) + return std::unexpected(Pending.error()); + auto DestinationOperand = Pending->addDef(Destination); + if (!DestinationOperand) + return std::unexpected(DestinationOperand.error()); + auto SourceOperand = Pending->addUse(Source); + if (!SourceOperand) + return std::unexpected(SourceOperand.error()); + if (Instruction.debugLocation()) { + auto Debug = Pending->setDebugLocation(*Instruction.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + return Pending->build(); +} + +[[nodiscard]] Expected<void> repairMappedOperand(MachineFunction &Function, + const TargetDescription &Target, + MachineInstr &Instruction, MachineOperand &Operand, + RegisterBankID RequiredBank, + std::vector<MappingMutation> &Mutations) { + const Register *Original = Operand.getIf<Register>(); + if (Original == nullptr || !Original->isVirtual() || Operand.isDef() == Operand.isUse()) { + return makeUnexpected(ErrorCode::VerificationError, + "register-bank repair requires a virtual def or use operand"); + } + const Register OriginalReg = *Original; + auto OriginalInfo = Function.registerInfo().virtualRegisterInfo(OriginalReg); + if (!OriginalInfo) + return std::unexpected(OriginalInfo.error()); + const std::optional<LowLevelType> &OriginalType = OriginalInfo->get().Type; + if (!OriginalType) { + return makeUnexpected(ErrorCode::VerificationError, + "register-bank repair requires a virtual-register type"); + } + if (Operand.subReg().value() != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "register-bank repair does not support subregister operands"); + } + + auto Fresh = Function.registerInfo().createVirtualRegister(*OriginalType); + if (!Fresh) + return std::unexpected(Fresh.error()); + auto SetBank = Function.registerInfo().setRegisterBank(*Fresh, RequiredBank); + if (!SetBank) { + auto Erased = Function.registerInfo().eraseVirtualRegister(*Fresh); + (void)Erased; + return std::unexpected(SetBank.error()); + } + + auto Copy = Operand.isDef() + ? buildBankCopy(Function, Target, Instruction, false, OriginalReg, *Fresh) + : buildBankCopy(Function, Target, Instruction, true, *Fresh, OriginalReg); + if (!Copy) { + auto Erased = Function.registerInfo().eraseVirtualRegister(*Fresh); + (void)Erased; + return std::unexpected(Copy.error()); + } + auto Replaced = Operand.setRegister(*Fresh); + if (!Replaced) { + if ((*Copy)->parent() != nullptr) { + auto Removed = (*Copy)->eraseFromParent(); + (void)Removed; + } + auto Erased = Function.registerInfo().eraseVirtualRegister(*Fresh); + (void)Erased; + return std::unexpected(Replaced.error()); + } + Mutations.push_back(MappingMutation{ + .Operand = &Operand, .Original = OriginalReg, .Fresh = *Fresh, .Copy = *Copy}); + return {}; +} + +[[nodiscard]] Expected<void> rollbackMappingMutations(MachineRegisterInfo &RegInfo, + std::vector<MappingMutation> &Mutations) { + for (auto Iterator = Mutations.rbegin(); Iterator != Mutations.rend(); ++Iterator) { + auto Restored = Iterator->Operand->setRegister(Iterator->Original); + if (!Restored) + return std::unexpected(Restored.error()); + if (Iterator->Copy != nullptr && Iterator->Copy->parent() != nullptr) { + auto Removed = Iterator->Copy->eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + } + auto Erased = RegInfo.eraseVirtualRegister(Iterator->Fresh); + if (!Erased) + return std::unexpected(Erased.error()); + } + Mutations.clear(); + return {}; +} + +[[nodiscard]] Expected<void> +applyInstructionMappings(MachineFunction &Function, const TargetDescription &Target, + const RegisterBankInfo &Info, + std::unordered_set<Register> &ConstrainedRegisters, + std::vector<MappingMutation> &Mutations, MachinePassResult &Result) { + MachineRegisterInfo &RegInfo = Function.registerInfo(); + const std::vector<MachineInstr *> Instructions = layoutInstructions(Function); + for (MachineInstr *Instruction : Instructions) { + auto Mapping = Info.getInstructionMapping(*Instruction, Function, Target); + if (!Mapping) + return std::unexpected(Mapping.error()); + if (Mapping->empty()) + continue; + if (Mapping->size() != Instruction->operandCount()) { + return makeUnexpected(ErrorCode::InternalError, + "target register-bank mapping has the wrong operand count"); + } + for (std::size_t Index = 0; Index < Mapping->size(); ++Index) { + const std::optional<RegisterBankID> &MappedBank = (*Mapping)[Index]; + if (!MappedBank) + continue; + const RegisterBankID RequiredBank = *MappedBank; + if (Target.registers().registerBank(RequiredBank) == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "target instruction mapping names an unknown register bank"); + } + MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "target register-bank mapping requires a virtual-register operand"); + } + auto RegisterInfo = RegInfo.virtualRegisterInfo(*Reg); + if (!RegisterInfo) + return std::unexpected(RegisterInfo.error()); + const std::optional<RegisterBankID> &CurrentBank = RegisterInfo->get().Bank; + if (!CurrentBank) { + auto SetBank = RegInfo.setRegisterBank(*Reg, RequiredBank); + if (!SetBank) + return std::unexpected(SetBank.error()); + ConstrainedRegisters.insert(*Reg); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Registers); + continue; + } + if (*CurrentBank == RequiredBank) { + ConstrainedRegisters.insert(*Reg); + continue; + } + if (!ConstrainedRegisters.contains(*Reg) && !RegisterInfo->get().RegClass) { + auto SetBank = RegInfo.setRegisterBank(*Reg, RequiredBank); + if (!SetBank) + return std::unexpected(SetBank.error()); + ConstrainedRegisters.insert(*Reg); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Registers); + continue; + } + auto Repaired = + repairMappedOperand(Function, Target, *Instruction, *Operand, RequiredBank, Mutations); + if (!Repaired) + return std::unexpected(Repaired.error()); + ConstrainedRegisters.insert(Mutations.back().Fresh); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + Result.Changes.add(MachinePassChange::Operands); + Result.Changes.add(MachinePassChange::Registers); + } + } + return {}; +} + +} // namespace + +const MachinePassContract &RegBankSelect::contract() const noexcept { + return bankSelectContract(); +} + +Expected<MachinePassResult> RegBankSelect::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + if (Function.properties().has(MachineFunctionProperty::FailedISel)) + return MachinePassResult{ + .Changed = false, + .Changes = {}, + .Preserved = PreservedAnalyses::all(), + }; + if (Info == nullptr) + return std::unexpected(passFailure("register-bank select has no target behavior interface", + ErrorCode::InvalidArgument)); + MachinePassResult Result; + MachineRegisterInfo &RegInfo = Function.registerInfo(); + const std::uint64_t RegisterConfigurationBefore = registerConfigurationFingerprint(RegInfo); + using RegisterConfiguration = + std::pair<std::optional<RegisterBankID>, std::optional<RegisterClassID>>; + std::vector<std::pair<Register, RegisterConfiguration>> Snapshot; + std::vector<PlannedRegisterBank> PlannedBanks; + std::unordered_set<Register> ConstrainedRegisters; + for (std::size_t Index = 0; Index < RegInfo.virtualRegisters().size(); ++Index) { + auto Reg = Register::makeVirtual(Index); + if (!Reg) + return std::unexpected(Reg.error()); + const VirtualRegisterInfo &StoredInfo = RegInfo.virtualRegisters()[Index]; + if (StoredInfo.IsTombstone) + continue; + Snapshot.emplace_back(*Reg, RegisterConfiguration{StoredInfo.Bank, StoredInfo.RegClass}); + auto InfoResult = RegInfo.virtualRegisterInfo(*Reg); + if (!InfoResult) + return std::unexpected(InfoResult.error()); + const VirtualRegisterInfo &InfoValue = InfoResult->get(); + const MachineInstr *Context = nullptr; + for (const MachineOperand &Operand : RegInfo.regOperands(*Reg)) { + Context = Operand.parent(); + if (Context != nullptr) + break; + } + auto Bank = Info->selectBank(Function, *Reg, InfoValue, Context, Target); + if (!Bank) + return std::unexpected(Bank.error()); + if (Target.registers().registerBank(Bank->Bank) == nullptr) + return std::unexpected(passFailure("target register-bank selector returned an unknown bank")); + if (InfoValue.Bank && *InfoValue.Bank != Bank->Bank) + return std::unexpected( + passFailure("target register-bank selector conflicts with an existing bank")); + if (InfoValue.RegClass && !Target.registers().bankCoversClass(Bank->Bank, *InfoValue.RegClass)) + return std::unexpected( + passFailure("existing register class is not covered by selected bank")); + PlannedBanks.push_back(PlannedRegisterBank{.Reg = *Reg, + .Bank = Bank->Bank, + .IsConstrained = Bank->IsConstrained || + InfoValue.RegClass.has_value()}); + } + + for (MachineInstr *Instruction : layoutInstructions(Function)) { + auto Validation = Info->validateInstruction(*Instruction, Function, Target); + if (!Validation) + return std::unexpected( + withErrorContext(Validation.error(), "register-bank mapping precondition failed: ")); + } + + for (const PlannedRegisterBank &Plan : PlannedBanks) { + const auto InfoResult = RegInfo.virtualRegisterInfo(Plan.Reg); + if (!InfoResult) + return std::unexpected(InfoResult.error()); + if (!InfoResult->get().Bank) { + auto SetBank = RegInfo.setRegisterBank(Plan.Reg, Plan.Bank); + if (!SetBank) + return std::unexpected(SetBank.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Registers); + } + if (Plan.IsConstrained) + ConstrainedRegisters.insert(Plan.Reg); + } + + std::vector<MappingMutation> MappingMutations; + const auto Restore = [&]() -> Expected<void> { + auto MappingRestore = rollbackMappingMutations(RegInfo, MappingMutations); + if (!MappingRestore) + return std::unexpected(MappingRestore.error()); + return restoreRegisterConfiguration(RegInfo, Snapshot); + }; + auto Mapped = applyInstructionMappings(Function, Target, *Info, ConstrainedRegisters, + MappingMutations, Result); + if (!Mapped) { + auto Restored = Restore(); + if (!Restored) + return std::unexpected(withErrorContext(Restored.error(), "register-bank rollback failed: ")); + return std::unexpected(withErrorContext(Mapped.error(), "register-bank mapping failed: ")); + } + for (MachineInstr *Instruction : layoutInstructions(Function)) { + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) { + auto Restored = Restore(); + if (!Restored) + return std::unexpected( + withErrorContext(Restored.error(), "register-bank rollback failed: ")); + return std::unexpected(withErrorContext( + Validation.error(), "register-bank selection produced invalid MachineIR: ")); + } + } + auto RegisterValidation = Target.validateRegisterInfo(RegInfo); + if (!RegisterValidation) { + auto Restored = Restore(); + if (!Restored) + return std::unexpected(withErrorContext(Restored.error(), "register-bank rollback failed: ")); + return std::unexpected(withErrorContext( + RegisterValidation.error(), "register-bank selection produced invalid registers: ")); + } + + if (registerConfigurationFingerprint(RegInfo) != RegisterConfigurationBefore) { + Result.Changed = true; + Result.Changes.add(MachinePassChange::Registers); + } + + Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/Utils.cpp b/backend/lib/CodeGen/GlobalISel/Utils.cpp new file mode 100644 index 0000000..f9d6b0a --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/Utils.cpp @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/Utils.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cassert> +#include <cstdint> +#include <numeric> +#include <optional> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.flags().has(MachineInstrFlag::FrameSetup) || + Instruction.flags().has(MachineInstrFlag::FrameDestroy) || + Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool wouldBeTriviallyDead(const MachineInstr &Instruction, + const InstructionDescriptor &Descriptor) noexcept { + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + case GenericOpcode::LOCAL_ESCAPE: + case GenericOpcode::FAKE_USE: + case GenericOpcode::LIFETIME_START: + case GenericOpcode::LIFETIME_END: + return false; + default: + break; + } + + const InstructionFlags Flags = Descriptor.Flags; + if (Flags.has(InstructionFlag::Phi)) + return true; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayLoad) || + Flags.has(InstructionFlag::MayStore) || Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || + Flags.has(InstructionFlag::MayRaiseFPException) || hasProtectedAttachment(Instruction)) + return false; + return true; +} + +[[nodiscard]] Expected<void> saveUsesAndErase(MachineInstr &Instruction, MachineRegisterInfo &MRI, + std::vector<MachineInstr *> &DeadChain) { + for (const MachineOperand &Operand : Instruction.operands()) { + if (!Operand.isRegister() || !Operand.isUse()) + continue; + const std::optional<Register> Reg = Operand.registerValue(); + if (!Reg || !Reg->isVirtual()) + continue; + MachineInstr *Definition = MRI.getVRegDef(*Reg); + if (Definition != nullptr && Definition != &Instruction) + DeadChain.push_back(Definition); + } + return Instruction.eraseFromParent(); +} + +[[nodiscard]] Expected<LowLevelType> makeVectorType(std::uint64_t ElementCount, bool IsScalable, + LowLevelType ElementType) { + if (!IsScalable && ElementCount == 1) + return ElementType; + return IsScalable ? LowLevelType::makeScalableVector(ElementCount, ElementType) + : LowLevelType::makeFixedVector(ElementCount, ElementType); +} + +[[nodiscard]] Expected<void> extractVectorParts(Register Reg, std::size_t ElementCount, + std::vector<Register> &Parts, + MachineIRBuilder &MIRBuilder, + MachineRegisterInfo &MRI) { + auto RegisterType = MRI.getType(Reg); + if (!RegisterType) + return std::unexpected(RegisterType.error()); + if (!RegisterType->isVector() || RegisterType->isScalableVector() || ElementCount == 0) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "vector part extraction requires a fixed vector and a nonzero element count"); + } + + const std::optional<LowLevelType> ElementTypeValue = RegisterType->scalarElementType(); + assert(ElementTypeValue && "validated vector type must have an element type"); + const LowLevelType ElementType = *ElementTypeValue; + auto NarrowType = makeVectorType(ElementCount, false, ElementType); + if (!NarrowType) + return std::unexpected(NarrowType.error()); + const std::size_t RegisterElementCount = RegisterType->minimumElementCount(); + const std::size_t LeftoverElementCount = RegisterElementCount % ElementCount; + const std::size_t NarrowPartCount = RegisterElementCount / ElementCount; + + if (LeftoverElementCount == 0) + return extractParts(Reg, *NarrowType, static_cast<int>(NarrowPartCount), Parts, MIRBuilder, + MRI); + + std::vector<Register> Elements; + auto Extracted = extractParts(Reg, ElementType, static_cast<int>(RegisterElementCount), Elements, + MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + + std::size_t Offset = 0; + for (std::size_t Index = 0; Index != NarrowPartCount; ++Index, Offset += ElementCount) { + std::vector<Register> Piece(Elements.begin() + static_cast<std::ptrdiff_t>(Offset), + Elements.begin() + + static_cast<std::ptrdiff_t>(Offset + ElementCount)); + auto Part = MRI.createVirtualRegister(*NarrowType); + if (!Part) + return std::unexpected(Part.error()); + auto Built = MIRBuilder.buildMergeLikeInstr(*Part, Piece); + if (!Built) + return std::unexpected(Built.error()); + Parts.push_back(*Part); + } + + if (LeftoverElementCount == 1) { + Parts.push_back(Elements[Offset]); + return {}; + } + + auto LeftoverType = LowLevelType::makeFixedVector(LeftoverElementCount, ElementType); + if (!LeftoverType) + return std::unexpected(LeftoverType.error()); + std::vector<Register> LeftoverElements(Elements.begin() + static_cast<std::ptrdiff_t>(Offset), + Elements.end()); + auto Leftover = MRI.createVirtualRegister(*LeftoverType); + if (!Leftover) + return std::unexpected(Leftover.error()); + auto Built = MIRBuilder.buildMergeLikeInstr(*Leftover, LeftoverElements); + if (!Built) + return std::unexpected(Built.error()); + Parts.push_back(*Leftover); + return {}; +} + +} // namespace + +std::optional<ValueAndVReg> getIConstantVRegValWithLookThrough(Register VReg, + const MachineRegisterInfo &MRI, + bool LookThroughInstrs) { + std::vector<std::pair<GenericOpcode, std::uint32_t>> SeenOpcodes; + const MachineInstr *Instruction = nullptr; + + while ((Instruction = MRI.getVRegDef(VReg)) != nullptr && + Instruction->opcode() != opcode(GenericOpcode::G_CONSTANT) && LookThroughInstrs) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction->opcode().value()); + switch (Generic) { + case GenericOpcode::G_ANYEXT: + return std::nullopt; + case GenericOpcode::G_TRUNC: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ZEXT: { + const MachineOperand *DestinationOperand = Instruction->operandAt(0); + const MachineOperand *SourceOperand = Instruction->operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Destination == nullptr || Source == nullptr) + return std::nullopt; + auto Type = MRI.getType(*Destination); + if (!Type) + return std::nullopt; + SeenOpcodes.emplace_back(Generic, Type->scalarBitWidth()); + VReg = *Source; + break; + } + case GenericOpcode::COPY: { + const MachineOperand *SourceOperand = Instruction->operandAt(1); + const Register *Source = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Source == nullptr || Source->isPhysical()) + return std::nullopt; + VReg = *Source; + break; + } + case GenericOpcode::G_INTTOPTR: { + const MachineOperand *SourceOperand = Instruction->operandAt(1); + const Register *Source = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Source == nullptr) + return std::nullopt; + VReg = *Source; + break; + } + default: + return std::nullopt; + } + } + + if (Instruction == nullptr || Instruction->opcode() != opcode(GenericOpcode::G_CONSTANT)) + return std::nullopt; + const MachineOperand *ValueOperand = Instruction->operandAt(1); + const auto *Immediate = + ValueOperand == nullptr ? nullptr : ValueOperand->getIf<IntegerImmediateOperand>(); + if (Immediate == nullptr) + return std::nullopt; + + APInt Value = Immediate->Value; + for (auto Iterator = SeenOpcodes.rbegin(); Iterator != SeenOpcodes.rend(); ++Iterator) { + Expected<APInt> Transformed = [&]() -> Expected<APInt> { + switch (Iterator->first) { + case GenericOpcode::G_TRUNC: + return Value.trunc(Iterator->second); + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_SEXT: + return Value.sext(Iterator->second); + case GenericOpcode::G_ZEXT: + return Value.zext(Iterator->second); + default: + return makeUnexpected(ErrorCode::InternalError, + "unexpected opcode in constant look-through chain"); + } + }(); + if (!Transformed) + return std::nullopt; + Value = std::move(*Transformed); + } + return ValueAndVReg{.Value = std::move(Value), .VReg = VReg}; +} + +bool isTriviallyDead(const MachineInstr &Instruction, const MachineRegisterInfo &MRI, + const TargetDescription &Target) noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + if (!Operand.isRegister() || !Operand.isDef()) + continue; + const std::optional<Register> Reg = Operand.registerValue(); + if (!Reg || Reg->isPhysical() || !MRI.useEmpty(*Reg, false)) + return false; + } + + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + return Descriptor != nullptr && wouldBeTriviallyDead(Instruction, *Descriptor); +} + +Expected<void> eraseInstrs(std::span<MachineInstr *const> Instructions, MachineRegisterInfo &MRI, + const TargetDescription &Target) { + std::vector<MachineInstr *> DeadChain; + for (MachineInstr *Instruction : Instructions) { + if (Instruction == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction erasure list contains a null instruction"); + auto Removed = saveUsesAndErase(*Instruction, MRI, DeadChain); + if (!Removed) + return std::unexpected(Removed.error()); + } + + while (!DeadChain.empty()) { + MachineInstr *Instruction = DeadChain.back(); + DeadChain.pop_back(); + if (Instruction->parent() == nullptr || !isTriviallyDead(*Instruction, MRI, Target)) + continue; + auto Removed = saveUsesAndErase(*Instruction, MRI, DeadChain); + if (!Removed) + return std::unexpected(Removed.error()); + } + return {}; +} + +Expected<void> eraseInstr(MachineInstr &Instruction, MachineRegisterInfo &MRI, + const TargetDescription &Target) { + MachineInstr *Instructions[] = {&Instruction}; + return eraseInstrs(Instructions, MRI, Target); +} + +Expected<void> extractParts(Register Reg, LowLevelType Type, int PartCount, + std::vector<Register> &Parts, MachineIRBuilder &MIRBuilder, + MachineRegisterInfo &MRI) { + std::vector<Register> NewParts; + if (PartCount < 0) + return makeUnexpected(ErrorCode::InvalidArgument, "part count cannot be negative"); + NewParts.reserve(static_cast<std::size_t>(PartCount)); + for (int Index = 0; Index != PartCount; ++Index) { + auto Part = MRI.createVirtualRegister(Type); + if (!Part) + return std::unexpected(Part.error()); + NewParts.push_back(*Part); + } + auto Built = MIRBuilder.buildUnmergeValues(NewParts, Reg); + if (!Built) + return std::unexpected(Built.error()); + Parts.insert(Parts.end(), NewParts.begin(), NewParts.end()); + return {}; +} + +Expected<bool> extractParts(Register Reg, LowLevelType RegisterType, LowLevelType MainType, + LowLevelType &LeftoverType, std::vector<Register> &Parts, + std::vector<Register> &LeftoverParts, MachineIRBuilder &MIRBuilder, + MachineRegisterInfo &MRI) { + if (LeftoverType.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "part extraction leftover type must be an output"); + } + + const std::uint64_t RegisterSize = RegisterType.minimumBitWidth(); + const std::uint64_t MainSize = MainType.minimumBitWidth(); + if (MainSize == 0 || MainSize > RegisterSize) { + return makeUnexpected(ErrorCode::InvalidArgument, + "part extraction requires a smaller nonzero main type"); + } + const std::size_t PartCount = RegisterSize / MainSize; + const std::uint64_t LeftoverSize = RegisterSize - PartCount * MainSize; + + if (LeftoverSize == 0) { + auto Extracted = + extractParts(Reg, MainType, static_cast<int>(PartCount), Parts, MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + return true; + } + + if (RegisterType.isVector() && MainType.isVector() && !RegisterType.isScalableVector() && + !MainType.isScalableVector()) { + const std::size_t RegisterElementCount = RegisterType.minimumElementCount(); + const std::size_t MainElementCount = MainType.minimumElementCount(); + const std::size_t LeftoverElementCount = RegisterElementCount % MainElementCount; + if (LeftoverElementCount > 1 && MainElementCount % LeftoverElementCount == 0 && + RegisterElementCount % LeftoverElementCount == 0 && + RegisterType.scalarBitWidth() == MainType.scalarBitWidth()) { + const std::optional<LowLevelType> ElementType = RegisterType.scalarElementType(); + assert(ElementType && "validated vector type must have an element type"); + auto Candidate = LowLevelType::makeFixedVector(LeftoverElementCount, *ElementType); + if (!Candidate) + return std::unexpected(Candidate.error()); + LeftoverType = *Candidate; + + std::vector<Register> Unmerged; + auto Extracted = extractParts(Reg, LeftoverType, + static_cast<int>(RegisterElementCount / LeftoverElementCount), + Unmerged, MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + + const std::size_t LeftoversPerMain = MainElementCount / LeftoverElementCount; + const std::size_t LeftoverValueCount = + (RegisterElementCount % MainElementCount) / LeftoverElementCount; + std::vector<Register> MergeValues; + for (std::size_t Index = 0; Index != Unmerged.size() - LeftoverValueCount; ++Index) { + MergeValues.push_back(Unmerged[Index]); + if (MergeValues.size() != LeftoversPerMain) + continue; + auto Part = MRI.createVirtualRegister(MainType); + if (!Part) + return std::unexpected(Part.error()); + auto Built = MIRBuilder.buildMergeLikeInstr(*Part, MergeValues); + if (!Built) + return std::unexpected(Built.error()); + Parts.push_back(*Part); + MergeValues.clear(); + } + LeftoverParts.insert(LeftoverParts.end(), + Unmerged.end() - static_cast<std::ptrdiff_t>(LeftoverValueCount), + Unmerged.end()); + return true; + } + } + + if (MainType.isVector()) { + std::vector<Register> RegisterPieces; + auto Extracted = + extractVectorParts(Reg, MainType.minimumElementCount(), RegisterPieces, MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + Parts.insert(Parts.end(), RegisterPieces.begin(), std::prev(RegisterPieces.end())); + LeftoverParts.push_back(RegisterPieces.back()); + auto Type = MRI.getType(LeftoverParts.front()); + if (!Type) + return std::unexpected(Type.error()); + LeftoverType = *Type; + return true; + } + + auto ScalarLeftover = LowLevelType::makeCategoryNeutralScalar(LeftoverSize); + if (!ScalarLeftover) + return std::unexpected(ScalarLeftover.error()); + LeftoverType = *ScalarLeftover; + for (std::size_t Index = 0; Index != PartCount; ++Index) { + auto Part = MRI.createVirtualRegister(MainType); + if (!Part) + return std::unexpected(Part.error()); + auto Built = MIRBuilder.buildExtract(*Part, Reg, MainSize * Index); + if (!Built) + return std::unexpected(Built.error()); + Parts.push_back(*Part); + } + for (std::uint64_t Offset = MainSize * PartCount; Offset < RegisterSize; Offset += LeftoverSize) { + auto Part = MRI.createVirtualRegister(LeftoverType); + if (!Part) + return std::unexpected(Part.error()); + auto Built = MIRBuilder.buildExtract(*Part, Reg, Offset); + if (!Built) + return std::unexpected(Built.error()); + LeftoverParts.push_back(*Part); + } + return true; +} + +Expected<LowLevelType> getLCMType(LowLevelType OriginalType, LowLevelType TargetType) { + if (OriginalType.minimumBitWidth() == TargetType.minimumBitWidth()) + return OriginalType; + if ((OriginalType.isScalableVector() && TargetType.isVector() && + !TargetType.isScalableVector()) || + (TargetType.isScalableVector() && OriginalType.isVector() && + !OriginalType.isScalableVector())) { + return makeUnexpected(ErrorCode::Unsupported, + "LCM type is undefined between fixed and scalable vectors"); + } + + if (OriginalType.isVector() && TargetType.isVector()) { + const LowLevelType OriginalElement = *OriginalType.scalarElementType(); + const LowLevelType TargetElement = *TargetType.scalarElementType(); + if (OriginalElement.minimumBitWidth() == TargetElement.minimumBitWidth()) { + const std::uint64_t ElementCount = + std::lcm(static_cast<std::uint64_t>(OriginalType.minimumElementCount()), + static_cast<std::uint64_t>(TargetType.minimumElementCount())); + return makeVectorType(ElementCount, OriginalType.isScalableVector(), OriginalElement); + } + const std::uint64_t Width = + std::lcm(OriginalType.minimumBitWidth(), TargetType.minimumBitWidth()); + return makeVectorType(Width / OriginalElement.minimumBitWidth(), + OriginalType.isScalableVector(), OriginalElement); + } + + if (OriginalType.isVector() || TargetType.isVector()) { + const LowLevelType VectorType = OriginalType.isVector() ? OriginalType : TargetType; + const LowLevelType ScalarType = OriginalType.isVector() ? TargetType : OriginalType; + const LowLevelType VectorElement = *VectorType.scalarElementType(); + const LowLevelType OriginalElement = + OriginalType.isVector() ? *OriginalType.scalarElementType() : OriginalType; + if (VectorElement.minimumBitWidth() == ScalarType.minimumBitWidth()) + return makeVectorType(VectorType.minimumElementCount(), VectorType.isScalableVector(), + OriginalElement); + const std::uint64_t Width = + std::lcm(VectorType.minimumBitWidth(), ScalarType.minimumBitWidth()); + return makeVectorType(Width / OriginalElement.minimumBitWidth(), VectorType.isScalableVector(), + OriginalElement); + } + + const std::uint64_t Width = + std::lcm(OriginalType.minimumBitWidth(), TargetType.minimumBitWidth()); + if (Width == OriginalType.minimumBitWidth()) + return OriginalType; + if (Width == TargetType.minimumBitWidth()) + return TargetType; + return LowLevelType::makeCategoryNeutralScalar(Width); +} + +Expected<LowLevelType> getGCDType(LowLevelType OriginalType, LowLevelType TargetType) { + if (OriginalType.minimumBitWidth() == TargetType.minimumBitWidth()) + return OriginalType; + if ((OriginalType.isScalableVector() && TargetType.isVector() && + !TargetType.isScalableVector()) || + (TargetType.isScalableVector() && OriginalType.isVector() && + !OriginalType.isScalableVector())) { + return makeUnexpected(ErrorCode::Unsupported, + "GCD type is undefined between fixed and scalable vectors"); + } + + if (OriginalType.isVector() && TargetType.isVector()) { + const LowLevelType OriginalElement = *OriginalType.scalarElementType(); + const std::uint64_t Width = + std::gcd(OriginalType.minimumBitWidth(), TargetType.minimumBitWidth()); + if (Width == OriginalElement.minimumBitWidth()) + return makeVectorType(1, OriginalType.isScalableVector(), OriginalElement); + if (Width < OriginalElement.minimumBitWidth()) + return LowLevelType::makeCategoryNeutralScalar(Width); + return makeVectorType(Width / OriginalElement.minimumBitWidth(), + OriginalType.isScalableVector(), OriginalElement); + } + + if (OriginalType.isVector() && OriginalType.scalarBitWidth() == TargetType.minimumBitWidth()) + return *OriginalType.scalarElementType(); + if (TargetType.isVector() && TargetType.scalarBitWidth() == OriginalType.minimumBitWidth()) + return OriginalType; + + const std::uint64_t OriginalScalarWidth = OriginalType.scalarBitWidth(); + const std::uint64_t TargetScalarWidth = TargetType.scalarBitWidth(); + return LowLevelType::makeCategoryNeutralScalar(std::gcd(OriginalScalarWidth, TargetScalarWidth)); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalObject.cpp b/backend/lib/CodeGen/GlobalObject.cpp new file mode 100644 index 0000000..aa8fd15 --- /dev/null +++ b/backend/lib/CodeGen/GlobalObject.cpp @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalObject.h" + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <memory> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] bool validText(const std::string &Text) noexcept { + return !Text.empty() && Text.find('\0') == std::string::npos && + Text.find('\n') == std::string::npos && Text.find('\r') == std::string::npos; +} + +[[nodiscard]] bool validValueSize(std::uint8_t Size) noexcept { + return Size == 1 || Size == 2 || Size == 4 || Size == 8; +} + +[[nodiscard]] Expected<std::uint64_t> addSize(std::uint64_t Left, std::uint64_t Right) { + std::uint64_t Result = 0; + if (__builtin_add_overflow(Left, Right, &Result)) + return makeUnexpected(ErrorCode::VerificationError, "global initializer size overflows"); + return Result; +} + +[[nodiscard]] Expected<std::uint64_t> multiplySize(std::uint64_t Left, std::uint64_t Right) { + std::uint64_t Result = 0; + if (__builtin_mul_overflow(Left, Right, &Result)) + return makeUnexpected(ErrorCode::VerificationError, "global initializer size overflows"); + return Result; +} + +[[nodiscard]] Expected<std::uint64_t> initializerSize(const GlobalInitializer &Initializer, + std::size_t Depth, std::size_t MaximumDepth) { + if (Depth > MaximumDepth) + return makeUnexpected(ErrorCode::VerificationError, "global initializer is too deeply nested"); + switch (Initializer.kind()) { + case GlobalInitializer::Kind::Invalid: + return makeUnexpected(ErrorCode::VerificationError, "global initializer is invalid"); + case GlobalInitializer::Kind::Integer: { + const auto *Value = Initializer.getIf<APInt>(); + if (Value == nullptr) + return makeUnexpected(ErrorCode::InternalError, "integer initializer storage is invalid"); + auto Verified = Value->verify(); + if (!Verified) + return std::unexpected(Verified.error()); + return (static_cast<std::uint64_t>(Value->bitWidth()) + 7U) / 8U; + } + case GlobalInitializer::Kind::Floating: { + const auto *Value = Initializer.getIf<APFloat>(); + if (Value == nullptr) + return makeUnexpected(ErrorCode::InternalError, "floating initializer storage is invalid"); + return (static_cast<std::uint64_t>(Value->bitWidth()) + 7U) / 8U; + } + case GlobalInitializer::Kind::Bytes: { + const auto *Value = Initializer.getIf<std::vector<std::byte>>(); + if (Value == nullptr || Value->empty()) + return makeUnexpected(ErrorCode::VerificationError, "byte initializer cannot be empty"); + return static_cast<std::uint64_t>(Value->size()); + } + case GlobalInitializer::Kind::ZeroFill: { + const auto *Value = Initializer.getIf<GlobalZeroFill>(); + if (Value == nullptr || Value->Size == 0) + return makeUnexpected(ErrorCode::VerificationError, "zero-fill initializer has zero size"); + return Value->Size; + } + case GlobalInitializer::Kind::Repeated: { + const auto *Value = Initializer.getIf<GlobalRepeatedInitializer>(); + if (Value == nullptr || Value->Count == 0 || !Value->Value) + return makeUnexpected(ErrorCode::VerificationError, "repeated initializer is invalid"); + auto ElementSize = initializerSize(*Value->Value, Depth + 1, MaximumDepth); + if (!ElementSize) + return std::unexpected(ElementSize.error()); + return multiplySize(*ElementSize, Value->Count); + } + case GlobalInitializer::Kind::Aggregate: { + const auto *Value = Initializer.getIf<GlobalAggregateInitializer>(); + if (Value == nullptr || Value->Elements.empty()) + return makeUnexpected(ErrorCode::VerificationError, "aggregate initializer cannot be empty"); + std::uint64_t Size = 0; + for (const GlobalInitializerRef &Element : Value->Elements) { + if (!Element) + return makeUnexpected(ErrorCode::VerificationError, "aggregate initializer is null"); + auto ElementSize = initializerSize(*Element, Depth + 1, MaximumDepth); + if (!ElementSize) + return std::unexpected(ElementSize.error()); + auto Updated = addSize(Size, *ElementSize); + if (!Updated) + return std::unexpected(Updated.error()); + Size = *Updated; + } + return Size; + } + case GlobalInitializer::Kind::SymbolAddress: { + const auto *Value = Initializer.getIf<GlobalSymbolAddress>(); + if (Value == nullptr || !validText(Value->Symbol) || !validValueSize(Value->Size)) { + return makeUnexpected(ErrorCode::VerificationError, "symbol-address initializer is invalid"); + } + return Value->Size; + } + case GlobalInitializer::Kind::SymbolDifference: { + const auto *Value = Initializer.getIf<GlobalSymbolDifference>(); + if (Value == nullptr || !validText(Value->Minuend) || !validText(Value->Subtrahend) || + !validValueSize(Value->Size)) { + return makeUnexpected(ErrorCode::VerificationError, + "symbol-difference initializer is invalid"); + } + return Value->Size; + } + case GlobalInitializer::Kind::Target: { + const auto *Value = Initializer.getIf<TargetConstant>(); + if (Value == nullptr) + return makeUnexpected(ErrorCode::InternalError, "target initializer storage is invalid"); + auto Emission = Value->emit(); + if (!Emission) + return std::unexpected(Emission.error()); + return static_cast<std::uint64_t>(Emission->Bytes.size()); + } + case GlobalInitializer::Kind::AlignmentPadding: { + const auto *Value = Initializer.getIf<GlobalAlignmentPadding>(); + if (Value == nullptr || Value->Size == 0) + return makeUnexpected(ErrorCode::VerificationError, "alignment padding has zero size"); + return Value->Size; + } + case GlobalInitializer::Kind::UndefinedStorage: { + const auto *Value = Initializer.getIf<GlobalUndefinedStorage>(); + if (Value == nullptr || Value->Size == 0) + return makeUnexpected(ErrorCode::VerificationError, "undefined storage has zero size"); + return Value->Size; + } + } + return makeUnexpected(ErrorCode::InternalError, "global initializer kind is invalid"); +} + +} // namespace + +GlobalInitializer GlobalInitializer::integer(APInt Value) { + return GlobalInitializer(std::move(Value)); +} + +GlobalInitializer GlobalInitializer::floating(APFloat Value) { + return GlobalInitializer(Value); +} + +Expected<GlobalInitializer> GlobalInitializer::bytes(std::vector<std::byte> Value) { + if (Value.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "byte initializer cannot be empty"); + return GlobalInitializer(std::move(Value)); +} + +Expected<GlobalInitializer> GlobalInitializer::zeroFill(std::uint64_t Size) { + if (Size == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "zero-fill initializer size must be nonzero"); + return GlobalInitializer(GlobalZeroFill{.Size = Size}); +} + +Expected<GlobalInitializer> GlobalInitializer::repeated(std::uint64_t Count, + GlobalInitializer Value) { + if (Count == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "initializer repeat count must be nonzero"); + auto Reference = std::make_shared<GlobalInitializer>(std::move(Value)); + auto Verification = Reference->verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return GlobalInitializer( + GlobalRepeatedInitializer{.Count = Count, .Value = std::move(Reference)}); +} + +Expected<GlobalInitializer> GlobalInitializer::aggregate(std::vector<GlobalInitializer> Elements) { + if (Elements.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "aggregate initializer cannot be empty"); + GlobalAggregateInitializer Result; + Result.Elements.reserve(Elements.size()); + for (GlobalInitializer &Element : Elements) { + auto Reference = std::make_shared<GlobalInitializer>(std::move(Element)); + auto Verification = Reference->verify(); + if (!Verification) + return std::unexpected(Verification.error()); + Result.Elements.push_back(std::move(Reference)); + } + return GlobalInitializer(std::move(Result)); +} + +Expected<GlobalInitializer> GlobalInitializer::symbolAddress(GlobalSymbolAddress Value) { + GlobalInitializer Result(std::move(Value)); + auto Verification = Result.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return Result; +} + +Expected<GlobalInitializer> GlobalInitializer::symbolDifference(GlobalSymbolDifference Value) { + GlobalInitializer Result(std::move(Value)); + auto Verification = Result.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return Result; +} + +Expected<GlobalInitializer> GlobalInitializer::target(TargetConstant Value) { + GlobalInitializer Result(std::move(Value)); + auto Verification = Result.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return Result; +} + +Expected<GlobalInitializer> GlobalInitializer::alignmentPadding(std::uint64_t Size, + std::uint8_t Fill) { + if (Size == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "alignment padding size must be nonzero"); + return GlobalInitializer(GlobalAlignmentPadding{.Size = Size, .Fill = Fill}); +} + +Expected<GlobalInitializer> GlobalInitializer::undefinedStorage(std::uint64_t Size) { + if (Size == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "undefined storage size must be nonzero"); + return GlobalInitializer(GlobalUndefinedStorage{.Size = Size}); +} + +GlobalInitializer::Kind GlobalInitializer::kind() const noexcept { + return static_cast<Kind>(Storage.index()); +} + +Expected<void> GlobalInitializer::verify(std::size_t MaximumDepth) const { + if (MaximumDepth == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "initializer depth limit cannot be zero"); + auto Size = initializerSize(*this, 1, MaximumDepth); + if (!Size) + return std::unexpected(Size.error()); + return {}; +} + +Expected<std::uint64_t> GlobalInitializer::storageSize(std::size_t MaximumDepth) const { + if (MaximumDepth == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "initializer depth limit cannot be zero"); + return initializerSize(*this, 1, MaximumDepth); +} + +Expected<void> GlobalObjectDesc::verify() const { + if (!validText(Symbol)) + return makeUnexpected(ErrorCode::VerificationError, "global object symbol is invalid"); + auto TypeVerification = ObjectType.verify(); + if (!TypeVerification) + return std::unexpected(TypeVerification.error()); + if (Linkage >= GlobalLinkage::Count || Visibility >= GlobalVisibility::Count || + TLSModel >= GlobalTLSModel::Count || UnnamedAddress >= GlobalUnnamedAddress::Count) { + return makeUnexpected(ErrorCode::VerificationError, "global object enum field is invalid"); + } + if (Section && !validText(*Section)) + return makeUnexpected(ErrorCode::VerificationError, "global object section is invalid"); + if (COMDAT && !validText(*COMDAT)) + return makeUnexpected(ErrorCode::VerificationError, "global object COMDAT is invalid"); + if (Size == 0 && !isDeclaration()) + return makeUnexpected(ErrorCode::VerificationError, "defined global object has zero size"); + if (!isDeclaration() && + Size > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) { + return makeUnexpected(ErrorCode::VerificationError, + "global object size exceeds the supported assembly range"); + } + if (Linkage == GlobalLinkage::Common && Initializer) + return makeUnexpected(ErrorCode::VerificationError, "common global cannot have an initializer"); + if (Initializer) { + auto InitializerSize = Initializer->storageSize(); + if (!InitializerSize) + return std::unexpected(InitializerSize.error()); + if (*InitializerSize != Size) { + return makeUnexpected(ErrorCode::VerificationError, + "global initializer size does not match its object size"); + } + } + std::unordered_set<std::string> AttributeNames; + for (const auto &[Name, Value] : TargetAttributes) { + if (!validText(Name) || Value.find('\0') != std::string::npos || + Value.find('\n') != std::string::npos || Value.find('\r') != std::string::npos || + !AttributeNames.insert(Name).second) { + return makeUnexpected(ErrorCode::VerificationError, + "global target attribute is invalid or duplicated"); + } + } + for (const std::string &Expression : DebugExpressions) { + if (!validText(Expression)) + return makeUnexpected(ErrorCode::VerificationError, "global debug expression is invalid"); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/InlineSpiller.cpp b/backend/lib/CodeGen/InlineSpiller.cpp new file mode 100644 index 0000000..4243e84 --- /dev/null +++ b/backend/lib/CodeGen/InlineSpiller.cpp @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachinePassManager.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/Spiller.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] Expected<void> clearDebugRegisterOperand(MachineOperand &Operand) { + RegisterOperandFlags Flags = Operand.registerFlags(); + Flags.set(RegisterOperandFlag::Undef); + auto SetReg = Operand.setRegister(Register{}); + if (!SetReg) + return std::unexpected(SetReg.error()); + auto SetSubReg = Operand.setSubReg(SubRegIndex{}); + if (!SetSubReg) + return std::unexpected(SetSubReg.error()); + return Operand.setRegisterFlags(Flags); +} + +} // namespace + +Expected<std::optional<std::vector<Register>>> Spiller::tryRematerialize(Register VirtualReg) { + if (!EnableRematerialization) + return std::optional<std::vector<Register>>{}; + const MachineOperand *Definition = Function->registerInfo().getUniqueVRegDef(VirtualReg); + const MachineInstr *DefiningInstruction = Definition == nullptr ? nullptr : Definition->parent(); + if (DefiningInstruction == nullptr || DefiningInstruction->parent() == nullptr) + return std::optional<std::vector<Register>>{}; + auto Rematerializable = + TargetInfo->isRematerializable(*DefiningInstruction, VirtualReg, *Function, *Target); + if (!Rematerializable) + return std::unexpected(Rematerializable.error()); + if (!*Rematerializable) + return std::optional<std::vector<Register>>{}; + + struct UseSite final { + MachineInstr *Instruction = nullptr; + std::vector<std::size_t> OperandIndices; + }; + std::vector<UseSite> Sites; + for (MachineBasicBlock *Block = Function->firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + UseSite Site{Instruction, {}}; + bool HasDefinition = false; + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + const MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || *Reg != VirtualReg) + continue; + HasDefinition |= Operand->isDef(); + if (Operand->isUse() && !Operand->isDebugUse()) + Site.OperandIndices.push_back(Index); + } + if (HasDefinition && !Site.OperandIndices.empty()) + return std::optional<std::vector<Register>>{}; + if (!Site.OperandIndices.empty()) + Sites.push_back(std::move(Site)); + } + } + if (Sites.empty()) + return std::optional<std::vector<Register>>{}; + + if (ActionAnalyses != nullptr) { + for (const UseSite &Site : Sites) { + auto Accepted = + ActionAnalyses->requestAction("rematerialize", *Function, Site.Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + return std::optional<std::vector<Register>>{}; + } + } + + std::vector<Register> NewRegisters; + NewRegisters.reserve(Sites.size()); + for (UseSite &Site : Sites) { + auto Child = Edit.createFrom(VirtualReg, false); + if (!Child) + return std::unexpected(Child.error()); + MachineIRBuilder Builder(*Function, *Target); + auto Positioned = Builder.setInsertPointBefore(*Site.Instruction); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Materialized = + TargetInfo->rematerialize(Builder, *DefiningInstruction, *Child, VirtualReg, *Target); + if (!Materialized) + return std::unexpected(Materialized.error()); + for (const std::size_t Index : Site.OperandIndices) { + MachineOperand *Operand = Site.Instruction->operandAt(Index); + auto Replaced = Operand->setRegister(*Child); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + if (Entry != nullptr) + ++Entry->Spill.RematerializationCount; + NewRegisters.push_back(*Child); + } + + bool HasNonDebugUse = false; + for (const MachineOperand &Operand : Function->registerInfo().useOperands(VirtualReg, false)) { + if (Operand.parent() != nullptr) { + HasNonDebugUse = true; + break; + } + } + if (!HasNonDebugUse) { + auto Removed = const_cast<MachineInstr *>(DefiningInstruction)->eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + } + return std::optional<std::vector<Register>>(std::move(NewRegisters)); +} + +Expected<std::vector<Register>> Spiller::spillAtInstructions(Register VirtualReg) { + auto ClassID = regalloc_detail::registerClassFor(*Function, VirtualReg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Validated = TargetInfo->validateSpillReload(*ClassID, *Target); + if (!Validated) { + std::string Message = + "target declares an allocatable spillable class without spill/reload support: "; + Message.append(Validated.error().message()); + return makeUnexpected(ErrorCode::InternalError, std::move(Message)); + } + auto StackSlot = Edit.getOrCreateSpillSlot(VirtualReg); + if (!StackSlot) + return std::unexpected(StackSlot.error()); + auto LiveInSpill = spillLiveIn(VirtualReg, *ClassID, *StackSlot); + if (!LiveInSpill) + return std::unexpected(LiveInSpill.error()); + + struct SpillSite final { + MachineInstr *Instruction = nullptr; + std::vector<std::size_t> OperandIndices; + std::vector<std::size_t> DebugIndices; + bool HasUse = false; + bool HasDef = false; + }; + std::vector<SpillSite> Sites; + for (MachineBasicBlock *Block = Function->firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + SpillSite Site; + Site.Instruction = Instruction; + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + const MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || *Reg != VirtualReg) + continue; + if (Operand->isDebugUse()) { + Site.DebugIndices.push_back(Index); + continue; + } + Site.OperandIndices.push_back(Index); + Site.HasUse |= Operand->isUse(); + Site.HasDef |= Operand->isDef(); + } + if (!Site.OperandIndices.empty() || !Site.DebugIndices.empty()) + Sites.push_back(std::move(Site)); + } + } + + std::vector<Register> NewRegisters; + if (ActionAnalyses != nullptr) { + for (const SpillSite &Site : Sites) { + if (Site.OperandIndices.empty()) + continue; + auto Accepted = ActionAnalyses->requestAction("split-live-range", *Function, + Site.Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "required live-range split action was rejected"); + } + } + } + for (SpillSite &Site : Sites) { + for (const std::size_t Index : Site.DebugIndices) { + MachineOperand *Operand = Site.Instruction->operandAt(Index); + auto Cleared = clearDebugRegisterOperand(*Operand); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + if (Site.OperandIndices.empty()) + continue; + + auto Child = Edit.createFrom(VirtualReg, false); + if (!Child) + return std::unexpected(Child.error()); + if (Site.HasUse) { + MachineIRBuilder Builder(*Function, *Target); + auto Positioned = Builder.setInsertPointBefore(*Site.Instruction); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Reload = TargetInfo->emitReload(Builder, *Child, *ClassID, *StackSlot, LaneMask::all()); + if (!Reload) + return std::unexpected(Reload.error()); + VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + if (Entry != nullptr) + ++Entry->Spill.ReloadCount; + } + for (const std::size_t Index : Site.OperandIndices) { + MachineOperand *Operand = Site.Instruction->operandAt(Index); + auto Replaced = Operand->setRegister(*Child); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + if (Site.HasDef) { + const InstructionDescriptor *Descriptor = + Target->instructionDescriptor(Site.Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Terminator)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "a terminator-defined value cannot be spilled on its outgoing edge"); + } + MachineIRBuilder Builder(*Function, *Target); + auto Positioned = Builder.setInsertPointAfter(*Site.Instruction); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Spill = TargetInfo->emitSpill(Builder, *Child, *ClassID, *StackSlot, LaneMask::all()); + if (!Spill) + return std::unexpected(Spill.error()); + VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + if (Entry != nullptr) + ++Entry->Spill.SpillCount; + } + NewRegisters.push_back(*Child); + } + return NewRegisters; +} + +Expected<void> Spiller::spillLiveIn(Register VirtualReg, RegisterClassID Class, + FrameIndex StackSlot) { + const auto LiveIn = std::ranges::find_if(Function->registerInfo().liveIns(), + [VirtualReg](const PhysicalRegisterLiveIn &Value) { + return Value.InitialVirtualRegister == VirtualReg; + }); + if (LiveIn == Function->registerInfo().liveIns().end()) + return {}; + MachineBasicBlock *EntryBlock = Function->firstBlock(); + if (EntryBlock == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "function live-in spill requires an entry block"); + } + MachineIRBuilder Builder(*Function, *Target); + Expected<void> Positioned = EntryBlock->firstInstruction() != nullptr + ? Builder.setInsertPointBefore(*EntryBlock->firstInstruction()) + : Builder.setInsertPoint(*EntryBlock); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Spill = TargetInfo->emitSpill(Builder, Register::fromPhysical(LiveIn->PhysReg), Class, + StackSlot, LaneMask::all()); + if (!Spill) + return std::unexpected(Spill.error()); + VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + if (Entry != nullptr) + ++Entry->Spill.SpillCount; + return {}; +} + +Expected<std::vector<Register>> Spiller::spill(Register VirtualReg) { + if (!VirtualReg.isVirtual() || !Function->registerInfo().isKnownVirtualRegister(VirtualReg)) + return makeUnexpected(ErrorCode::InvalidArgument, "Spiller requires a known virtual register"); + if (!Mapping->isSpillable(VirtualReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "register pressure requires spilling a non-spillable value"); + } + if (ActionAnalyses != nullptr) { + const MachineOperand *Definition = Function->registerInfo().getUniqueVRegDef(VirtualReg); + const MachineInstr *Instruction = Definition == nullptr ? nullptr : Definition->parent(); + auto Accepted = ActionAnalyses->requestAction("spill", *Function, + Instruction == nullptr ? MachineInstrStableID{} + : Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "required spill action was rejected"); + } + } + auto Rematerialized = tryRematerialize(VirtualReg); + if (!Rematerialized) + return std::unexpected(Rematerialized.error()); + if (*Rematerialized) + return std::move(**Rematerialized); + return spillAtInstructions(VirtualReg); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/InstructionDescriptor.cpp b/backend/lib/CodeGen/InstructionDescriptor.cpp new file mode 100644 index 0000000..027f738 --- /dev/null +++ b/backend/lib/CodeGen/InstructionDescriptor.cpp @@ -0,0 +1,769 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/InstructionDescriptor.h" + +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] bool isRegisterOnly(const OperandKindSet &Kinds) noexcept { + OperandKindSet RegisterKind{MachineOperand::Kind::Register}; + return Kinds == RegisterKind; +} + +[[nodiscard]] bool isDefRole(RegisterOperandRole Role) noexcept { + return Role == RegisterOperandRole::Def || Role == RegisterOperandRole::DefUse; +} + +[[nodiscard]] bool isUseRole(RegisterOperandRole Role) noexcept { + return Role == RegisterOperandRole::Use || Role == RegisterOperandRole::DefUse; +} + +[[nodiscard]] bool formsDefUsePair(RegisterOperandRole Left, RegisterOperandRole Right) noexcept { + return (Left == RegisterOperandRole::Def && Right == RegisterOperandRole::Use) || + (Left == RegisterOperandRole::Use && Right == RegisterOperandRole::Def); +} + +[[nodiscard]] bool isPureDef(const MachineOperand &Operand) noexcept { + return Operand.isRegister() && Operand.isDef() && !Operand.isUse(); +} + +[[nodiscard]] bool isPureUse(const MachineOperand &Operand) noexcept { + return Operand.isRegister() && Operand.isUse() && !Operand.isDef(); +} + +[[nodiscard]] Expected<void> validateDescriptorStructure(const InstructionDescriptor &Descriptor) { + if (!Descriptor.InstructionOpcode.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "instruction descriptor opcode is invalid"); + if (Descriptor.Name.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "instruction descriptor name is empty"); + if (Descriptor.NumExplicitDefs > Descriptor.Operands.size()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction descriptor explicit def count is out of range"); + } + + bool SawOptional = false; + std::optional<std::size_t> VariadicBegin; + std::optional<std::size_t> VariadicEnd; + for (std::size_t Index = 0; Index < Descriptor.Operands.size(); ++Index) { + const OperandDescriptor &Operand = Descriptor.Operands[Index]; + if (Operand.Kinds.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "operand descriptor must accept at least one operand kind"); + } + + const bool IsRegister = Operand.Kinds.contains(MachineOperand::Kind::Register); + if (IsRegister != (Operand.RegisterRole != RegisterOperandRole::None)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register-capable operand descriptor must declare a register role"); + } + if (isDefRole(Operand.RegisterRole) && !isRegisterOnly(Operand.Kinds)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register definition descriptor must accept only registers"); + } + if (!IsRegister && + (Operand.RegClass || Operand.RegBank || Operand.EarlyClobber || Operand.AllowSubReg)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "non-register operand descriptor has register constraints"); + } + if (Operand.TypeConstraint) { + if (!isRegisterOnly(Operand.Kinds) || Operand.TypeConstraint->Kinds.empty() || + Operand.TypeConstraint->Kinds.contains(LowLevelType::Kind::Invalid)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "typed operand descriptor requires a nonempty register type set"); + } + } + if (Operand.EarlyClobber && !isDefRole(Operand.RegisterRole)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "early-clobber operand descriptor must define a register"); + } + if (Descriptor.Variadic && Operand.Optional) { + return makeUnexpected(ErrorCode::InvalidArgument, + "variadic instruction descriptor cannot contain optional operands"); + } + if (Operand.Optional) + SawOptional = true; + else if (SawOptional && !Operand.VariadicGroup) { + return makeUnexpected(ErrorCode::InvalidArgument, + "optional operand descriptors must form a trailing suffix"); + } + + if (Operand.VariadicGroup && (Operand.Optional || Operand.DefaultTiedOperand)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "variadic operand descriptor cannot be optional or have a default tie"); + } + if (Operand.VariadicGroup) { + if (!Descriptor.Variadic) { + return makeUnexpected(ErrorCode::InvalidArgument, + "non-variadic instruction descriptor has a variadic group"); + } + if (!VariadicBegin) + VariadicBegin = Index; + VariadicEnd = Index + 1; + } else if (VariadicEnd) { + const bool LaterGroup = + std::ranges::any_of(Descriptor.Operands.begin() + static_cast<std::ptrdiff_t>(Index), + Descriptor.Operands.end(), [](const OperandDescriptor &Candidate) { + return Candidate.VariadicGroup; + }); + if (LaterGroup) { + return makeUnexpected(ErrorCode::InvalidArgument, + "variadic operand group descriptors must be contiguous"); + } + } + } + + if (Descriptor.Variadic && !VariadicBegin) { + return makeUnexpected(ErrorCode::InvalidArgument, + "variadic instruction descriptor requires a variadic group"); + } + if (!Descriptor.Variadic && Descriptor.MinimumVariadicRepetitions != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "non-variadic instruction has a minimum variadic repetition count"); + } + + bool SawUse = false; + std::size_t MinimumDefs = 0; + for (std::size_t Index = 0; Index < Descriptor.Operands.size(); ++Index) { + const OperandDescriptor &Operand = Descriptor.Operands[Index]; + const std::size_t Repetitions = + Operand.VariadicGroup ? Descriptor.MinimumVariadicRepetitions : 1; + for (std::size_t Repeat = 0; Repeat < Repetitions; ++Repeat) { + if (isDefRole(Operand.RegisterRole)) { + if (SawUse) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register definition appears after an instruction use"); + } + ++MinimumDefs; + } else { + SawUse = true; + } + } + } + if (Descriptor.NumExplicitDefs != MinimumDefs) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction descriptor explicit def count does not match its shape"); + } + + for (std::size_t Index = 0; Index < Descriptor.Operands.size(); ++Index) { + const OperandDescriptor &Operand = Descriptor.Operands[Index]; + if (!Operand.DefaultTiedOperand) + continue; + const std::size_t OtherIndex = *Operand.DefaultTiedOperand; + if (OtherIndex >= Descriptor.Operands.size() || OtherIndex == Index || + Descriptor.Operands[OtherIndex].DefaultTiedOperand != Index) { + return makeUnexpected(ErrorCode::InvalidArgument, + "default operand tie must be reciprocal and in range"); + } + if (Operand.Optional || Descriptor.Operands[OtherIndex].Optional || + !isRegisterOnly(Operand.Kinds) || !isRegisterOnly(Descriptor.Operands[OtherIndex].Kinds) || + !formsDefUsePair(Operand.RegisterRole, Descriptor.Operands[OtherIndex].RegisterRole)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "default operand tie must connect required register def and use"); + } + } + + std::unordered_set<NamedOperandID> OperandNames; + for (const OperandDescriptor &Operand : Descriptor.Operands) { + if (Operand.Name && !OperandNames.insert(*Operand.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction descriptor has duplicate named operands"); + } + } + + std::unordered_set<PhysicalRegister> ImplicitDefs; + for (const ImplicitRegisterDescriptor &Implicit : Descriptor.ImplicitDefs) { + if (!Implicit.Reg.isValid() || !ImplicitDefs.insert(Implicit.Reg).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "implicit definitions must name unique physical registers"); + } + } + std::unordered_set<PhysicalRegister> ImplicitUses; + for (const ImplicitRegisterDescriptor &Implicit : Descriptor.ImplicitUses) { + if (!Implicit.Reg.isValid() || Implicit.EarlyClobber || + !ImplicitUses.insert(Implicit.Reg).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "implicit uses must name unique physical registers and cannot " + "early-clobber"); + } + } + + const bool IsBranch = Descriptor.Flags.has(InstructionFlag::Branch); + const bool IsTerminator = Descriptor.Flags.has(InstructionFlag::Terminator); + if ((IsBranch || Descriptor.Flags.has(InstructionFlag::Return) || + Descriptor.Flags.has(InstructionFlag::Barrier) || + Descriptor.Flags.has(InstructionFlag::IndirectBranch)) && + !IsTerminator) { + return makeUnexpected(ErrorCode::InvalidArgument, + "branch, return, and barrier descriptors must be terminators"); + } + if (Descriptor.Flags.has(InstructionFlag::IndirectBranch) && !IsBranch) { + return makeUnexpected(ErrorCode::InvalidArgument, + "indirect branch descriptor must also be a branch"); + } + + const bool IsPseudo = Descriptor.Flags.has(InstructionFlag::Pseudo); + const bool IsMeta = Descriptor.Flags.has(InstructionFlag::Meta); + if (IsMeta && (!IsPseudo || Descriptor.Flags.has(InstructionFlag::ProducesCode))) { + return makeUnexpected(ErrorCode::InvalidArgument, + "meta instruction must be a non-emitting pseudo"); + } + if (!IsPseudo && Descriptor.Owner != PseudoOwner::None) { + return makeUnexpected(ErrorCode::InvalidArgument, + "non-pseudo instruction cannot have a pseudo owner"); + } + if (IsPseudo && !IsMeta && Descriptor.Owner == PseudoOwner::None) { + return makeUnexpected(ErrorCode::InvalidArgument, + "executable pseudo must declare its expansion owner"); + } + if (Descriptor.Flags.has(InstructionFlag::TerminalEmission) && + (!IsPseudo || Descriptor.Owner != PseudoOwner::AsmPrinter || + !Descriptor.Flags.has(InstructionFlag::ProducesCode))) { + return makeUnexpected(ErrorCode::InvalidArgument, + "terminal emission pseudo must produce code and be owned by AsmPrinter"); + } + if (Descriptor.RequiresMemoryOperands && !Descriptor.Flags.has(InstructionFlag::MayLoad) && + !Descriptor.Flags.has(InstructionFlag::MayStore)) { + return makeUnexpected(ErrorCode::InvalidArgument, "MMO requirement needs mayLoad or mayStore"); + } + + return {}; +} + +struct OperandShape final { + std::size_t VariadicBegin = 0; + std::size_t VariadicEnd = 0; + std::size_t Repetitions = 0; +}; + +[[nodiscard]] Expected<OperandShape> resolveOperandShape(const InstructionDescriptor &Descriptor, + std::size_t OperandCount) { + if (!Descriptor.Variadic) { + std::size_t RequiredCount = Descriptor.Operands.size(); + while (RequiredCount != 0 && Descriptor.Operands[RequiredCount - 1].Optional) + --RequiredCount; + if (OperandCount < RequiredCount || OperandCount > Descriptor.Operands.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction has an invalid explicit operand count"); + } + return OperandShape{}; + } + + const auto Begin = std::ranges::find_if( + Descriptor.Operands, [](const OperandDescriptor &Operand) { return Operand.VariadicGroup; }); + const auto End = std::ranges::find_if_not( + Begin, Descriptor.Operands.end(), + [](const OperandDescriptor &Operand) { return Operand.VariadicGroup; }); + const std::size_t VariadicBegin = + static_cast<std::size_t>(std::distance(Descriptor.Operands.begin(), Begin)); + const std::size_t VariadicEnd = + static_cast<std::size_t>(std::distance(Descriptor.Operands.begin(), End)); + const std::size_t GroupSize = VariadicEnd - VariadicBegin; + const std::size_t FixedCount = Descriptor.Operands.size() - GroupSize; + if (GroupSize == 0 || OperandCount < FixedCount) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction has an invalid variadic operand count"); + } + const std::size_t VariableCount = OperandCount - FixedCount; + if (VariableCount % GroupSize != 0 || + VariableCount / GroupSize < Descriptor.MinimumVariadicRepetitions) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction has an incomplete or undersized variadic operand group"); + } + return OperandShape{VariadicBegin, VariadicEnd, VariableCount / GroupSize}; +} + +[[nodiscard]] const OperandDescriptor &descriptorForOperand(const InstructionDescriptor &Descriptor, + const OperandShape &Shape, + std::size_t Index) { + if (!Descriptor.Variadic) + return Descriptor.Operands[Index]; + if (Index < Shape.VariadicBegin) + return Descriptor.Operands[Index]; + const std::size_t GroupSize = Shape.VariadicEnd - Shape.VariadicBegin; + const std::size_t VariableCount = Shape.Repetitions * GroupSize; + if (Index < Shape.VariadicBegin + VariableCount) { + return Descriptor.Operands[Shape.VariadicBegin + (Index - Shape.VariadicBegin) % GroupSize]; + } + return Descriptor.Operands[Shape.VariadicEnd + (Index - Shape.VariadicBegin - VariableCount)]; +} + +} // namespace + +Expected<std::vector<const OperandDescriptor *>> +resolveExplicitOperandDescriptors(const InstructionDescriptor &Descriptor, + std::size_t OperandCount) { + auto Shape = resolveOperandShape(Descriptor, OperandCount); + if (!Shape) + return std::unexpected(Shape.error()); + std::vector<const OperandDescriptor *> Result; + Result.reserve(OperandCount); + for (std::size_t Index = 0; Index < OperandCount; ++Index) + Result.push_back(&descriptorForOperand(Descriptor, *Shape, Index)); + return Result; +} + +OperandKindSet::OperandKindSet(std::initializer_list<MachineOperand::Kind> Kinds) noexcept { + for (const MachineOperand::Kind Kind : Kinds) + add(Kind); +} + +LowLevelTypeKindSet::LowLevelTypeKindSet(std::initializer_list<LowLevelType::Kind> Kinds) noexcept { + for (const LowLevelType::Kind Kind : Kinds) + add(Kind); +} + +InstructionFlags::InstructionFlags(std::initializer_list<InstructionFlag> Flags) noexcept { + for (const InstructionFlag Flag : Flags) + set(Flag); +} + +Expected<InstructionSet> +InstructionSet::create(std::vector<InstructionDescriptor> Descriptors, + std::vector<ImmediatePredicate> ImmediatePredicates) { + if (Descriptors.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction set requires at least one descriptor"); + } + + InstructionSet Result(std::move(Descriptors), std::move(ImmediatePredicates)); + + std::unordered_set<std::string> PredicateNames; + for (std::size_t Index = 0; Index < Result.ImmediatePredicates.size(); ++Index) { + const ImmediatePredicate &Predicate = Result.ImmediatePredicates[Index]; + if (Predicate.Name.empty() || !Predicate.Predicate) { + return makeUnexpected(ErrorCode::InvalidArgument, + "immediate predicate requires a name and callable"); + } + if (!Result.PredicateToIndex.emplace(Predicate.ID, Index).second) { + return makeUnexpected(ErrorCode::InvalidArgument, "duplicate immediate predicate ID"); + } + if (!PredicateNames.insert(Predicate.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, "duplicate immediate predicate name"); + } + } + + for (std::size_t Index = 0; Index < Result.Descriptors.size(); ++Index) { + const InstructionDescriptor &Descriptor = Result.Descriptors[Index]; + const auto Validation = validateDescriptorStructure(Descriptor); + if (!Validation) + return std::unexpected(Validation.error()); + if (!Result.OpcodeToIndex.emplace(Descriptor.InstructionOpcode, Index).second) + return makeUnexpected(ErrorCode::InvalidArgument, "duplicate instruction opcode"); + if (!Result.NameToIndex.emplace(Descriptor.Name, Index).second) + return makeUnexpected(ErrorCode::InvalidArgument, "duplicate instruction name"); + + for (const OperandDescriptor &Operand : Descriptor.Operands) { + if (Operand.ImmediatePredicate && + Result.immediatePredicate(*Operand.ImmediatePredicate) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "operand descriptor references an unknown immediate predicate"); + } + } + } + + return Result; +} + +const InstructionDescriptor *InstructionSet::descriptor(Opcode InstructionOpcode) const noexcept { + const auto Position = OpcodeToIndex.find(InstructionOpcode); + if (Position == OpcodeToIndex.end()) + return nullptr; + return &Descriptors[Position->second]; +} + +const InstructionDescriptor *InstructionSet::descriptor(std::string_view Name) const noexcept { + const auto Position = NameToIndex.find(std::string(Name)); + if (Position == NameToIndex.end()) + return nullptr; + return &Descriptors[Position->second]; +} + +Expected<void> InstructionSet::validateExplicitOperands(const InstructionDescriptor &Descriptor, + std::span<const MachineOperand> Operands, + const MachineRegisterInfo &RegInfo) const { + auto Shape = resolveOperandShape(Descriptor, Operands.size()); + if (!Shape) + return std::unexpected(Shape.error()); + + bool SawUse = false; + std::unordered_map<std::uint16_t, LowLevelType> EqualityGroups; + for (std::size_t Index = 0; Index < Operands.size(); ++Index) { + const MachineOperand &Operand = Operands[Index]; + if (Operand.isImplicit()) { + return makeUnexpected(ErrorCode::VerificationError, + "explicit operand sequence contains an implicit register"); + } + const OperandDescriptor &OperandDesc = descriptorForOperand(Descriptor, *Shape, Index); + const auto Validation = validateOperand(OperandDesc, Operand, RegInfo); + if (!Validation) { + return makeUnexpected(Validation.error().code(), + "operand " + std::to_string(Index) + " of '" + Descriptor.Name + + "': " + std::string(Validation.error().message())); + } + if (Operand.isRegister()) { + if (isDefRole(OperandDesc.RegisterRole)) { + if (SawUse) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction register definition appears after a use"); + } + } else { + SawUse = true; + } + } else { + SawUse = true; + } + + if (!OperandDesc.TypeConstraint) + continue; + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "typed instruction operand must use a virtual register"); + } + auto Info = RegInfo.virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + const std::optional<LowLevelType> &RegisterType = Info->get().Type; + if (!RegisterType) { + return makeUnexpected(ErrorCode::VerificationError, + "typed instruction operand virtual register has no LowLevelType"); + } + const LowLevelType Type = RegisterType.value(); + if (!OperandDesc.TypeConstraint->Kinds.contains(Type.kind())) { + return makeUnexpected( + ErrorCode::VerificationError, + "instruction operand LowLevelType kind is not permitted by its descriptor"); + } + if (OperandDesc.TypeConstraint->EqualityGroup) { + const auto [Position, Inserted] = + EqualityGroups.emplace(*OperandDesc.TypeConstraint->EqualityGroup, Type); + if (!Inserted && Position->second != Type) { + return makeUnexpected( + ErrorCode::VerificationError, + "instruction operands in one LowLevelType group have different types"); + } + } + } + if (Descriptor.VerifyExplicitOperands != nullptr) + return Descriptor.VerifyExplicitOperands(Descriptor.InstructionOpcode, Operands, RegInfo); + return {}; +} + +Expected<void> InstructionSet::validateExplicitOperands(Opcode InstructionOpcode, + std::span<const MachineOperand> Operands, + const MachineRegisterInfo &RegInfo) const { + const InstructionDescriptor *Descriptor = descriptor(InstructionOpcode); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "instruction opcode is not registered"); + } + return validateExplicitOperands(*Descriptor, Operands, RegInfo); +} + +Expected<void> InstructionSet::validateOperandTies(const InstructionDescriptor &Descriptor, + std::span<const MachineOperand> ExplicitOperands, + std::span<const MachineOperandTie> Ties) const { + std::vector<std::optional<std::size_t>> Partners(ExplicitOperands.size()); + for (const MachineOperandTie Tie : Ties) { + if (Tie.DefOperand >= ExplicitOperands.size() || Tie.UseOperand >= ExplicitOperands.size() || + Tie.DefOperand == Tie.UseOperand || !isPureDef(ExplicitOperands[Tie.DefOperand]) || + !isPureUse(ExplicitOperands[Tie.UseOperand])) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction tie must connect an in-range pure register def and use"); + } + if (Partners[Tie.DefOperand] || Partners[Tie.UseOperand]) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction operand participates in duplicate or conflicting ties"); + } + Partners[Tie.DefOperand] = Tie.UseOperand; + Partners[Tie.UseOperand] = Tie.DefOperand; + } + + for (std::size_t Index = 0; Index < Descriptor.Operands.size(); ++Index) { + const OperandDescriptor &Operand = Descriptor.Operands[Index]; + if (Operand.DefaultTiedOperand && + (Index >= Partners.size() || Partners[Index] != Operand.DefaultTiedOperand)) { + return makeUnexpected(ErrorCode::VerificationError, + "required descriptor tie is not materialized"); + } + } + return {}; +} + +Expected<void> InstructionSet::validateOperandTies(Opcode InstructionOpcode, + std::span<const MachineOperand> ExplicitOperands, + std::span<const MachineOperandTie> Ties) const { + const InstructionDescriptor *Descriptor = descriptor(InstructionOpcode); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "instruction opcode is not registered"); + } + return validateOperandTies(*Descriptor, ExplicitOperands, Ties); +} + +Expected<void> InstructionSet::validateImplicitOperands( + const InstructionDescriptor &Descriptor, + std::span<const MachineOperand *const> ImplicitOperands) const { + const std::size_t DefaultCount = Descriptor.ImplicitDefs.size() + Descriptor.ImplicitUses.size(); + if (ImplicitOperands.size() < DefaultCount || + (!Descriptor.AllowExtraImplicitOperands && ImplicitOperands.size() != DefaultCount)) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction implicit operand count does not match its descriptor"); + } + + std::size_t Index = 0; + auto ValidateDefault = [&](const ImplicitRegisterDescriptor &Implicit, + bool IsDef) -> Expected<void> { + const MachineOperand *Operand = ImplicitOperands[Index++]; + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || Reg->physicalRegister() != Implicit.Reg || Operand->isDef() != IsDef || + Operand->isUse() == IsDef || !Operand->isImplicit() || + Operand->registerFlags().has(RegisterOperandFlag::EarlyClobber) != Implicit.EarlyClobber || + Operand->registerFlags().has(RegisterOperandFlag::Renamable) != Implicit.Renamable) { + return makeUnexpected(ErrorCode::VerificationError, + "default implicit register operand does not match its descriptor"); + } + return {}; + }; + + for (const ImplicitRegisterDescriptor &Implicit : Descriptor.ImplicitDefs) { + auto Validation = ValidateDefault(Implicit, true); + if (!Validation) + return std::unexpected(Validation.error()); + } + for (const ImplicitRegisterDescriptor &Implicit : Descriptor.ImplicitUses) { + auto Validation = ValidateDefault(Implicit, false); + if (!Validation) + return std::unexpected(Validation.error()); + } + for (; Index < ImplicitOperands.size(); ++Index) { + const MachineOperand *Operand = ImplicitOperands[Index]; + if (Operand == nullptr || !Operand->isRegister() || !Operand->isImplicit()) { + return makeUnexpected(ErrorCode::VerificationError, + "dynamic implicit operand must be an implicit register"); + } + } + return {}; +} + +Expected<void> InstructionSet::validateImplicitOperands( + Opcode InstructionOpcode, std::span<const MachineOperand *const> ImplicitOperands) const { + const InstructionDescriptor *Descriptor = descriptor(InstructionOpcode); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "instruction opcode is not registered"); + } + return validateImplicitOperands(*Descriptor, ImplicitOperands); +} + +Expected<void> +InstructionSet::validateMemOperands(const InstructionDescriptor &Descriptor, + std::span<const MachineMemOperand *const> MemOperands) const { + const bool MayLoad = Descriptor.Flags.has(InstructionFlag::MayLoad); + const bool MayStore = Descriptor.Flags.has(InstructionFlag::MayStore); + if (Descriptor.RequiresMemoryOperands && MemOperands.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "memory instruction is missing MachineMemOperand"); + } + if (!MayLoad && !MayStore && !MemOperands.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "non-memory instruction carries MachineMemOperand"); + } + for (const MachineMemOperand *MemOperand : MemOperands) { + if (MemOperand == nullptr || (MemOperand->isLoad() && !MayLoad) || + (MemOperand->isStore() && !MayStore)) { + return makeUnexpected(ErrorCode::VerificationError, + "MachineMemOperand access conflicts with instruction descriptor"); + } + } + return {}; +} + +Expected<void> +InstructionSet::validateMemOperands(Opcode InstructionOpcode, + std::span<const MachineMemOperand *const> MemOperands) const { + const InstructionDescriptor *Descriptor = descriptor(InstructionOpcode); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "instruction opcode is not registered"); + } + return validateMemOperands(*Descriptor, MemOperands); +} + +Expected<void> InstructionSet::validateOperand(const OperandDescriptor &Descriptor, + const MachineOperand &Operand, + const MachineRegisterInfo &RegInfo) const { + if (!Descriptor.Kinds.contains(Operand.kind())) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction operand kind does not match its descriptor"); + } + if (!Descriptor.AllowAnyTargetFlags && + (Operand.targetFlags() & ~Descriptor.AllowedTargetFlags) != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction operand uses unregistered target flags"); + } + + if (Operand.isRegister()) { + const bool IsDef = Operand.isDef(); + const bool IsUse = Operand.isUse(); + const bool ExpectedDef = isDefRole(Descriptor.RegisterRole); + const bool ExpectedUse = isUseRole(Descriptor.RegisterRole); + if (Descriptor.RegisterRole == RegisterOperandRole::Any) { + if (!IsDef && !IsUse) { + return makeUnexpected(ErrorCode::VerificationError, + "unclassified register operand must define or use its register"); + } + } else if (IsDef != ExpectedDef || IsUse != ExpectedUse) { + return makeUnexpected(ErrorCode::VerificationError, + "register operand def/use role does not match its descriptor"); + } + if (Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) != Descriptor.EarlyClobber) { + return makeUnexpected(ErrorCode::VerificationError, + "register operand early-clobber does not match its descriptor"); + } + if (!Descriptor.AllowSubReg && Operand.subReg().value() != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "register operand has a disallowed subregister index"); + } + + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr) { + return makeUnexpected(ErrorCode::InternalError, "register operand has inconsistent storage"); + } + if (Reg->isVirtual()) { + auto Info = RegInfo.virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + } + } + + if (Descriptor.ImmediatePredicate && Operand.kind() == MachineOperand::Kind::Immediate) { + const ImmediatePredicate *Predicate = immediatePredicate(*Descriptor.ImmediatePredicate); + if (Predicate == nullptr || !Predicate->Predicate(Operand)) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction operand fails its immediate predicate"); + } + } + + return {}; +} + +Expected<void> InstructionSet::validateInstruction(const MachineInstr &Instruction, + const MachineRegisterInfo &RegInfo) const { + const InstructionDescriptor *Descriptor = descriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "instruction opcode is not registered"); + + const auto AllOperands = Instruction.operands(); + std::size_t ExplicitCount = 0; + while (ExplicitCount < AllOperands.size() && !AllOperands[ExplicitCount].isImplicit()) + ++ExplicitCount; + for (std::size_t Index = ExplicitCount; Index < AllOperands.size(); ++Index) { + if (!AllOperands[Index].isImplicit()) { + return makeUnexpected(ErrorCode::VerificationError, + "explicit operand appears after implicit operands"); + } + } + + const auto ExplicitValidation = + validateExplicitOperands(*Descriptor, AllOperands.first(ExplicitCount), RegInfo); + if (!ExplicitValidation) + return std::unexpected(ExplicitValidation.error()); + + std::vector<MachineOperandTie> Ties; + for (std::size_t Index = 0; Index < ExplicitCount; ++Index) { + const MachineOperand &Operand = AllOperands[Index]; + if (!Operand.tiedOperandIndex()) + continue; + const std::size_t OtherIndex = *Operand.tiedOperandIndex(); + if (OtherIndex >= ExplicitCount || AllOperands[OtherIndex].tiedOperandIndex() != Index) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction operand tie is out of range or not reciprocal"); + } + if (Index > OtherIndex) + continue; + if (isPureUse(Operand) && isPureDef(AllOperands[OtherIndex])) + Ties.push_back(MachineOperandTie{OtherIndex, Index}); + else + Ties.push_back(MachineOperandTie{Index, OtherIndex}); + } + + auto TieValidation = validateOperandTies(*Descriptor, AllOperands.first(ExplicitCount), Ties); + if (!TieValidation) + return std::unexpected(TieValidation.error()); + + std::vector<const MachineOperand *> ImplicitOperands; + ImplicitOperands.reserve(AllOperands.size() - ExplicitCount); + for (std::size_t Index = ExplicitCount; Index < AllOperands.size(); ++Index) { + if (AllOperands[Index].tiedOperandIndex()) { + return makeUnexpected(ErrorCode::VerificationError, + "implicit register operand cannot participate in a tie"); + } + ImplicitOperands.push_back(&AllOperands[Index]); + } + + auto ImplicitValidation = validateImplicitOperands(*Descriptor, ImplicitOperands); + if (!ImplicitValidation) + return std::unexpected(ImplicitValidation.error()); + auto MemoryValidation = validateMemOperands(*Descriptor, Instruction.memOperands()); + if (!MemoryValidation) + return std::unexpected(MemoryValidation.error()); + if (Descriptor->VerifyInstructionSemantics != nullptr) + return Descriptor->VerifyInstructionSemantics(Instruction, RegInfo); + return {}; +} + +Expected<std::vector<MachineOperand>> +InstructionSet::createDefaultImplicitOperands(const InstructionDescriptor &Descriptor) const { + std::vector<MachineOperand> Result; + Result.reserve(Descriptor.ImplicitDefs.size() + Descriptor.ImplicitUses.size()); + + auto AppendImplicit = [&Result](const ImplicitRegisterDescriptor &Implicit, + bool IsDef) -> Expected<void> { + RegisterOperandFlags Flags{ + IsDef ? RegisterOperandFlag::Define : RegisterOperandFlag::Use, + RegisterOperandFlag::Implicit, + }; + if (Implicit.EarlyClobber) + Flags.set(RegisterOperandFlag::EarlyClobber); + if (Implicit.Renamable) + Flags.set(RegisterOperandFlag::Renamable); + + auto Operand = MachineOperand::registerOperand(Register::fromPhysical(Implicit.Reg), Flags); + if (!Operand) + return std::unexpected(Operand.error()); + Result.push_back(std::move(*Operand)); + return {}; + }; + + for (const ImplicitRegisterDescriptor &Implicit : Descriptor.ImplicitDefs) { + auto AppendResult = AppendImplicit(Implicit, true); + if (!AppendResult) + return std::unexpected(AppendResult.error()); + } + for (const ImplicitRegisterDescriptor &Implicit : Descriptor.ImplicitUses) { + auto AppendResult = AppendImplicit(Implicit, false); + if (!AppendResult) + return std::unexpected(AppendResult.error()); + } + return Result; +} + +const ImmediatePredicate * +InstructionSet::immediatePredicate(ImmediatePredicateID ID) const noexcept { + const auto Position = PredicateToIndex.find(ID); + if (Position == PredicateToIndex.end()) + return nullptr; + return &ImmediatePredicates[Position->second]; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LiveInterval.cpp b/backend/lib/CodeGen/LiveInterval.cpp new file mode 100644 index 0000000..22e0164 --- /dev/null +++ b/backend/lib/CodeGen/LiveInterval.cpp @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LiveInterval.h" + +#include <algorithm> +#include <cstddef> +#include <limits> +#include <ranges> +#include <unordered_set> + +namespace rcc { + +const LiveSegment *LiveRange::segmentAt(SlotIndex Position) const noexcept { + const auto Candidate = std::ranges::upper_bound( + Segments, Position, {}, [](const LiveSegment &Segment) { return Segment.Start; }); + if (Candidate == Segments.begin()) + return nullptr; + const LiveSegment &Previous = *std::prev(Candidate); + return Previous.contains(Position) ? &Previous : nullptr; +} + +const LiveValueNumber *LiveRange::value(LiveValueNumberID ID) const noexcept { + if (!ID.isValid() || ID.value() > Values.size()) + return nullptr; + const LiveValueNumber &Result = Values[ID.value() - 1U]; + return Result.ID == ID ? &Result : nullptr; +} + +const LiveValueNumber *LiveRange::valueAt(SlotIndex Position) const noexcept { + const LiveSegment *Segment = segmentAt(Position); + return Segment == nullptr ? nullptr : value(Segment->Value); +} + +bool LiveRange::liveAt(SlotIndex Position) const noexcept { + return segmentAt(Position) != nullptr; +} + +bool LiveRange::overlaps(const LiveRange &Other) const noexcept { + std::size_t Left = 0; + std::size_t Right = 0; + while (Left < Segments.size() && Right < Other.Segments.size()) { + if (Segments[Left].overlaps(Other.Segments[Right])) + return true; + if (Segments[Left].End <= Other.Segments[Right].Start) + ++Left; + else + ++Right; + } + return false; +} + +Expected<LiveValueNumberID> LiveRange::createValue(SlotIndex Definition, LiveValueState State, + std::optional<MachineInstrStableID> Origin) { + if (!Definition.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "live value requires a valid definition"); + if (Values.size() >= std::numeric_limits<std::uint32_t>::max()) + return makeUnexpected(ErrorCode::Unsupported, "live value-number space is exhausted"); + const LiveValueNumberID ID = + LiveValueNumberID::fromValue(static_cast<std::uint32_t>(Values.size() + 1U)); + Values.push_back({ID, Definition, State, Origin}); + return ID; +} + +Expected<void> LiveRange::addSegment(LiveSegment Segment) { + if (!Segment.Start.isValid() || !Segment.End.isValid() || Segment.empty() || + value(Segment.Value) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "live segment requires ordered indexes and a known value number"); + } + std::vector<LiveSegment> Candidate = Segments; + Candidate.push_back(Segment); + std::ranges::sort(Candidate, [](const LiveSegment &Left, const LiveSegment &Right) { + if (Left.Start != Right.Start) + return Left.Start < Right.Start; + return Left.End < Right.End; + }); + for (std::size_t Index = 1; Index < Candidate.size(); ++Index) { + if (Candidate[Index].Start < Candidate[Index - 1U].End && + Candidate[Index].Value != Candidate[Index - 1U].Value) { + return makeUnexpected(ErrorCode::VerificationError, + "live segments with different values overlap"); + } + } + Segments = std::move(Candidate); + mergeAdjacent(); + return {}; +} + +Expected<void> LiveRange::extend(LiveValueNumberID Value, SlotIndex Start, SlotIndex End) { + return addSegment({Start, End, Value}); +} + +Expected<void> LiveRange::remove(SlotIndex Start, SlotIndex End) { + if (!Start.isValid() || !End.isValid() || !(Start < End)) { + return makeUnexpected(ErrorCode::InvalidArgument, "removed live range must be nonempty"); + } + std::vector<LiveSegment> Result; + for (const LiveSegment &Segment : Segments) { + if (!(Segment.Start < End && Start < Segment.End)) { + Result.push_back(Segment); + continue; + } + if (Segment.Start < Start) + Result.push_back({Segment.Start, Start, Segment.Value}); + if (End < Segment.End) + Result.push_back({End, Segment.End, Segment.Value}); + } + Segments = std::move(Result); + mergeAdjacent(); + return {}; +} + +Expected<void> LiveRange::shrinkToUses(std::span<const SlotIndex> Uses) { + std::vector<LiveSegment> Result; + for (const LiveSegment &Segment : Segments) { + const LiveValueNumber *Value = value(Segment.Value); + if (Value == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "live segment references an unknown value number"); + std::optional<SlotIndex> LastUse; + for (const SlotIndex Use : Uses) { + if (Segment.contains(Use) && (!LastUse || *LastUse < Use)) + LastUse = Use; + } + if (!LastUse) { + if (Value->State == LiveValueState::Unused) + Result.push_back(Segment); + continue; + } + SlotIndex NewEnd = LastUse->deadSlot(); + if (Segment.End < NewEnd) + NewEnd = Segment.End; + if (Segment.Start < NewEnd) + Result.push_back({Segment.Start, NewEnd, Segment.Value}); + } + Segments = std::move(Result); + mergeAdjacent(); + return verify(); +} + +Expected<void> LiveRange::verify() const { + std::unordered_set<LiveValueNumberID> IDs; + for (std::size_t Index = 0; Index < Values.size(); ++Index) { + const LiveValueNumber &Value = Values[Index]; + if (!Value.ID.isValid() || Value.ID.value() != Index + 1U || !IDs.insert(Value.ID).second || + !Value.Definition.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "live range contains an invalid value number"); + } + } + for (std::size_t Index = 0; Index < Segments.size(); ++Index) { + const LiveSegment &Segment = Segments[Index]; + if (!Segment.Start.isValid() || !Segment.End.isValid() || Segment.empty() || + value(Segment.Value) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "live range contains an invalid segment"); + } + if (Index != 0 && Segments[Index - 1U].End > Segment.Start) { + return makeUnexpected(ErrorCode::VerificationError, + "live range segments are not ordered and disjoint"); + } + } + return {}; +} + +void LiveRange::mergeAdjacent() { + if (Segments.empty()) + return; + std::vector<LiveSegment> Result; + Result.reserve(Segments.size()); + Result.push_back(Segments.front()); + for (std::size_t Index = 1; Index < Segments.size(); ++Index) { + LiveSegment &Previous = Result.back(); + const LiveSegment &Current = Segments[Index]; + if (Current.Value == Previous.Value && Current.Start <= Previous.End) { + if (Previous.End < Current.End) + Previous.End = Current.End; + continue; + } + Result.push_back(Current); + } + Segments = std::move(Result); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LiveIntervals.cpp b/backend/lib/CodeGen/LiveIntervals.cpp new file mode 100644 index 0000000..dc25af8 --- /dev/null +++ b/backend/lib/CodeGen/LiveIntervals.cpp @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LiveIntervals.h" + +#include "rcc/CodeGen/LiveRegUnits.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <vector> + +namespace rcc { + +namespace { +[[nodiscard]] bool maskIntersects(LaneMask Access, std::optional<LaneMask> Filter) noexcept { + return !Filter || Access.intersects(*Filter); +} + +struct InstructionRangeEvent final { + bool HasUse = false; + bool HasNormalDef = false; + bool HasEarlyDef = false; + bool HasRegMaskClobber = false; +}; + +[[nodiscard]] Expected<InstructionRangeEvent> +virtualRangeEvent(const MachineInstr &Instruction, Register Reg, std::optional<LaneMask> Filter, + const MachineFunction &Function, const TargetDescription &Target) { + InstructionRangeEvent Event; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *OperandReg = Operand.getIf<Register>(); + if (OperandReg == nullptr || *OperandReg != Reg) + continue; + auto Lanes = virtualOperandLanes(Operand, Function, Target); + if (!Lanes) + return std::unexpected(Lanes.error()); + if (!maskIntersects(*Lanes, Filter)) + continue; + Event.HasUse |= Operand.isUse() && !Operand.isDebugUse() && + !Operand.registerFlags().has(RegisterOperandFlag::Undef); + if (Operand.isDef()) { + if (Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber)) + Event.HasEarlyDef = true; + else + Event.HasNormalDef = true; + } + } + return Event; +} + +[[nodiscard]] Expected<InstructionRangeEvent> unitRangeEvent(const MachineInstr &Instruction, + RegisterUnitID Unit, + std::optional<LaneMask> Filter, + const TargetDescription &Target) { + InstructionRangeEvent Event; + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.isRegister()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isPhysical()) + continue; + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + const auto Position = std::ranges::find_if( + *Units, [Unit](const PhysicalUnitAccess &Access) { return Access.Unit == Unit; }); + if (Position == Units->end() || !maskIntersects(Position->Lanes, Filter)) + continue; + Event.HasUse |= Operand.isUse() && !Operand.isDebugUse() && + !Operand.registerFlags().has(RegisterOperandFlag::Undef); + if (Operand.isDef()) { + if (Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber)) + Event.HasEarlyDef = true; + else + Event.HasNormalDef = true; + } + continue; + } + if (const auto *MaskOperand = Operand.getIf<RegisterMaskOperand>(); + MaskOperand != nullptr && MaskOperand->Mask) { + for (const PhysicalRegisterDesc &Description : Target.registers().physicalRegisters()) { + if (!MaskOperand->Mask->clobbers(Description.Reg)) + continue; + if (std::ranges::any_of(Description.Units, [Unit, Filter](const RegisterUnitLane &Access) { + return Access.Unit == Unit && maskIntersects(Access.Lanes, Filter); + })) { + Event.HasRegMaskClobber = true; + break; + } + } + continue; + } + if (const auto *MaskOperand = Operand.getIf<LiveOutRegisterMaskOperand>(); + MaskOperand != nullptr && MaskOperand->Mask) { + for (const PhysicalRegisterDesc &Description : Target.registers().physicalRegisters()) { + if (!MaskOperand->Mask->preserves(Description.Reg)) + continue; + if (std::ranges::any_of(Description.Units, [Unit, Filter](const RegisterUnitLane &Access) { + return Access.Unit == Unit && maskIntersects(Access.Lanes, Filter); + })) { + Event.HasUse = true; + break; + } + } + } + } + return Event; +} + +template <typename EventFn, typename LiveOutFn> +[[nodiscard]] Expected<LiveRange> +buildLiveRange(MachineFunction &Function, const SlotIndexesAnalysis::Result &Indexes, + const EventFn &EventForInstruction, const LiveOutFn &IsLiveOut) { + LiveRange Range; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + std::optional<SlotIndex> End; + bool SawLaterDefinition = false; + if (IsLiveOut(*Block)) + End = Indexes.blockEnd(*Block); + + for (MachineInstr *Instruction = Block->lastInstruction(); Instruction != nullptr; + Instruction = Instruction->previous()) { + auto Event = EventForInstruction(*Instruction); + if (!Event) + return std::unexpected(Event.error()); + if (!Event->HasUse && !Event->HasNormalDef && !Event->HasEarlyDef && + !Event->HasRegMaskClobber) { + continue; + } + + const SlotIndex InstructionSlot = Indexes.instructionIndex(*Instruction); + if (!InstructionSlot.isValid()) { + return makeUnexpected(ErrorCode::InternalError, + "live-range construction has no instruction SlotIndex"); + } + auto AddDefinition = [&](SlotIndex DefSlot, bool IsRegMask) -> Expected<void> { + const bool HasLiveValue = End && DefSlot < *End; + if (IsRegMask && !HasLiveValue) { + End.reset(); + SawLaterDefinition = true; + return {}; + } + const LiveValueState State = IsRegMask ? LiveValueState::Redefined + : HasLiveValue ? LiveValueState::Normal + : SawLaterDefinition ? LiveValueState::Redefined + : LiveValueState::Unused; + auto Value = Range.createValue(DefSlot, State, Instruction->stableID()); + if (!Value) + return std::unexpected(Value.error()); + const SlotIndex SegmentEnd = HasLiveValue ? *End : InstructionSlot.deadSlot(); + if (DefSlot < SegmentEnd) { + auto Added = Range.addSegment({DefSlot, SegmentEnd, *Value}); + if (!Added) + return std::unexpected(Added.error()); + } + End.reset(); + SawLaterDefinition = true; + return {}; + }; + + if (Event->HasNormalDef) { + auto Added = AddDefinition(InstructionSlot.registerSlot(), false); + if (!Added) + return std::unexpected(Added.error()); + } + if (Event->HasUse) { + const SlotIndex UseEnd = + Event->HasNormalDef ? InstructionSlot.registerSlot() : InstructionSlot.deadSlot(); + if (!End || *End < UseEnd) + End = UseEnd; + } + if (Event->HasEarlyDef) { + auto Added = AddDefinition(InstructionSlot.earlyClobberSlot(), false); + if (!Added) + return std::unexpected(Added.error()); + } else if (Event->HasRegMaskClobber) { + auto Added = AddDefinition(InstructionSlot.earlyClobberSlot(), true); + if (!Added) + return std::unexpected(Added.error()); + } + } + + if (End) { + const SlotIndex Start = Indexes.blockStart(*Block); + if (!Start.isValid() || !(*End > Start)) { + return makeUnexpected(ErrorCode::InternalError, + "live-in range has invalid block boundary indexes"); + } + auto Value = Range.createValue(Start, LiveValueState::Phi); + if (!Value) + return std::unexpected(Value.error()); + auto Added = Range.addSegment({Start, *End, *Value}); + if (!Added) + return std::unexpected(Added.error()); + } + } + auto Verification = Range.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return Range; +} + +[[nodiscard]] Expected<LaneMask> virtualRegisterFullLanes(Register Reg, + const MachineFunction &Function, + const TargetDescription &Target) { + return virtualRegisterLanes(Reg, Function, Target); +} + +[[nodiscard]] Expected<std::vector<LaneMask>> +partitionVirtualRegisterLanes(Register Reg, MachineFunction &Function, + const TargetDescription &Target) { + auto Full = virtualRegisterFullLanes(Reg, Function, Target); + if (!Full) + return std::unexpected(Full.error()); + std::vector<LaneMask> AccessMasks; + for (const MachineOperand &Operand : Function.registerInfo().regNoDebugOperands(Reg)) { + auto Lanes = virtualOperandLanes(Operand, Function, Target); + if (!Lanes) + return std::unexpected(Lanes.error()); + if (*Lanes != *Full && std::ranges::find(AccessMasks, *Lanes) == AccessMasks.end()) + AccessMasks.push_back(*Lanes); + } + if (AccessMasks.empty()) + return {}; + + std::vector<LaneMask> Groups{*Full}; + for (const LaneMask Access : AccessMasks) { + std::vector<LaneMask> Refined; + for (const LaneMask Group : Groups) { + const LaneMask Intersection = Group & Access; + const LaneMask Difference = Group & ~Access; + if (!Intersection.empty()) + Refined.push_back(Intersection); + if (!Difference.empty()) + Refined.push_back(Difference); + } + Groups = std::move(Refined); + } + std::ranges::sort( + Groups, [](LaneMask Left, LaneMask Right) { return Left.rawValue() < Right.rawValue(); }); + return Groups; +} +} // namespace + +const LiveInterval *LiveIntervalsAnalysis::Result::interval(Register Reg) const noexcept { + const auto Position = Intervals.find(Reg); + return Position == Intervals.end() ? nullptr : &Position->second; +} + +LiveInterval *LiveIntervalsAnalysis::Result::interval(Register Reg) noexcept { + const auto Position = Intervals.find(Reg); + return Position == Intervals.end() ? nullptr : &Position->second; +} + +const LiveRange *LiveIntervalsAnalysis::Result::regUnitRange(RegisterUnitID Unit) const noexcept { + const auto Position = RegisterUnitRanges.find(Unit); + return Position == RegisterUnitRanges.end() ? nullptr : &Position->second; +} + +bool LiveIntervalsAnalysis::Result::interferes(Register Left, Register Right) const noexcept { + const LiveInterval *LeftInterval = interval(Left); + const LiveInterval *RightInterval = interval(Right); + return LeftInterval != nullptr && RightInterval != nullptr && + LeftInterval->overlaps(*RightInterval); +} + +bool LiveIntervalsAnalysis::Result::regMaskClobbers(const LiveInterval &Interval, + PhysicalRegister Candidate) const noexcept { + for (const RegMaskPosition &Position : RegMasks) { + if (Position.Mask && Position.Mask->clobbers(Candidate) && + Interval.MainRange.liveAt(Position.Position)) { + return true; + } + } + return false; +} + +Expected<void> LiveIntervalsAnalysis::Result::shrinkToUses(Register Reg, + std::span<const SlotIndex> Uses) { + LiveInterval *Value = interval(Reg); + if (Value == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "virtual register has no live interval"); + auto Main = Value->MainRange.shrinkToUses(Uses); + if (!Main) + return std::unexpected(Main.error()); + for (LiveSubRange &SubRange : Value->SubRanges) { + auto Shrunk = SubRange.Range.shrinkToUses(Uses); + if (!Shrunk) + return std::unexpected(Shrunk.error()); + } + return {}; +} + +bool LiveIntervalsAnalysis::Result::invalidate(const MachineFunction &, + const PreservedAnalyses &Preserved) const noexcept { + return !Preserved.preserves(MachineFunctionAnalysisManager::id<LiveIntervalsAnalysis>()); +} + +Expected<LiveIntervalsAnalysis::Result> +LiveIntervalsAnalysis::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Indexes = Analyses.getResult<SlotIndexesAnalysis>(Function, Target); + if (!Indexes) + return std::unexpected(Indexes.error()); + auto Variables = Analyses.getResult<LiveVariablesAnalysis>(Function, Target); + if (!Variables) + return std::unexpected(Variables.error()); + + Result ResultValue; + ResultValue.Indexes = **Indexes; + for (std::size_t Index = 0; Index < Function.registerInfo().virtualRegisters().size(); ++Index) { + auto RegValue = Register::makeVirtual(Index); + if (!RegValue || !Function.registerInfo().isKnownVirtualRegister(*RegValue)) + continue; + const Register Reg = *RegValue; + auto MainRange = buildLiveRange( + Function, ResultValue.Indexes, + [&Function, &Target, Reg](const MachineInstr &Instruction) { + return virtualRangeEvent(Instruction, Reg, std::nullopt, Function, Target); + }, + [&Variables, Reg](const MachineBasicBlock &Block) { + const MachineBlockLiveness *Value = (*Variables)->block(Block); + return Value != nullptr && Value->VirtualLiveOut.intersects(Reg); + }); + if (!MainRange) + return std::unexpected(MainRange.error()); + if (MainRange->empty()) + continue; + + LiveInterval Interval; + Interval.Reg = Reg; + Interval.MainRange = std::move(*MainRange); + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + // Preserve the target-provided hint order. MachineRegisterInfo already de-duplicates + // hints, and their order is the deterministic preference order consumed by allocators. + Interval.AllocationHints = Info->get().AllocationHints; + + std::uint64_t Weight = 1; + for (const MachineOperand &Operand : Function.registerInfo().useOperands(Reg, false)) { + if (Operand.parent() == nullptr || Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + Weight = Weight > std::numeric_limits<std::uint64_t>::max() - 10U + ? std::numeric_limits<std::uint64_t>::max() + : Weight + 10U; + } + Interval.SpillWeight = Weight; + + auto LaneGroups = partitionVirtualRegisterLanes(Reg, Function, Target); + if (!LaneGroups) + return std::unexpected(LaneGroups.error()); + for (const LaneMask Lanes : *LaneGroups) { + auto SubRange = buildLiveRange( + Function, ResultValue.Indexes, + [&Function, &Target, Reg, Lanes](const MachineInstr &Instruction) { + return virtualRangeEvent(Instruction, Reg, Lanes, Function, Target); + }, + [&Variables, Reg, Lanes](const MachineBasicBlock &Block) { + const MachineBlockLiveness *Value = (*Variables)->block(Block); + return Value != nullptr && Value->VirtualLiveOut.intersects(Reg, Lanes); + }); + if (!SubRange) + return std::unexpected(SubRange.error()); + if (!SubRange->empty()) + Interval.SubRanges.push_back({Lanes, std::move(*SubRange)}); + } + ResultValue.Intervals.emplace(Reg, std::move(Interval)); + } + + for (const RegisterUnitDesc &Unit : Target.registers().registerUnits()) { + auto Range = buildLiveRange( + Function, ResultValue.Indexes, + [&Target, Unit](const MachineInstr &Instruction) { + return unitRangeEvent(Instruction, Unit.ID, std::nullopt, Target); + }, + [&Variables, Unit](const MachineBasicBlock &Block) { + const MachineBlockLiveness *Value = (*Variables)->block(Block); + return Value != nullptr && Value->PhysicalLiveOut.intersects(Unit.ID); + }); + if (!Range) + return std::unexpected(Range.error()); + if (!Range->empty()) + ResultValue.RegisterUnitRanges.emplace(Unit.ID, std::move(*Range)); + } + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + const auto *MaskOperand = Operand.getIf<RegisterMaskOperand>(); + if (MaskOperand == nullptr) + continue; + if (!MaskOperand->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "live intervals encountered a null register mask"); + ResultValue.RegMasks.push_back( + {ResultValue.Indexes.instructionIndex(*Instruction).earlyClobberSlot(), + Instruction->stableID(), MaskOperand->Mask}); + } + } + } + return ResultValue; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LivePhysRegs.cpp b/backend/lib/CodeGen/LivePhysRegs.cpp new file mode 100644 index 0000000..9bbc069 --- /dev/null +++ b/backend/lib/CodeGen/LivePhysRegs.cpp @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LivePhysRegs.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <cstddef> +#include <ranges> +#include <vector> + +namespace rcc { + +namespace { +struct OperandFlagUpdate final { + MachineOperand *Operand = nullptr; + RegisterOperandFlags Flags; +}; + +[[nodiscard]] bool intersects(const LiveRegUnits &Live, + const std::vector<PhysicalUnitAccess> &Units) noexcept { + return std::ranges::any_of(Units, [&Live](const PhysicalUnitAccess &Unit) { + return Live.intersects(Unit.Unit, Unit.Lanes); + }); +} + +void addUnits(LiveRegUnits &Live, const std::vector<PhysicalUnitAccess> &Units) { + for (const PhysicalUnitAccess &Unit : Units) + Live.add(Unit.Unit, Unit.Lanes); +} + +void removeUnits(LiveRegUnits &Live, const std::vector<PhysicalUnitAccess> &Units) { + for (const PhysicalUnitAccess &Unit : Units) + Live.remove(Unit.Unit, Unit.Lanes); +} + +[[nodiscard]] OperandFlagUpdate &getFlagUpdate(std::vector<OperandFlagUpdate> &Updates, + MachineOperand &Operand) { + const auto Position = std::ranges::find_if( + Updates, [&Operand](const OperandFlagUpdate &Update) { return Update.Operand == &Operand; }); + if (Position != Updates.end()) + return *Position; + Updates.push_back({&Operand, Operand.registerFlags()}); + return Updates.back(); +} + +[[nodiscard]] Expected<std::vector<OperandFlagUpdate>> +buildBlockLivenessRepairPlan(MachineBasicBlock &Block, const TargetDescription &Target, + const MachineBlockLiveness &Boundary) { + LiveRegUnits Live = Boundary.PhysicalLiveOut; + std::vector<OperandFlagUpdate> Updates; + for (MachineInstr *Instruction = Block.lastInstruction(); Instruction != nullptr; + Instruction = Instruction->previous()) { + for (const MachineOperand &Operand : Instruction->operands()) { + const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); + if (Mask == nullptr) + continue; + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a null live-out mask"); + } + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (Mask->Mask->preserves(Register.Reg)) + addPhysicalRegister(Live, Register); + } + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister()) + continue; + OperandFlagUpdate &Update = getFlagUpdate(Updates, *Operand); + Update.Flags.clear(RegisterOperandFlag::Dead); + if (!Operand->isDef()) + continue; + const Register *Reg = Operand->getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(*Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + if (!intersects(Live, *Units)) + Update.Flags.set(RegisterOperandFlag::Dead); + } + + for (const MachineOperand &Operand : Instruction->operands()) { + if (Operand.isRegister() && Operand.isDef()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + removeUnits(Live, *Units); + continue; + } + const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + if (Mask == nullptr) + continue; + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA liveness repair encountered a null register mask"); + } + LiveRegUnits Clobbers; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (Mask->Mask->clobbers(Register.Reg)) + addPhysicalRegister(Clobbers, Register); + } + Live.subtract(Clobbers); + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister()) + continue; + OperandFlagUpdate &Update = getFlagUpdate(Updates, *Operand); + Update.Flags.clear(RegisterOperandFlag::Kill); + if (!Operand->isUse() || Operand->isDebugUse() || + Operand->registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + const Register *Reg = Operand->getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + if (isReservedPhysicalRegister(*Reg, Block.function(), Target)) + continue; + auto Units = physicalOperandUnits(*Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + if (!intersects(Live, *Units)) + Update.Flags.set(RegisterOperandFlag::Kill); + addUnits(Live, *Units); + } + } + if (Live != Boundary.PhysicalLiveIn) { + return makeUnexpected(ErrorCode::InternalError, + "post-RA local liveness repair changed the block boundary"); + } + return Updates; +} +} // namespace + +Expected<bool> repairPostRAMachineBlockLiveness(std::span<MachineBasicBlock *const> Blocks, + const TargetDescription &Target, + const LiveVariablesAnalysis::Result &Liveness) { + std::vector<OperandFlagUpdate> Updates; + for (MachineBasicBlock *Block : Blocks) { + const MachineBlockLiveness *Boundary = Liveness.block(*Block); + if (Boundary == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "post-RA scheduler lost a block liveness boundary"); + } + auto BlockUpdates = buildBlockLivenessRepairPlan(*Block, Target, *Boundary); + if (!BlockUpdates) + return std::unexpected(BlockUpdates.error()); + Updates.insert(Updates.end(), BlockUpdates->begin(), BlockUpdates->end()); + } + bool Changed = false; + for (const OperandFlagUpdate &Update : Updates) { + Changed |= Update.Operand->registerFlags() != Update.Flags; + auto Applied = Update.Operand->setRegisterFlags(Update.Flags); + if (!Applied) + return std::unexpected(Applied.error()); + } + return Changed; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LiveRangeEdit.cpp b/backend/lib/CodeGen/LiveRangeEdit.cpp new file mode 100644 index 0000000..23a7b3d --- /dev/null +++ b/backend/lib/CodeGen/LiveRangeEdit.cpp @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LiveRangeEdit.h" + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +namespace rcc { + +Expected<Register> LiveRangeEdit::createFrom(Register Parent, bool Spillable, + std::optional<Register> Origin) { + auto ParentInfo = Function->registerInfo().virtualRegisterInfo(Parent); + if (!ParentInfo) + return std::unexpected(ParentInfo.error()); + const VirtualRegisterInfo &Source = ParentInfo->get(); + Expected<Register> Created = Source.Type + ? Function->registerInfo().createVirtualRegister(*Source.Type) + : Function->registerInfo().createVirtualRegister(); + if (!Created) + return std::unexpected(Created.error()); + const Register Child = *Created; + auto Fail = [&](const Error &Failure) -> Expected<Register> { + auto Erased = Function->registerInfo().eraseVirtualRegister(Child); + if (!Erased) + return makeUnexpected(ErrorCode::InternalError, + "split virtual-register setup and rollback both failed"); + return std::unexpected(Failure); + }; + if (Source.Bank) { + auto Set = Function->registerInfo().setRegisterBank(Child, *Source.Bank); + if (!Set) + return Fail(Set.error()); + } + if (Source.RegClass) { + auto Set = Function->registerInfo().setRegisterClass(Child, *Source.RegClass); + if (!Set) + return Fail(Set.error()); + } + for (const PhysicalRegister Hint : Source.AllocationHints) { + auto Added = Function->registerInfo().addAllocationHint(Child, Hint); + if (!Added) + return Fail(Added.error()); + } + if (Source.TargetFlags != 0) { + auto Set = Function->registerInfo().setTargetRegisterFlags(Child, Source.TargetFlags); + if (!Set) + return Fail(Set.error()); + } + const VirtRegMapEntry *ParentEntry = Mapping->entry(Parent); + const Register Root = Origin.value_or( + ParentEntry != nullptr && ParentEntry->Origin ? *ParentEntry->Origin : Parent); + const std::optional<FrameIndex> ParentStackSlot = + ParentEntry == nullptr ? std::nullopt : ParentEntry->Spill.StackSlot; + const LaneMask ParentSpillLanes = + ParentEntry == nullptr ? LaneMask::all() : ParentEntry->Spill.Lanes; + auto Added = Mapping->add(Child, Spillable, Parent, Root); + if (!Added) + return Fail(Added.error()); + auto Split = Mapping->markSplit(Child, Parent, Root); + if (!Split) + return Fail(Split.error()); + if (ParentStackSlot) { + auto SharedSlot = Mapping->assignStackSlot(Child, *ParentStackSlot, ParentSpillLanes); + if (!SharedSlot) + return Fail(SharedSlot.error()); + } + return Child; +} + +Expected<FrameIndex> LiveRangeEdit::getOrCreateSpillSlot(Register VirtualReg) { + if (const auto Existing = Mapping->stackSlot(VirtualReg)) + return *Existing; + const VirtRegMapEntry *Entry = Mapping->entry(VirtualReg); + const Register Original = Entry != nullptr && Entry->Origin ? *Entry->Origin : VirtualReg; + if (Original != VirtualReg) { + if (const auto Existing = Mapping->stackSlot(Original)) { + auto Shared = Mapping->assignStackSlot(VirtualReg, *Existing); + if (!Shared) + return std::unexpected(Shared.error()); + return *Existing; + } + } + auto ClassID = regalloc_detail::registerClassFor(*Function, Original); + if (!ClassID) + return std::unexpected(ClassID.error()); + const RegisterClassDesc *Class = Target->registers().registerClass(*ClassID); + if (Class == nullptr || Class->SpillSizeInBits == 0 || Class->SpillAlignmentInBits == 0 || + Class->SpillSizeInBits % 8U != 0 || Class->SpillAlignmentInBits % 8U != 0) { + return makeUnexpected(ErrorCode::InternalError, + "allocatable register class has no valid spill size and alignment"); + } + auto Align = Alignment::create(Class->SpillAlignmentInBits / 8U); + if (!Align) + return std::unexpected(Align.error()); + StackObjectDesc Description; + Description.Kind = StackObjectKind::SpillSlot; + Description.Size = Class->SpillSizeInBits / 8U; + Description.Align = *Align; + auto Slot = Function->frameInfo().createObject(std::move(Description)); + if (!Slot) + return std::unexpected(Slot.error()); + auto AssignedOriginal = Mapping->assignStackSlot(Original, *Slot); + if (!AssignedOriginal) + return std::unexpected(AssignedOriginal.error()); + if (Original != VirtualReg) { + auto AssignedChild = Mapping->assignStackSlot(VirtualReg, *Slot); + if (!AssignedChild) + return std::unexpected(AssignedChild.error()); + } + return *Slot; +} + +Expected<void> LiveRangeEdit::eraseVirtualRegister(Register VirtualReg) { + if (!Function->registerInfo().useEmpty(VirtualReg)) + return makeUnexpected(ErrorCode::InternalError, + "cannot erase a split virtual register with remaining uses"); + for (const MachineOperand &Operand : Function->registerInfo().defOperands(VirtualReg)) { + if (Operand.parent() != nullptr) + return makeUnexpected(ErrorCode::InternalError, + "cannot erase a split virtual register with remaining definitions"); + } + return Function->registerInfo().eraseVirtualRegister(VirtualReg); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LiveRegMatrix.cpp b/backend/lib/CodeGen/LiveRegMatrix.cpp new file mode 100644 index 0000000..64c7046 --- /dev/null +++ b/backend/lib/CodeGen/LiveRegMatrix.cpp @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LiveRegMatrix.h" + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <bit> +#include <cstdint> +#include <optional> +#include <ranges> +#include <span> +#include <tuple> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] std::optional<LaneMask> relevantLanes(const LiveInterval &Interval, + RegisterUnitLane Unit) noexcept { + if (Interval.SubRanges.empty()) + return Unit.Lanes; + LaneMask Result; + for (const LiveSubRange &SubRange : Interval.SubRanges) { + if (SubRange.Lanes.intersects(Unit.Lanes)) + Result = Result | (SubRange.Lanes & Unit.Lanes); + } + if (Result.empty()) + return std::nullopt; + return Result; +} + +[[nodiscard]] LaneMask normalizeUnitLanes(LaneMask UnitViewLanes, LaneMask RegisterLanes) noexcept { + std::uint64_t View = UnitViewLanes.rawValue(); + const std::uint64_t Selected = View & RegisterLanes.rawValue(); + std::uint64_t Normalized = 0; + std::uint32_t UnitLane = 0; + while (View != 0) { + const std::uint64_t RegisterLane = std::uint64_t{1} << std::countr_zero(View); + if ((Selected & RegisterLane) != 0) + Normalized |= std::uint64_t{1} << UnitLane; + View &= View - 1U; + ++UnitLane; + } + return LaneMask::fromRaw(Normalized); +} + +[[nodiscard]] LaneMask intervalUnitLanes(const LiveInterval &Interval, + RegisterUnitLane Unit) noexcept { + if (Interval.SubRanges.empty()) + return normalizeUnitLanes(Unit.Lanes, Unit.Lanes); + LaneMask Result; + for (const LiveSubRange &SubRange : Interval.SubRanges) + Result = Result | normalizeUnitLanes(Unit.Lanes, SubRange.Lanes); + return Result; +} + +[[nodiscard]] bool intervalLiveAtFixedRegister(const LiveInterval &Interval, + const PhysicalRegisterDesc &Candidate, + const PhysicalRegisterDesc &Fixed, + SlotIndex Position) noexcept { + for (const RegisterUnitLane CandidateUnit : Candidate.Units) { + const auto FixedUnit = + std::ranges::find(Fixed.Units, CandidateUnit.Unit, &RegisterUnitLane::Unit); + if (FixedUnit == Fixed.Units.end()) + continue; + const LaneMask FixedLanes = normalizeUnitLanes(FixedUnit->Lanes, FixedUnit->Lanes); + if (Interval.SubRanges.empty()) { + if (normalizeUnitLanes(CandidateUnit.Lanes, CandidateUnit.Lanes).intersects(FixedLanes) && + Interval.MainRange.liveAt(Position)) { + return true; + } + continue; + } + for (const LiveSubRange &SubRange : Interval.SubRanges) { + if (normalizeUnitLanes(CandidateUnit.Lanes, SubRange.Lanes).intersects(FixedLanes) && + SubRange.Range.liveAt(Position)) { + return true; + } + } + } + return false; +} + +[[nodiscard]] bool rangeForUnitOverlaps(const LiveInterval &Interval, RegisterUnitLane Unit, + const LiveRange &Other) noexcept { + if (!relevantLanes(Interval, Unit)) + return false; + if (Interval.SubRanges.empty()) + return Interval.MainRange.overlaps(Other); + for (const LiveSubRange &SubRange : Interval.SubRanges) { + if (SubRange.Lanes.intersects(Unit.Lanes) && SubRange.Range.overlaps(Other)) + return true; + } + return false; +} + +} // namespace + +bool regalloc_detail::intervalsOverlapOnPhysicalRegisters( + const LiveInterval &Left, PhysicalRegister LeftReg, const LiveInterval &Right, + PhysicalRegister RightReg, const TargetRegisterInfo &Registers) noexcept { + const PhysicalRegisterDesc *LeftDescription = Registers.physicalRegister(LeftReg); + const PhysicalRegisterDesc *RightDescription = Registers.physicalRegister(RightReg); + if (LeftDescription == nullptr || RightDescription == nullptr) + return true; + for (const RegisterUnitLane LeftUnit : LeftDescription->Units) { + for (const RegisterUnitLane RightUnit : RightDescription->Units) { + if (LeftUnit.Unit != RightUnit.Unit) + continue; + const LaneMask LeftUnitLanes = intervalUnitLanes(Left, LeftUnit); + const LaneMask RightUnitLanes = intervalUnitLanes(Right, RightUnit); + if (!LeftUnitLanes.intersects(RightUnitLanes)) + continue; + if (Left.SubRanges.empty() && Right.SubRanges.empty()) + return Left.MainRange.overlaps(Right.MainRange); + const auto LeftRanges = Left.SubRanges.empty() + ? std::span<const LiveSubRange>{} + : std::span<const LiveSubRange>(Left.SubRanges); + const auto RightRanges = Right.SubRanges.empty() + ? std::span<const LiveSubRange>{} + : std::span<const LiveSubRange>(Right.SubRanges); + if (LeftRanges.empty()) { + for (const LiveSubRange &RightRange : RightRanges) { + const LaneMask RightRangeLanes = normalizeUnitLanes(RightUnit.Lanes, RightRange.Lanes); + if (LeftUnitLanes.intersects(RightRangeLanes) && + Left.MainRange.overlaps(RightRange.Range)) + return true; + } + continue; + } + if (RightRanges.empty()) { + for (const LiveSubRange &LeftRange : LeftRanges) { + const LaneMask LeftRangeLanes = normalizeUnitLanes(LeftUnit.Lanes, LeftRange.Lanes); + if (LeftRangeLanes.intersects(RightUnitLanes) && + LeftRange.Range.overlaps(Right.MainRange)) + return true; + } + continue; + } + for (const LiveSubRange &LeftRange : LeftRanges) { + const LaneMask LeftRangeLanes = normalizeUnitLanes(LeftUnit.Lanes, LeftRange.Lanes); + if (LeftRangeLanes.empty()) + continue; + for (const LiveSubRange &RightRange : RightRanges) { + const LaneMask RightRangeLanes = normalizeUnitLanes(RightUnit.Lanes, RightRange.Lanes); + if (LeftRangeLanes.intersects(RightRangeLanes) && + LeftRange.Range.overlaps(RightRange.Range)) + return true; + } + } + } + } + return false; +} + +bool LiveRegMatrix::intervalOverlapsUnit(const LiveInterval &Interval, RegisterUnitLane Unit, + const LiveRange &UnitRange) const noexcept { + return rangeForUnitOverlaps(Interval, Unit, UnitRange); +} + +bool LiveRegMatrix::intervalsOverlapOnCandidates(const LiveInterval &Left, PhysicalRegister LeftReg, + const LiveInterval &Right, + PhysicalRegister RightReg) const noexcept { + return regalloc_detail::intervalsOverlapOnPhysicalRegisters(Left, LeftReg, Right, RightReg, + Target->registers()); +} + +std::vector<RegisterInterference> +LiveRegMatrix::collectInterferences(Register VirtualReg, PhysicalRegister PhysicalReg) const { + std::vector<RegisterInterference> Result; + const LiveInterval *Interval = Intervals->interval(VirtualReg); + const PhysicalRegisterDesc *Candidate = Target->registers().physicalRegister(PhysicalReg); + if (Interval == nullptr || Candidate == nullptr) + return Result; + + for (const MachineBasicBlock *Block = Function->firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const SlotIndex Position = Intervals->slot(*Instruction); + if (!Position.isValid()) + continue; + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isPhysical()) + continue; + const auto Fixed = Reg->physicalRegister(); + if (!Fixed) + continue; + const PhysicalRegister FixedReg = Fixed.value(); + if (!Target->registers().regsOverlap(FixedReg, PhysicalReg)) + continue; + const PhysicalRegisterDesc *FixedDescription = + Target->registers().physicalRegister(FixedReg); + if (FixedDescription == nullptr) + continue; + const SlotIndex OperandPosition = + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) + ? Position.earlyClobberSlot() + : Position.registerSlot(); + if (intervalLiveAtFixedRegister(*Interval, *Candidate, *FixedDescription, + OperandPosition)) { + Result.push_back(RegisterInterference{RegisterInterferenceKind::FixedRegister, + std::nullopt, FixedReg, std::nullopt, + Instruction->stableID()}); + } + } + } + } + + for (const RegMaskPosition &Mask : Intervals->regMaskPositions()) { + if (Mask.Mask && Mask.Mask->clobbers(PhysicalReg) && + Interval->MainRange.liveAt(Mask.Position)) { + Result.push_back(RegisterInterference{RegisterInterferenceKind::RegisterMask, std::nullopt, + std::nullopt, std::nullopt, Mask.Instruction}); + } + } + + for (const RegisterUnitLane Unit : Candidate->Units) { + const LiveRange *FixedRange = Intervals->regUnitRange(Unit.Unit); + if (FixedRange != nullptr && intervalOverlapsUnit(*Interval, Unit, *FixedRange)) { + Result.push_back(RegisterInterference{RegisterInterferenceKind::RegisterUnit, std::nullopt, + std::nullopt, Unit.Unit, std::nullopt}); + } + } + + for (const auto &[OtherReg, OtherPhysical] : OrderedAssignments) { + if (OtherReg == VirtualReg) + continue; + const LiveInterval *OtherInterval = Intervals->interval(OtherReg); + if (OtherInterval != nullptr && + intervalsOverlapOnCandidates(*Interval, PhysicalReg, *OtherInterval, OtherPhysical)) { + Result.push_back(RegisterInterference{RegisterInterferenceKind::VirtualRegister, OtherReg, + std::nullopt, std::nullopt, std::nullopt}); + } + } + + std::ranges::sort( + Result, [](const RegisterInterference &Left, const RegisterInterference &Right) { + return std::tuple{Left.Kind, Left.VirtualReg.value_or(Register{}).opaqueValue(), + Left.FixedReg.value_or(PhysicalRegister{}).number(), + Left.Unit ? Left.Unit->value() : 0U, + Left.Instruction ? Left.Instruction->value() : 0ULL} < + std::tuple{Right.Kind, Right.VirtualReg.value_or(Register{}).opaqueValue(), + Right.FixedReg.value_or(PhysicalRegister{}).number(), + Right.Unit ? Right.Unit->value() : 0U, + Right.Instruction ? Right.Instruction->value() : 0ULL}; + }); + Result.erase(std::unique(Result.begin(), Result.end(), + [](const RegisterInterference &Left, const RegisterInterference &Right) { + return Left.Kind == Right.Kind && + Left.VirtualReg == Right.VirtualReg && + Left.FixedReg == Right.FixedReg && Left.Unit == Right.Unit && + Left.Instruction == Right.Instruction; + }), + Result.end()); + return Result; +} + +RegisterInterference LiveRegMatrix::checkInterference(Register VirtualReg, + PhysicalRegister PhysicalReg) const noexcept { + const std::vector<RegisterInterference> Values = collectInterferences(VirtualReg, PhysicalReg); + if (Values.empty()) + return {}; + return Values.front(); +} + +Expected<void> LiveRegMatrix::assign(Register VirtualReg, PhysicalRegister PhysicalReg) { + if (!VirtualReg.isVirtual() || Intervals->interval(VirtualReg) == nullptr || + Target->registers().physicalRegister(PhysicalReg) == nullptr) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "LiveRegMatrix assignment requires a known interval and physical register"); + } + if (regalloc_detail::isUnavailablePhysicalRegister(*Function, Target->registers(), PhysicalReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "LiveRegMatrix cannot assign an unavailable physical register"); + } + auto Info = Function->registerInfo().virtualRegisterInfo(VirtualReg); + if (!Info) + return std::unexpected(Info.error()); + const auto ClassID = Info->get().RegClass; + if (ClassID) { + if (!Target->registers().registerClassContains(ClassID.value(), PhysicalReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "LiveRegMatrix assignment is outside the virtual register class"); + } + } + if (!regalloc_detail::candidateSupportsSubRegisters(*Function, Target->registers(), VirtualReg, + PhysicalReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "LiveRegMatrix assignment lacks a required subregister view"); + } + if (Assignments.contains(VirtualReg)) + return makeUnexpected(ErrorCode::InvalidArgument, "virtual register is already assigned"); + const RegisterInterference Interference = checkInterference(VirtualReg, PhysicalReg); + if (Interference.Kind != RegisterInterferenceKind::Free) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "physical register assignment has live interference"); + } + Assignments.emplace(VirtualReg, PhysicalReg); + OrderedAssignments.emplace_back(VirtualReg, PhysicalReg); + std::ranges::sort(OrderedAssignments, [](const auto &Left, const auto &Right) { + return Left.first.opaqueValue() < Right.first.opaqueValue(); + }); + return {}; +} + +Expected<void> LiveRegMatrix::unassign(Register VirtualReg) { + if (!Assignments.erase(VirtualReg)) + return makeUnexpected(ErrorCode::InvalidArgument, "virtual register is not assigned"); + std::erase_if(OrderedAssignments, + [VirtualReg](const auto &Entry) { return Entry.first == VirtualReg; }); + return {}; +} + +std::optional<PhysicalRegister> LiveRegMatrix::assignment(Register VirtualReg) const noexcept { + const auto Position = Assignments.find(VirtualReg); + if (Position == Assignments.end()) + return std::nullopt; + return Position->second; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LiveRegUnits.cpp b/backend/lib/CodeGen/LiveRegUnits.cpp new file mode 100644 index 0000000..935478e --- /dev/null +++ b/backend/lib/CodeGen/LiveRegUnits.cpp @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LiveRegUnits.h" + +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <unordered_map> +#include <vector> + +namespace rcc { + +namespace { +template <typename KeyT> +void addLanes(std::unordered_map<KeyT, LaneMask> &Values, KeyT Key, LaneMask Lanes) { + if (Lanes.empty()) + return; + const auto Position = Values.find(Key); + if (Position == Values.end()) { + Values.emplace(Key, Lanes); + return; + } + Position->second |= Lanes; +} + +template <typename KeyT> +void removeLanes(std::unordered_map<KeyT, LaneMask> &Values, KeyT Key, LaneMask Lanes) { + const auto Position = Values.find(Key); + if (Position == Values.end() || Lanes.empty()) + return; + Position->second &= ~Lanes; + if (Position->second.empty()) + Values.erase(Position); +} + +template <typename KeyT> +[[nodiscard]] LaneMask lanesFor(const std::unordered_map<KeyT, LaneMask> &Values, + KeyT Key) noexcept { + const auto Position = Values.find(Key); + return Position == Values.end() ? LaneMask{} : Position->second; +} + +template <typename KeyT> +[[nodiscard]] bool containsLanes(const std::unordered_map<KeyT, LaneMask> &Values, KeyT Key, + LaneMask Lanes) noexcept { + const LaneMask Present = lanesFor(Values, Key); + return !Present.empty() && Present.contains(Lanes); +} + +template <typename KeyT> +[[nodiscard]] bool intersectsLanes(const std::unordered_map<KeyT, LaneMask> &Values, KeyT Key, + LaneMask Lanes) noexcept { + return lanesFor(Values, Key).intersects(Lanes); +} + +template <typename KeyT> +void uniteLanes(std::unordered_map<KeyT, LaneMask> &Values, + const std::unordered_map<KeyT, LaneMask> &Other) { + for (const auto &[Key, Lanes] : Other) + addLanes(Values, Key, Lanes); +} + +template <typename KeyT> +void subtractLaneSets(std::unordered_map<KeyT, LaneMask> &Values, + const std::unordered_map<KeyT, LaneMask> &Other) { + for (const auto &[Key, Lanes] : Other) + removeLanes(Values, Key, Lanes); +} + +} // namespace + +LaneMask normalizeUnitLanes(LaneMask UnitViewLanes, LaneMask RegisterLanes) noexcept { + std::uint64_t View = UnitViewLanes.rawValue(); + const std::uint64_t Selected = View & RegisterLanes.rawValue(); + std::uint64_t Normalized = 0; + std::uint32_t UnitLane = 0; + while (View != 0) { + const std::uint64_t RegisterLane = std::uint64_t{1} << std::countr_zero(View); + if ((Selected & RegisterLane) != 0) + Normalized |= std::uint64_t{1} << UnitLane; + View &= View - 1U; + ++UnitLane; + } + return LaneMask::fromRaw(Normalized); +} + +LaneMask denormalizeUnitLanes(LaneMask UnitLanes, LaneMask UnitViewLanes) noexcept { + std::uint64_t View = UnitViewLanes.rawValue(); + const std::uint64_t Selected = UnitLanes.rawValue(); + std::uint64_t RegisterLanes = 0; + std::uint32_t UnitLane = 0; + while (View != 0) { + const std::uint64_t RegisterLane = std::uint64_t{1} << std::countr_zero(View); + if ((Selected & (std::uint64_t{1} << UnitLane)) != 0) + RegisterLanes |= RegisterLane; + View &= View - 1U; + ++UnitLane; + } + return LaneMask::fromRaw(RegisterLanes); +} + +LaneMask physicalRegisterLanes(const PhysicalRegisterDesc &Description) noexcept { + LaneMask Result; + for (const RegisterUnitLane &Unit : Description.Units) + Result = Result | Unit.Lanes; + return Result; +} + +LiveRegUnits physicalRegisterCoverage(const PhysicalRegisterDesc &Description, LaneMask Requested) { + LiveRegUnits Result; + for (const RegisterUnitLane &Unit : Description.Units) + Result.add(Unit.Unit, normalizeUnitLanes(Unit.Lanes, Requested)); + return Result; +} + +bool isSubset(const LiveRegUnits &Subset, const LiveRegUnits &Superset) noexcept { + return std::ranges::all_of(Subset.entries(), [&Superset](const auto &Entry) { + return Superset.contains(Entry.first, Entry.second); + }); +} + +std::size_t laneCount(const LiveRegUnits &Set) noexcept { + std::size_t Result = 0; + for (const auto &[Unit, Lanes] : Set.entries()) { + (void)Unit; + Result += static_cast<std::size_t>(std::popcount(Lanes.rawValue())); + } + return Result; +} + +[[nodiscard]] Expected<LaneMask> virtualRegisterLanes(Register Reg, const MachineFunction &Function, + const TargetDescription &Target) { + if (!Reg.isVirtual()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "virtual lane query requires a virtual register"); + } + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + const std::optional<RegisterClassID> ClassID = Info->get().RegClass; + if (!ClassID) + return LaneMask::all(); + const RegisterClassDesc *Class = Target.registers().registerClass(*ClassID); + if (Class == nullptr || Class->Lanes.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered an unknown or empty register class"); + } + return Class->Lanes; +} + +[[nodiscard]] Expected<LaneMask> virtualOperandLanes(const MachineOperand &Operand, + const MachineFunction &Function, + const TargetDescription &Target) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "virtual lane query requires a virtual-register operand"); + } + if (Operand.subReg().value() != 0) { + const SubRegIndexDesc *SubReg = Target.registers().subRegIndex(Operand.subReg()); + if (SubReg == nullptr || SubReg->Lanes.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered an unknown or empty subregister lane mask"); + } + return SubReg->Lanes; + } + + return virtualRegisterLanes(*Reg, Function, Target); +} + +[[nodiscard]] Expected<std::vector<PhysicalUnitAccess>> +physicalOperandUnits(const MachineOperand &Operand, const TargetDescription &Target) { + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Original = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Original) { + return makeUnexpected(ErrorCode::InvalidArgument, + "physical unit query requires a physical-register operand"); + } + + PhysicalRegister Effective = *Original; + if (Operand.subReg().value() != 0) { + const std::optional<PhysicalRegister> SubRegister = + Target.registers().subRegister(*Original, Operand.subReg()); + if (!SubRegister) { + return makeUnexpected(ErrorCode::VerificationError, + "physical register has no requested subregister projection"); + } + Effective = *SubRegister; + } + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(Effective); + if (Description == nullptr || Description->Units.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered an unknown or unitless physical register"); + } + + std::vector<PhysicalUnitAccess> Result; + Result.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) { + if (Unit.Lanes.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "physical register unit has an empty lane mask"); + } + Result.push_back({Unit.Unit, normalizeUnitLanes(Unit.Lanes, LaneMask::all())}); + } + return Result; +} + +void addPhysicalRegister(LiveRegUnits &Set, const PhysicalRegisterDesc &Description, + LaneMask Requested) { + Set.unite(physicalRegisterCoverage(Description, Requested)); +} + +[[nodiscard]] bool isReservedPhysicalRegister(Register Reg, const MachineFunction &Function, + const TargetDescription &Target) noexcept { + const std::optional<PhysicalRegister> Physical = Reg.physicalRegister(); + return Physical && (Function.registerInfo().isReserved(*Physical) || + Target.registers().isReservedRegister(*Physical)); +} + +[[nodiscard]] Expected<void> addRegisterMaskDefs(LiveRegUnits &Set, const RegisterMask &Mask, + const TargetDescription &Target) { + for (const PhysicalRegisterDesc &Description : Target.registers().physicalRegisters()) { + if (Mask.clobbers(Description.Reg)) + addPhysicalRegister(Set, Description); + } + return {}; +} + +[[nodiscard]] Expected<void> addRegisterMaskUses(LiveRegUnits &Set, const RegisterMask &Mask, + const TargetDescription &Target) { + for (const PhysicalRegisterDesc &Description : Target.registers().physicalRegisters()) { + if (Mask.preserves(Description.Reg)) + addPhysicalRegister(Set, Description); + } + return {}; +} +LaneMask LiveRegUnits::lanes(RegisterUnitID Unit) const noexcept { + return lanesFor(Values, Unit); +} + +bool LiveRegUnits::contains(RegisterUnitID Unit, LaneMask Lanes) const noexcept { + return containsLanes(Values, Unit, Lanes); +} + +bool LiveRegUnits::intersects(RegisterUnitID Unit, LaneMask Lanes) const noexcept { + return intersectsLanes(Values, Unit, Lanes); +} + +void LiveRegUnits::add(RegisterUnitID Unit, LaneMask Lanes) { + if (Unit.value() != 0 && !Lanes.empty()) + addLanes(Values, Unit, Lanes); +} + +void LiveRegUnits::remove(RegisterUnitID Unit, LaneMask Lanes) { + removeLanes(Values, Unit, Lanes); +} + +void LiveRegUnits::unite(const LiveRegUnits &Other) { + uniteLanes(Values, Other.Values); +} + +void LiveRegUnits::subtract(const LiveRegUnits &Other) { + subtractLaneSets(Values, Other.Values); +} + +[[nodiscard]] Expected<bool> LiveRegUnits::available(PhysicalRegister Reg, + const TargetDescription &Target) const { + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(Reg); + if (Description == nullptr || Description->Units.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "register availability query requires a physical register with units"); + } + const LiveRegUnits Coverage = physicalRegisterCoverage(*Description); + return std::ranges::none_of(Coverage.entries(), [this](const auto &Entry) { + return intersects(Entry.first, Entry.second); + }); +} + +[[nodiscard]] Expected<void> LiveRegUnits::accumulateUsedDefed(const MachineInstr &Instruction, + LiveRegUnits &Modified, + LiveRegUnits &Used, + const TargetDescription &Target) { + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.isDebugUse()) + continue; + if (Operand.isRegister()) { + const std::optional<Register> Reg = Operand.registerValue(); + if (!Reg || !Reg->isPhysical() || Reg->isNoRegister()) + continue; + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + for (const PhysicalUnitAccess &Access : *Units) { + if (Operand.isDef()) + Modified.add(Access.Unit, Access.Lanes); + if (Operand.isUse()) + Used.add(Access.Unit, Access.Lanes); + } + continue; + } + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + if (auto Added = addRegisterMaskDefs(Modified, *Mask->Mask, Target); !Added) + return std::unexpected(Added.error()); + continue; + } + if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + if (auto Added = addRegisterMaskUses(Used, *Mask->Mask, Target); !Added) + return std::unexpected(Added.error()); + } + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LiveVariables.cpp b/backend/lib/CodeGen/LiveVariables.cpp new file mode 100644 index 0000000..4f65f72 --- /dev/null +++ b/backend/lib/CodeGen/LiveVariables.cpp @@ -0,0 +1,1053 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LiveVariables.h" + +#include "rcc/CodeGen/LiveInterval.h" +#include "rcc/CodeGen/LiveIntervals.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/ReachingDefAnalysis.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/SlotIndexes.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/VirtRegMap.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { + +namespace { +template <typename KeyT> +void addLanes(std::unordered_map<KeyT, LaneMask> &Values, KeyT Key, LaneMask Lanes) { + if (Lanes.empty()) + return; + const auto Position = Values.find(Key); + if (Position == Values.end()) { + Values.emplace(Key, Lanes); + return; + } + Position->second |= Lanes; +} + +template <typename KeyT> +void removeLanes(std::unordered_map<KeyT, LaneMask> &Values, KeyT Key, LaneMask Lanes) { + const auto Position = Values.find(Key); + if (Position == Values.end() || Lanes.empty()) + return; + Position->second &= ~Lanes; + if (Position->second.empty()) + Values.erase(Position); +} + +template <typename KeyT> +[[nodiscard]] LaneMask lanesFor(const std::unordered_map<KeyT, LaneMask> &Values, + KeyT Key) noexcept { + const auto Position = Values.find(Key); + return Position == Values.end() ? LaneMask{} : Position->second; +} + +template <typename KeyT> +[[nodiscard]] bool containsLanes(const std::unordered_map<KeyT, LaneMask> &Values, KeyT Key, + LaneMask Lanes) noexcept { + const LaneMask Present = lanesFor(Values, Key); + return !Present.empty() && Present.contains(Lanes); +} + +template <typename KeyT> +[[nodiscard]] bool intersectsLanes(const std::unordered_map<KeyT, LaneMask> &Values, KeyT Key, + LaneMask Lanes) noexcept { + return lanesFor(Values, Key).intersects(Lanes); +} + +template <typename KeyT> +void uniteLanes(std::unordered_map<KeyT, LaneMask> &Values, + const std::unordered_map<KeyT, LaneMask> &Other) { + for (const auto &[Key, Lanes] : Other) + addLanes(Values, Key, Lanes); +} + +template <typename KeyT> +void subtractLaneSets(std::unordered_map<KeyT, LaneMask> &Values, + const std::unordered_map<KeyT, LaneMask> &Other) { + for (const auto &[Key, Lanes] : Other) + removeLanes(Values, Key, Lanes); +} + +[[nodiscard]] Expected<void> addOperandUse(RegisterLaneSet &VirtualUses, LiveRegUnits &PhysicalUses, + const MachineOperand &Operand, + const MachineFunction &Function, + const TargetDescription &Target) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister() || Operand.isDebugUse() || + Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + return {}; + } + if (Reg->isVirtual()) { + auto Lanes = virtualOperandLanes(Operand, Function, Target); + if (!Lanes) + return std::unexpected(Lanes.error()); + VirtualUses.add(*Reg, *Lanes); + return {}; + } + if (isReservedPhysicalRegister(*Reg, Function, Target)) + return {}; + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + for (const PhysicalUnitAccess &Unit : *Units) + PhysicalUses.add(Unit.Unit, Unit.Lanes); + return {}; +} + +[[nodiscard]] Expected<void> addOperandDef(RegisterLaneSet &VirtualDefs, LiveRegUnits &PhysicalDefs, + const MachineOperand &Operand, + const MachineFunction &Function, + const TargetDescription &Target) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Lanes = virtualOperandLanes(Operand, Function, Target); + if (!Lanes) + return std::unexpected(Lanes.error()); + VirtualDefs.add(*Reg, *Lanes); + return {}; + } + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + for (const PhysicalUnitAccess &Unit : *Units) + PhysicalDefs.add(Unit.Unit, Unit.Lanes); + return {}; +} + +struct LivenessEdgeKey final { + const MachineBasicBlock *Predecessor = nullptr; + const MachineBasicBlock *Successor = nullptr; + + friend bool operator==(const LivenessEdgeKey &, const LivenessEdgeKey &) = default; +}; + +struct LivenessEdgeKeyHash final { + [[nodiscard]] std::size_t operator()(const LivenessEdgeKey &Key) const noexcept { + const std::size_t Left = std::hash<const MachineBasicBlock *>{}(Key.Predecessor); + const std::size_t Right = std::hash<const MachineBasicBlock *>{}(Key.Successor); + return Left ^ (Right + 0x9e3779b9U + (Left << 6U) + (Left >> 2U)); + } +}; + +struct EdgeLivenessUses final { + RegisterLaneSet Virtual; + LiveRegUnits Physical; +}; + +[[nodiscard]] Expected<LiveVariablesAnalysis::Result> +computeLiveVariables(const MachineFunction &Function, const TargetDescription &Target) { + LiveVariablesAnalysis::Result Result; + std::vector<const MachineBasicBlock *> Blocks; + std::unordered_map<LivenessEdgeKey, EdgeLivenessUses, LivenessEdgeKeyHash> EdgeUses; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + Blocks.push_back(Block); + (void)Result.getOrCreateBlock(*Block); + } + + for (const MachineBasicBlock *Block : Blocks) { + MachineBlockLiveness &Liveness = Result.getOrCreateBlock(*Block); + if (Block == Function.firstBlock()) { + for (const PhysicalRegisterLiveIn &LiveIn : Function.registerInfo().liveIns()) { + const PhysicalRegisterDesc *Description = + Target.registers().physicalRegister(LiveIn.PhysReg); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "function live-in references an unknown physical register"); + } + addPhysicalRegister(Liveness.PhysicalUses, *Description); + if (LiveIn.InitialVirtualRegister) { + auto Lanes = virtualRegisterLanes(*LiveIn.InitialVirtualRegister, Function, Target); + if (!Lanes) + return std::unexpected(Lanes.error()); + Liveness.VirtualDefs.add(*LiveIn.InitialVirtualRegister, *Lanes); + } + } + } + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered an unknown instruction descriptor"); + } + const bool IsPhi = Descriptor->Flags.has(InstructionFlag::Phi); + if (IsPhi) { + if (Instruction->operandCount() < 3 || (Instruction->operandCount() - 1U) % 2U != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered a malformed PHI instruction"); + } + std::unordered_set<const MachineBasicBlock *> IncomingBlocks; + for (std::size_t Index = 1; Index < Instruction->operandCount(); Index += 2U) { + const MachineOperand *Incoming = Instruction->operandAt(Index); + const MachineOperand *PredecessorOperand = Instruction->operandAt(Index + 1U); + const auto *Predecessor = PredecessorOperand == nullptr + ? nullptr + : PredecessorOperand->getIf<BasicBlockOperand>(); + if (Incoming == nullptr || Predecessor == nullptr || Predecessor->Block == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered a malformed PHI incoming pair"); + } + const MachineBasicBlock *IncomingBlock = Predecessor->Block; + const bool HasPredecessor = std::ranges::find(Block->predecessors(), IncomingBlock) != + Block->predecessors().end(); + const bool HasSuccessor = std::ranges::any_of( + IncomingBlock->successors(), + [Block](const MachineSuccessor &Successor) { return Successor.Block == Block; }); + if (&IncomingBlock->function() != &Function || !IncomingBlock->isLinked() || + !HasPredecessor || !HasSuccessor || !IncomingBlocks.insert(IncomingBlock).second) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered a PHI incoming without a unique CFG edge"); + } + EdgeLivenessUses &Uses = EdgeUses[{IncomingBlock, Block}]; + auto Added = addOperandUse(Uses.Virtual, Uses.Physical, *Incoming, Function, Target); + if (!Added) + return std::unexpected(Added.error()); + } + } + + RegisterLaneSet InstructionVirtualUses; + RegisterLaneSet InstructionVirtualDefs; + LiveRegUnits InstructionPhysicalUses; + LiveRegUnits InstructionPhysicalDefs; + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + const MachineOperand &Operand = *Instruction->operandAt(Index); + if (Operand.isRegister()) { + if (Operand.isUse() && (!IsPhi || Index == 0)) { + auto Use = addOperandUse(InstructionVirtualUses, InstructionPhysicalUses, Operand, + Function, Target); + if (!Use) + return std::unexpected(Use.error()); + } + if (Operand.isDef()) { + auto Def = addOperandDef(InstructionVirtualDefs, InstructionPhysicalDefs, Operand, + Function, Target); + if (!Def) + return std::unexpected(Def.error()); + } + continue; + } + if (const auto *MaskOperand = Operand.getIf<RegisterMaskOperand>(); + MaskOperand != nullptr) { + if (!MaskOperand->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered a null register mask"); + auto Added = addRegisterMaskDefs(InstructionPhysicalDefs, *MaskOperand->Mask, Target); + if (!Added) + return std::unexpected(Added.error()); + continue; + } + if (const auto *MaskOperand = Operand.getIf<LiveOutRegisterMaskOperand>(); + MaskOperand != nullptr) { + if (!MaskOperand->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered a null live-out register mask"); + auto Added = addRegisterMaskUses(InstructionPhysicalUses, *MaskOperand->Mask, Target); + if (!Added) + return std::unexpected(Added.error()); + } + } + + RegisterLaneSet NewVirtualUses = InstructionVirtualUses; + NewVirtualUses.subtract(Liveness.VirtualDefs); + Liveness.VirtualUses.unite(NewVirtualUses); + Liveness.VirtualDefs.unite(InstructionVirtualDefs); + + LiveRegUnits NewPhysicalUses = InstructionPhysicalUses; + NewPhysicalUses.subtract(Liveness.PhysicalDefs); + Liveness.PhysicalUses.unite(NewPhysicalUses); + Liveness.PhysicalDefs.unite(InstructionPhysicalDefs); + } + } + + bool Changed = true; + while (Changed) { + Changed = false; + for (const MachineBasicBlock *Block : Blocks | std::views::reverse) { + MachineBlockLiveness &Liveness = Result.getOrCreateBlock(*Block); + RegisterLaneSet VirtualOut; + LiveRegUnits PhysicalOut; + for (const MachineSuccessor &Successor : Block->successors()) { + if (Successor.Block == nullptr || Result.block(*Successor.Block) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness encountered an invalid CFG successor"); + } + const MachineBlockLiveness &SuccessorLiveness = *Result.block(*Successor.Block); + VirtualOut.unite(SuccessorLiveness.VirtualLiveIn); + PhysicalOut.unite(SuccessorLiveness.PhysicalLiveIn); + const auto Edge = EdgeUses.find({Block, Successor.Block}); + if (Edge != EdgeUses.end()) { + VirtualOut.unite(Edge->second.Virtual); + PhysicalOut.unite(Edge->second.Physical); + } + } + + RegisterLaneSet VirtualIn = VirtualOut; + VirtualIn.subtract(Liveness.VirtualDefs); + VirtualIn.unite(Liveness.VirtualUses); + LiveRegUnits PhysicalIn = PhysicalOut; + PhysicalIn.subtract(Liveness.PhysicalDefs); + PhysicalIn.unite(Liveness.PhysicalUses); + + if (VirtualOut != Liveness.VirtualLiveOut || VirtualIn != Liveness.VirtualLiveIn || + PhysicalOut != Liveness.PhysicalLiveOut || PhysicalIn != Liveness.PhysicalLiveIn) { + Liveness.VirtualLiveOut = std::move(VirtualOut); + Liveness.VirtualLiveIn = std::move(VirtualIn); + Liveness.PhysicalLiveOut = std::move(PhysicalOut); + Liveness.PhysicalLiveIn = std::move(PhysicalIn); + Changed = true; + } + } + } + return Result; +} +[[nodiscard]] Expected<std::vector<MachineBasicBlockLiveIn>> +canonicalPhysicalLiveIns(const LiveRegUnits &Units, const TargetDescription &Target) { + struct Candidate final { + const PhysicalRegisterDesc *Description = nullptr; + LaneMask RegisterLanes; + LiveRegUnits Coverage; + std::size_t ExtraLanes = 0; + bool Selected = false; + }; + + std::vector<Candidate> Candidates; + for (const PhysicalRegisterDesc &Description : Target.registers().physicalRegisters()) { + const LaneMask FullLanes = physicalRegisterLanes(Description); + std::uint64_t SelectableLanes = 0; + LaneMask RelevantLanes; + for (const RegisterUnitLane &Unit : Description.Units) { + RelevantLanes = RelevantLanes | denormalizeUnitLanes(Units.lanes(Unit.Unit), Unit.Lanes); + } + std::uint64_t PendingLanes = RelevantLanes.rawValue(); + while (PendingLanes != 0) { + const std::uint64_t Lane = std::uint64_t{1} << std::countr_zero(PendingLanes); + const LiveRegUnits Coverage = physicalRegisterCoverage(Description, LaneMask::fromRaw(Lane)); + if (isSubset(Coverage, Units)) + SelectableLanes |= Lane; + PendingLanes &= PendingLanes - 1U; + } + if (SelectableLanes == 0) + continue; + Candidate Value; + Value.Description = &Description; + Value.RegisterLanes = LaneMask::fromRaw(SelectableLanes); + Value.Coverage = physicalRegisterCoverage(Description, Value.RegisterLanes); + Value.ExtraLanes = static_cast<std::size_t>(std::popcount(FullLanes.rawValue()) - + std::popcount(SelectableLanes)); + Candidates.push_back(std::move(Value)); + } + + LiveRegUnits Covered; + std::vector<MachineBasicBlockLiveIn> Result; + while (Covered != Units) { + Candidate *Best = nullptr; + std::size_t BestNewLanes = 0; + std::size_t BestCoverage = 0; + for (Candidate &Value : Candidates) { + if (Value.Selected) + continue; + LiveRegUnits NewCoverage = Value.Coverage; + NewCoverage.subtract(Covered); + const std::size_t NewLanes = laneCount(NewCoverage); + if (NewLanes == 0) + continue; + const std::size_t Coverage = laneCount(Value.Coverage); + if (Best == nullptr || NewLanes > BestNewLanes || + (NewLanes == BestNewLanes && Value.ExtraLanes < Best->ExtraLanes) || + (NewLanes == BestNewLanes && Value.ExtraLanes == Best->ExtraLanes && + Coverage > BestCoverage) || + (NewLanes == BestNewLanes && Value.ExtraLanes == Best->ExtraLanes && + Coverage == BestCoverage && + Value.Description->Reg.number() < Best->Description->Reg.number())) { + Best = &Value; + BestNewLanes = NewLanes; + BestCoverage = Coverage; + } + } + if (Best == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "physical live-in units cannot be represented by target registers"); + } + Best->Selected = true; + Covered.unite(Best->Coverage); + const LaneMask FullLanes = physicalRegisterLanes(*Best->Description); + const LaneMask Lanes = Best->RegisterLanes == FullLanes ? LaneMask::all() : Best->RegisterLanes; + Result.push_back({Best->Description->Reg, Lanes}); + } + std::ranges::sort(Result, + [](const MachineBasicBlockLiveIn &Left, const MachineBasicBlockLiveIn &Right) { + return Left.PhysReg.number() < Right.PhysReg.number(); + }); + return Result; +} + +[[nodiscard]] Expected<LiveRegUnits> liveInUnits(std::span<const MachineBasicBlockLiveIn> LiveIns, + const TargetDescription &Target) { + LiveRegUnits Result; + for (const MachineBasicBlockLiveIn &LiveIn : LiveIns) { + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(LiveIn.PhysReg); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "block live-in references an unknown physical register"); + } + Result.unite(physicalRegisterCoverage(*Description, LiveIn.Lanes)); + } + return Result; +} + +struct OperandFlagPlan final { + MachineInstrStableID Instruction; + std::size_t OperandIndex = 0; + RegisterOperandFlags Original; + RegisterOperandFlags Rebuilt; + bool KillIsValid = false; + bool DeadIsValid = false; +}; + +struct BlockLivenessPlan final { + MachineBlockStableID Block; + std::vector<MachineBasicBlockLiveIn> OriginalLiveIns; + std::vector<MachineBasicBlockLiveIn> RebuiltLiveIns; +}; + +struct LivenessRebuildPlan final { + std::vector<BlockLivenessPlan> Blocks; + std::vector<OperandFlagPlan> Operands; +}; + +[[nodiscard]] Expected<bool> liveSetIntersects(const MachineOperand &Operand, + const RegisterLaneSet &Virtual, + const LiveRegUnits &Physical, + const MachineFunction &Function, + const TargetDescription &Target) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return false; + if (Reg->isVirtual()) { + auto Lanes = virtualOperandLanes(Operand, Function, Target); + if (!Lanes) + return std::unexpected(Lanes.error()); + return Virtual.intersects(*Reg, *Lanes); + } + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + return std::ranges::any_of(*Units, [&Physical](const PhysicalUnitAccess &Unit) { + return Physical.intersects(Unit.Unit, Unit.Lanes); + }); +} + +[[nodiscard]] Expected<void> removeOperandFromLiveSet(const MachineOperand &Operand, + RegisterLaneSet &Virtual, + LiveRegUnits &Physical, + const MachineFunction &Function, + const TargetDescription &Target) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Lanes = virtualOperandLanes(Operand, Function, Target); + if (!Lanes) + return std::unexpected(Lanes.error()); + Virtual.remove(*Reg, *Lanes); + return {}; + } + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + for (const PhysicalUnitAccess &Unit : *Units) + Physical.remove(Unit.Unit, Unit.Lanes); + return {}; +} + +[[nodiscard]] Expected<void> addOperandToLiveSet(const MachineOperand &Operand, + RegisterLaneSet &Virtual, LiveRegUnits &Physical, + const MachineFunction &Function, + const TargetDescription &Target) { + return addOperandUse(Virtual, Physical, Operand, Function, Target); +} + +[[nodiscard]] Expected<LiveRegUnits> requiredLocalPhysicalLiveIns(const MachineBasicBlock &Block, + const MachineFunction &Function, + const TargetDescription &Target) { + RegisterLaneSet Virtual; + LiveRegUnits Physical; + for (const MachineInstr *Instruction = Block.lastInstruction(); Instruction != nullptr; + Instruction = Instruction->previous()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness verification encountered an unknown instruction descriptor"); + } + const bool IsPhi = Descriptor->Flags.has(InstructionFlag::Phi); + for (const MachineOperand &Operand : Instruction->operands()) { + const auto *MaskOperand = Operand.getIf<LiveOutRegisterMaskOperand>(); + if (MaskOperand == nullptr) + continue; + if (!MaskOperand->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness verification encountered a null live-out register mask"); + } + auto Added = addRegisterMaskUses(Physical, *MaskOperand->Mask, Target); + if (!Added) + return std::unexpected(Added.error()); + } + + for (const MachineOperand &Operand : Instruction->operands()) { + if (Operand.isRegister() && Operand.isDef()) { + auto Removed = removeOperandFromLiveSet(Operand, Virtual, Physical, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + continue; + } + const auto *MaskOperand = Operand.getIf<RegisterMaskOperand>(); + if (MaskOperand == nullptr) + continue; + if (!MaskOperand->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness verification encountered a null register mask"); + } + LiveRegUnits Clobbers; + auto Added = addRegisterMaskDefs(Clobbers, *MaskOperand->Mask, Target); + if (!Added) + return std::unexpected(Added.error()); + Physical.subtract(Clobbers); + } + + if (IsPhi) + continue; + for (const MachineOperand &Operand : Instruction->operands()) { + if (!Operand.isRegister() || !Operand.isUse()) + continue; + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && Reg->isPhysical() && + isReservedPhysicalRegister(*Reg, Function, Target)) { + continue; + } + auto Added = addOperandToLiveSet(Operand, Virtual, Physical, Function, Target); + if (!Added) + return std::unexpected(Added.error()); + } + } + return Physical; +} + +[[nodiscard]] Expected<LivenessRebuildPlan> +buildLivenessRebuildPlan(const MachineFunction &Function, const TargetDescription &Target) { + auto Variables = computeLiveVariables(Function, Target); + if (!Variables) + return std::unexpected(Variables.error()); + + LivenessRebuildPlan Plan; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + const MachineBlockLiveness *BlockVariables = Variables->block(*Block); + if (BlockVariables == nullptr) + return makeUnexpected(ErrorCode::InternalError, "liveness result omitted a machine block"); + auto NewLiveIns = canonicalPhysicalLiveIns(BlockVariables->PhysicalLiveIn, Target); + if (!NewLiveIns) + return std::unexpected(NewLiveIns.error()); + Plan.Blocks.push_back( + {Block->stableID(), + std::vector<MachineBasicBlockLiveIn>(Block->liveIns().begin(), Block->liveIns().end()), + std::move(*NewLiveIns)}); + + RegisterLaneSet Virtual = BlockVariables->VirtualLiveOut; + LiveRegUnits Physical = BlockVariables->PhysicalLiveOut; + for (const MachineInstr *Instruction = Block->lastInstruction(); Instruction != nullptr; + Instruction = Instruction->previous()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness rebuild encountered an unknown instruction descriptor"); + } + const bool IsPhi = Descriptor->Flags.has(InstructionFlag::Phi); + for (const MachineOperand &Operand : Instruction->operands()) { + const auto *MaskOperand = Operand.getIf<LiveOutRegisterMaskOperand>(); + if (MaskOperand == nullptr) + continue; + if (!MaskOperand->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "liveness rebuild encountered a null live-out register mask"); + auto Added = addRegisterMaskUses(Physical, *MaskOperand->Mask, Target); + if (!Added) + return std::unexpected(Added.error()); + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + const MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister() || !Operand->isDef()) + continue; + auto IsLive = liveSetIntersects(*Operand, Virtual, Physical, Function, Target); + if (!IsLive) + return std::unexpected(IsLive.error()); + RegisterOperandFlags Flags = Operand->registerFlags(); + Flags.clear(RegisterOperandFlag::Dead); + if (!*IsLive) + Flags.set(RegisterOperandFlag::Dead); + Plan.Operands.push_back( + {Instruction->stableID(), Index, Operand->registerFlags(), Flags, false, !*IsLive}); + } + + for (const MachineOperand &Operand : Instruction->operands()) { + if (Operand.isRegister() && Operand.isDef()) { + auto Removed = removeOperandFromLiveSet(Operand, Virtual, Physical, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + continue; + } + const auto *MaskOperand = Operand.getIf<RegisterMaskOperand>(); + if (MaskOperand == nullptr) + continue; + if (!MaskOperand->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "liveness rebuild encountered a null register mask"); + LiveRegUnits Clobbers; + auto Added = addRegisterMaskDefs(Clobbers, *MaskOperand->Mask, Target); + if (!Added) + return std::unexpected(Added.error()); + Physical.subtract(Clobbers); + } + + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + const MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isRegister() || !Operand->isUse()) + continue; + if (IsPhi) { + RegisterOperandFlags Flags = Operand->registerFlags(); + Flags.clear(RegisterOperandFlag::Kill); + const auto Existing = std::ranges::find_if( + Plan.Operands, [Instruction, Index](const OperandFlagPlan &Entry) { + return Entry.Instruction == Instruction->stableID() && Entry.OperandIndex == Index; + }); + if (Existing == Plan.Operands.end()) { + Plan.Operands.push_back( + {Instruction->stableID(), Index, Operand->registerFlags(), Flags, false, false}); + } else { + Existing->Rebuilt.clear(RegisterOperandFlag::Kill); + Existing->KillIsValid = false; + } + continue; + } + const Register OperandRegister = *Operand->registerValue(); + const auto PhysicalRegister = OperandRegister.physicalRegister(); + const PhysicalRegisterDesc *PhysicalDescription = + PhysicalRegister ? Target.registers().physicalRegister(*PhysicalRegister) : nullptr; + const bool IsSemanticUse = + !Operand->isDebugUse() && !Operand->registerFlags().has(RegisterOperandFlag::Undef) && + !OperandRegister.isNoRegister() && + (PhysicalDescription == nullptr || !PhysicalDescription->IsConstant); + bool IsKill = false; + if (IsSemanticUse) { + auto IsLive = liveSetIntersects(*Operand, Virtual, Physical, Function, Target); + if (!IsLive) + return std::unexpected(IsLive.error()); + IsKill = !*IsLive; + } + RegisterOperandFlags Flags = Operand->registerFlags(); + Flags.clear(RegisterOperandFlag::Kill); + if (IsKill) + Flags.set(RegisterOperandFlag::Kill); + + const auto Existing = + std::ranges::find_if(Plan.Operands, [Instruction, Index](const OperandFlagPlan &Entry) { + return Entry.Instruction == Instruction->stableID() && Entry.OperandIndex == Index; + }); + if (Existing == Plan.Operands.end()) { + Plan.Operands.push_back( + {Instruction->stableID(), Index, Operand->registerFlags(), Flags, IsKill, false}); + } else { + Existing->Rebuilt.clear(RegisterOperandFlag::Kill); + if (IsKill) + Existing->Rebuilt.set(RegisterOperandFlag::Kill); + Existing->KillIsValid = IsKill; + } + if (IsSemanticUse) { + auto Added = addOperandToLiveSet(*Operand, Virtual, Physical, Function, Target); + if (!Added) + return std::unexpected(Added.error()); + } + } + } + + if (Block == Function.firstBlock()) { + for (const PhysicalRegisterLiveIn &LiveIn : Function.registerInfo().liveIns()) { + const PhysicalRegisterDesc *Description = + Target.registers().physicalRegister(LiveIn.PhysReg); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "liveness rebuild encountered an unknown function live-in"); + } + addPhysicalRegister(Physical, *Description); + if (LiveIn.InitialVirtualRegister) { + auto Lanes = virtualRegisterLanes(*LiveIn.InitialVirtualRegister, Function, Target); + if (!Lanes) + return std::unexpected(Lanes.error()); + Virtual.remove(*LiveIn.InitialVirtualRegister, *Lanes); + } + } + } + + if (Virtual != BlockVariables->VirtualLiveIn || Physical != BlockVariables->PhysicalLiveIn) { + return makeUnexpected(ErrorCode::InternalError, + "reverse liveness reconstruction disagrees with block dataflow"); + } + } + return Plan; +} + +struct ResolvedBlockLivenessPlan final { + MachineBasicBlock *Block = nullptr; + const BlockLivenessPlan *Plan = nullptr; +}; + +struct ResolvedOperandFlagPlan final { + MachineOperand *Operand = nullptr; + const OperandFlagPlan *Plan = nullptr; +}; + +struct ResolvedLivenessRebuildPlan final { + std::vector<ResolvedBlockLivenessPlan> Blocks; + std::vector<ResolvedOperandFlagPlan> Operands; +}; + +[[nodiscard]] Expected<ResolvedLivenessRebuildPlan> +resolveLivenessPlan(MachineFunction &Function, const LivenessRebuildPlan &Plan) { + ResolvedLivenessRebuildPlan Result; + Result.Blocks.reserve(Plan.Blocks.size()); + for (const BlockLivenessPlan &BlockPlan : Plan.Blocks) { + const auto Position = + std::ranges::find_if(Function.allBlocks(), [&BlockPlan](const MachineBasicBlock *Block) { + return Block != nullptr && Block->stableID() == BlockPlan.Block; + }); + if (Position == Function.allBlocks().end() || !(*Position)->isLinked()) { + return makeUnexpected(ErrorCode::InternalError, + "liveness rebuild plan references an unavailable machine block"); + } + Result.Blocks.push_back({*Position, &BlockPlan}); + } + + Result.Operands.reserve(Plan.Operands.size()); + for (const OperandFlagPlan &OperandPlan : Plan.Operands) { + const auto Position = std::ranges::find_if( + Function.allInstructions(), [&OperandPlan](const MachineInstr *Instruction) { + return Instruction != nullptr && Instruction->stableID() == OperandPlan.Instruction; + }); + if (Position == Function.allInstructions().end() || !(*Position)->isLinked()) { + return makeUnexpected(ErrorCode::InternalError, + "liveness rebuild plan references an unavailable instruction"); + } + MachineOperand *Operand = (*Position)->operandAt(OperandPlan.OperandIndex); + if (Operand == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "liveness rebuild plan references an unavailable operand"); + } + Result.Operands.push_back({Operand, &OperandPlan}); + } + return Result; +} + +[[nodiscard]] Expected<void> restoreLivenessPlan(const ResolvedLivenessRebuildPlan &Plan) { + for (const ResolvedBlockLivenessPlan &Resolved : Plan.Blocks) { + Resolved.Block->clearLiveIns(); + for (const MachineBasicBlockLiveIn &LiveIn : Resolved.Plan->OriginalLiveIns) { + auto Restored = Resolved.Block->addLiveIn(LiveIn.PhysReg, LiveIn.Lanes); + if (!Restored) + return std::unexpected(Restored.error()); + } + } + for (const ResolvedOperandFlagPlan &Resolved : Plan.Operands) { + auto Restored = Resolved.Operand->setRegisterFlags(Resolved.Plan->Original); + if (!Restored) + return std::unexpected(Restored.error()); + } + return {}; +} + +[[nodiscard]] Expected<bool> applyLivenessPlan(MachineFunction &Function, + const LivenessRebuildPlan &Plan) { + auto Resolved = resolveLivenessPlan(Function, Plan); + if (!Resolved) + return std::unexpected(Resolved.error()); + + bool Changed = false; + for (const ResolvedBlockLivenessPlan &BlockPlan : Resolved->Blocks) { + Changed |= BlockPlan.Plan->OriginalLiveIns != BlockPlan.Plan->RebuiltLiveIns; + BlockPlan.Block->clearLiveIns(); + for (const MachineBasicBlockLiveIn &LiveIn : BlockPlan.Plan->RebuiltLiveIns) { + auto Added = BlockPlan.Block->addLiveIn(LiveIn.PhysReg, LiveIn.Lanes); + if (!Added) { + auto Restored = restoreLivenessPlan(*Resolved); + if (!Restored) + return makeUnexpected(ErrorCode::InternalError, + "liveness rebuild failed and rollback also failed"); + return std::unexpected(Added.error()); + } + } + } + for (const ResolvedOperandFlagPlan &OperandPlan : Resolved->Operands) { + Changed |= OperandPlan.Plan->Original != OperandPlan.Plan->Rebuilt; + auto Updated = OperandPlan.Operand->setRegisterFlags(OperandPlan.Plan->Rebuilt); + if (!Updated) { + auto Restored = restoreLivenessPlan(*Resolved); + if (!Restored) + return makeUnexpected(ErrorCode::InternalError, + "liveness rebuild failed and rollback also failed"); + return std::unexpected(Updated.error()); + } + } + return Changed; +} +} // namespace + +LaneMask RegisterLaneSet::lanes(Register Reg) const noexcept { + return lanesFor(Values, Reg); +} + +bool RegisterLaneSet::contains(Register Reg, LaneMask Lanes) const noexcept { + return containsLanes(Values, Reg, Lanes); +} + +bool RegisterLaneSet::intersects(Register Reg, LaneMask Lanes) const noexcept { + return intersectsLanes(Values, Reg, Lanes); +} + +void RegisterLaneSet::add(Register Reg, LaneMask Lanes) { + if (!Reg.isNoRegister()) + addLanes(Values, Reg, Lanes); +} + +void RegisterLaneSet::remove(Register Reg, LaneMask Lanes) { + removeLanes(Values, Reg, Lanes); +} + +void RegisterLaneSet::unite(const RegisterLaneSet &Other) { + uniteLanes(Values, Other.Values); +} + +void RegisterLaneSet::subtract(const RegisterLaneSet &Other) { + subtractLaneSets(Values, Other.Values); +} +const MachineBlockLiveness * +LiveVariablesAnalysis::Result::block(const MachineBasicBlock &Block) const noexcept { + const auto Position = Blocks.find(&Block); + return Position == Blocks.end() ? nullptr : &Position->second; +} + +MachineBlockLiveness & +LiveVariablesAnalysis::Result::getOrCreateBlock(const MachineBasicBlock &Block) { + return Blocks[&Block]; +} + +bool LiveVariablesAnalysis::Result::isLiveIn(const MachineBasicBlock &Block, Register Reg, + LaneMask Lanes) const noexcept { + const MachineBlockLiveness *Value = block(Block); + return Value != nullptr && Value->VirtualLiveIn.intersects(Reg, Lanes); +} + +bool LiveVariablesAnalysis::Result::isLiveOut(const MachineBasicBlock &Block, Register Reg, + LaneMask Lanes) const noexcept { + const MachineBlockLiveness *Value = block(Block); + return Value != nullptr && Value->VirtualLiveOut.intersects(Reg, Lanes); +} + +bool LiveVariablesAnalysis::Result::isUnitLiveIn(const MachineBasicBlock &Block, + RegisterUnitID Unit, + LaneMask Lanes) const noexcept { + const MachineBlockLiveness *Value = block(Block); + return Value != nullptr && Value->PhysicalLiveIn.intersects(Unit, Lanes); +} + +bool LiveVariablesAnalysis::Result::isUnitLiveOut(const MachineBasicBlock &Block, + RegisterUnitID Unit, + LaneMask Lanes) const noexcept { + const MachineBlockLiveness *Value = block(Block); + return Value != nullptr && Value->PhysicalLiveOut.intersects(Unit, Lanes); +} + +bool LiveVariablesAnalysis::Result::invalidate(const MachineFunction &, + const PreservedAnalyses &Preserved) const noexcept { + return !Preserved.preserves(MachineFunctionAnalysisManager::id<LiveVariablesAnalysis>()); +} + +Expected<LiveVariablesAnalysis::Result> +LiveVariablesAnalysis::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + return computeLiveVariables(Function, Target); +} +const MachinePassContract &RebuildMachineLivenessPass::contract() const noexcept { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "RebuildMachineLiveness", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {MachineFunctionProperty::TracksLiveness}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = std::nullopt, + }; + Value.DeclaredChanges = {MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +Expected<MachinePassResult> RebuildMachineLivenessPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + auto Applied = rebuildMachineLiveness(Function, Target); + if (!Applied) + return std::unexpected(Applied.error()); + + MachinePassResult Result; + Result.Changed = *Applied; + if (Result.Changed) + Result.Changes.add(MachinePassChange::Operands); + Result.Preserved = PreservedAnalyses::all(); + return Result; +} + +Expected<bool> rebuildMachineLiveness(MachineFunction &Function, const TargetDescription &Target) { + auto Plan = buildLivenessRebuildPlan(Function, Target); + if (!Plan) + return std::unexpected(Plan.error()); + return applyLivenessPlan(Function, *Plan); +} + +Expected<void> verifyTrackedMachineLiveness(const MachineFunction &Function, + const TargetDescription &Target) { + auto Plan = buildLivenessRebuildPlan(Function, Target); + if (!Plan) + return std::unexpected(Plan.error()); + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Actual = liveInUnits(Block->liveIns(), Target); + if (!Actual) + return std::unexpected(Actual.error()); + auto Expected = requiredLocalPhysicalLiveIns(*Block, Function, Target); + if (!Expected) + return std::unexpected(Expected.error()); + if (!isSubset(*Expected, *Actual)) { + const auto Missing = std::ranges::find_if(Expected->entries(), [&Actual](const auto &Entry) { + return !Actual->contains(Entry.first, Entry.second); + }); + if (Missing != Expected->entries().end()) { + return makeUnexpected(ErrorCode::VerificationError, + "TracksLiveness block " + std::to_string(Block->stableID().value()) + + " is missing required physical register unit " + + std::to_string(Missing->first.value())); + } + return makeUnexpected(ErrorCode::VerificationError, + "TracksLiveness block is missing required physical live-ins"); + } + } + for (const OperandFlagPlan &OperandPlan : Plan->Operands) { + if (OperandPlan.Original.has(RegisterOperandFlag::Kill) && !OperandPlan.KillIsValid) { + return makeUnexpected(ErrorCode::VerificationError, + "TracksLiveness operand has a stale kill flag"); + } + if (OperandPlan.Original.has(RegisterOperandFlag::Dead) && !OperandPlan.DeadIsValid) { + return makeUnexpected(ErrorCode::VerificationError, + "TracksLiveness operand has a stale dead flag"); + } + } + return {}; +} + +Expected<void> registerStage11MachineAnalyses(MachineFunctionAnalysisManager &Analyses) { + const std::array IDs{ + MachineFunctionAnalysisManager::id<SlotIndexesAnalysis>(), + MachineFunctionAnalysisManager::id<LiveVariablesAnalysis>(), + MachineFunctionAnalysisManager::id<LiveIntervalsAnalysis>(), + MachineFunctionAnalysisManager::id<VirtRegMapAnalysis>(), + MachineFunctionAnalysisManager::id<ReachingDefAnalysis>(), + }; + const std::size_t Registered = static_cast<std::size_t>( + std::ranges::count_if(IDs, [&Analyses](AnalysisID ID) { return Analyses.isRegistered(ID); })); + if (Registered == IDs.size()) + return {}; + if (Registered != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "standard liveness analyses are only partially registered"); + } + + auto Candidate = Analyses.forkEmpty(); + if (!Candidate) + return std::unexpected(Candidate.error()); + MachineFunctionAnalysisManager &Staging = **Candidate; + auto Slots = Staging.registerAnalysis<SlotIndexesAnalysis>({ + .Name = "SlotIndexes", + .Dependencies = {}, + .InvalidatedBy = {MachinePassChange::Instructions, MachinePassChange::BlockLayout}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!Slots) + return std::unexpected(Slots.error()); + auto Variables = Staging.registerAnalysis<LiveVariablesAnalysis>({ + .Name = "LiveVariables", + .Dependencies = {}, + .InvalidatedBy = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers, MachinePassChange::CFG, + MachinePassChange::TargetState}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!Variables) + return std::unexpected(Variables.error()); + auto Intervals = Staging.registerAnalysis<LiveIntervalsAnalysis>({ + .Name = "LiveIntervals", + .Dependencies = {MachineFunctionAnalysisManager::id<SlotIndexesAnalysis>(), + MachineFunctionAnalysisManager::id<LiveVariablesAnalysis>()}, + .InvalidatedBy = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers, MachinePassChange::CFG, + MachinePassChange::BlockLayout, MachinePassChange::TargetState}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!Intervals) + return std::unexpected(Intervals.error()); + auto Mapping = Staging.registerAnalysis<VirtRegMapAnalysis>({ + .Name = "VirtRegMap", + .Dependencies = {}, + .InvalidatedBy = MachinePassChanges::all(), + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!Mapping) + return std::unexpected(Mapping.error()); + auto ReachingDefinitions = Staging.registerAnalysis<ReachingDefAnalysis>({ + .Name = "ReachingDefAnalysis", + .Dependencies = {}, + .InvalidatedBy = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers, MachinePassChange::CFG, + MachinePassChange::BlockLayout, MachinePassChange::Frame, + MachinePassChange::TargetState}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!ReachingDefinitions) + return std::unexpected(ReachingDefinitions.error()); + return Analyses.replaceRegistrations(std::move(Staging)); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/LowLevelType.cpp b/backend/lib/CodeGen/LowLevelType.cpp new file mode 100644 index 0000000..50ee0e6 --- /dev/null +++ b/backend/lib/CodeGen/LowLevelType.cpp @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/LowLevelType.h" + +#include <array> +#include <string> + +namespace rcc { +namespace { + +[[nodiscard]] LowLevelType::Kind getVectorKind(LowLevelType ElementType) noexcept { + switch (ElementType.kind()) { + case LowLevelType::Kind::CategoryNeutralScalar: + return LowLevelType::Kind::Vector; + case LowLevelType::Kind::Pointer: + return LowLevelType::Kind::VectorPointer; + case LowLevelType::Kind::Invalid: + case LowLevelType::Kind::Token: + case LowLevelType::Kind::Vector: + case LowLevelType::Kind::VectorPointer: + return LowLevelType::Kind::Invalid; + } + + return LowLevelType::Kind::Invalid; +} + +} // namespace + +Expected<LowLevelType> LowLevelType::makeCategoryNeutralScalar(std::uint64_t BitWidth) { + if (BitWidth == 0 || BitWidth > MaxBitWidth) { + return makeUnexpected(ErrorCode::InvalidArgument, + "LowLevelType bit width must be in the range [1, 65535]"); + } + + return LowLevelType( + encode(Kind::CategoryNeutralScalar, static_cast<std::uint32_t>(BitWidth), 0, 0, false)); +} + +Expected<LowLevelType> LowLevelType::makePointer(std::uint64_t AddressSpace, + std::uint64_t BitWidth) { + if (BitWidth == 0 || BitWidth > MaxBitWidth) { + return makeUnexpected(ErrorCode::InvalidArgument, + "LowLevelType pointer bit width must be in the range [1, 65535]"); + } + if (AddressSpace > MaxAddressSpace) { + return makeUnexpected(ErrorCode::InvalidArgument, + "LowLevelType address space must fit in 24 bits"); + } + + return LowLevelType(encode(Kind::Pointer, static_cast<std::uint32_t>(BitWidth), 0, + static_cast<std::uint32_t>(AddressSpace), false)); +} + +Expected<LowLevelType> LowLevelType::makeFixedVector(std::uint64_t ElementCount, + LowLevelType ElementType) { + return makeVector(ElementCount, false, ElementType); +} + +Expected<LowLevelType> LowLevelType::makeScalableVector(std::uint64_t MinimumElementCount, + LowLevelType ElementType) { + return makeVector(MinimumElementCount, true, ElementType); +} + +Expected<LowLevelType> LowLevelType::makeVector(std::uint64_t ElementCount, bool IsScalable, + LowLevelType ElementType) { + if (ElementCount == 0 || ElementCount > MaxElementCount) { + return makeUnexpected(ErrorCode::InvalidArgument, + "LowLevelType vector element count must be in the range [1, 65535]"); + } + if (!IsScalable && ElementCount == 1) { + return makeUnexpected(ErrorCode::InvalidArgument, + "a fixed LowLevelType vector must contain at least two elements"); + } + + const Kind VectorKind = getVectorKind(ElementType); + if (VectorKind == Kind::Invalid) { + return makeUnexpected(ErrorCode::InvalidArgument, + "LowLevelType vector elements must be non-vector value types"); + } + + std::uint32_t Auxiliary = 0; + if (ElementType.isPointer()) + Auxiliary = *ElementType.addressSpace(); + + return LowLevelType(encode(VectorKind, ElementType.scalarBitWidth(), + static_cast<std::uint32_t>(ElementCount), Auxiliary, IsScalable)); +} + +std::optional<std::uint32_t> LowLevelType::addressSpace() const noexcept { + if (!isPointerOrPointerVector()) + return std::nullopt; + return auxiliaryValue(); +} + +std::optional<LowLevelType> LowLevelType::scalarElementType() const noexcept { + switch (kind()) { + case Kind::Vector: + return LowLevelType(encode(Kind::CategoryNeutralScalar, scalarBitWidth(), 0, 0, false)); + case Kind::VectorPointer: + return LowLevelType(encode(Kind::Pointer, scalarBitWidth(), 0, auxiliaryValue(), false)); + case Kind::Invalid: + case Kind::Token: + case Kind::CategoryNeutralScalar: + case Kind::Pointer: + return std::nullopt; + } + + return std::nullopt; +} + +std::string LowLevelType::toString() const { + if (!isValid()) + return "LLT_invalid"; + if (isToken()) + return "s0"; + + std::string ScalarSpelling; + switch (kind()) { + case Kind::CategoryNeutralScalar: + case Kind::Vector: + ScalarSpelling = "s" + std::to_string(scalarBitWidth()); + break; + case Kind::Pointer: + case Kind::VectorPointer: + ScalarSpelling = + "p" + std::to_string(auxiliaryValue()) + ":" + std::to_string(scalarBitWidth()); + break; + case Kind::Invalid: + case Kind::Token: + break; + } + + if (!isVector()) + return ScalarSpelling; + + std::string Result = "<"; + if (isScalableVector()) + Result += "vscale x "; + Result += std::to_string(minimumElementCount()); + Result += " x "; + Result += ScalarSpelling; + Result += ">"; + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MIRParser/MILexer.cpp b/backend/lib/CodeGen/MIRParser/MILexer.cpp new file mode 100644 index 0000000..830c548 --- /dev/null +++ b/backend/lib/CodeGen/MIRParser/MILexer.cpp @@ -0,0 +1,964 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "MILexer.h" + +#include <algorithm> +#include <array> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <iomanip> +#include <optional> +#include <ranges> +#include <sstream> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +namespace rcc::mir { +namespace { + +template <std::size_t Size> +[[nodiscard]] consteval std::array<std::string_view, Size> +sortSpellings(std::array<std::string_view, Size> Spellings) { + for (std::size_t Left = 0; Left < Spellings.size(); ++Left) { + for (std::size_t Right = Left + 1; Right < Spellings.size(); ++Right) { + if (Spellings[Right] < Spellings[Left]) + std::swap(Spellings[Left], Spellings[Right]); + } + } + return Spellings; +} + +template <std::size_t Size> +[[nodiscard]] consteval bool +hasUniqueSpellings(const std::array<std::string_view, Size> &Spellings) { + for (std::size_t Index = 1; Index < Spellings.size(); ++Index) { + if (Spellings[Index - 1] == Spellings[Index]) + return false; + } + return true; +} + +constexpr auto Keywords = sortSpellings(std::array{ + std::string_view{"addrspace"}, + std::string_view{"adjust_cfa_offset"}, + std::string_view{"afn"}, + std::string_view{"align"}, + std::string_view{"arcp"}, + std::string_view{"basealign"}, + std::string_view{"bb_id"}, + std::string_view{"bbsections"}, + std::string_view{"bfloat"}, + std::string_view{"blockaddress"}, + std::string_view{"call-entry"}, + std::string_view{"call-frame-size"}, + std::string_view{"cfi-type"}, + std::string_view{"constant-pool"}, + std::string_view{"contract"}, + std::string_view{"custom"}, + std::string_view{"dead"}, + std::string_view{"deactivation-symbol"}, + std::string_view{"debug-instr-number"}, + std::string_view{"debug-location"}, + std::string_view{"debug-use"}, + std::string_view{"dbg-instr-ref"}, + std::string_view{"def"}, + std::string_view{"def_cfa"}, + std::string_view{"def_cfa_offset"}, + std::string_view{"def_cfa_register"}, + std::string_view{"dereferenceable"}, + std::string_view{"disjoint"}, + std::string_view{"distinct"}, + std::string_view{"double"}, + std::string_view{"early-clobber"}, + std::string_view{"ehfunclet-entry"}, + std::string_view{"ehscope-entry"}, + std::string_view{"escape"}, + std::string_view{"exact"}, + std::string_view{"float"}, + std::string_view{"floatpred"}, + std::string_view{"fp128"}, + std::string_view{"frame-destroy"}, + std::string_view{"frame-setup"}, + std::string_view{"got"}, + std::string_view{"half"}, + std::string_view{"heap-alloc-marker"}, + std::string_view{"implicit"}, + std::string_view{"implicit-def"}, + std::string_view{"inbounds"}, + std::string_view{"inlineasm-br-indirect-target"}, + std::string_view{"internal"}, + std::string_view{"intpred"}, + std::string_view{"intrinsic"}, + std::string_view{"invariant"}, + std::string_view{"ir-block-address-taken"}, + std::string_view{"jump-table"}, + std::string_view{"killed"}, + std::string_view{"landing-pad"}, + std::string_view{"lanemask"}, + std::string_view{"liveins"}, + std::string_view{"liveout"}, + std::string_view{"llvm_def_aspace_cfa"}, + std::string_view{"llvm_register_pair"}, + std::string_view{"llvm_vector_offset"}, + std::string_view{"llvm_vector_register_mask"}, + std::string_view{"llvm_vector_registers"}, + std::string_view{"machine-block-address-taken"}, + std::string_view{"mmra"}, + std::string_view{"negate_ra_sign_state"}, + std::string_view{"negate_ra_sign_state_with_pc"}, + std::string_view{"ninf"}, + std::string_view{"nnan"}, + std::string_view{"noconvergent"}, + std::string_view{"nofpexcept"}, + std::string_view{"non-temporal"}, + std::string_view{"nneg"}, + std::string_view{"nsw"}, + std::string_view{"nsz"}, + std::string_view{"nusw"}, + std::string_view{"nuw"}, + std::string_view{"offset"}, + std::string_view{"pcsections"}, + std::string_view{"post-instr-symbol"}, + std::string_view{"ppc_fp128"}, + std::string_view{"pre-instr-symbol"}, + std::string_view{"reassoc"}, + std::string_view{"rel_offset"}, + std::string_view{"remember_state"}, + std::string_view{"renamable"}, + std::string_view{"register"}, + std::string_view{"restore"}, + std::string_view{"restore_state"}, + std::string_view{"same_value"}, + std::string_view{"samesign"}, + std::string_view{"shufflemask"}, + std::string_view{"stack"}, + std::string_view{"successors"}, + std::string_view{"target-flags"}, + std::string_view{"target-index"}, + std::string_view{"tied-def"}, + std::string_view{"undef"}, + std::string_view{"undefined"}, + std::string_view{"unknown-address"}, + std::string_view{"unknown-size"}, + std::string_view{"unpredictable"}, + std::string_view{"volatile"}, + std::string_view{"window_save"}, + std::string_view{"x86_fp80"}, +}); + +constexpr auto MetadataKeywords = sortSpellings(std::array{ + std::string_view{"!DIExpression"}, + std::string_view{"!DILocation"}, + std::string_view{"!alias.scope"}, + std::string_view{"!noalias"}, + std::string_view{"!noalias.addrspace"}, + std::string_view{"!range"}, + std::string_view{"!tbaa"}, +}); + +static_assert(hasUniqueSpellings(Keywords)); +static_assert(hasUniqueSpellings(MetadataKeywords)); + +[[nodiscard]] constexpr bool isASCIIAlpha(char Character) noexcept { + return (Character >= 'a' && Character <= 'z') || (Character >= 'A' && Character <= 'Z'); +} + +[[nodiscard]] constexpr bool isASCIIDigit(char Character) noexcept { + return Character >= '0' && Character <= '9'; +} + +[[nodiscard]] constexpr bool isASCIIHexDigit(char Character) noexcept { + return isASCIIDigit(Character) || (Character >= 'a' && Character <= 'f') || + (Character >= 'A' && Character <= 'F'); +} + +[[nodiscard]] constexpr unsigned hexDigitValue(char Character) noexcept { + if (isASCIIDigit(Character)) + return static_cast<unsigned>(Character - '0'); + if (Character >= 'a' && Character <= 'f') + return static_cast<unsigned>(Character - 'a' + 10); + return static_cast<unsigned>(Character - 'A' + 10); +} + +[[nodiscard]] constexpr bool isIdentifierCharacter(char Character) noexcept { + return isASCIIAlpha(Character) || isASCIIDigit(Character) || Character == '_' || + Character == '-' || Character == '.' || Character == '$'; +} + +[[nodiscard]] constexpr bool isRegisterCharacter(char Character) noexcept { + return isIdentifierCharacter(Character) && Character != '.'; +} + +[[nodiscard]] constexpr bool isHorizontalWhitespace(char Character) noexcept { + return Character == ' ' || Character == '\t'; +} + +[[nodiscard]] constexpr bool isNewlineCharacter(char Character) noexcept { + return Character == '\n' || Character == '\r'; +} + +[[nodiscard]] constexpr bool isHexFloatingPointPrefix(char Character) noexcept { + return Character == 'H' || Character == 'K' || Character == 'L' || Character == 'M' || + Character == 'R'; +} + +[[nodiscard]] std::string unexpectedCharacterMessage(unsigned char Character) { + if (Character >= 0x20U && Character <= 0x7EU) { + return "unexpected character '" + std::string(1, static_cast<char>(Character)) + "'"; + } + std::ostringstream Message; + Message << "unexpected byte 0x" << std::uppercase << std::hex << std::setw(2) << std::setfill('0') + << static_cast<unsigned>(Character); + return Message.str(); +} + +} // namespace + +std::string_view getMITokenKindName(MITokenKind Kind) noexcept { + switch (Kind) { + case MITokenKind::EndOfFile: + return "end-of-file"; + case MITokenKind::Error: + return "error"; + case MITokenKind::Newline: + return "newline"; + case MITokenKind::Comma: + return "comma"; + case MITokenKind::Equal: + return "equal"; + case MITokenKind::Underscore: + return "underscore"; + case MITokenKind::Colon: + return "colon"; + case MITokenKind::DoubleColon: + return "double-colon"; + case MITokenKind::Dot: + return "dot"; + case MITokenKind::Exclaim: + return "exclaim"; + case MITokenKind::LeftParen: + return "left-paren"; + case MITokenKind::RightParen: + return "right-paren"; + case MITokenKind::LeftBrace: + return "left-brace"; + case MITokenKind::RightBrace: + return "right-brace"; + case MITokenKind::Plus: + return "plus"; + case MITokenKind::Minus: + return "minus"; + case MITokenKind::Less: + return "less"; + case MITokenKind::Greater: + return "greater"; + case MITokenKind::Keyword: + return "keyword"; + case MITokenKind::MetadataKeyword: + return "metadata-keyword"; + case MITokenKind::Identifier: + return "identifier"; + case MITokenKind::NamedRegister: + return "named-register"; + case MITokenKind::NamedVirtualRegister: + return "named-virtual-register"; + case MITokenKind::MachineBasicBlockLabel: + return "machine-basic-block-label"; + case MITokenKind::MachineBasicBlock: + return "machine-basic-block"; + case MITokenKind::StackObject: + return "stack-object"; + case MITokenKind::FixedStackObject: + return "fixed-stack-object"; + case MITokenKind::NamedGlobalValue: + return "named-global-value"; + case MITokenKind::GlobalValue: + return "global-value"; + case MITokenKind::ExternalSymbol: + return "external-symbol"; + case MITokenKind::MCSymbol: + return "mc-symbol"; + case MITokenKind::IntegerLiteral: + return "integer-literal"; + case MITokenKind::FloatingPointLiteral: + return "floating-point-literal"; + case MITokenKind::HexLiteral: + return "hex-literal"; + case MITokenKind::VirtualRegister: + return "virtual-register"; + case MITokenKind::ConstantPoolItem: + return "constant-pool-item"; + case MITokenKind::JumpTableIndex: + return "jump-table-index"; + case MITokenKind::NamedIRBlock: + return "named-ir-block"; + case MITokenKind::IRBlock: + return "ir-block"; + case MITokenKind::NamedIRValue: + return "named-ir-value"; + case MITokenKind::IRValue: + return "ir-value"; + case MITokenKind::QuotedIRValue: + return "quoted-ir-value"; + case MITokenKind::SubRegisterIndex: + return "subregister-index"; + case MITokenKind::StringConstant: + return "string-constant"; + } + return "unknown"; +} + +bool isMIKeyword(std::string_view Spelling) noexcept { + return std::ranges::binary_search(Keywords, Spelling); +} + +bool isMIMetadataKeyword(std::string_view Spelling) noexcept { + return std::ranges::binary_search(MetadataKeywords, Spelling); +} + +std::string_view MIToken::stringValue() const noexcept { + if (const auto *View = std::get_if<std::string_view>(&StringValue)) + return *View; + if (const auto *Owned = std::get_if<std::string>(&StringValue)) + return *Owned; + return {}; +} + +bool MIToken::isRegister() const noexcept { + return Kind == MITokenKind::NamedRegister || Kind == MITokenKind::NamedVirtualRegister || + Kind == MITokenKind::VirtualRegister || Kind == MITokenKind::Underscore; +} + +bool MIToken::isRegisterFlag() const noexcept { + return isKeyword("implicit") || isKeyword("implicit-def") || isKeyword("def") || + isKeyword("dead") || isKeyword("killed") || isKeyword("undef") || isKeyword("internal") || + isKeyword("early-clobber") || isKeyword("debug-use") || isKeyword("renamable"); +} + +bool MIToken::isMemoryOperandFlag() const noexcept { + return isKeyword("volatile") || isKeyword("non-temporal") || isKeyword("dereferenceable") || + isKeyword("invariant") || Kind == MITokenKind::StringConstant; +} + +MIToken MILexer::makeToken(MITokenKind Kind, TextRange Range, std::string_view Spelling, + std::string_view IntegerValue) { + return MIToken(Kind, Range, Spelling, {}, IntegerValue); +} + +MIToken MILexer::makeStringToken(MITokenKind Kind, TextRange Range, std::string_view Spelling, + std::string_view StringValue, std::string_view IntegerValue) { + return MIToken(Kind, Range, Spelling, StringValue, IntegerValue); +} + +MIToken MILexer::makeOwnedStringToken(MITokenKind Kind, TextRange Range, std::string_view Spelling, + std::string StringValue, std::string_view IntegerValue) { + return MIToken(Kind, Range, Spelling, std::move(StringValue), IntegerValue); +} + +class MILexer::Impl final { +public: + Impl(std::string SourceName, std::string SourceText, MILexerOptions Options) + : Source(std::move(SourceName), std::move(SourceText)), Options(Options) {} + + [[nodiscard]] MIToken next() { + if (Failed) + return token(MITokenKind::EndOfFile, Offset, Offset); + + if (auto TriviaError = skipTrivia()) + return std::move(*TriviaError); + + if (atEnd()) + return token(MITokenKind::EndOfFile, Offset, Offset); + + const std::size_t Begin = Offset; + MIToken Result = lexToken(); + if (Result.kind() == MITokenKind::Error) + return Result; + + if (Options.MaximumTokenLength != 0 && + Result.range().End.Offset - Result.range().Begin.Offset > Options.MaximumTokenLength) { + return fail(Begin, Offset, "machine instruction token exceeds the configured length limit"); + } + if (Options.MaximumTokens != 0 && TokenCount >= Options.MaximumTokens) { + return fail(Begin, Offset, "machine instruction token count exceeds the configured limit"); + } + ++TokenCount; + return Result; + } + + [[nodiscard]] const TextSource &source() const noexcept { + return Source; + } + + [[nodiscard]] std::span<const TextDiagnostic> diagnostics() const noexcept { + return Diagnostics; + } + + [[nodiscard]] bool failed() const noexcept { + return Failed; + } + + [[nodiscard]] std::size_t tokenCount() const noexcept { + return TokenCount; + } + +private: + struct QuotedValue final { + std::size_t End = 0; + std::string Value; + }; + + [[nodiscard]] std::string_view text() const noexcept { + return Source.text(); + } + + [[nodiscard]] bool atEnd(std::size_t Lookahead = 0) const noexcept { + return Offset >= text().size() || Lookahead >= text().size() - Offset; + } + + [[nodiscard]] char peek(std::size_t Lookahead = 0) const noexcept { + if (Offset >= text().size() || Lookahead >= text().size() - Offset) + return '\0'; + return text()[Offset + Lookahead]; + } + + [[nodiscard]] bool startsWith(std::string_view Prefix) const noexcept { + return text().substr(Offset).starts_with(Prefix); + } + + [[nodiscard]] std::string_view slice(std::size_t Begin, std::size_t End) const noexcept { + assert(Begin <= End && End <= text().size()); + return text().substr(Begin, End - Begin); + } + + [[nodiscard]] MIToken token(MITokenKind Kind, std::size_t Begin, std::size_t End, + std::string_view IntegerValue = {}) const { + return MILexer::makeToken(Kind, Source.range(Begin, End), slice(Begin, End), IntegerValue); + } + + [[nodiscard]] MIToken stringToken(MITokenKind Kind, std::size_t Begin, std::size_t End, + std::string_view Value, + std::string_view IntegerValue = {}) const { + return MILexer::makeStringToken(Kind, Source.range(Begin, End), slice(Begin, End), Value, + IntegerValue); + } + + [[nodiscard]] MIToken ownedStringToken(MITokenKind Kind, std::size_t Begin, std::size_t End, + std::string Value, + std::string_view IntegerValue = {}) const { + return MILexer::makeOwnedStringToken(Kind, Source.range(Begin, End), slice(Begin, End), + std::move(Value), IntegerValue); + } + + [[nodiscard]] MIToken fail(std::size_t Begin, std::size_t End, std::string Message) { + End = std::clamp(End, Begin, text().size()); + Diagnostics.push_back(TextDiagnostic{ + .Severity = DiagnosticSeverity::Error, + .Category = "mir-lexer", + .Message = std::move(Message), + .SourceName = std::string(Source.name()), + .Range = Source.range(Begin, End), + }); + Failed = true; + return token(MITokenKind::Error, Begin, End); + } + + [[nodiscard]] std::optional<MIToken> skipTrivia() { + while (true) { + while (!atEnd() && isHorizontalWhitespace(peek())) + ++Offset; + + if (peek() == ';') { + while (!atEnd() && !isNewlineCharacter(peek())) + ++Offset; + continue; + } + + if (!startsWith("/*")) + return std::nullopt; + + const std::size_t Begin = Offset; + Offset += 2; + while (!atEnd() && !startsWith("*/")) + ++Offset; + if (atEnd()) { + return fail(Begin, Offset, + "end of machine instruction input reached before the closing '*/'"); + } + Offset += 2; + } + } + + [[nodiscard]] std::optional<QuotedValue> scanQuotedString(std::size_t QuoteOffset) { + assert(QuoteOffset < text().size() && text()[QuoteOffset] == '"'); + std::size_t Cursor = QuoteOffset + 1; + while (Cursor < text().size() && text()[Cursor] != '"') { + if (isNewlineCharacter(text()[Cursor])) { + Offset = Cursor; + static_cast<void>(fail(QuoteOffset, Cursor, + "end of machine instruction reached before the closing '\"'")); + return std::nullopt; + } + ++Cursor; + } + if (Cursor == text().size()) { + Offset = Cursor; + static_cast<void>( + fail(QuoteOffset, Cursor, "end of machine instruction reached before the closing '\"'")); + return std::nullopt; + } + + std::string Value; + Value.reserve(Cursor - QuoteOffset - 1); + for (std::size_t Index = QuoteOffset + 1; Index < Cursor; ++Index) { + const char Character = text()[Index]; + if (Character == '\\' && Index + 1 < Cursor && text()[Index + 1] == '\\') { + Value.push_back('\\'); + ++Index; + continue; + } + if (Character == '\\' && Index + 2 < Cursor && isASCIIHexDigit(text()[Index + 1]) && + isASCIIHexDigit(text()[Index + 2])) { + const unsigned Decoded = + hexDigitValue(text()[Index + 1]) * 16U + hexDigitValue(text()[Index + 2]); + Value.push_back(static_cast<char>(Decoded)); + Index += 2; + continue; + } + Value.push_back(Character); + } + return QuotedValue{.End = Cursor + 1, .Value = std::move(Value)}; + } + + [[nodiscard]] MIToken lexName(MITokenKind Kind, std::size_t PrefixLength) { + const std::size_t Begin = Offset; + Offset += PrefixLength; + if (peek() == '"') { + auto Quoted = scanQuotedString(Offset); + if (!Quoted) + return token(MITokenKind::Error, Begin, Offset); + Offset = Quoted->End; + return ownedStringToken(Kind, Begin, Offset, std::move(Quoted->Value)); + } + + const std::size_t ValueBegin = Offset; + while (!atEnd() && isIdentifierCharacter(peek())) + ++Offset; + return stringToken(Kind, Begin, Offset, slice(ValueBegin, Offset)); + } + + [[nodiscard]] std::optional<MIToken> lexMachineBasicBlock() { + const bool IsReference = startsWith("%bb."); + if (!IsReference && !startsWith("bb.")) + return std::nullopt; + + const std::size_t Begin = Offset; + const std::size_t PrefixLength = IsReference ? 4 : 3; + Offset += PrefixLength; + if (!isASCIIDigit(peek())) { + return fail(Offset, std::min(Offset + 1, text().size()), + "expected a decimal block number after '" + + std::string(IsReference ? "%bb.'" : "bb.'")); + } + + const std::size_t NumberBegin = Offset; + while (!atEnd() && isASCIIDigit(peek())) + ++Offset; + const std::string_view Number = slice(NumberBegin, Offset); + + if (peek() != '.') { + return stringToken(IsReference ? MITokenKind::MachineBasicBlock + : MITokenKind::MachineBasicBlockLabel, + Begin, Offset, {}, Number); + } + + ++Offset; + const std::size_t NameBegin = Offset; + if (peek() == '"') { + auto Quoted = scanQuotedString(Offset); + if (!Quoted) + return token(MITokenKind::Error, Begin, Offset); + Offset = Quoted->End; + return ownedStringToken(IsReference ? MITokenKind::MachineBasicBlock + : MITokenKind::MachineBasicBlockLabel, + Begin, Offset, std::move(Quoted->Value), Number); + } + while (!atEnd() && isIdentifierCharacter(peek())) + ++Offset; + return stringToken(IsReference ? MITokenKind::MachineBasicBlock + : MITokenKind::MachineBasicBlockLabel, + Begin, Offset, slice(NameBegin, Offset), Number); + } + + [[nodiscard]] std::optional<MIToken> lexIndex(std::string_view Prefix, MITokenKind Kind, + bool HasOptionalName) { + if (!startsWith(Prefix) || !isASCIIDigit(peek(Prefix.size()))) + return std::nullopt; + + const std::size_t Begin = Offset; + Offset += Prefix.size(); + const std::size_t NumberBegin = Offset; + while (!atEnd() && isASCIIDigit(peek())) + ++Offset; + const std::string_view Number = slice(NumberBegin, Offset); + + if (!HasOptionalName) + return token(Kind, Begin, Offset, Number); + + if (peek() != '.') + return stringToken(Kind, Begin, Offset, {}, Number); + + ++Offset; + const std::size_t NameBegin = Offset; + if (peek() == '"') { + auto Quoted = scanQuotedString(Offset); + if (!Quoted) + return token(MITokenKind::Error, Begin, Offset); + Offset = Quoted->End; + return ownedStringToken(Kind, Begin, Offset, std::move(Quoted->Value), Number); + } + while (!atEnd() && isIdentifierCharacter(peek())) + ++Offset; + return stringToken(Kind, Begin, Offset, slice(NameBegin, Offset), Number); + } + + [[nodiscard]] std::optional<MIToken> + lexIRReference(std::string_view Prefix, MITokenKind NumericKind, MITokenKind NamedKind) { + if (!startsWith(Prefix)) + return std::nullopt; + if (isASCIIDigit(peek(Prefix.size()))) + return lexIndex(Prefix, NumericKind, false); + return lexName(NamedKind, Prefix.size()); + } + + [[nodiscard]] std::optional<MIToken> lexRegister() { + if (peek() != '%' && peek() != '$') + return std::nullopt; + + const std::size_t Begin = Offset; + const char Prefix = peek(); + ++Offset; + if (Prefix == '%' && isASCIIDigit(peek())) { + const std::size_t NumberBegin = Offset; + while (!atEnd() && isASCIIDigit(peek())) + ++Offset; + return token(MITokenKind::VirtualRegister, Begin, Offset, slice(NumberBegin, Offset)); + } + + if (Prefix == '%' && !isRegisterCharacter(peek())) { + Offset = Begin; + return std::nullopt; + } + + const std::size_t NameBegin = Offset; + while (!atEnd() && isRegisterCharacter(peek())) + ++Offset; + return stringToken(Prefix == '%' ? MITokenKind::NamedVirtualRegister + : MITokenKind::NamedRegister, + Begin, Offset, slice(NameBegin, Offset)); + } + + [[nodiscard]] std::optional<MIToken> lexGlobalValue() { + if (peek() != '@') + return std::nullopt; + if (!isASCIIDigit(peek(1))) + return lexName(MITokenKind::NamedGlobalValue, 1); + + const std::size_t Begin = Offset++; + const std::size_t NumberBegin = Offset; + while (!atEnd() && isASCIIDigit(peek())) + ++Offset; + return token(MITokenKind::GlobalValue, Begin, Offset, slice(NumberBegin, Offset)); + } + + [[nodiscard]] std::optional<MIToken> lexMCSymbol() { + constexpr std::string_view Prefix = "<mcsymbol "; + if (!startsWith(Prefix)) + return std::nullopt; + + const std::size_t Begin = Offset; + Offset += Prefix.size(); + if (peek() != '"') { + const std::size_t NameBegin = Offset; + while (!atEnd() && isIdentifierCharacter(peek())) + ++Offset; + const std::string_view Name = slice(NameBegin, Offset); + if (peek() != '>') { + return fail(Begin, Offset, "expected the '<mcsymbol ...' operand to be closed by '>'"); + } + ++Offset; + return stringToken(MITokenKind::MCSymbol, Begin, Offset, Name); + } + + auto Quoted = scanQuotedString(Offset); + if (!Quoted) + return token(MITokenKind::Error, Begin, Offset); + Offset = Quoted->End; + if (peek() != '>') { + return fail(Begin, Offset, "expected the '<mcsymbol ...' operand to be closed by '>'"); + } + ++Offset; + return ownedStringToken(MITokenKind::MCSymbol, Begin, Offset, std::move(Quoted->Value)); + } + + [[nodiscard]] std::optional<MIToken> lexFloatHexBits() { + if (!startsWith("f0x") && !startsWith("f0X")) + return std::nullopt; + std::size_t End = Offset + 3; + while (End < text().size() && isASCIIHexDigit(text()[End])) + ++End; + if (End == Offset + 3) + return std::nullopt; + const std::size_t Begin = Offset; + Offset = End; + return token(MITokenKind::FloatingPointLiteral, Begin, Offset); + } + + [[nodiscard]] std::optional<MIToken> lexHexadecimalLiteral() { + if (peek() != '0' || (peek(1) != 'x' && peek(1) != 'X')) + return std::nullopt; + + std::size_t End = Offset + 2; + bool IsFloatingPoint = false; + if (End < text().size() && isHexFloatingPointPrefix(text()[End])) { + IsFloatingPoint = true; + ++End; + } + const std::size_t DigitsBegin = End; + while (End < text().size() && isASCIIHexDigit(text()[End])) + ++End; + if (End == DigitsBegin) + return std::nullopt; + + const std::size_t Begin = Offset; + Offset = End; + return token(IsFloatingPoint ? MITokenKind::FloatingPointLiteral : MITokenKind::HexLiteral, + Begin, Offset); + } + + [[nodiscard]] std::optional<MIToken> lexNumericalLiteral() { + if (!isASCIIDigit(peek()) && (peek() != '-' || !isASCIIDigit(peek(1)))) { + return std::nullopt; + } + + const std::size_t Begin = Offset; + if (peek() == '-') + ++Offset; + while (!atEnd() && isASCIIDigit(peek())) + ++Offset; + + if (peek() != '.') { + return token(MITokenKind::IntegerLiteral, Begin, Offset, slice(Begin, Offset)); + } + + ++Offset; + while (!atEnd() && isASCIIDigit(peek())) + ++Offset; + if ((peek() == 'e' || peek() == 'E') && + (isASCIIDigit(peek(1)) || ((peek(1) == '-' || peek(1) == '+') && isASCIIDigit(peek(2))))) { + ++Offset; + if (peek() == '-' || peek() == '+') + ++Offset; + while (!atEnd() && isASCIIDigit(peek())) + ++Offset; + } + return token(MITokenKind::FloatingPointLiteral, Begin, Offset); + } + + [[nodiscard]] std::optional<MIToken> lexExclaim() { + if (peek() != '!') + return std::nullopt; + + const std::size_t Begin = Offset++; + if (isASCIIDigit(peek()) || !isIdentifierCharacter(peek())) + return token(MITokenKind::Exclaim, Begin, Offset); + + while (!atEnd() && isIdentifierCharacter(peek())) + ++Offset; + const std::string_view Spelling = slice(Begin, Offset); + if (!isMIMetadataKeyword(Spelling)) { + return fail(Begin, Offset, "use of unknown metadata keyword '" + std::string(Spelling) + "'"); + } + return stringToken(MITokenKind::MetadataKeyword, Begin, Offset, Spelling); + } + + [[nodiscard]] std::optional<MIToken> lexSymbol() { + const std::size_t Begin = Offset; + if (startsWith("::")) { + Offset += 2; + return token(MITokenKind::DoubleColon, Begin, Offset); + } + + MITokenKind Kind = MITokenKind::Error; + switch (peek()) { + case ',': + Kind = MITokenKind::Comma; + break; + case '=': + Kind = MITokenKind::Equal; + break; + case ':': + Kind = MITokenKind::Colon; + break; + case '.': + Kind = MITokenKind::Dot; + break; + case '(': + Kind = MITokenKind::LeftParen; + break; + case ')': + Kind = MITokenKind::RightParen; + break; + case '{': + Kind = MITokenKind::LeftBrace; + break; + case '}': + Kind = MITokenKind::RightBrace; + break; + case '+': + Kind = MITokenKind::Plus; + break; + case '-': + Kind = MITokenKind::Minus; + break; + case '<': + Kind = MITokenKind::Less; + break; + case '>': + Kind = MITokenKind::Greater; + break; + default: + return std::nullopt; + } + ++Offset; + return token(Kind, Begin, Offset); + } + + [[nodiscard]] std::optional<MIToken> lexNewline() { + if (!isNewlineCharacter(peek())) + return std::nullopt; + const std::size_t Begin = Offset++; + if (text()[Begin] == '\r' && peek() == '\n') + ++Offset; + return token(MITokenKind::Newline, Begin, Offset); + } + + [[nodiscard]] std::optional<MIToken> lexQuotedIRValue() { + if (peek() != '`') + return std::nullopt; + + const std::size_t Begin = Offset++; + const std::size_t ValueBegin = Offset; + while (!atEnd() && peek() != '`' && !isNewlineCharacter(peek())) + ++Offset; + if (atEnd() || isNewlineCharacter(peek())) { + return fail(Begin, Offset, "end of machine instruction reached before the closing '`'"); + } + const std::size_t ValueEnd = Offset++; + return stringToken(MITokenKind::QuotedIRValue, Begin, Offset, slice(ValueBegin, ValueEnd)); + } + + [[nodiscard]] MIToken lexToken() { + if (auto Result = lexMachineBasicBlock()) + return std::move(*Result); + if (auto Result = lexFloatHexBits()) + return std::move(*Result); + + if (isASCIIAlpha(peek()) || peek() == '_') { + const std::size_t Begin = Offset; + while (!atEnd() && isIdentifierCharacter(peek())) + ++Offset; + const std::string_view Value = slice(Begin, Offset); + if (Value == "_") + return token(MITokenKind::Underscore, Begin, Offset); + return stringToken(isMIKeyword(Value) ? MITokenKind::Keyword : MITokenKind::Identifier, Begin, + Offset, Value); + } + + if (auto Result = lexIndex("%jump-table.", MITokenKind::JumpTableIndex, false)) + return std::move(*Result); + if (auto Result = lexIndex("%stack.", MITokenKind::StackObject, true)) + return std::move(*Result); + if (auto Result = lexIndex("%fixed-stack.", MITokenKind::FixedStackObject, false)) + return std::move(*Result); + if (auto Result = lexIndex("%const.", MITokenKind::ConstantPoolItem, false)) + return std::move(*Result); + if (startsWith("%subreg.")) + return lexName(MITokenKind::SubRegisterIndex, 8); + if (auto Result = lexIRReference("%ir-block.", MITokenKind::IRBlock, MITokenKind::NamedIRBlock)) + return std::move(*Result); + if (auto Result = lexIRReference("%ir.", MITokenKind::IRValue, MITokenKind::NamedIRValue)) + return std::move(*Result); + if (auto Result = lexRegister()) + return std::move(*Result); + if (auto Result = lexGlobalValue()) + return std::move(*Result); + if (peek() == '&') + return lexName(MITokenKind::ExternalSymbol, 1); + if (auto Result = lexMCSymbol()) + return std::move(*Result); + if (auto Result = lexHexadecimalLiteral()) + return std::move(*Result); + if (auto Result = lexNumericalLiteral()) + return std::move(*Result); + if (auto Result = lexExclaim()) + return std::move(*Result); + if (auto Result = lexSymbol()) + return std::move(*Result); + if (auto Result = lexNewline()) + return std::move(*Result); + if (auto Result = lexQuotedIRValue()) + return std::move(*Result); + if (peek() == '"') + return lexName(MITokenKind::StringConstant, 0); + + const std::size_t Begin = Offset++; + return fail(Begin, Offset, + unexpectedCharacterMessage(static_cast<unsigned char>(text()[Begin]))); + } + + TextSource Source; + MILexerOptions Options; + std::size_t Offset = 0; + std::size_t TokenCount = 0; + bool Failed = false; + std::vector<TextDiagnostic> Diagnostics; +}; + +MILexer::MILexer(std::string SourceName, std::string SourceText, MILexerOptions Options) + : Implementation( + std::make_unique<Impl>(std::move(SourceName), std::move(SourceText), Options)) {} + +MILexer::MILexer(MILexer &&) noexcept = default; +MILexer &MILexer::operator=(MILexer &&) noexcept = default; +MILexer::~MILexer() = default; + +MIToken MILexer::next() { + assert(Implementation != nullptr); + return Implementation->next(); +} + +const TextSource &MILexer::source() const noexcept { + assert(Implementation != nullptr); + return Implementation->source(); +} + +std::span<const TextDiagnostic> MILexer::diagnostics() const noexcept { + assert(Implementation != nullptr); + return Implementation->diagnostics(); +} + +bool MILexer::failed() const noexcept { + assert(Implementation != nullptr); + return Implementation->failed(); +} + +std::size_t MILexer::tokenCount() const noexcept { + assert(Implementation != nullptr); + return Implementation->tokenCount(); +} + +} // namespace rcc::mir diff --git a/backend/lib/CodeGen/MIRParser/MILexer.h b/backend/lib/CodeGen/MIRParser/MILexer.h new file mode 100644 index 0000000..e3d50e7 --- /dev/null +++ b/backend/lib/CodeGen/MIRParser/MILexer.h @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_LIB_CODEGEN_MIRPARSER_MILEXER_H +#define RCC_LIB_CODEGEN_MIRPARSER_MILEXER_H + +#include "rcc/Support/TextDiagnostic.h" + +#include <cstddef> +#include <cstdint> +#include <memory> +#include <span> +#include <string> +#include <string_view> +#include <utility> +#include <variant> + +namespace rcc::mir { + +enum class MITokenKind : std::uint8_t { + EndOfFile, + Error, + Newline, + + Comma, + Equal, + Underscore, + Colon, + DoubleColon, + Dot, + Exclaim, + LeftParen, + RightParen, + LeftBrace, + RightBrace, + Plus, + Minus, + Less, + Greater, + + Keyword, + MetadataKeyword, + Identifier, + NamedRegister, + NamedVirtualRegister, + MachineBasicBlockLabel, + MachineBasicBlock, + StackObject, + FixedStackObject, + NamedGlobalValue, + GlobalValue, + ExternalSymbol, + MCSymbol, + IntegerLiteral, + FloatingPointLiteral, + HexLiteral, + VirtualRegister, + ConstantPoolItem, + JumpTableIndex, + NamedIRBlock, + IRBlock, + NamedIRValue, + IRValue, + QuotedIRValue, + SubRegisterIndex, + StringConstant, +}; + +[[nodiscard]] std::string_view getMITokenKindName(MITokenKind Kind) noexcept; +[[nodiscard]] bool isMIKeyword(std::string_view Spelling) noexcept; +[[nodiscard]] bool isMIMetadataKeyword(std::string_view Spelling) noexcept; + +class MIToken final { +public: + [[nodiscard]] MITokenKind kind() const noexcept { + return Kind; + } + + [[nodiscard]] const TextRange &range() const noexcept { + return Range; + } + + [[nodiscard]] std::string_view spelling() const noexcept { + return Spelling; + } + + [[nodiscard]] bool hasStringValue() const noexcept { + return !std::holds_alternative<std::monostate>(StringValue); + } + + [[nodiscard]] std::string_view stringValue() const noexcept; + + [[nodiscard]] bool hasIntegerValue() const noexcept { + return !IntegerValue.empty(); + } + + [[nodiscard]] std::string_view integerValue() const noexcept { + return IntegerValue; + } + + [[nodiscard]] bool is(MITokenKind Expected) const noexcept { + return Kind == Expected; + } + + [[nodiscard]] bool isKeyword(std::string_view Expected) const noexcept { + return Kind == MITokenKind::Keyword && stringValue() == Expected; + } + + [[nodiscard]] bool isMetadataKeyword(std::string_view Expected) const noexcept { + return Kind == MITokenKind::MetadataKeyword && stringValue() == Expected; + } + + [[nodiscard]] bool isNewlineOrEnd() const noexcept { + return Kind == MITokenKind::Newline || Kind == MITokenKind::EndOfFile; + } + + [[nodiscard]] bool isErrorOrEnd() const noexcept { + return Kind == MITokenKind::Error || Kind == MITokenKind::EndOfFile; + } + + [[nodiscard]] bool isRegister() const noexcept; + [[nodiscard]] bool isRegisterFlag() const noexcept; + [[nodiscard]] bool isMemoryOperandFlag() const noexcept; + +private: + using StringValueStorage = std::variant<std::monostate, std::string_view, std::string>; + + MIToken(MITokenKind Kind, TextRange Range, std::string_view Spelling, + StringValueStorage StringValue = {}, std::string_view IntegerValue = {}) + : Kind(Kind), Range(Range), Spelling(Spelling), StringValue(std::move(StringValue)), + IntegerValue(IntegerValue) {} + + MITokenKind Kind = MITokenKind::Error; + TextRange Range; + std::string_view Spelling; + StringValueStorage StringValue; + std::string_view IntegerValue; + + friend class MILexer; +}; + +struct MILexerOptions final { + std::size_t MaximumTokenLength = std::size_t{1} << 24U; + std::size_t MaximumTokens = std::size_t{1} << 24U; +}; + +class MILexer final { +public: + MILexer(std::string SourceName, std::string SourceText, MILexerOptions Options = {}); + MILexer(const MILexer &) = delete; + MILexer &operator=(const MILexer &) = delete; + MILexer(MILexer &&) noexcept; + MILexer &operator=(MILexer &&) noexcept; + ~MILexer(); + + [[nodiscard]] MIToken next(); + + [[nodiscard]] const TextSource &source() const noexcept; + [[nodiscard]] std::span<const TextDiagnostic> diagnostics() const noexcept; + [[nodiscard]] bool failed() const noexcept; + [[nodiscard]] std::size_t tokenCount() const noexcept; + +private: + [[nodiscard]] static MIToken makeToken(MITokenKind Kind, TextRange Range, + std::string_view Spelling, + std::string_view IntegerValue = {}); + [[nodiscard]] static MIToken makeStringToken(MITokenKind Kind, TextRange Range, + std::string_view Spelling, + std::string_view StringValue, + std::string_view IntegerValue = {}); + [[nodiscard]] static MIToken makeOwnedStringToken(MITokenKind Kind, TextRange Range, + std::string_view Spelling, + std::string StringValue, + std::string_view IntegerValue = {}); + + class Impl; + std::unique_ptr<Impl> Implementation; +}; + +} // namespace rcc::mir + +#endif // RCC_LIB_CODEGEN_MIRPARSER_MILEXER_H diff --git a/backend/lib/CodeGen/MIRParser/MIParser.cpp b/backend/lib/CodeGen/MIRParser/MIParser.cpp new file mode 100644 index 0000000..a98dbd3 --- /dev/null +++ b/backend/lib/CodeGen/MIRParser/MIParser.cpp @@ -0,0 +1,2931 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MIRParser/MIParser.h" + +#include "MILexer.h" +#include "rcc/AsmParser/LLParser.h" +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InlineAsm.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MIRPrinter.h" +#include "rcc/CodeGen/MIRYamlMapping.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMIRTextInfo.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <charconv> +#include <cmath> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <limits> +#include <memory> +#include <optional> +#include <span> +#include <string> +#include <string_view> +#include <system_error> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc::mir { +namespace { + +[[nodiscard]] bool parseUnsignedDecimal(std::string_view Spelling, std::uint64_t &Value) { + if (Spelling.empty() || Spelling.front() == '-') + return false; + const char *Begin = Spelling.data(); + const char *End = Begin + Spelling.size(); + const auto Result = std::from_chars(Begin, End, Value, 10); + return Result.ec == std::errc{} && Result.ptr == End; +} + +[[nodiscard]] bool parseSignedDecimal(std::string_view Spelling, std::int64_t &Value) { + if (Spelling.empty()) + return false; + const char *Begin = Spelling.data(); + const char *End = Begin + Spelling.size(); + const auto Result = std::from_chars(Begin, End, Value, 10); + return Result.ec == std::errc{} && Result.ptr == End; +} + +[[nodiscard]] bool parseHexadecimal(std::string_view Spelling, std::uint64_t &Value) { + if (Spelling.size() < 3 || Spelling[0] != '0' || (Spelling[1] != 'x' && Spelling[1] != 'X')) + return false; + const char *Begin = Spelling.data() + 2; + const char *End = Spelling.data() + Spelling.size(); + const auto Result = std::from_chars(Begin, End, Value, 16); + return Result.ec == std::errc{} && Result.ptr == End; +} + +[[nodiscard]] bool parseUnsignedDigits(std::string_view Spelling, std::uint64_t &Value) { + if (Spelling.empty()) + return false; + const char *Begin = Spelling.data(); + const char *End = Begin + Spelling.size(); + const auto Result = std::from_chars(Begin, End, Value, 10); + return Result.ec == std::errc{} && Result.ptr == End; +} + +[[nodiscard]] std::optional<FloatSemantics> floatSemanticsForKeyword(std::string_view Keyword) { + if (Keyword == "half") + return FloatSemantics::IEEEHalf; + if (Keyword == "bfloat") + return FloatSemantics::BFloat; + if (Keyword == "float") + return FloatSemantics::IEEESingle; + if (Keyword == "double") + return FloatSemantics::IEEEDouble; + if (Keyword == "x86_fp80") + return FloatSemantics::X87DoubleExtended; + if (Keyword == "fp128") + return FloatSemantics::IEEEQuad; + if (Keyword == "ppc_fp128") + return FloatSemantics::PPCDoubleDouble; + return std::nullopt; +} + +[[nodiscard]] bool isFloatingSemanticsKeyword(const MIToken &Token) { + return Token.kind() == MITokenKind::Keyword && + floatSemanticsForKeyword(Token.stringValue()).has_value(); +} + +[[nodiscard]] bool isCFIOperationToken(const MIToken &Token) { + return Token.isKeyword("same_value") || Token.isKeyword("remember_state") || + Token.isKeyword("restore_state") || Token.isKeyword("offset") || + Token.isKeyword("llvm_def_aspace_cfa") || Token.isKeyword("def_cfa_register") || + Token.isKeyword("def_cfa_offset") || Token.isKeyword("def_cfa") || + Token.isKeyword("rel_offset") || Token.isKeyword("adjust_cfa_offset") || + Token.isKeyword("escape") || Token.isKeyword("restore") || Token.isKeyword("undefined") || + Token.isKeyword("register") || Token.isKeyword("window_save") || + Token.isKeyword("negate_ra_sign_state") || + Token.isKeyword("negate_ra_sign_state_with_pc") || Token.isKeyword("llvm_register_pair") || + Token.isKeyword("llvm_vector_registers") || Token.isKeyword("llvm_vector_offset") || + Token.isKeyword("llvm_vector_register_mask"); +} + +[[nodiscard]] Expected<APInt> parseIntegerImmediateValue(std::uint32_t BitWidth, + std::string_view Spelling) { + if (BitWidth == 0 || BitWidth > APInt::MaxBitWidth) + return makeUnexpected(ErrorCode::ParseError, "typed integer width is out of range"); + + const bool Negative = Spelling.starts_with('-'); + if (Negative) + Spelling.remove_prefix(1); + if (Spelling.empty()) + return makeUnexpected(ErrorCode::ParseError, "typed integer has no digits"); + + std::vector<std::uint64_t> Words((static_cast<std::size_t>(BitWidth) + 63U) / 64U, 0); + for (char Character : Spelling) { + if (Character < '0' || Character > '9') + return makeUnexpected(ErrorCode::ParseError, "typed integer contains a non-decimal digit"); + std::uint64_t Carry = static_cast<unsigned>(Character - '0'); + for (std::uint64_t &Word : Words) { + const unsigned __int128 Product = static_cast<unsigned __int128>(Word) * 10U + Carry; + Word = static_cast<std::uint64_t>(Product); + Carry = static_cast<std::uint64_t>(Product >> 64U); + } + } + + // LLVM parses the arbitrary-precision literal first and then truncates it to + // the declared integer width. Keep exactly that modulo-2^N bit pattern. + if (BitWidth % 64U != 0) + Words.back() &= (std::uint64_t{1} << (BitWidth % 64U)) - 1U; + if (Negative) { + for (std::uint64_t &Word : Words) + Word = ~Word; + std::uint64_t Carry = 1; + for (std::uint64_t &Word : Words) { + const std::uint64_t Previous = Word; + Word += Carry; + Carry = Carry != 0 && Word < Previous ? 1 : 0; + } + if (BitWidth % 64U != 0) + Words.back() &= (std::uint64_t{1} << (BitWidth % 64U)) - 1U; + } + return APInt::create(BitWidth, Words); +} + +[[nodiscard]] std::optional<std::uint64_t> minimumTypeSizeInBits(LowLevelType Type) noexcept { + if (!Type.isValid() || Type.isToken()) + return std::nullopt; + const std::uint64_t ScalarSize = Type.scalarBitWidth(); + if (!Type.isVector()) + return ScalarSize; + const std::uint64_t ElementCount = Type.minimumElementCount(); + if (ScalarSize != 0 && ElementCount > std::numeric_limits<std::uint64_t>::max() / ScalarSize) + return std::nullopt; + return ScalarSize * ElementCount; +} + +[[nodiscard]] std::optional<AtomicOrdering> atomicOrdering(std::string_view Name) noexcept { + if (Name == "unordered") + return AtomicOrdering::Unordered; + if (Name == "monotonic") + return AtomicOrdering::Monotonic; + if (Name == "acquire") + return AtomicOrdering::Acquire; + if (Name == "release") + return AtomicOrdering::Release; + if (Name == "acq_rel") + return AtomicOrdering::AcquireRelease; + if (Name == "seq_cst") + return AtomicOrdering::SequentiallyConsistent; + return std::nullopt; +} + +[[nodiscard]] bool isInstructionFlag(const MIToken &Token) { + return Token.isKeyword("frame-setup") || Token.isKeyword("frame-destroy") || + Token.isKeyword("nnan") || Token.isKeyword("ninf") || Token.isKeyword("nsz") || + Token.isKeyword("arcp") || Token.isKeyword("contract") || Token.isKeyword("afn") || + Token.isKeyword("reassoc") || Token.isKeyword("nuw") || Token.isKeyword("nsw") || + Token.isKeyword("nusw") || Token.isKeyword("exact") || Token.isKeyword("nofpexcept") || + Token.isKeyword("unpredictable") || Token.isKeyword("noconvergent") || + Token.isKeyword("nneg") || Token.isKeyword("disjoint") || Token.isKeyword("samesign") || + Token.isKeyword("inbounds"); +} + +[[nodiscard]] std::optional<MachineInstrFlag> instructionFlag(std::string_view Name) { + if (Name == "frame-setup") + return MachineInstrFlag::FrameSetup; + if (Name == "frame-destroy") + return MachineInstrFlag::FrameDestroy; + if (Name == "nnan") + return MachineInstrFlag::FastMathNoNaNs; + if (Name == "ninf") + return MachineInstrFlag::FastMathNoInfs; + if (Name == "nsz") + return MachineInstrFlag::FastMathNoSignedZeros; + if (Name == "arcp") + return MachineInstrFlag::FastMathAllowReciprocal; + if (Name == "contract") + return MachineInstrFlag::FastMathAllowContract; + if (Name == "afn") + return MachineInstrFlag::FastMathApproxFunc; + if (Name == "reassoc") + return MachineInstrFlag::FastMathAllowReassoc; + if (Name == "nuw") + return MachineInstrFlag::NoUnsignedWrap; + if (Name == "nsw") + return MachineInstrFlag::NoSignedWrap; + if (Name == "nusw") + return MachineInstrFlag::NoUSWrap; + if (Name == "exact") + return MachineInstrFlag::Exact; + if (Name == "nofpexcept") + return MachineInstrFlag::NoFPExcept; + if (Name == "unpredictable") + return MachineInstrFlag::Unpredictable; + if (Name == "noconvergent") + return MachineInstrFlag::NoConvergent; + if (Name == "nneg") + return MachineInstrFlag::NonNeg; + if (Name == "disjoint") + return MachineInstrFlag::Disjoint; + if (Name == "samesign") + return MachineInstrFlag::SameSign; + if (Name == "inbounds") + return MachineInstrFlag::InBounds; + return std::nullopt; +} + +[[nodiscard]] std::optional<IntegerPredicate> integerPredicate(std::string_view Name) { + if (Name == "eq") + return IntegerPredicate::Equal; + if (Name == "ne") + return IntegerPredicate::NotEqual; + if (Name == "sgt") + return IntegerPredicate::SignedGreaterThan; + if (Name == "sge") + return IntegerPredicate::SignedGreaterEqual; + if (Name == "slt") + return IntegerPredicate::SignedLessThan; + if (Name == "sle") + return IntegerPredicate::SignedLessEqual; + if (Name == "ugt") + return IntegerPredicate::UnsignedGreaterThan; + if (Name == "uge") + return IntegerPredicate::UnsignedGreaterEqual; + if (Name == "ult") + return IntegerPredicate::UnsignedLessThan; + if (Name == "ule") + return IntegerPredicate::UnsignedLessEqual; + return std::nullopt; +} + +[[nodiscard]] std::optional<FloatingPredicate> floatingPredicate(std::string_view Name) { + if (Name == "false") + return FloatingPredicate::AlwaysFalse; + if (Name == "oeq") + return FloatingPredicate::OrderedEqual; + if (Name == "ogt") + return FloatingPredicate::OrderedGreaterThan; + if (Name == "oge") + return FloatingPredicate::OrderedGreaterEqual; + if (Name == "olt") + return FloatingPredicate::OrderedLessThan; + if (Name == "ole") + return FloatingPredicate::OrderedLessEqual; + if (Name == "one") + return FloatingPredicate::OrderedNotEqual; + if (Name == "ord") + return FloatingPredicate::Ordered; + if (Name == "uno") + return FloatingPredicate::Unordered; + if (Name == "ueq") + return FloatingPredicate::UnorderedEqual; + if (Name == "ugt") + return FloatingPredicate::UnorderedGreaterThan; + if (Name == "uge") + return FloatingPredicate::UnorderedGreaterEqual; + if (Name == "ult") + return FloatingPredicate::UnorderedLessThan; + if (Name == "ule") + return FloatingPredicate::UnorderedLessEqual; + if (Name == "une") + return FloatingPredicate::UnorderedNotEqual; + if (Name == "true") + return FloatingPredicate::AlwaysTrue; + return std::nullopt; +} + +} // namespace + +class MIParser::Impl final { +public: + Impl(MachineFunction &Function, const TargetDescription &Target, std::string SourceName, + std::string SourceText, MIParserContext Context, MIParserOptions Options) + : Function(Function), Target(Target), + Lexer(std::move(SourceName), std::move(SourceText), + MILexerOptions{.MaximumTokenLength = Options.MaximumTokenLength, + .MaximumTokens = Options.MaximumTokens}), + Context(std::move(Context)), Options(Options) {} + + [[nodiscard]] Expected<void> parse(); + + [[nodiscard]] const TextSource &source() const noexcept { + return Lexer.source(); + } + + [[nodiscard]] std::span<const TextDiagnostic> diagnostics() const noexcept { + return Diagnostics; + } + + [[nodiscard]] bool failed() const noexcept { + return !Diagnostics.empty(); + } + +private: + struct ParsedOperand final { + MachineOperand Operand; + std::optional<std::size_t> TiedDef; + TextRange Range; + }; + + [[nodiscard]] const MIToken &token() const noexcept { + return Tokens[Position]; + } + + [[nodiscard]] bool at(MITokenKind Kind) const noexcept { + return token().is(Kind); + } + + [[nodiscard]] bool nextIs(MITokenKind Kind) const noexcept { + return Position + 1U < Tokens.size() && Tokens[Position + 1U].is(Kind); + } + + [[nodiscard]] bool atKeyword(std::string_view Keyword) const noexcept { + return token().isKeyword(Keyword); + } + + void advance() noexcept { + if (!at(MITokenKind::EndOfFile)) + ++Position; + } + + [[nodiscard]] bool consume(MITokenKind Kind) noexcept { + if (!at(Kind)) + return false; + advance(); + return true; + } + + [[nodiscard]] bool expect(MITokenKind Kind, std::string_view Description); + [[nodiscard]] bool fail(const MIToken &At, std::string Message); + [[nodiscard]] bool fail(std::string Message) { + return fail(token(), std::move(Message)); + } + [[nodiscard]] Expected<void> failureResult() const; + [[nodiscard]] bool importError(const MIToken &At, const Error &Failure); + [[nodiscard]] bool tokenize(); + + [[nodiscard]] bool parseUnsigned(const MIToken &Token, std::uint64_t &Value); + [[nodiscard]] bool parseUnsigned32(const MIToken &Token, std::uint32_t &Value); + [[nodiscard]] bool parseSigned64(const MIToken &Token, std::int64_t &Value); + [[nodiscard]] bool parseOptionalOffset(std::int64_t &Offset); + [[nodiscard]] bool parseAlignment(Alignment &Result); + [[nodiscard]] bool parseLowLevelType(LowLevelType &Result); + [[nodiscard]] bool parseScalarLowLevelType(const MIToken &Token, LowLevelType &Result); + + [[nodiscard]] bool parseFirstPass(); + [[nodiscard]] bool parseSecondPass(); + [[nodiscard]] bool parseBlockDefinition(bool Create, std::uint32_t &ID); + [[nodiscard]] bool parseBlockAttributes(MachineBasicBlock *Block); + [[nodiscard]] bool parseBlockLiveIns(MachineBasicBlock &Block); + [[nodiscard]] bool parseBlockSuccessors(MachineBasicBlock &Block); + [[nodiscard]] bool parseInstruction(MachineBasicBlock &Block, bool InBundle, + MachineInstr *PreviousInstruction, MachineInstr *&Result); + [[nodiscard]] bool inferSuccessors(); + + [[nodiscard]] bool parseRegister(Register &Result); + [[nodiscard]] bool parseRegisterOperand(bool IsDef, ParsedOperand &Result); + [[nodiscard]] bool parseTargetFlags(std::uint32_t &Flags); + [[nodiscard]] bool parseOperand(bool IsDef, ParsedOperand &Result); + [[nodiscard]] bool parseInlineAsmExtraInfoOperand(ParsedOperand &Result); + [[nodiscard]] bool parseInlineAsmOperandFlag(ParsedOperand &Result); + [[nodiscard]] bool parseTypedIntegerOperand(ParsedOperand &Result); + [[nodiscard]] bool parseFloatingImmediateOperand(ParsedOperand &Result); + [[nodiscard]] bool parseMetadataReference(MachineMetadataReference &Result); + [[nodiscard]] bool parseMetadataOperand(ParsedOperand &Result); + [[nodiscard]] bool + parseInlineDILocation(std::vector<MachineDILocationFrame> &InlineChain, + std::optional<MachineMetadataReference> &TerminalInlinedAt); + [[nodiscard]] bool parseCFIRegister(DwarfRegisterNumber &Result); + [[nodiscard]] bool parseCFISigned(std::int64_t &Result); + [[nodiscard]] bool parseCFIUnsigned(std::uint32_t &Result); + [[nodiscard]] bool parseCFIOperand(ParsedOperand &Result); + [[nodiscard]] bool parseMemoryBase(MemoryBase &Result, std::int64_t &Offset); + [[nodiscard]] bool parseMemoryOperand(const MachineMemOperand *&Result, Opcode InstructionOpcode); + [[nodiscard]] bool parsePredicateOperand(ParsedOperand &Result, bool IsFloating); + [[nodiscard]] bool parseShuffleMaskOperand(ParsedOperand &Result); + [[nodiscard]] bool parseDebugInstrRefOperand(ParsedOperand &Result); + [[nodiscard]] bool parseLaneMaskOperand(ParsedOperand &Result); + [[nodiscard]] bool parseIntrinsicOperand(ParsedOperand &Result); + [[nodiscard]] bool parseTargetIndexOperand(ParsedOperand &Result); + [[nodiscard]] bool parseLiveOutMaskOperand(ParsedOperand &Result); + [[nodiscard]] bool parseCustomMaskOperand(ParsedOperand &Result); + [[nodiscard]] bool parseBlockAddressOperand(ParsedOperand &Result); + + [[nodiscard]] Register getOrCreateVirtualRegister(const MIToken &Token); + [[nodiscard]] MachineBasicBlock *blockFor(const MIToken &Token); + [[nodiscard]] std::optional<FrameIndex> frameIndexFor(const MIToken &Token, bool Fixed); + [[nodiscard]] std::optional<ConstantPoolIndex> constantPoolIndexFor(const MIToken &Token); + [[nodiscard]] std::optional<JumpTableIndex> jumpTableIndexFor(const MIToken &Token); + [[nodiscard]] std::string symbolName(const MIToken &Token) const; + + MachineFunction &Function; + const TargetDescription &Target; + MILexer Lexer; + MIParserContext Context; + MIParserOptions Options; + std::vector<MIToken> Tokens; + std::size_t Position = 0; + std::vector<TextDiagnostic> Diagnostics; + std::unordered_map<std::uint32_t, MachineBasicBlock *> Blocks; + std::unordered_map<std::uint32_t, Register> NumericVirtualRegisters; + std::unordered_map<std::string, Register> NamedVirtualRegisters; + std::unordered_set<MachineBasicBlock *> ExplicitSuccessorBlocks; + std::size_t ParsedInstructionCount = 0; + bool ParseAttempted = false; +}; + +bool MIParser::Impl::expect(MITokenKind Kind, std::string_view Description) { + if (consume(Kind)) + return false; + return fail("expected " + std::string(Description) + ", found " + + std::string(getMITokenKindName(token().kind()))); +} + +bool MIParser::Impl::fail(const MIToken &At, std::string Message) { + if (Diagnostics.empty()) { + Diagnostics.push_back(TextDiagnostic{ + .Severity = DiagnosticSeverity::Error, + .Category = "mir-parser", + .Message = std::move(Message), + .SourceName = std::string(Lexer.source().name()), + .Range = At.range(), + }); + } + return true; +} + +Expected<void> MIParser::Impl::failureResult() const { + const std::string Message = + Diagnostics.empty() ? "machine instruction parsing failed" : Diagnostics.front().Message; + return makeUnexpected(ErrorCode::ParseError, Message); +} + +bool MIParser::Impl::importError(const MIToken &At, const Error &Failure) { + return fail(At, std::string(Failure.message())); +} + +bool MIParser::Impl::tokenize() { + while (true) { + MIToken Token = Lexer.next(); + const MITokenKind Kind = Token.kind(); + Tokens.push_back(std::move(Token)); + if (Kind == MITokenKind::Error) { + const auto LexerDiagnostics = Lexer.diagnostics(); + Diagnostics.assign(LexerDiagnostics.begin(), LexerDiagnostics.end()); + return true; + } + if (Kind == MITokenKind::EndOfFile) + return false; + } +} + +bool MIParser::Impl::parseUnsigned(const MIToken &Token, std::uint64_t &Value) { + if (Token.kind() == MITokenKind::HexLiteral) + return !parseHexadecimal(Token.spelling(), Value); + if (Token.hasIntegerValue()) + return !parseUnsignedDecimal(Token.integerValue(), Value); + return true; +} + +bool MIParser::Impl::parseUnsigned32(const MIToken &Token, std::uint32_t &Value) { + std::uint64_t Wide = 0; + if (parseUnsigned(Token, Wide) || Wide > std::numeric_limits<std::uint32_t>::max()) + return true; + Value = static_cast<std::uint32_t>(Wide); + return false; +} + +bool MIParser::Impl::parseSigned64(const MIToken &Token, std::int64_t &Value) { + if (Token.kind() != MITokenKind::IntegerLiteral) + return true; + if (parseSignedDecimal(Token.integerValue(), Value)) + return false; + std::uint64_t Unsigned = 0; + if (!parseUnsignedDecimal(Token.integerValue(), Unsigned)) + return true; + Value = std::bit_cast<std::int64_t>(Unsigned); + return false; +} + +bool MIParser::Impl::parseOptionalOffset(std::int64_t &Offset) { + Offset = 0; + if (!at(MITokenKind::Plus) && !at(MITokenKind::Minus)) + return false; + const bool Negative = at(MITokenKind::Minus); + advance(); + std::uint64_t Magnitude = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Magnitude) || + Magnitude > (std::uint64_t{1} << 63U)) { + return fail("expected a 64-bit integer offset"); + } + if (Negative) { + if (Magnitude == (std::uint64_t{1} << 63U)) + Offset = std::numeric_limits<std::int64_t>::min(); + else + Offset = -static_cast<std::int64_t>(Magnitude); + } else { + if (Magnitude > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return fail("positive offset exceeds signed 64-bit range"); + Offset = static_cast<std::int64_t>(Magnitude); + } + advance(); + return false; +} + +bool MIParser::Impl::parseAlignment(Alignment &Result) { + advance(); + std::uint64_t Bytes = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Bytes)) + return fail("expected an unsigned alignment"); + auto Parsed = Alignment::create(Bytes); + if (!Parsed) + return importError(token(), Parsed.error()); + Result = *Parsed; + advance(); + return false; +} + +bool MIParser::Impl::parseScalarLowLevelType(const MIToken &TypeToken, LowLevelType &Result) { + if (TypeToken.kind() != MITokenKind::Identifier) + return fail(TypeToken, "expected a scalar low-level type"); + std::string_view Spelling = TypeToken.stringValue(); + enum class ScalarKind : std::uint8_t { CategoryNeutral, Pointer }; + ScalarKind Kind = ScalarKind::CategoryNeutral; + std::string_view Digits; + if (Spelling.starts_with("bf")) { + Digits = Spelling.substr(2); + } else if (Spelling.starts_with('s') || Spelling.starts_with('i') || Spelling.starts_with('f')) { + Digits = Spelling.substr(1); + } else if (Spelling.starts_with('p')) { + Kind = ScalarKind::Pointer; + Digits = Spelling.substr(1); + } else { + return fail(TypeToken, "unknown scalar low-level type '" + std::string(Spelling) + "'"); + } + + std::uint64_t Value = 0; + if (!parseUnsignedDigits(Digits, Value)) + return fail(TypeToken, "expected integers after 's'/'i'/'f'/'bf'/'p' type identifier"); + + Expected<LowLevelType> Created = makeUnexpected(ErrorCode::ParseError, "invalid low-level type"); + switch (Kind) { + case ScalarKind::CategoryNeutral: + if (Value == 0) { + Result = LowLevelType::token(); + return false; + } + Created = LowLevelType::makeCategoryNeutralScalar(Value); + break; + case ScalarKind::Pointer: + if (!Context.PointerSizeInBits) + return fail(TypeToken, "pointer low-level type requires an explicit pointer size"); + if (auto PointerSize = Context.PointerSizeInBits(static_cast<std::uint32_t>(Value))) + Created = LowLevelType::makePointer(Value, *PointerSize); + else + return importError(TypeToken, PointerSize.error()); + break; + } + if (!Created) + return importError(TypeToken, Created.error()); + Result = *Created; + return false; +} + +bool MIParser::Impl::parseLowLevelType(LowLevelType &Result) { + const MIToken Begin = token(); + const auto HasScalarPrefix = [](std::string_view Spelling) { + return Spelling.starts_with('s') || Spelling.starts_with('i') || Spelling.starts_with('f') || + Spelling.starts_with("bf") || Spelling.starts_with('p'); + }; + if (at(MITokenKind::Identifier)) { + if (!HasScalarPrefix(token().stringValue())) { + return fail(Begin, "expected tN, pA, <M x tN>, <M x pA>, <vscale x M x tN>, or " + "<vscale x M x pA> for GlobalISel type, where t = {'s', 'i', 'f', 'bf'}"); + } + if (parseScalarLowLevelType(token(), Result)) + return true; + advance(); + return false; + } + if (!consume(MITokenKind::Less)) { + return fail(Begin, "expected tN, pA, <M x tN>, <M x pA>, <vscale x M x tN>, or " + "<vscale x M x pA> for GlobalISel type, where t = {'s', 'i', 'f', 'bf'}"); + } + + bool Scalable = false; + if (at(MITokenKind::Identifier) && token().stringValue() == "vscale") { + Scalable = true; + advance(); + if (!at(MITokenKind::Identifier) || token().stringValue() != "x") { + return fail("expected <vscale x M x tN>, where t = {'s', 'i', 'f', 'bf', 'p'}"); + } + advance(); + } + + const auto VectorError = [this, &Begin, Scalable] { + if (Scalable) { + return fail(Begin, "expected <vscale x M x tN> for vector type, where t = " + "{'s', 'i', 'f', 'bf', 'p'}"); + } + return fail(Begin, "expected <M x tN> for vector type, where t = {'s', 'i', 'f', 'bf', 'p'}"); + }; + + std::uint64_t ElementCount = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), ElementCount)) + return VectorError(); + advance(); + if (!at(MITokenKind::Identifier) || token().stringValue() != "x") + return VectorError(); + advance(); + + if (!at(MITokenKind::Identifier) || !HasScalarPrefix(token().stringValue())) + return VectorError(); + LowLevelType ElementType; + if (parseScalarLowLevelType(token(), ElementType)) + return true; + if (ElementType.isToken()) + return fail("token cannot be a vector element type"); + advance(); + if (!consume(MITokenKind::Greater)) + return VectorError(); + + Expected<LowLevelType> Created = Scalable + ? LowLevelType::makeScalableVector(ElementCount, ElementType) + : LowLevelType::makeFixedVector(ElementCount, ElementType); + if (!Created) + return importError(Begin, Created.error()); + Result = *Created; + return false; +} + +bool MIParser::Impl::parseBlockAttributes(MachineBasicBlock *Block) { + std::unordered_set<std::string> Seen; + while (!at(MITokenKind::RightParen)) { + const MIToken AttributeToken = token(); + if (atKeyword("machine-block-address-taken") || atKeyword("landing-pad") || + atKeyword("inlineasm-br-indirect-target") || atKeyword("ehfunclet-entry") || + atKeyword("ehscope-entry")) { + const std::string Name(AttributeToken.stringValue()); + if (!Seen.insert(Name).second) + return fail("duplicate basic block attribute '" + Name + "'"); + if (Block != nullptr) { + if (Name == "machine-block-address-taken") + Block->attributes().set(MachineBasicBlockAttribute::MachineBlockAddressTaken); + else if (Name == "landing-pad") + Block->attributes().set(MachineBasicBlockAttribute::LandingPad); + else if (Name == "inlineasm-br-indirect-target") + Block->attributes().set(MachineBasicBlockAttribute::InlineAsmBrIndirectTarget); + else if (Name == "ehfunclet-entry") + Block->attributes().set(MachineBasicBlockAttribute::EHFuncletEntry); + else + Block->attributes().set(MachineBasicBlockAttribute::EHScopeEntry); + } + advance(); + } else if (atKeyword("ir-block-address-taken")) { + if (!Seen.insert("ir-block-address-taken").second) + return fail("duplicate basic block attribute 'ir-block-address-taken'"); + advance(); + if (!at(MITokenKind::IRBlock) && !at(MITokenKind::NamedIRBlock)) + return fail("expected an IR block after 'ir-block-address-taken'"); + if (Block != nullptr) { + Block->attributes().set(MachineBasicBlockAttribute::IRBlockAddressTaken); + auto SetName = Block->attributes().setIRBlockName(symbolName(token())); + if (!SetName) + return importError(token(), SetName.error()); + } + advance(); + } else if (at(MITokenKind::IRBlock) || at(MITokenKind::NamedIRBlock)) { + if (!Seen.insert("ir-block").second) + return fail("duplicate IR block attribute"); + if (Context.HasIRBlock && !Context.HasIRBlock(symbolName(token()))) + return fail(token(), "use of undefined IR block '" + std::string(token().spelling()) + "'"); + if (Block != nullptr) { + auto SetName = Block->attributes().setIRBlockName(symbolName(token())); + if (!SetName) + return importError(token(), SetName.error()); + } + advance(); + } else if (atKeyword("align")) { + if (!Seen.insert("align").second) + return fail("duplicate basic block alignment"); + Alignment Align; + if (parseAlignment(Align)) + return true; + if (Block != nullptr) + Block->attributes().setAlignment(Align); + } else if (atKeyword("bbsections")) { + if (!Seen.insert("bbsections").second) + return fail("duplicate basic block section"); + advance(); + MachineBasicBlockSectionID Section; + std::uint32_t Number = 0; + if (at(MITokenKind::IntegerLiteral) && !parseUnsigned32(token(), Number)) + Section = MachineBasicBlockSectionID::numbered(Number); + else if (at(MITokenKind::Identifier) && token().stringValue() == "Exception") + Section = MachineBasicBlockSectionID::exception(); + else if (at(MITokenKind::Identifier) && token().stringValue() == "Cold") + Section = MachineBasicBlockSectionID::cold(); + else + return fail("unknown basic block section ID"); + if (Block != nullptr) + Block->attributes().setSectionID(Section); + advance(); + } else if (atKeyword("bb_id")) { + if (!Seen.insert("bb_id").second) + return fail("duplicate basic block profile ID"); + advance(); + std::uint32_t BaseID = 0; + std::uint32_t CloneID = 0; + if (at(MITokenKind::FloatingPointLiteral)) { + const std::string_view Spelling = token().spelling(); + const std::size_t Dot = Spelling.find('.'); + std::uint64_t Base = 0; + std::uint64_t Clone = 0; + if (Dot == std::string_view::npos || !parseUnsignedDecimal(Spelling.substr(0, Dot), Base) || + !parseUnsignedDecimal(Spelling.substr(Dot + 1), Clone) || + Base > std::numeric_limits<std::uint32_t>::max() || + Clone > std::numeric_limits<std::uint32_t>::max()) { + return fail("invalid basic block profile ID"); + } + BaseID = static_cast<std::uint32_t>(Base); + CloneID = static_cast<std::uint32_t>(Clone); + advance(); + } else { + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), BaseID)) + return fail("invalid basic block base ID"); + advance(); + if (consume(MITokenKind::Dot) || consume(MITokenKind::Comma)) { + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), CloneID)) + return fail("invalid basic block clone ID"); + advance(); + } + } + if (Block != nullptr) + Block->attributes().setProfileID({BaseID, CloneID}); + } else if (atKeyword("call-frame-size")) { + if (!Seen.insert("call-frame-size").second) + return fail("duplicate basic block call frame size"); + advance(); + std::uint32_t Size = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Size)) + return fail("invalid basic block call frame size"); + if (Block != nullptr) + Block->attributes().setCallFrameSize(Size); + advance(); + } else { + return fail("unknown basic block attribute '" + std::string(token().spelling()) + "'"); + } + + if (!consume(MITokenKind::Comma)) + break; + } + return expect(MITokenKind::RightParen, "')'"); +} + +bool MIParser::Impl::parseBlockDefinition(bool Create, std::uint32_t &ID) { + const MIToken Label = token(); + if (!at(MITokenKind::MachineBasicBlockLabel) || parseUnsigned32(Label, ID)) + return fail("invalid machine basic block label"); + advance(); + + MachineBasicBlock *Block = nullptr; + if (Create) { + if (Blocks.size() >= Options.MaximumBlocks) + return fail(Label, "machine basic block count exceeds parser limit"); + if (Blocks.contains(ID)) + return fail(Label, "redefinition of machine basic block with id #" + std::to_string(ID)); + if (!Label.stringValue().empty() && Context.HasIRBlock && + !Context.HasIRBlock(Label.stringValue())) { + return fail(Label, "basic block '" + std::string(Label.stringValue()) + + "' is not defined in the function '" + std::string(Function.name()) + + "'"); + } + auto Created = Function.createBasicBlock(); + if (!Created) + return importError(Label, Created.error()); + Block = *Created; + auto Appended = Function.appendBlock(*Block); + if (!Appended) + return importError(Label, Appended.error()); + if (!Label.stringValue().empty()) { + auto SetName = Block->attributes().setName(std::string(Label.stringValue())); + if (!SetName) + return importError(Label, SetName.error()); + } + Blocks.emplace(ID, Block); + } else { + const auto Found = Blocks.find(ID); + if (Found == Blocks.end()) + return fail(Label, "use of undefined machine basic block #" + std::to_string(ID)); + Block = Found->second; + } + + if (consume(MITokenKind::LeftParen) && parseBlockAttributes(Create ? Block : nullptr)) + return true; + return expect(MITokenKind::Colon, "':' after the basic block definition"); +} + +bool MIParser::Impl::parseFirstPass() { + Position = 0; + bool AtLineStart = true; + std::size_t BraceDepth = 0; + while (!at(MITokenKind::EndOfFile)) { + if (consume(MITokenKind::Newline)) { + AtLineStart = true; + continue; + } + if (at(MITokenKind::MachineBasicBlockLabel)) { + if (!AtLineStart || token().range().Begin.Column != 1) + return fail("basic block definition must be at the start of a line"); + std::uint32_t ID = 0; + if (parseBlockDefinition(true, ID)) + return true; + AtLineStart = false; + continue; + } + if (at(MITokenKind::LeftBrace)) + ++BraceDepth; + else if (at(MITokenKind::RightBrace)) { + if (BraceDepth == 0) + return fail("extraneous closing instruction bundle brace"); + --BraceDepth; + } + AtLineStart = false; + advance(); + } + if (BraceDepth != 0) + return fail("unterminated instruction bundle"); + return false; +} + +Register MIParser::Impl::getOrCreateVirtualRegister(const MIToken &Token) { + if (Token.kind() == MITokenKind::VirtualRegister) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) + return {}; + if (const auto Found = NumericVirtualRegisters.find(ID); + Found != NumericVirtualRegisters.end()) { + return Found->second; + } + auto Created = Function.registerInfo().createVirtualRegister(); + if (!Created) { + static_cast<void>(importError(Token, Created.error())); + return {}; + } + NumericVirtualRegisters.emplace(ID, *Created); + return *Created; + } + + const std::string Name(Token.stringValue()); + if (const auto Found = NamedVirtualRegisters.find(Name); Found != NamedVirtualRegisters.end()) + return Found->second; + auto Created = Function.registerInfo().createVirtualRegister(); + if (!Created) { + static_cast<void>(importError(Token, Created.error())); + return {}; + } + auto Info = Function.registerInfo().virtualRegisterInfo(*Created); + if (!Info) { + static_cast<void>(importError(Token, Info.error())); + return {}; + } + Info->get().Name = Name; + NamedVirtualRegisters.emplace(Name, *Created); + return *Created; +} + +bool MIParser::Impl::parseRegister(Register &Result) { + const MIToken RegisterToken = token(); + if (consume(MITokenKind::Underscore)) { + Result = {}; + return false; + } + if (at(MITokenKind::NamedRegister)) { + if (RegisterToken.stringValue() == "noreg") { + Result = {}; + advance(); + return false; + } + const PhysicalRegisterDesc *Description = + Target.registers().physicalRegister(RegisterToken.stringValue()); + if (Description == nullptr) + return fail("unknown physical register '$" + std::string(RegisterToken.stringValue()) + "'"); + Result = Register::fromPhysical(Description->Reg); + advance(); + return false; + } + if (at(MITokenKind::VirtualRegister) || at(MITokenKind::NamedVirtualRegister)) { + Result = getOrCreateVirtualRegister(RegisterToken); + if (Result.isNoRegister()) + return true; + advance(); + return false; + } + return fail("expected a machine register"); +} + +bool MIParser::Impl::parseRegisterOperand(bool IsDef, ParsedOperand &Result) { + const MIToken Begin = token(); + RegisterOperandFlags Flags{IsDef ? RegisterOperandFlag::Define : RegisterOperandFlag::Use}; + std::unordered_set<std::string> SeenFlags; + while (token().isRegisterFlag()) { + const std::string Name(token().stringValue()); + if (!SeenFlags.insert(Name).second) + return fail("duplicate '" + Name + "' register flag"); + if (Name == "implicit") + Flags.set(RegisterOperandFlag::Implicit); + else if (Name == "implicit-def") { + Flags.clear(RegisterOperandFlag::Use); + Flags.set(RegisterOperandFlag::Define); + Flags.set(RegisterOperandFlag::Implicit); + IsDef = true; + } else if (Name == "def") { + Flags.clear(RegisterOperandFlag::Use); + Flags.set(RegisterOperandFlag::Define); + IsDef = true; + } else if (Name == "dead") + Flags.set(RegisterOperandFlag::Dead); + else if (Name == "killed") + Flags.set(RegisterOperandFlag::Kill); + else if (Name == "undef") + Flags.set(RegisterOperandFlag::Undef); + else if (Name == "internal") + Flags.set(RegisterOperandFlag::InternalRead); + else if (Name == "early-clobber") + Flags.set(RegisterOperandFlag::EarlyClobber); + else if (Name == "debug-use") + Flags.set(RegisterOperandFlag::Debug); + else if (Name == "renamable") + Flags.set(RegisterOperandFlag::Renamable); + advance(); + } + + Register Reg; + if (parseRegister(Reg)) + return true; + + SubRegIndex SubReg; + if (consume(MITokenKind::Dot)) { + if (!Reg.isVirtual() || !at(MITokenKind::Identifier)) + return fail("subregister index requires a virtual register"); + const SubRegIndexDesc *Description = Target.registers().subRegIndex(token().stringValue()); + if (Description == nullptr) + return fail("unknown subregister index '" + std::string(token().stringValue()) + "'"); + SubReg = Description->ID; + advance(); + } + + if (consume(MITokenKind::Colon)) { + if (!Reg.isVirtual()) + return fail("register class or bank requires a virtual register"); + auto InfoResult = Function.registerInfo().virtualRegisterInfo(Reg); + if (!InfoResult) + return importError(Begin, InfoResult.error()); + VirtualRegisterInfo &Info = InfoResult->get(); + if (consume(MITokenKind::Underscore)) { + if (Info.RegClass || Info.Bank) + return fail("conflicting generic virtual register constraint"); + } else if (at(MITokenKind::Identifier)) { + if (const RegisterClassDesc *Class = + Target.registers().registerClass(token().stringValue())) { + if ((Info.RegClass && *Info.RegClass != Class->ID) || + (Info.Bank && *Info.Bank != Class->DefaultBank)) + return fail("conflicting virtual register class"); + auto SetClass = Function.registerInfo().setRegisterClass(Reg, Class->ID); + if (!SetClass) + return importError(token(), SetClass.error()); + auto SetBank = Function.registerInfo().setRegisterBank(Reg, Class->DefaultBank); + if (!SetBank) + return importError(token(), SetBank.error()); + } else if (const RegisterBankDesc *Bank = + Target.registers().registerBank(token().stringValue())) { + const RegisterClassDesc *ExistingClass = + Info.RegClass ? Target.registers().registerClass(*Info.RegClass) : nullptr; + if ((Info.Bank && *Info.Bank != Bank->ID) || + (Info.RegClass && (ExistingClass == nullptr || ExistingClass->DefaultBank != Bank->ID))) + return fail("conflicting virtual register bank"); + auto SetBank = Function.registerInfo().setRegisterBank(Reg, Bank->ID); + if (!SetBank) + return importError(token(), SetBank.error()); + } else { + return fail("unknown register class or bank '" + std::string(token().stringValue()) + "'"); + } + advance(); + } else { + return fail("expected '_', register class, or register bank name"); + } + } + + std::optional<std::size_t> TiedDef; + if (consume(MITokenKind::LeftParen)) { + if (atKeyword("tied-def")) { + if (IsDef) + return fail("'tied-def' is only valid on register uses"); + advance(); + std::uint64_t Index = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Index) || + Index > std::numeric_limits<std::size_t>::max()) { + return fail("invalid tied-def operand index"); + } + TiedDef = static_cast<std::size_t>(Index); + advance(); + if (expect(MITokenKind::RightParen, "')' after tied-def")) + return true; + } else { + if (!Reg.isVirtual()) + return fail("low-level type is only valid on a virtual register"); + LowLevelType Type; + if (parseLowLevelType(Type)) + return true; + if (expect(MITokenKind::RightParen, "')' after the low-level type")) + return true; + auto InfoResult = Function.registerInfo().virtualRegisterInfo(Reg); + if (!InfoResult) + return importError(Begin, InfoResult.error()); + VirtualRegisterInfo &Info = InfoResult->get(); + if (Info.Type && *Info.Type != Type) + return fail(Begin, "conflicting virtual register low-level type"); + auto SetType = Function.registerInfo().setType(Reg, Type); + if (!SetType) + return importError(Begin, SetType.error()); + } + } + + if (IsDef && Reg.isVirtual()) { + auto InfoResult = Function.registerInfo().virtualRegisterInfo(Reg); + if (!InfoResult) + return importError(Begin, InfoResult.error()); + const VirtualRegisterInfo &Info = InfoResult->get(); + if (!Info.RegClass && !Info.Type) + return fail(Begin, "generic virtual register definition requires a low-level type"); + } + + if (Flags.has(RegisterOperandFlag::Define) && Flags.has(RegisterOperandFlag::Kill)) + return fail(Begin, "defined register operand cannot be killed"); + if (!Flags.has(RegisterOperandFlag::Define) && Flags.has(RegisterOperandFlag::Dead)) + return fail(Begin, "register use cannot be dead"); + + auto Operand = MachineOperand::registerOperand(Reg, Flags, SubReg); + if (!Operand) + return importError(Begin, Operand.error()); + Result = + ParsedOperand{std::move(*Operand), TiedDef, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +MachineBasicBlock *MIParser::Impl::blockFor(const MIToken &Token) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) { + static_cast<void>(fail(Token, "invalid machine basic block reference")); + return nullptr; + } + const auto Found = Blocks.find(ID); + if (Found == Blocks.end()) { + static_cast<void>(fail(Token, "use of undefined machine basic block #" + std::to_string(ID))); + return nullptr; + } + if (!Token.stringValue().empty()) { + const std::optional<std::string_view> Name = Found->second->attributes().name(); + if (!Name || *Name != Token.stringValue()) { + static_cast<void>(fail(Token, "machine basic block reference has a mismatched name")); + return nullptr; + } + } + return Found->second; +} + +std::optional<FrameIndex> MIParser::Impl::frameIndexFor(const MIToken &Token, bool Fixed) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) { + static_cast<void>(fail(Token, "invalid stack object reference")); + return std::nullopt; + } + const auto &Slots = Fixed ? Context.FixedStackObjects : Context.StackObjects; + for (const MIFrameObjectSlot &Slot : Slots) { + if (Slot.TextID != ID) + continue; + if (!Token.stringValue().empty() && (!Slot.Name || *Slot.Name != Token.stringValue())) { + static_cast<void>(fail(Token, "stack object reference has a mismatched name")); + return std::nullopt; + } + if (Function.frameInfo().object(Slot.Index) == nullptr) { + static_cast<void>(fail(Token, "stack object slot maps to a missing frame object")); + return std::nullopt; + } + return Slot.Index; + } + static_cast<void>(fail(Token, std::string("use of undefined ") + + (Fixed ? "fixed stack" : "stack") + " object #" + + std::to_string(ID))); + return std::nullopt; +} + +std::optional<ConstantPoolIndex> MIParser::Impl::constantPoolIndexFor(const MIToken &Token) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) { + static_cast<void>(fail(Token, "invalid constant pool reference")); + return std::nullopt; + } + for (const MIConstantPoolSlot &Slot : Context.ConstantPoolItems) { + if (Slot.TextID == ID && Function.constantPool().entry(Slot.Index) != nullptr) + return Slot.Index; + } + static_cast<void>(fail(Token, "use of undefined constant pool item #" + std::to_string(ID))); + return std::nullopt; +} + +std::optional<JumpTableIndex> MIParser::Impl::jumpTableIndexFor(const MIToken &Token) { + std::uint32_t ID = 0; + if (parseUnsigned32(Token, ID)) { + static_cast<void>(fail(Token, "invalid jump table reference")); + return std::nullopt; + } + for (const MIJumpTableSlot &Slot : Context.JumpTables) { + if (Slot.TextID == ID && (Function.jumpTableInfo().table(Slot.Index) != nullptr || + (Slot.Index.value() < Function.jumpTableInfo().tables().size() && + Function.jumpTableInfo().tables()[Slot.Index.value()].isPending()))) + return Slot.Index; + } + static_cast<void>(fail(Token, "use of undefined jump table #" + std::to_string(ID))); + return std::nullopt; +} + +std::string MIParser::Impl::symbolName(const MIToken &Token) const { + if (Token.hasStringValue()) + return std::string(Token.stringValue()); + return std::string(Token.integerValue()); +} + +bool MIParser::Impl::parseTargetFlags(std::uint32_t &Flags) { + Flags = 0; + if (!atKeyword("target-flags")) + return false; + advance(); + if (expect(MITokenKind::LeftParen, "'(' after target-flags")) + return true; + if (!at(MITokenKind::Identifier)) + return fail("expected a target flag name"); + + const std::string First(token().stringValue()); + if (const auto Direct = Target.mirText().directOperandFlag(First)) + Flags = *Direct; + else if (const auto Bit = Target.mirText().bitmaskOperandFlag(First)) + Flags = *Bit; + else + return fail("unknown target operand flag '" + First + "'"); + advance(); + + std::unordered_set<std::string> Seen{First}; + while (consume(MITokenKind::Comma)) { + if (!at(MITokenKind::Identifier)) + return fail("expected a target bit flag name"); + const std::string Name(token().stringValue()); + const auto Bit = Target.mirText().bitmaskOperandFlag(Name); + if (!Bit) + return fail("unknown target bit operand flag '" + Name + "'"); + if (!Seen.insert(Name).second || (Flags & *Bit) == *Bit) + return fail("duplicate target operand flag '" + Name + "'"); + Flags |= *Bit; + advance(); + } + if (expect(MITokenKind::RightParen, "')' after target flags")) + return true; + const auto Valid = Target.mirText().validateOperandFlags(Flags); + return !Valid && importError(token(), Valid.error()); +} + +bool MIParser::Impl::parseTypedIntegerOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + const std::string_view TypeSpelling = Begin.stringValue(); + if (!TypeSpelling.starts_with('i')) + return fail(Begin, "typed integer operand must use an 'iN' type"); + std::uint64_t WideBitWidth = 0; + if (!parseUnsignedDigits(TypeSpelling.substr(1), WideBitWidth) || + WideBitWidth > APInt::MaxBitWidth) { + return fail(Begin, "typed integer operand has an invalid bit width"); + } + const auto BitWidth = static_cast<std::uint32_t>(WideBitWidth); + advance(); + + std::string_view ValueSpelling; + if (at(MITokenKind::IntegerLiteral)) { + ValueSpelling = token().integerValue(); + } else if (at(MITokenKind::Identifier) && token().stringValue() == "true") { + ValueSpelling = "1"; + } else if (at(MITokenKind::Identifier) && token().stringValue() == "false") { + ValueSpelling = "0"; + } else { + return fail("expected a decimal integer, true, or false after the typed integer type"); + } + + auto Value = parseIntegerImmediateValue(BitWidth, ValueSpelling); + if (!Value) + return importError(token(), Value.error()); + advance(); + Result = ParsedOperand{ + MachineOperand::integerImmediate(std::move(*Value)), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseFloatingImmediateOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + const auto Semantics = floatSemanticsForKeyword(Begin.stringValue()); + if (!Semantics) + return fail(Begin, "unknown floating-point immediate type"); + advance(); + if (!at(MITokenKind::FloatingPointLiteral) && !at(MITokenKind::HexLiteral)) + return fail("expected a floating-point literal"); + + const MIToken Literal = token(); + std::string ConstantText(Begin.stringValue()); + ConstantText.push_back(' '); + ConstantText.append(Literal.spelling()); + auto Constant = parseLLVMConstant(ConstantText); + if (!Constant) + return fail(Literal, std::string(Constant.error().message())); + if (!Constant->FloatingValue || Constant->FloatingValue->semantics() != *Semantics) + return fail(Literal, "floating-point literal produced the wrong semantics"); + advance(); + Result = ParsedOperand{ + MachineOperand::floatingImmediate(*Constant->FloatingValue), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseMetadataReference(MachineMetadataReference &Result) { + const MIToken Begin = token(); + if (expect(MITokenKind::Exclaim, "'!' before a metadata slot")) + return true; + std::uint32_t TextID = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), TextID)) + return fail("expected a 32-bit metadata slot number"); + + for (const MIMetadataSlot &Slot : Context.Metadata) { + if (Slot.TextID != TextID) + continue; + if (!Slot.Reference.isValid()) + return fail(Begin, "metadata slot maps to an invalid reference"); + Result = Slot.Reference; + advance(); + return false; + } + if (Function.metadata().node(TextID) != nullptr) { + Result = MachineMetadataReference::machineFunction(TextID); + advance(); + return false; + } + return fail(Begin, "use of undefined metadata slot !" + std::to_string(TextID)); +} + +bool MIParser::Impl::parseMetadataOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + MachineMetadataValue Value; + if (at(MITokenKind::Exclaim)) { + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + auto Created = MachineMetadataValue::reference(Reference); + if (!Created) + return importError(Begin, Created.error()); + Value = std::move(*Created); + } else if (at(MITokenKind::MetadataKeyword) && token().isMetadataKeyword("!DIExpression")) { + advance(); + if (expect(MITokenKind::LeftParen, "'(' after !DIExpression")) + return true; + std::vector<std::uint64_t> Elements; + while (!at(MITokenKind::RightParen)) { + if (at(MITokenKind::Identifier)) { + const auto Opcode = parseDIExpressionOpcodeName(token().stringValue()); + const auto Encoding = parseDIExpressionAttributeEncodingName(token().stringValue()); + if (!Opcode && !Encoding) + return fail(invalidDIExpressionElementMessage(token().stringValue())); + Elements.push_back(Opcode ? *Opcode : *Encoding); + advance(); + } else if (at(MITokenKind::IntegerLiteral)) { + std::int64_t Signed = 0; + if (parseSigned64(token(), Signed)) + return fail("debug expression element exceeds 64 bits"); + Elements.push_back(std::bit_cast<std::uint64_t>(Signed)); + advance(); + } else { + return fail("expected a debug expression opcode or integer"); + } + if (!consume(MITokenKind::Comma)) + break; + } + if (expect(MITokenKind::RightParen, "')' after !DIExpression")) + return true; + Value = MachineMetadataValue::expression(MachineDIExpression(std::move(Elements))); + } else { + return fail("expected a metadata reference or !DIExpression"); + } + + auto Operand = MachineOperand::metadata(std::move(Value)); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseInlineDILocation( + std::vector<MachineDILocationFrame> &InlineChain, + std::optional<MachineMetadataReference> &TerminalInlinedAt) { + if (!at(MITokenKind::MetadataKeyword) || !token().isMetadataKeyword("!DILocation")) { + return fail("expected an inline !DILocation"); + } + advance(); + if (expect(MITokenKind::LeftParen, "'(' after !DILocation")) + return true; + + MachineDILocationFrame Frame; + bool HasLine = false; + bool HasScope = false; + std::vector<MachineDILocationFrame> NestedChain; + std::optional<MachineMetadataReference> NestedTerminal; + std::unordered_set<std::string> SeenFields; + while (!at(MITokenKind::RightParen)) { + if (!at(MITokenKind::Identifier)) + return fail("expected a DILocation field name"); + const MIToken FieldToken = token(); + const std::string Field(token().stringValue()); + if (!SeenFields.insert(Field).second) + return fail("duplicate DILocation field '" + Field + "'"); + advance(); + if (expect(MITokenKind::Colon, "':' after a DILocation field name")) + return true; + + if (Field == "line") { + std::uint32_t Line = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Line)) + return fail("expected an unsigned 32-bit DILocation line"); + Frame.Line = Line; + HasLine = true; + advance(); + } else if (Field == "column") { + std::uint64_t Column = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Column) || + Column > std::numeric_limits<std::uint16_t>::max()) { + return fail("expected an unsigned 16-bit DILocation column"); + } + Frame.Column = static_cast<std::uint16_t>(Column); + advance(); + } else if (Field == "scope") { + MachineMetadataReference Scope; + if (parseMetadataReference(Scope)) + return true; + if (Scope.scope() != MachineMetadataScope::EmbeddedIR) + return fail(FieldToken, "DILocation scope must reference embedded IR metadata"); + Frame.Scope = Scope; + HasScope = true; + } else if (Field == "inlinedAt") { + if (at(MITokenKind::MetadataKeyword) && token().isMetadataKeyword("!DILocation")) { + if (parseInlineDILocation(NestedChain, NestedTerminal)) + return true; + } else { + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + if (Reference.scope() != MachineMetadataScope::EmbeddedIR) { + return fail(FieldToken, "DILocation inlinedAt must reference embedded IR metadata"); + } + NestedTerminal = Reference; + } + } else if (Field == "isImplicitCode") { + if (!at(MITokenKind::Identifier) || + (token().stringValue() != "true" && token().stringValue() != "false")) { + return fail("expected 'true' or 'false' for DILocation isImplicitCode"); + } + Frame.IsImplicitCode = token().stringValue() == "true"; + advance(); + } else if (Field == "atomGroup") { + std::uint64_t Group = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Group) || + Group > MachineDILocation::MaxAtomGroup) { + return fail("DILocation atomGroup exceeds 61 bits"); + } + Frame.AtomGroup = Group; + advance(); + } else if (Field == "atomRank") { + std::uint64_t Rank = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Rank) || Rank > 7) { + return fail("DILocation atomRank exceeds three bits"); + } + Frame.AtomRank = static_cast<std::uint8_t>(Rank); + advance(); + } else { + return fail(FieldToken, "unknown DILocation field '" + Field + "'"); + } + + if (!consume(MITokenKind::Comma)) + break; + } + if (expect(MITokenKind::RightParen, "')' after !DILocation")) + return true; + if (!HasLine) + return fail("DILocation requires a line field"); + if (!HasScope) + return fail("DILocation requires a scope field"); + + InlineChain.push_back(Frame); + InlineChain.insert(InlineChain.end(), NestedChain.begin(), NestedChain.end()); + TerminalInlinedAt = NestedTerminal; + return false; +} + +bool MIParser::Impl::parseCFIRegister(DwarfRegisterNumber &Result) { + if (!at(MITokenKind::NamedRegister)) + return fail("expected a physical register in the CFI instruction"); + const MIToken RegisterToken = token(); + Register Reg; + if (parseRegister(Reg) || !Reg.isPhysical()) + return true; + const auto Physical = Reg.physicalRegister(); + if (!Physical) + return fail(RegisterToken, "CFI register is not a physical register"); + const auto Dwarf = Target.mirText().dwarfRegister(*Physical); + if (!Dwarf) + return fail(RegisterToken, "physical register has no canonical DWARF register number"); + Result = DwarfRegisterNumber::fromValue(*Dwarf); + return false; +} + +bool MIParser::Impl::parseCFISigned(std::int64_t &Result) { + if (!at(MITokenKind::IntegerLiteral) || parseSigned64(token(), Result)) + return fail("expected a signed 64-bit CFI integer"); + advance(); + return false; +} + +bool MIParser::Impl::parseCFIUnsigned(std::uint32_t &Result) { + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Result)) + return fail("expected an unsigned 32-bit CFI integer"); + advance(); + return false; +} + +bool MIParser::Impl::parseCFIOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + const std::string Operation(token().stringValue()); + advance(); + + const auto Comma = [this]() { return expect(MITokenKind::Comma, "',' in CFI instruction"); }; + DwarfRegisterNumber Register; + DwarfRegisterNumber OtherRegister; + std::int64_t Offset = 0; + std::optional<MachineCFIInstruction> Instruction; + if (Operation == "same_value") { + if (parseCFIRegister(Register)) + return true; + Instruction = MachineCFIInstruction::sameValue(Register); + } else if (Operation == "remember_state") { + Instruction = MachineCFIInstruction::rememberState(); + } else if (Operation == "restore_state") { + Instruction = MachineCFIInstruction::restoreState(); + } else if (Operation == "offset" || Operation == "rel_offset") { + if (parseCFIRegister(Register) || Comma() || parseCFISigned(Offset)) + return true; + Instruction = Operation == "offset" ? MachineCFIInstruction::offset(Register, Offset) + : MachineCFIInstruction::relativeOffset(Register, Offset); + } else if (Operation == "def_cfa_register") { + if (parseCFIRegister(Register)) + return true; + Instruction = MachineCFIInstruction::defCFARegister(Register); + } else if (Operation == "def_cfa_offset" || Operation == "adjust_cfa_offset") { + if (parseCFISigned(Offset)) + return true; + Instruction = Operation == "def_cfa_offset" ? MachineCFIInstruction::defCFAOffset(Offset) + : MachineCFIInstruction::adjustCFAOffset(Offset); + } else if (Operation == "def_cfa") { + if (parseCFIRegister(Register) || Comma() || parseCFISigned(Offset)) + return true; + Instruction = MachineCFIInstruction::defCFA(Register, Offset); + } else if (Operation == "llvm_def_aspace_cfa") { + std::uint32_t AddressSpace = 0; + if (parseCFIRegister(Register) || Comma() || parseCFISigned(Offset) || Comma() || + parseCFIUnsigned(AddressSpace)) { + return true; + } + Instruction = MachineCFIInstruction::llvmDefAddressSpaceCFA(Register, Offset, AddressSpace); + } else if (Operation == "restore" || Operation == "undefined") { + if (parseCFIRegister(Register)) + return true; + Instruction = Operation == "restore" ? MachineCFIInstruction::restore(Register) + : MachineCFIInstruction::undefined(Register); + } else if (Operation == "register") { + if (parseCFIRegister(Register) || Comma() || parseCFIRegister(OtherRegister)) + return true; + Instruction = MachineCFIInstruction::registerRule(Register, OtherRegister); + } else if (Operation == "window_save") { + Instruction = MachineCFIInstruction::windowSave(); + } else if (Operation == "negate_ra_sign_state") { + Instruction = MachineCFIInstruction::negateReturnAddressState(); + } else if (Operation == "negate_ra_sign_state_with_pc") { + Instruction = MachineCFIInstruction::negateReturnAddressStateWithPC(); + } else if (Operation == "llvm_register_pair") { + DwarfRegisterNumber First; + DwarfRegisterNumber Second; + std::uint32_t FirstSize = 0; + std::uint32_t SecondSize = 0; + if (parseCFIRegister(Register) || Comma() || parseCFIRegister(First) || Comma() || + parseCFIUnsigned(FirstSize) || Comma() || parseCFIRegister(Second) || Comma() || + parseCFIUnsigned(SecondSize)) { + return true; + } + Instruction = MachineCFIInstruction::llvmRegisterPair({.Register = Register, + .FirstRegister = First, + .FirstSizeInBits = FirstSize, + .SecondRegister = Second, + .SecondSizeInBits = SecondSize}); + } else if (Operation == "llvm_vector_registers") { + if (parseCFIRegister(Register) || Comma()) + return true; + std::vector<MachineCFIVectorRegisterLane> Registers; + do { + DwarfRegisterNumber VectorRegister; + std::uint32_t Lane = 0; + std::uint32_t Size = 0; + if (parseCFIRegister(VectorRegister) || Comma() || parseCFIUnsigned(Lane) || Comma() || + parseCFIUnsigned(Size)) { + return true; + } + Registers.push_back({.Register = VectorRegister, .Lane = Lane, .SizeInBits = Size}); + if (!at(MITokenKind::Comma)) + break; + advance(); + if (!at(MITokenKind::NamedRegister)) + return fail("expected another vector CFI register tuple"); + } while (true); + auto Created = MachineCFIInstruction::llvmVectorRegisters( + {.Register = Register, .VectorRegisters = std::move(Registers)}); + if (!Created) + return importError(Begin, Created.error()); + Instruction = std::move(*Created); + } else if (Operation == "llvm_vector_offset") { + DwarfRegisterNumber MaskRegister; + std::uint32_t RegisterSize = 0; + std::uint32_t MaskSize = 0; + if (parseCFIRegister(Register) || Comma() || parseCFIUnsigned(RegisterSize) || Comma() || + parseCFIRegister(MaskRegister) || Comma() || parseCFIUnsigned(MaskSize) || Comma() || + parseCFISigned(Offset)) { + return true; + } + Instruction = MachineCFIInstruction::llvmVectorOffset({.Register = Register, + .RegisterSizeInBits = RegisterSize, + .MaskRegister = MaskRegister, + .MaskRegisterSizeInBits = MaskSize, + .Offset = Offset}); + } else if (Operation == "llvm_vector_register_mask") { + DwarfRegisterNumber SpillRegister; + DwarfRegisterNumber MaskRegister; + std::uint32_t SpillLaneSize = 0; + std::uint32_t MaskSize = 0; + if (parseCFIRegister(Register) || Comma() || parseCFIRegister(SpillRegister) || Comma() || + parseCFIUnsigned(SpillLaneSize) || Comma() || parseCFIRegister(MaskRegister) || Comma() || + parseCFIUnsigned(MaskSize)) { + return true; + } + Instruction = + MachineCFIInstruction::llvmVectorRegisterMask({.Register = Register, + .SpillRegister = SpillRegister, + .SpillRegisterLaneSizeInBits = SpillLaneSize, + .MaskRegister = MaskRegister, + .MaskRegisterSizeInBits = MaskSize}); + } else if (Operation == "escape") { + std::vector<std::uint8_t> Bytes; + while (true) { + std::uint64_t Byte = 0; + if (!at(MITokenKind::HexLiteral) || parseUnsigned(token(), Byte) || + Byte > std::numeric_limits<std::uint8_t>::max()) { + return fail("CFI escape element must be an 8-bit hexadecimal literal"); + } + Bytes.push_back(static_cast<std::uint8_t>(Byte)); + advance(); + if (!at(MITokenKind::Comma) || !nextIs(MITokenKind::HexLiteral)) + break; + advance(); + } + auto Created = MachineCFIInstruction::escape(std::move(Bytes)); + if (!Created) + return importError(Begin, Created.error()); + Instruction = std::move(*Created); + } else { + return fail(Begin, "unsupported CFI operation '" + Operation + "'"); + } + + auto Index = Function.cfiInfo().addInstruction(std::move(*Instruction)); + if (!Index) + return importError(Begin, Index.error()); + Result = ParsedOperand{ + MachineOperand::cfiIndex(*Index), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseMemoryBase(MemoryBase &Result, std::int64_t &Offset) { + const MIToken BaseToken = token(); + switch (BaseToken.kind()) { + case MITokenKind::StackObject: + case MITokenKind::FixedStackObject: { + const auto Index = frameIndexFor(BaseToken, BaseToken.kind() == MITokenKind::FixedStackObject); + if (!Index) + return true; + Result = MemoryBase::frameIndex(*Index); + advance(); + break; + } + case MITokenKind::ConstantPoolItem: { + const auto Index = constantPoolIndexFor(BaseToken); + if (!Index) + return true; + Result = MemoryBase::constantPool(*Index); + advance(); + break; + } + case MITokenKind::JumpTableIndex: { + const auto Index = jumpTableIndexFor(BaseToken); + if (!Index) + return true; + Result = MemoryBase::jumpTable(*Index); + advance(); + break; + } + case MITokenKind::GlobalValue: + case MITokenKind::NamedGlobalValue: + Result = MemoryBase::globalSymbol(symbolName(BaseToken)); + advance(); + break; + case MITokenKind::ExternalSymbol: + Result = MemoryBase::externalSymbol(symbolName(BaseToken)); + advance(); + break; + case MITokenKind::IRValue: + case MITokenKind::NamedIRValue: + Result = MemoryBase::irValue(symbolName(BaseToken)); + advance(); + break; + case MITokenKind::QuotedIRValue: { + auto Constant = parseLLVMConstant(BaseToken.stringValue()); + if (!Constant) + return importError(BaseToken, Constant.error()); + auto Base = MemoryBase::irConstant(std::move(*Constant)); + if (!Base) + return importError(BaseToken, Base.error()); + Result = std::move(*Base); + advance(); + break; + } + case MITokenKind::Keyword: + if (BaseToken.isKeyword("unknown-address")) { + Result = MemoryBase(); + advance(); + } else if (BaseToken.isKeyword("stack")) { + Result = MemoryBase::stack(); + advance(); + } else if (BaseToken.isKeyword("got")) { + Result = MemoryBase::got(); + advance(); + } else if (BaseToken.isKeyword("constant-pool")) { + Result = MemoryBase::constantPoolPseudo(); + advance(); + } else if (BaseToken.isKeyword("jump-table")) { + Result = MemoryBase::jumpTablePseudo(); + advance(); + } else if (BaseToken.isKeyword("call-entry")) { + advance(); + if (at(MITokenKind::GlobalValue) || at(MITokenKind::NamedGlobalValue)) { + Result = MemoryBase::callEntryGlobal(symbolName(token())); + } else if (at(MITokenKind::ExternalSymbol)) { + Result = MemoryBase::callEntryExternal(symbolName(token())); + } else { + return fail("call-entry memory base requires a global or external symbol"); + } + advance(); + } else if (BaseToken.isKeyword("custom")) { + advance(); + if (!at(MITokenKind::Identifier) && !at(MITokenKind::StringConstant)) + return fail("custom memory base requires a target spelling"); + Result = MemoryBase::custom(std::string(token().stringValue())); + advance(); + } else { + return fail(BaseToken, "unsupported machine memory base"); + } + break; + default: + return fail(BaseToken, "expected a machine memory base"); + } + return parseOptionalOffset(Offset); +} + +bool MIParser::Impl::parseMemoryOperand(const MachineMemOperand *&Result, + Opcode InstructionOpcode) { + const MIToken Begin = token(); + if (expect(MITokenKind::LeftParen, "'(' before a machine memory operand")) + return true; + + MachineMemOperandDesc Description; + std::unordered_set<std::string> SeenFlags; + while (token().isMemoryOperandFlag()) { + const std::string Name(token().stringValue()); + if (!SeenFlags.insert(Name).second) + return fail("duplicate machine memory operand flag '" + Name + "'"); + if (atKeyword("volatile")) + Description.Accesses.set(MemoryAccessFlag::Volatile); + else if (atKeyword("non-temporal")) + Description.Accesses.set(MemoryAccessFlag::NonTemporal); + else if (atKeyword("dereferenceable")) + Description.Accesses.set(MemoryAccessFlag::Dereferenceable); + else if (atKeyword("invariant")) + Description.Accesses.set(MemoryAccessFlag::Invariant); + else { + const auto Flag = Target.mirText().machineMemOperandFlag(token().stringValue()); + if (!Flag) + return fail("unknown target machine memory operand flag '" + Name + "'"); + if ((Description.TargetFlags & *Flag) != 0) + return fail("overlapping target machine memory operand flag '" + Name + "'"); + Description.TargetFlags |= *Flag; + } + advance(); + } + + if (!at(MITokenKind::Identifier) || + (token().stringValue() != "load" && token().stringValue() != "store")) { + return fail("expected load or store in a machine memory operand"); + } + if (token().stringValue() == "load") + Description.Accesses.set(MemoryAccessFlag::Load); + else + Description.Accesses.set(MemoryAccessFlag::Store); + advance(); + if (at(MITokenKind::Identifier) && token().stringValue() == "store") { + if (Description.Accesses.has(MemoryAccessFlag::Store)) + return fail("duplicate store memory operation"); + Description.Accesses.set(MemoryAccessFlag::Store); + advance(); + } + + if (at(MITokenKind::Identifier) && token().stringValue() == "syncscope") { + advance(); + if (expect(MITokenKind::LeftParen, "'(' after syncscope")) + return true; + if (!at(MITokenKind::StringConstant)) + return fail("syncscope requires a quoted scope name"); + const std::string_view Name = token().stringValue(); + auto Scope = std::ranges::find_if( + Context.SyncScopes, [Name](const MISyncScopeSlot &Slot) { return Slot.Name == Name; }); + if (Scope == Context.SyncScopes.end() && Context.InternSyncScope) { + const MIToken NameToken = token(); + auto ID = Context.InternSyncScope(Name); + if (!ID) + return importError(NameToken, ID.error()); + Context.SyncScopes.push_back({.Name = std::string(Name), .ID = *ID}); + Scope = std::prev(Context.SyncScopes.end()); + } + if (Scope == Context.SyncScopes.end()) + return fail("unknown synchronization scope '" + std::string(Name) + "'"); + Description.SyncScope = Scope->ID; + advance(); + if (expect(MITokenKind::RightParen, "')' after syncscope")) + return true; + } + + std::size_t OrderingCount = 0; + while (at(MITokenKind::Identifier)) { + const auto Ordering = atomicOrdering(token().stringValue()); + if (!Ordering) + break; + if (OrderingCount == 0) + Description.SuccessOrdering = *Ordering; + else if (OrderingCount == 1) + Description.FailureOrdering = *Ordering; + else + return fail("machine memory operand has more than two atomic orderings"); + ++OrderingCount; + advance(); + } + const bool IsLoad = Description.Accesses.has(MemoryAccessFlag::Load); + const bool IsStore = Description.Accesses.has(MemoryAccessFlag::Store); + if (OrderingCount == 1) { + if (InstructionOpcode == opcode(GenericOpcode::G_ATOMIC_CMPXCHG) || + InstructionOpcode == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) { + Description.AtomicKind = AtomicOperation::CompareExchange; + } else { + Description.AtomicKind = IsLoad && IsStore + ? AtomicOperation::ReadModifyWrite + : (IsLoad ? AtomicOperation::Load : AtomicOperation::Store); + } + } else if (OrderingCount == 2) + Description.AtomicKind = AtomicOperation::CompareExchange; + + std::uint64_t MinimumBytes = 0; + if (at(MITokenKind::IntegerLiteral)) { + if (parseUnsigned(token(), MinimumBytes) || MinimumBytes == 0) + return fail("machine memory operand size must be a nonzero byte count"); + auto Size = MemorySize::fixed(MinimumBytes); + if (!Size) + return importError(token(), Size.error()); + Description.Size = *Size; + advance(); + } else if (atKeyword("unknown-size")) { + advance(); + } else if (consume(MITokenKind::LeftParen)) { + LowLevelType Type; + if (parseLowLevelType(Type)) + return true; + if (expect(MITokenKind::RightParen, "')' after the memory low-level type")) + return true; + const auto MinimumBits = minimumTypeSizeInBits(Type); + if (!MinimumBits || *MinimumBits == 0) + return fail(Begin, "memory low-level type has no representable size"); + MinimumBytes = (*MinimumBits + 7U) / 8U; + auto Size = Type.isScalableVector() ? MemorySize::scalable(MinimumBytes) + : MemorySize::fixed(MinimumBytes); + if (!Size) + return importError(Begin, Size.error()); + Description.Size = *Size; + Description.MemoryType = Type; + } else { + return fail("expected a memory size, unknown-size, or memory low-level type"); + } + + if (MinimumBytes != 0) { + if (MinimumBytes > (std::uint64_t{1} << 63U)) + return fail(Begin, "machine memory operand size has no representable default alignment"); + auto DefaultAlignment = Alignment::create(std::bit_ceil(MinimumBytes)); + if (!DefaultAlignment) + return importError(Begin, DefaultAlignment.error()); + Description.Align = *DefaultAlignment; + } + + if (at(MITokenKind::Identifier)) { + const std::string_view ExpectedPreposition = + IsLoad && IsStore ? "on" : (IsLoad ? "from" : "into"); + if (token().stringValue() != ExpectedPreposition) { + return fail("expected '" + std::string(ExpectedPreposition) + + "' before the machine memory base"); + } + advance(); + if (parseMemoryBase(Description.Base, Description.Offset)) + return true; + if (const GenericConstantData *Constant = Description.Base.constantValue()) + Description.AddressSpace = Constant->Type.AddressSpace; + } + + Alignment BaseAlignment = Description.Align; + std::unordered_set<std::string> SeenOptions; + while (consume(MITokenKind::Comma)) { + const std::string Name(token().spelling()); + if (!SeenOptions.insert(Name).second) + return fail("duplicate machine memory operand option '" + Name + "'"); + if (atKeyword("align")) { + Alignment ParsedAlignment; + if (parseAlignment(ParsedAlignment)) + return true; + if ((static_cast<std::uint64_t>(Description.Offset) & (ParsedAlignment.bytes() - 1U)) != 0) { + return fail("specified alignment is more aligned than offset"); + } + BaseAlignment = ParsedAlignment; + } else if (atKeyword("basealign")) { + if (parseAlignment(BaseAlignment)) + return true; + } else if (atKeyword("addrspace")) { + advance(); + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Description.AddressSpace) || + Description.AddressSpace > LowLevelType::MaxAddressSpace) { + return fail("invalid machine memory operand address space"); + } + advance(); + } else if (at(MITokenKind::MetadataKeyword)) { + const std::string MetadataKind(token().stringValue()); + advance(); + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + if (MetadataKind == "!tbaa") + Description.TBAA = Reference; + else if (MetadataKind == "!alias.scope") + Description.AliasScope = Reference; + else if (MetadataKind == "!noalias") + Description.NoAlias = Reference; + else if (MetadataKind == "!noalias.addrspace") + Description.NoAliasAddressSpace = Reference; + else if (MetadataKind == "!range") + Description.RangeMetadata = Reference; + else + return fail("unsupported machine memory operand metadata attachment"); + } else { + return fail("unknown machine memory operand option '" + Name + "'"); + } + } + + std::uint64_t EffectiveAlignment = BaseAlignment.bytes(); + if (Description.Offset != 0) { + const auto OffsetBits = static_cast<std::uint64_t>(Description.Offset); + const std::uint64_t OffsetAlignment = std::uint64_t{1} << std::countr_zero(OffsetBits); + EffectiveAlignment = std::min(EffectiveAlignment, OffsetAlignment); + } + Description.Align = *Alignment::create(EffectiveAlignment); + if (BaseAlignment != Description.Align) + Description.BaseAlign = BaseAlignment; + else + Description.BaseAlign.reset(); + + if (expect(MITokenKind::RightParen, "')' after the machine memory operand")) + return true; + + auto Created = Function.createMemOperand(std::move(Description)); + if (!Created) + return importError(Begin, Created.error()); + Result = *Created; + return false; +} + +bool MIParser::Impl::parsePredicateOperand(ParsedOperand &Result, bool IsFloating) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after predicate kind")) + return true; + if (!at(MITokenKind::Identifier)) + return fail("expected a predicate spelling"); + + Expected<MachineOperand> Operand = IsFloating ? [&]() -> Expected<MachineOperand> { + const auto Predicate = floatingPredicate(token().stringValue()); + if (!Predicate) { + return makeUnexpected(ErrorCode::ParseError, "invalid floating-point predicate"); + } + return MachineOperand::floatingPredicate(*Predicate); + }() + : [&]() -> Expected<MachineOperand> { + const auto Predicate = integerPredicate(token().stringValue()); + if (!Predicate) { + return makeUnexpected(ErrorCode::ParseError, "invalid integer predicate"); + } + return MachineOperand::integerPredicate(*Predicate); + }(); + if (!Operand) + return importError(token(), Operand.error()); + advance(); + if (expect(MITokenKind::RightParen, "')' after predicate")) + return true; + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseShuffleMaskOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after shufflemask")) + return true; + std::vector<std::int32_t> Elements; + while (!at(MITokenKind::RightParen)) { + if (atKeyword("undef")) { + Elements.push_back(ShuffleMaskOperand::Undef); + advance(); + } else { + std::int64_t Value = 0; + if (!at(MITokenKind::IntegerLiteral) || parseSigned64(token(), Value) || + Value < std::numeric_limits<std::int32_t>::min() || + Value > std::numeric_limits<std::int32_t>::max()) { + return fail("shuffle mask element must be a 32-bit integer or undef"); + } + Elements.push_back(static_cast<std::int32_t>(Value)); + advance(); + } + if (!consume(MITokenKind::Comma)) + break; + } + if (expect(MITokenKind::RightParen, "')' after shufflemask")) + return true; + auto Operand = MachineOperand::shuffleMask(std::move(Elements)); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseDebugInstrRefOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after dbg-instr-ref")) + return true; + std::uint64_t Instruction = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), Instruction)) + return fail("invalid debug instruction number"); + advance(); + if (expect(MITokenKind::Comma, "',' in dbg-instr-ref")) + return true; + std::uint64_t OperandNumber = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned(token(), OperandNumber)) + return fail("invalid debug operand number"); + advance(); + if (expect(MITokenKind::RightParen, "')' after dbg-instr-ref")) + return true; + auto Operand = MachineOperand::debugInstrRef(Instruction, OperandNumber); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseLaneMaskOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after lanemask")) + return true; + std::uint64_t Value = 0; + if (parseUnsigned(token(), Value)) + return fail("invalid 64-bit lane mask"); + advance(); + if (expect(MITokenKind::RightParen, "')' after lanemask")) + return true; + Result = ParsedOperand{MachineOperand::laneMask(LaneMask::fromRaw(Value)), + std::nullopt, + {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseIntrinsicOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after intrinsic")) + return true; + if (!at(MITokenKind::NamedGlobalValue)) + return fail("expected a named intrinsic"); + const auto ID = Target.mirText().intrinsic(token().stringValue()); + if (!ID) + return fail("unknown intrinsic '" + std::string(token().stringValue()) + "'"); + advance(); + if (expect(MITokenKind::RightParen, "')' after intrinsic")) + return true; + auto Operand = MachineOperand::intrinsic(*ID); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseTargetIndexOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after target-index")) + return true; + if (!at(MITokenKind::Identifier)) + return fail("expected a target index name"); + const auto Index = Target.mirText().targetIndex(token().stringValue()); + if (!Index) + return fail("unknown target index '" + std::string(token().stringValue()) + "'"); + advance(); + if (expect(MITokenKind::RightParen, "')' after target index")) + return true; + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + Result = ParsedOperand{MachineOperand::targetIndex(*Index, Offset), + std::nullopt, + {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseCustomMaskOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after CustomRegMask")) + return true; + std::uint32_t MaximumRegister = 0; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) + MaximumRegister = std::max(MaximumRegister, Register.Reg.number()); + std::vector<std::uint64_t> Words(MaximumRegister / 64U + 1U, 0); + if (!at(MITokenKind::RightParen)) { + do { + Register Reg; + if (!at(MITokenKind::NamedRegister) || parseRegister(Reg) || !Reg.isPhysical()) + return fail("CustomRegMask requires physical registers"); + const auto Physical = Reg.physicalRegister(); + if (!Physical) + return fail("CustomRegMask requires physical registers"); + const std::uint32_t Number = Physical->number(); + Words[Number / 64U] |= std::uint64_t{1} << (Number % 64U); + } while (consume(MITokenKind::Comma)); + } + if (expect(MITokenKind::RightParen, "')' after CustomRegMask")) + return true; + auto Operand = + MachineOperand::registerMask(std::make_shared<const RegisterMask>(std::move(Words))); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseLiveOutMaskOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after liveout")) + return true; + std::uint32_t MaximumRegister = 0; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) + MaximumRegister = std::max(MaximumRegister, Register.Reg.number()); + std::vector<std::uint64_t> Words(MaximumRegister / 64U + 1U, 0); + do { + Register Reg; + if (!at(MITokenKind::NamedRegister) || parseRegister(Reg) || !Reg.isPhysical()) + return fail("liveout requires physical registers"); + const auto Physical = Reg.physicalRegister(); + if (!Physical) + return fail("liveout requires physical registers"); + const std::uint32_t Number = Physical->number(); + Words[Number / 64U] |= std::uint64_t{1} << (Number % 64U); + } while (consume(MITokenKind::Comma)); + if (expect(MITokenKind::RightParen, "')' after liveout")) + return true; + auto Operand = + MachineOperand::liveOutRegisterMask(std::make_shared<const RegisterMask>(std::move(Words))); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseBlockAddressOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + advance(); + if (expect(MITokenKind::LeftParen, "'(' after blockaddress")) + return true; + if (!at(MITokenKind::GlobalValue) && !at(MITokenKind::NamedGlobalValue)) + return fail("blockaddress requires a function reference"); + const std::string FunctionName = symbolName(token()); + advance(); + if (expect(MITokenKind::Comma, "',' in blockaddress")) + return true; + if (!at(MITokenKind::IRBlock) && !at(MITokenKind::NamedIRBlock)) + return fail("blockaddress requires an IR block reference"); + const std::string BlockName = symbolName(token()); + advance(); + if (expect(MITokenKind::RightParen, "')' after blockaddress")) + return true; + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + auto Operand = MachineOperand::blockAddress(FunctionName, BlockName, Offset); + if (!Operand) + return importError(Begin, Operand.error()); + Result = ParsedOperand{ + std::move(*Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseOperand(bool IsDef, ParsedOperand &Result) { + const MIToken Begin = token(); + if (token().isRegister() || token().isRegisterFlag()) + return parseRegisterOperand(IsDef, Result); + + std::uint32_t TargetFlags = 0; + if (parseTargetFlags(TargetFlags)) + return true; + const MIToken OperandToken = token(); + if (isFloatingSemanticsKeyword(OperandToken)) { + if (parseFloatingImmediateOperand(Result)) + return true; + Result.Operand.setTargetFlags(TargetFlags); + return false; + } + if (OperandToken.kind() == MITokenKind::Exclaim || + (OperandToken.kind() == MITokenKind::MetadataKeyword && + OperandToken.isMetadataKeyword("!DIExpression"))) { + if (parseMetadataOperand(Result)) + return true; + Result.Operand.setTargetFlags(TargetFlags); + return false; + } + if (isCFIOperationToken(OperandToken)) { + if (parseCFIOperand(Result)) + return true; + Result.Operand.setTargetFlags(TargetFlags); + return false; + } + MachineOperand Operand = MachineOperand::immediate(0); + bool HasOperand = true; + + switch (OperandToken.kind()) { + case MITokenKind::IntegerLiteral: { + std::int64_t Value = 0; + if (parseSigned64(OperandToken, Value)) + return fail("integer literal is too large for a machine immediate"); + Operand = MachineOperand::immediate(Value, TargetFlags); + advance(); + break; + } + case MITokenKind::MachineBasicBlock: { + MachineBasicBlock *Block = blockFor(OperandToken); + if (Block == nullptr) + return true; + Operand = MachineOperand::basicBlock(*Block, TargetFlags); + advance(); + break; + } + case MITokenKind::StackObject: + case MITokenKind::FixedStackObject: { + const bool Fixed = OperandToken.kind() == MITokenKind::FixedStackObject; + const auto Index = frameIndexFor(OperandToken, Fixed); + if (!Index) + return true; + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + Operand = MachineOperand::frameIndex(*Index, Offset, TargetFlags); + break; + } + case MITokenKind::ConstantPoolItem: { + const auto Index = constantPoolIndexFor(OperandToken); + if (!Index) + return true; + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + Operand = MachineOperand::constantPool(*Index, Offset, TargetFlags); + break; + } + case MITokenKind::JumpTableIndex: { + const auto Index = jumpTableIndexFor(OperandToken); + if (!Index) + return true; + Operand = MachineOperand::jumpTable(*Index, TargetFlags); + advance(); + break; + } + case MITokenKind::NamedGlobalValue: + case MITokenKind::GlobalValue: { + const std::string Name = symbolName(OperandToken); + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + const GlobalValueProperties Properties = Context.GlobalValuePropertiesForName + ? Context.GlobalValuePropertiesForName(Name) + : GlobalValueProperties{}; + auto Created = MachineOperand::globalSymbol(Name, Offset, TargetFlags, Properties); + if (!Created) + return importError(OperandToken, Created.error()); + Operand = std::move(*Created); + break; + } + case MITokenKind::ExternalSymbol: { + const std::string Name = symbolName(OperandToken); + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + auto Created = MachineOperand::externalSymbol(Name, Offset, TargetFlags); + if (!Created) + return importError(OperandToken, Created.error()); + Operand = std::move(*Created); + break; + } + case MITokenKind::MCSymbol: { + const std::string Name = symbolName(OperandToken); + advance(); + std::int64_t Offset = 0; + if (parseOptionalOffset(Offset)) + return true; + auto Created = MachineOperand::mcSymbol(Name, Offset, TargetFlags); + if (!Created) + return importError(OperandToken, Created.error()); + Operand = std::move(*Created); + break; + } + case MITokenKind::SubRegisterIndex: { + const SubRegIndexDesc *Description = Target.registers().subRegIndex(OperandToken.stringValue()); + if (Description == nullptr) + return fail("unknown subregister index '" + std::string(OperandToken.stringValue()) + "'"); + Operand = + MachineOperand::immediate(static_cast<std::int64_t>(Description->ID.value()), TargetFlags); + advance(); + break; + } + case MITokenKind::Identifier: { + if (const auto Mask = Target.mirText().registerMask(OperandToken.stringValue())) { + auto Created = MachineOperand::registerMask(Mask, TargetFlags); + if (!Created) + return importError(OperandToken, Created.error()); + Operand = std::move(*Created); + advance(); + } else if (OperandToken.stringValue() == "CustomRegMask") { + if (TargetFlags != 0) + return fail("custom register masks cannot have target flags"); + return parseCustomMaskOperand(Result); + } else if (OperandToken.stringValue().starts_with('i')) { + if (parseTypedIntegerOperand(Result)) + return true; + Result.Operand.setTargetFlags(TargetFlags); + return false; + } else { + return fail("unknown named machine operand '" + std::string(OperandToken.stringValue()) + + "'"); + } + break; + } + case MITokenKind::Keyword: + HasOperand = false; + break; + default: + return fail("expected a machine operand"); + } + + if (!HasOperand) { + bool Failed = false; + if (OperandToken.isKeyword("intpred")) + Failed = parsePredicateOperand(Result, false); + else if (OperandToken.isKeyword("floatpred")) + Failed = parsePredicateOperand(Result, true); + else if (OperandToken.isKeyword("shufflemask")) + Failed = parseShuffleMaskOperand(Result); + else if (OperandToken.isKeyword("dbg-instr-ref")) + Failed = parseDebugInstrRefOperand(Result); + else if (OperandToken.isKeyword("lanemask")) + Failed = parseLaneMaskOperand(Result); + else if (OperandToken.isKeyword("intrinsic")) + Failed = parseIntrinsicOperand(Result); + else if (OperandToken.isKeyword("target-index")) + Failed = parseTargetIndexOperand(Result); + else if (OperandToken.isKeyword("liveout")) + Failed = parseLiveOutMaskOperand(Result); + else if (OperandToken.isKeyword("blockaddress")) + Failed = parseBlockAddressOperand(Result); + else + return fail("unsupported machine operand spelling '" + std::string(OperandToken.spelling()) + + "'"); + if (!Failed) + Result.Operand.setTargetFlags(TargetFlags); + return Failed; + } + + Result = + ParsedOperand{std::move(Operand), std::nullopt, {Begin.range().Begin, token().range().Begin}}; + return false; +} + +bool MIParser::Impl::parseInlineAsmExtraInfoOperand(ParsedOperand &Result) { + const MIToken Begin = token(); + InlineAsmExtraInfo ExtraInfo; + while (at(MITokenKind::Identifier)) { + const auto Bits = InlineAsmExtraInfo::namedFlagBits(token().stringValue()); + if (!Bits) { + return fail("unknown inline asm extra info flag '" + std::string(token().stringValue()) + + "'"); + } + ExtraInfo = InlineAsmExtraInfo(ExtraInfo.rawValue() | *Bits); + advance(); + } + Result = ParsedOperand{ + MachineOperand::immediate(static_cast<std::int64_t>(ExtraInfo.rawValue())), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseInlineAsmOperandFlag(ParsedOperand &Result) { + const MIToken Begin = token(); + const std::string KindName(token().stringValue()); + const auto Kind = InlineAsmOperandFlag::kindForName(KindName); + if (!Kind) + return fail("unknown inline asm operand kind '" + KindName + "'"); + advance(); + + InlineAsmOperandFlag Flag(*Kind); + if (at(MITokenKind::Identifier) && token().stringValue() == "tiedto") { + advance(); + if (!consume(MITokenKind::Colon)) + return fail("expected ':' after 'tiedto'"); + if (!at(MITokenKind::NamedRegister)) + return fail("expected '$N' operand number after 'tiedto:'"); + std::uint64_t OperandNumber = 0; + if (!parseUnsignedDecimal(token().stringValue(), OperandNumber) || + OperandNumber > std::numeric_limits<std::uint32_t>::max() || + !Flag.setMatchingOperand(static_cast<std::uint32_t>(OperandNumber))) { + return fail("invalid operand number in tiedto constraint"); + } + advance(); + } else if (consume(MITokenKind::Colon)) { + if (!at(MITokenKind::Identifier)) + return fail("expected register class or memory constraint name after ':'"); + const std::string Constraint(token().stringValue()); + if (*Kind == InlineAsmOperandKind::Memory) { + const auto MemoryConstraint = InlineAsmOperandFlag::memoryConstraintForName(Constraint); + if (!MemoryConstraint) + return fail("unknown memory constraint '" + Constraint + "'"); + if (!Flag.setMemoryConstraint(*MemoryConstraint)) + return fail("invalid memory constraint encoding"); + } else if (*Kind == InlineAsmOperandKind::RegisterDef || + *Kind == InlineAsmOperandKind::RegisterUse || + *Kind == InlineAsmOperandKind::RegisterDefEarlyClobber) { + const RegisterClassDesc *RegisterClass = Target.registers().registerClass(Constraint); + if (RegisterClass == nullptr) + return fail("unknown register class '" + Constraint + "'"); + if (!Flag.setRegisterClass(RegisterClass->ID.value())) + return fail("register class cannot be encoded in an inline asm operand flag"); + } + advance(); + } + + Result = ParsedOperand{ + MachineOperand::immediate(static_cast<std::int64_t>(Flag.rawValue())), + std::nullopt, + {Begin.range().Begin, token().range().Begin}, + }; + return false; +} + +bool MIParser::Impl::parseBlockLiveIns(MachineBasicBlock &Block) { + advance(); + if (expect(MITokenKind::Colon, "':' after liveins")) + return true; + if (token().isNewlineOrEnd()) + return false; + do { + if (!at(MITokenKind::NamedRegister)) + return fail("expected a physical register in liveins"); + Register Reg; + if (parseRegister(Reg) || !Reg.isPhysical()) + return true; + const auto Physical = Reg.physicalRegister(); + if (!Physical) + return fail("expected a physical register in liveins"); + LaneMask Lanes = LaneMask::all(); + if (consume(MITokenKind::Colon)) { + std::uint64_t Raw = 0; + if (parseUnsigned(token(), Raw)) + return fail("invalid live-in lane mask"); + Lanes = LaneMask::fromRaw(Raw); + advance(); + } + auto Added = Block.addLiveIn(*Physical, Lanes); + if (!Added) + return importError(token(), Added.error()); + } while (consume(MITokenKind::Comma)); + return false; +} + +bool MIParser::Impl::parseBlockSuccessors(MachineBasicBlock &Block) { + advance(); + if (expect(MITokenKind::Colon, "':' after successors")) + return true; + ExplicitSuccessorBlocks.insert(&Block); + if (token().isNewlineOrEnd()) + return false; + + std::vector<MachineBasicBlock *> Successors; + std::vector<BranchProbability> Probabilities; + do { + if (!at(MITokenKind::MachineBasicBlock)) + return fail("expected a machine basic block reference"); + MachineBasicBlock *Successor = blockFor(token()); + if (Successor == nullptr) + return true; + advance(); + BranchProbability Probability = BranchProbability::zero(); + if (consume(MITokenKind::LeftParen)) { + std::uint32_t Weight = 0; + if (parseUnsigned32(token(), Weight)) + return fail("expected unsigned integer"); + Probability = BranchProbability::fromRaw(Weight); + advance(); + if (expect(MITokenKind::RightParen, "')' after successor weight")) + return true; + } + Successors.push_back(Successor); + Probabilities.push_back(Probability); + } while (consume(MITokenKind::Comma)); + + BranchProbability::normalizeProbabilities(Probabilities); + for (std::size_t Index = 0; Index < Successors.size(); ++Index) { + auto Added = + Function.cfgUpdater().addSuccessor(Block, *Successors[Index], Probabilities[Index]); + if (!Added) + return importError(token(), Added.error()); + } + return false; +} + +bool MIParser::Impl::parseInstruction(MachineBasicBlock &Block, bool InBundle, + MachineInstr *PreviousInstruction, MachineInstr *&Result) { + if (ParsedInstructionCount >= Options.MaximumInstructions) + return fail("machine instruction count exceeds parser limit"); + + std::vector<ParsedOperand> Operands; + while (token().isRegister() || token().isRegisterFlag()) { + ParsedOperand Operand{MachineOperand::immediate(0), std::nullopt, {}}; + if (parseRegisterOperand(true, Operand)) + return true; + Operands.push_back(std::move(Operand)); + if (!consume(MITokenKind::Comma)) + break; + } + if (!Operands.empty() && expect(MITokenKind::Equal, "'=' after register definitions")) + return true; + + MachineInstrFlags Flags; + while (isInstructionFlag(token())) { + const auto Flag = instructionFlag(token().stringValue()); + if (!Flag) + return fail("unknown instruction flag"); + if (Flags.has(*Flag)) + return fail("duplicate instruction flag '" + std::string(token().stringValue()) + "'"); + Flags.set(*Flag); + advance(); + } + if (!at(MITokenKind::Identifier)) + return fail("expected a machine instruction name"); + const MIToken OpcodeToken = token(); + const InstructionDescriptor *Descriptor = + Target.instructions().descriptor(OpcodeToken.stringValue()); + if (Descriptor == nullptr) + return fail("unknown machine instruction '" + std::string(OpcodeToken.stringValue()) + "'"); + advance(); + const bool IsInlineAsm = Descriptor->Name == "INLINEASM" || Descriptor->Name == "INLINEASM_BR"; + + while (!token().isNewlineOrEnd() && !at(MITokenKind::DoubleColon) && + !at(MITokenKind::LeftBrace) && !atKeyword("debug-instr-number") && + !atKeyword("debug-location") && !atKeyword("pre-instr-symbol") && + !atKeyword("post-instr-symbol") && !atKeyword("heap-alloc-marker") && + !atKeyword("pcsections") && !atKeyword("mmra") && !atKeyword("cfi-type") && + !atKeyword("deactivation-symbol")) { + if (Operands.size() >= Options.MaximumOperandsPerInstruction) + return fail("instruction operand count exceeds parser limit"); + ParsedOperand Operand{MachineOperand::immediate(0), std::nullopt, {}}; + const bool IsInlineAsmExtraInfo = IsInlineAsm && + Operands.size() == InlineAsmExtraInfoOperandIndex && + at(MITokenKind::Identifier); + const bool IsInlineAsmOperandFlag = IsInlineAsm && + Operands.size() >= InlineAsmFirstConstraintOperandIndex && + at(MITokenKind::Identifier); + if (IsInlineAsmExtraInfo ? parseInlineAsmExtraInfoOperand(Operand) + : (IsInlineAsmOperandFlag ? parseInlineAsmOperandFlag(Operand) + : parseOperand(false, Operand))) + return true; + Operands.push_back(std::move(Operand)); + if (token().isNewlineOrEnd() || at(MITokenKind::DoubleColon) || at(MITokenKind::LeftBrace) || + atKeyword("debug-instr-number") || atKeyword("debug-location")) + break; + if (expect(MITokenKind::Comma, "',' before the next machine operand")) + return true; + } + + std::uint32_t DebugNumber = 0; + std::optional<MachineDebugLocation> DebugLocation; + std::optional<std::string> PreInstrSymbol; + std::optional<std::string> PostInstrSymbol; + std::optional<MachineMetadataReference> HeapAllocMarker; + std::optional<MachineMetadataReference> PCSections; + std::optional<MachineMetadataReference> MMRA; + std::optional<std::uint32_t> CFIType; + std::optional<std::string> DeactivationSymbol; + std::unordered_set<std::string> SeenAttachments; + while (atKeyword("debug-instr-number") || atKeyword("debug-location") || + atKeyword("pre-instr-symbol") || atKeyword("post-instr-symbol") || + atKeyword("heap-alloc-marker") || atKeyword("pcsections") || atKeyword("mmra") || + atKeyword("cfi-type") || atKeyword("deactivation-symbol")) { + const std::string Name(token().stringValue()); + if (!SeenAttachments.insert(Name).second) + return fail("duplicate instruction attachment '" + Name + "'"); + advance(); + if (Name == "debug-instr-number") { + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), DebugNumber) || + DebugNumber == 0) { + return fail("invalid debug instruction number"); + } + advance(); + } else if (Name == "debug-location") { + if (at(MITokenKind::MetadataKeyword) && token().isMetadataKeyword("!DILocation")) { + std::vector<MachineDILocationFrame> InlineChain; + std::optional<MachineMetadataReference> TerminalInlinedAt; + if (parseInlineDILocation(InlineChain, TerminalInlinedAt)) + return true; + auto Location = MachineDILocation::create(std::move(InlineChain), TerminalInlinedAt); + if (!Location) + return importError(OpcodeToken, Location.error()); + DebugLocation = MachineDebugLocation::inlineLocation(std::move(*Location)); + } else { + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + auto Created = MachineDebugLocation::reference(Reference); + if (!Created) + return importError(OpcodeToken, Created.error()); + DebugLocation = std::move(*Created); + } + } else if (Name == "pre-instr-symbol" || Name == "post-instr-symbol") { + if (!at(MITokenKind::MCSymbol)) + return fail("expected an MC symbol after '" + Name + "'"); + if (Name == "pre-instr-symbol") + PreInstrSymbol = symbolName(token()); + else + PostInstrSymbol = symbolName(token()); + advance(); + } else if (Name == "heap-alloc-marker" || Name == "pcsections" || Name == "mmra") { + MachineMetadataReference Reference; + if (parseMetadataReference(Reference)) + return true; + if (Name == "heap-alloc-marker") + HeapAllocMarker = Reference; + else if (Name == "pcsections") + PCSections = Reference; + else + MMRA = Reference; + } else if (Name == "cfi-type") { + std::uint32_t Type = 0; + if (!at(MITokenKind::IntegerLiteral) || parseUnsigned32(token(), Type) || Type == 0) + return fail("invalid nonzero CFI type"); + CFIType = Type; + advance(); + } else { + if (!at(MITokenKind::GlobalValue) && !at(MITokenKind::NamedGlobalValue)) + return fail("expected a global value after 'deactivation-symbol'"); + DeactivationSymbol = symbolName(token()); + advance(); + } + static_cast<void>(consume(MITokenKind::Comma)); + } + std::vector<const MachineMemOperand *> MemOperands; + if (consume(MITokenKind::DoubleColon)) { + while (!token().isNewlineOrEnd() && !at(MITokenKind::LeftBrace)) { + if (MemOperands.size() >= Options.MaximumMemoryOperandsPerInstruction) + return fail("machine memory operand count exceeds parser limit"); + const MachineMemOperand *MemOperand = nullptr; + if (parseMemoryOperand(MemOperand, Descriptor->InstructionOpcode)) + return true; + MemOperands.push_back(MemOperand); + if (token().isNewlineOrEnd() || at(MITokenKind::LeftBrace)) + break; + if (expect(MITokenKind::Comma, "',' before the next machine memory operand")) + return true; + } + } + + auto Created = Function.createInstruction(Descriptor->InstructionOpcode); + if (!Created) + return importError(OpcodeToken, Created.error()); + MachineInstr *Instruction = *Created; + Instruction->setFlags(Flags); + for (ParsedOperand &Parsed : Operands) { + auto Added = Instruction->addOperand(std::move(Parsed.Operand)); + if (!Added) + return importError(OpcodeToken, Added.error()); + } + for (std::size_t Index = 0; Index < Operands.size(); ++Index) { + if (!Operands[Index].TiedDef) + continue; + if (*Operands[Index].TiedDef >= Operands.size()) + return fail(OpcodeToken, "tied-def operand index is out of range"); + auto Tied = Instruction->tieOperands(*Operands[Index].TiedDef, Index); + if (!Tied) + return importError(OpcodeToken, Tied.error()); + } + for (const MachineMemOperand *MemOperand : MemOperands) { + auto Added = Instruction->addMemOperand(*MemOperand); + if (!Added) + return importError(OpcodeToken, Added.error()); + } + if (DebugNumber != 0) { + auto SetNumber = Instruction->setDebugInstrNumber(DebugNumber); + if (!SetNumber) + return importError(OpcodeToken, SetNumber.error()); + } + if (DebugLocation) { + auto SetLocation = Instruction->setDebugLocation(std::move(*DebugLocation)); + if (!SetLocation) + return importError(OpcodeToken, SetLocation.error()); + } + if (PreInstrSymbol) { + auto SetSymbol = Instruction->setPreInstrSymbol(std::move(*PreInstrSymbol)); + if (!SetSymbol) + return importError(OpcodeToken, SetSymbol.error()); + } + if (PostInstrSymbol) { + auto SetSymbol = Instruction->setPostInstrSymbol(std::move(*PostInstrSymbol)); + if (!SetSymbol) + return importError(OpcodeToken, SetSymbol.error()); + } + if (HeapAllocMarker) { + auto SetMarker = Instruction->setHeapAllocMarker(*HeapAllocMarker); + if (!SetMarker) + return importError(OpcodeToken, SetMarker.error()); + } + if (PCSections) { + auto SetSections = Instruction->setPCSections(*PCSections); + if (!SetSections) + return importError(OpcodeToken, SetSections.error()); + } + if (MMRA) { + auto SetMMRA = Instruction->setMMRA(*MMRA); + if (!SetMMRA) + return importError(OpcodeToken, SetMMRA.error()); + } + if (CFIType) { + auto SetType = Instruction->setCFIType(*CFIType); + if (!SetType) + return importError(OpcodeToken, SetType.error()); + } + if (DeactivationSymbol) { + auto SetSymbol = Instruction->setDeactivationSymbol(std::move(*DeactivationSymbol)); + if (!SetSymbol) + return importError(OpcodeToken, SetSymbol.error()); + } + if (InBundle) { + Flags.set(MachineInstrFlag::BundledWithPred); + Instruction->setFlags(Flags); + if (PreviousInstruction != nullptr) { + MachineInstrFlags PreviousFlags = PreviousInstruction->flags(); + PreviousFlags.set(MachineInstrFlag::BundledWithSucc); + PreviousInstruction->setFlags(PreviousFlags); + } + } + auto Appended = Block.appendInstruction(*Instruction); + if (!Appended) + return importError(OpcodeToken, Appended.error()); + + ++ParsedInstructionCount; + Result = Instruction; + return false; +} + +bool MIParser::Impl::parseSecondPass() { + Position = 0; + while (consume(MITokenKind::Newline)) { + } + while (!at(MITokenKind::EndOfFile)) { + if (!at(MITokenKind::MachineBasicBlockLabel)) + return fail("expected a machine basic block definition"); + std::uint32_t ID = 0; + if (parseBlockDefinition(false, ID)) + return true; + MachineBasicBlock *Block = Blocks.at(ID); + + while (consume(MITokenKind::Newline)) { + } + while (atKeyword("liveins") || atKeyword("successors")) { + if ((atKeyword("liveins") && parseBlockLiveIns(*Block)) || + (atKeyword("successors") && parseBlockSuccessors(*Block))) { + return true; + } + if (!token().isNewlineOrEnd()) + return fail("expected a line break after the block list"); + while (consume(MITokenKind::Newline)) { + } + } + + bool InBundle = false; + MachineInstr *PreviousInstruction = nullptr; + while (!at(MITokenKind::MachineBasicBlockLabel) && !at(MITokenKind::EndOfFile)) { + if (consume(MITokenKind::Newline)) + continue; + if (consume(MITokenKind::RightBrace)) { + if (!InBundle) + return fail("closing brace without an active instruction bundle"); + InBundle = false; + continue; + } + + MachineInstr *Instruction = nullptr; + if (parseInstruction(*Block, InBundle, PreviousInstruction, Instruction)) + return true; + PreviousInstruction = Instruction; + if (consume(MITokenKind::LeftBrace)) { + if (InBundle) + return fail("nested instruction bundles are not allowed"); + MachineInstrFlags BundleFlags = Instruction->flags(); + BundleFlags.set(MachineInstrFlag::BundledWithSucc); + Instruction->setFlags(BundleFlags); + InBundle = true; + } + if (!token().isNewlineOrEnd() && !at(MITokenKind::RightBrace)) + return fail("expected a line break after the machine instruction"); + } + if (InBundle) + return fail("unterminated instruction bundle"); + } + return false; +} + +bool MIParser::Impl::inferSuccessors() { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (ExplicitSuccessorBlocks.contains(Block)) + continue; + + std::vector<MachineBasicBlock *> Successors; + bool BlocksFallthrough = false; + auto Guessed = guessSuccessors(*Block, Target, Successors, BlocksFallthrough); + if (!Guessed) + return importError(token(), Guessed.error()); + if (BlocksFallthrough && Block->next() != nullptr && + std::ranges::find(Successors, Block->next()) == Successors.end()) + Successors.push_back(Block->next()); + if (Successors.empty()) + continue; + const auto Probability = + BranchProbability::fromFraction(1U, static_cast<std::uint32_t>(Successors.size())); + if (!Probability) + return importError(token(), Probability.error()); + for (MachineBasicBlock *Successor : Successors) { + auto Added = Function.cfgUpdater().addSuccessor(*Block, *Successor, *Probability); + if (!Added) + return importError(token(), Added.error()); + } + } + return false; +} + +Expected<void> MIParser::Impl::parse() { + if (ParseAttempted) + return makeUnexpected(ErrorCode::InvalidArgument, "MI parser can only be run once"); + ParseAttempted = true; + if (Function.blockCount() != 0 || !Function.allInstructions().empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "MI parser requires a staging machine function with an empty body"); + } + for (const MIVirtualRegisterSlot &Slot : Context.VirtualRegisters) { + if (!Slot.Reg.isVirtual() || !Function.registerInfo().isKnownVirtualRegister(Slot.Reg)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "MI parser virtual-register slot maps to an unknown register"); + } + if (!NumericVirtualRegisters.emplace(Slot.TextID, Slot.Reg).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "MI parser virtual-register text IDs must be unique"); + } + } + if (tokenize()) + return failureResult(); + if (parseFirstPass() || parseSecondPass() || inferSuccessors()) + return failureResult(); + + Function.renumberBlocks(); + if (Options.VerifyAfterParse) { + auto Verified = + MachineVerifier::verify(Function, Target, + MachineVerifierOptions{.Mode = MachineVerifierMode::Generic, + .Stage = std::nullopt, + .VerifyProperties = false}); + if (!Verified) { + static_cast<void>(fail(token(), "parsed machine function failed verification: " + + std::string(Verified.error().message()))); + return failureResult(); + } + } + return {}; +} + +MIParser::MIParser(MachineFunction &Function, const TargetDescription &Target, + std::string SourceName, std::string SourceText, MIParserContext Context, + MIParserOptions Options) + : Implementation(std::make_unique<Impl>(Function, Target, std::move(SourceName), + std::move(SourceText), std::move(Context), Options)) {} + +MIParser::MIParser(MIParser &&) noexcept = default; + +MIParser &MIParser::operator=(MIParser &&) noexcept = default; + +MIParser::~MIParser() = default; + +Expected<void> MIParser::parse() { + return Implementation->parse(); +} + +const TextSource &MIParser::source() const noexcept { + return Implementation->source(); +} + +std::span<const TextDiagnostic> MIParser::diagnostics() const noexcept { + return Implementation->diagnostics(); +} + +bool MIParser::failed() const noexcept { + return Implementation->failed(); +} + +} // namespace rcc::mir diff --git a/backend/lib/CodeGen/MIRParser/MIRParser.cpp b/backend/lib/CodeGen/MIRParser/MIRParser.cpp new file mode 100644 index 0000000..944d0c9 --- /dev/null +++ b/backend/lib/CodeGen/MIRParser/MIRParser.cpp @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MIRParser/MIRParser.h" + +#include "rcc/CodeGen/MIRYamlMapping.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineConstantPool.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionAttributes.h" +#include "rcc/CodeGen/MachineFunctionProperties.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineMetadata.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMIRTextInfo.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/IR/DataLayout.h" + +#include <algorithm> +#include <array> +#include <cstddef> +#include <cstdint> +#include <initializer_list> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <string_view> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc::mir { + +bool MIRDocumentParser::fail(const TextRange &Range, std::string Message, std::string Category) { + if (Result.Diagnostics.empty()) { + Result.Diagnostics.push_back(TextDiagnostic{ + .Severity = DiagnosticSeverity::Error, + .Category = std::move(Category), + .Message = std::move(Message), + .SourceName = std::string(Result.Source.name()), + .Range = Range, + }); + } + return true; +} + +bool MIRDocumentParser::importError(const TextRange &Range, const Error &Failure, + std::string Category) { + return fail(Range, std::string(Failure.message()), std::move(Category)); +} + +const yaml::Node *MIRDocumentParser::mappingValue(const yaml::Node &Mapping, + std::string_view Key) const noexcept { + const auto Value = Mapping.mappingValue(Key); + return Value ? node(*Value) : nullptr; +} + +std::string_view MIRDocumentParser::effectiveDataLayout() const noexcept { + if (!Options.DataLayout.empty()) + return Options.DataLayout; + if (ActiveDocument != nullptr) { + if (const EmbeddedIRReferenceIndex *Embedded = ActiveDocument->embeddedIR()) { + if (const auto DataLayout = Embedded->dataLayout()) + return *DataLayout; + } + } + return {}; +} + +Expected<DataLayout> MIRDocumentParser::dataLayout() const { + return DataLayout::parse(effectiveDataLayout()); +} + +bool MIRDocumentParser::requireKind(const yaml::Node &Node, yaml::NodeKind Kind, + std::string_view Description) { + if (Node.kind() == Kind) + return false; + return fail(Node.range(), std::string(Description) + " has the wrong YAML node kind"); +} + +bool MIRDocumentParser::validateKeys(const yaml::Node &Mapping, + std::span<const std::string_view> Keys, + std::string_view Description) { + for (const yaml::MappingEntry &Entry : Mapping.mapping()) { + if (std::ranges::find(Keys, Entry.Key) == Keys.end()) { + return fail(Entry.KeyRange, + "unknown " + std::string(Description) + " key '" + Entry.Key + "'"); + } + } + return false; +} + +bool MIRDocumentParser::scalar(const yaml::Node &Node, std::string_view &Value, + std::string_view Description) { + if (requireKind(Node, yaml::NodeKind::Scalar, Description)) + return true; + Value = Node.scalar(); + return false; +} + +MIRDocumentParseResult MIRDocumentParser::run() { + yaml::ParseResult YAMLResult = yaml::parse(std::string(Result.Source.name()), + std::string(Result.Source.text()), Options.YAML); + if (!YAMLResult.succeeded()) { + const auto Diagnostics = YAMLResult.diagnostics(); + Result.Diagnostics.assign(Diagnostics.begin(), Diagnostics.end()); + return std::move(Result); + } + Stream = YAMLResult.stream(); + if (Stream == nullptr || Stream->documents().empty()) { + static_cast<void>(fail(Result.Source.range(0, 0), "MIR stream contains no YAML documents")); + return std::move(Result); + } + + MIRDocument Staging; + ActiveDocument = &Staging; + const auto IsEmptyDocument = [this](std::size_t Index) { + const yaml::Node *Root = node(Stream->documents()[Index].Root); + return Root != nullptr && Root->kind() == yaml::NodeKind::Null && + Root->range().Begin.Offset == Root->range().End.Offset; + }; + std::size_t FirstContent = 0; + while (FirstContent < Stream->documents().size() && IsEmptyDocument(FirstContent)) + ++FirstContent; + if (FirstContent == Stream->documents().size()) { + static_cast<void>(fail(Stream->documents().front().Range, + "MIR stream must contain at least one machine function document")); + return std::move(Result); + } + + std::size_t FirstFunction = FirstContent; + const yaml::Node *FirstRoot = node(Stream->documents()[FirstContent].Root); + if (FirstRoot != nullptr && FirstRoot->kind() == yaml::NodeKind::Scalar) { + EmbeddedIRIndexResult Embedded = indexEmbeddedIR( + std::string(Result.Source.name()) + ":embedded-ir", std::string(FirstRoot->scalar())); + if (!Embedded.succeeded()) { + const auto Diagnostics = Embedded.diagnostics(); + if (Diagnostics.empty()) { + static_cast<void>( + fail(FirstRoot->range(), "embedded LLVM IR indexing failed", "mir-embedded-ir")); + } else { + const TextDiagnostic &Diagnostic = Diagnostics.front(); + const TextRange Range = scalarDiagnosticRange(*FirstRoot, Diagnostic.Range); + static_cast<void>(fail(Range, Diagnostic.Message, Diagnostic.Category)); + } + return std::move(Result); + } + Staging.EmbeddedIR = std::move(Embedded).takeIndex(); + FirstFunction = FirstContent + 1; + } + + if (!effectiveDataLayout().empty()) { + auto Layout = dataLayout(); + if (!Layout) { + static_cast<void>( + importError(Stream->documents()[FirstContent].Range, Layout.error(), "mir-data-layout")); + return std::move(Result); + } + auto PointerSize = Layout->getPointerSizeInBits(); + if (!PointerSize) { + static_cast<void>(importError(Stream->documents()[FirstContent].Range, PointerSize.error(), + "mir-data-layout")); + return std::move(Result); + } + } + + bool SawFunction = false; + for (std::size_t Index = FirstFunction; Index < Stream->documents().size(); ++Index) { + if (IsEmptyDocument(Index)) + continue; + const yaml::Node *Root = node(Stream->documents()[Index].Root); + if (Root == nullptr || requireKind(*Root, yaml::NodeKind::Mapping, "machine function") || + parseFunction(*Root, Staging)) { + return std::move(Result); + } + SawFunction = true; + } + if (!SawFunction) { + static_cast<void>(fail(Stream->documents()[FirstContent].Range, + "MIR stream must contain at least one machine function document")); + return std::move(Result); + } + + auto UnitResult = Staging.Unit.verify(); + if (!UnitResult) { + static_cast<void>( + fail(Stream->documents().back().Range, "parsed compilation unit failed verification: " + + std::string(UnitResult.error().message()))); + return std::move(Result); + } + ActiveDocument = nullptr; + Result.Document = std::move(Staging); + return std::move(Result); +} + +bool MIRDocumentParser::parseFunction(const yaml::Node &Root, MIRDocument &Document) { + constexpr std::array Keys{ + std::string_view{"name"}, + std::string_view{"alignment"}, + std::string_view{"exposesReturnsTwice"}, + std::string_view{"legalized"}, + std::string_view{"regBankSelected"}, + std::string_view{"selected"}, + std::string_view{"failedISel"}, + std::string_view{"tracksRegLiveness"}, + std::string_view{"hasWinCFI"}, + std::string_view{"noPhis"}, + std::string_view{"isSSA"}, + std::string_view{"noVRegs"}, + std::string_view{"hasFakeUses"}, + std::string_view{"callsEHReturn"}, + std::string_view{"callsUnwindInit"}, + std::string_view{"hasEHContTarget"}, + std::string_view{"hasEHScopes"}, + std::string_view{"hasEHFunclets"}, + std::string_view{"isOutlined"}, + std::string_view{"debugInstrRef"}, + std::string_view{"failsVerification"}, + std::string_view{"tracksDebugUserValues"}, + std::string_view{"registers"}, + std::string_view{"liveins"}, + std::string_view{"calleeSavedRegisters"}, + std::string_view{"frameInfo"}, + std::string_view{"fixedStack"}, + std::string_view{"stack"}, + std::string_view{"entry_values"}, + std::string_view{"callSites"}, + std::string_view{"debugValueSubstitutions"}, + std::string_view{"constants"}, + std::string_view{"machineFunctionInfo"}, + std::string_view{"jumpTable"}, + std::string_view{"machineMetadataNodes"}, + std::string_view{"calledGlobals"}, + std::string_view{"prefetch-targets"}, + std::string_view{"body"}, + }; + const yaml::Node *NameNode = mappingValue(Root, "name"); + if (NameNode == nullptr) + return fail(Root.range(), "missing required key 'name'"); + if (validateKeys(Root, Keys, "machine function")) + return true; + std::string_view Name; + if (scalar(*NameNode, Name, "machine function name")) + return true; + + const EmbeddedIRFunction *IRFunction = nullptr; + if (Document.EmbeddedIR) { + if (Name.empty()) { + std::size_t AnonymousOrdinal = 0; + for (std::size_t Index = 0; Index < Document.Unit.functionCount(); ++Index) { + const MachineFunction *Existing = Document.Unit.functionAt(Index); + if (Existing != nullptr && Existing->name().empty()) + ++AnonymousOrdinal; + } + IRFunction = Document.EmbeddedIR->unnamedFunction(AnonymousOrdinal); + } else { + IRFunction = Document.EmbeddedIR->function(Name); + } + if (IRFunction == nullptr) + return fail(NameNode->range(), + "function '" + std::string(Name) + "' isn't defined in the provided LLVM IR"); + } + + if (!Name.empty() && Document.Unit.function(Name) != nullptr) + return fail(NameNode->range(), "redefinition of machine function '" + std::string(Name) + "'"); + + auto Created = Document.Unit.createMachineFunction(std::string(Name)); + if (!Created) + return importError(NameNode->range(), Created.error()); + MachineFunction &Function = **Created; + ActiveFunction = &Function; + if (IRFunction != nullptr && IRFunction->hasMinSize()) + Function.attributes().set(MachineFunctionAttribute::MinSize); + + MIRFunctionDocumentInfo DocumentInfo{ + .Function = Function.stableID(), + .NoPHIs = std::nullopt, + .IsSSA = std::nullopt, + .NoVRegs = std::nullopt, + .HasFakeUses = std::nullopt, + }; + MIParserContext Context; + if (IRFunction != nullptr) { + Context.HasIRBlock = [IRFunction](std::string_view BlockName) { + return IRFunction->block(BlockName) != nullptr; + }; + } + if (Document.EmbeddedIR) { + const EmbeddedIRReferenceIndex *Embedded = &*Document.EmbeddedIR; + Context.GlobalValuePropertiesForName = [Embedded](std::string_view GlobalName) { + return Embedded->globalObjectProperties(GlobalName).value_or(GlobalValueProperties{}); + }; + } + + std::vector<PendingJumpTable> PendingJumpTables; + if (parseFunctionScalars(Root, Function, DocumentInfo) || + parseMetadata(Root, Function, Context) || parseRegisters(Root, Function, Context) || + parseStackObjects(Root, Function, Context) || parseConstants(Root, Function, Context) || + reserveJumpTables(Root, Function, Context, PendingJumpTables) || + parseBody(Root, Function, Context) || defineJumpTables(Function, PendingJumpTables) || + parseFrameState(Root, Function, Context) || + recomputeProperties(Root, Function, DocumentInfo) || parseTargetState(Root, Function) || + parseFunctionRecords(Root, Function) || validateDocumentReferences(Function)) { + return true; + } + + if (Options.VerifyMachineFunctions) { + auto Verified = MachineVerifier::verify(Function, Target); + if (!Verified) + return importError(Root.range(), Verified.error(), "mir-verifier"); + } + Document.FunctionInfo.push_back(DocumentInfo); + ActiveFunction = nullptr; + return false; +} + +} // namespace rcc::mir + +namespace rcc::mir { + +MIRDocument::MIRDocument() = default; +MIRDocument::MIRDocument(CompilationUnit Unit) : Unit(std::move(Unit)) {} +MIRDocument::MIRDocument(MIRDocument &&) noexcept = default; +MIRDocument &MIRDocument::operator=(MIRDocument &&) noexcept = default; +MIRDocument::~MIRDocument() = default; + +MIRDocumentParseResult::MIRDocumentParseResult(MIRDocumentParseResult &&) noexcept = default; +MIRDocumentParseResult & +MIRDocumentParseResult::operator=(MIRDocumentParseResult &&) noexcept = default; +MIRDocumentParseResult::~MIRDocumentParseResult() = default; + +MIRDocumentParseResult parseMIRDocument(const TargetDescription &Target, std::string SourceName, + std::string SourceText, MIRDocumentParserOptions Options) { + return MIRDocumentParser(Target, std::move(SourceName), std::move(SourceText), std::move(Options)) + .run(); +} + +} // namespace rcc::mir diff --git a/backend/lib/CodeGen/MIRPrinter.cpp b/backend/lib/CodeGen/MIRPrinter.cpp new file mode 100644 index 0000000..774f322 --- /dev/null +++ b/backend/lib/CodeGen/MIRPrinter.cpp @@ -0,0 +1,3227 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MIRPrinter.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InlineAsm.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MIRYamlMapping.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFIInfo.h" +#include "rcc/CodeGen/MachineCompilationStatus.h" +#include "rcc/CodeGen/MachineConstantPool.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionAttributes.h" +#include "rcc/CodeGen/MachineFunctionProperties.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineMetadata.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMIRTextInfo.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/IR/DataLayout.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <charconv> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <string_view> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <variant> +#include <vector> + +namespace rcc::mir { +namespace { + +enum class YAMLQuoting : std::uint8_t { + None, + Single, + Double, +}; + +[[nodiscard]] bool isASCIIAlphanumeric(unsigned char Character) noexcept { + return (Character >= '0' && Character <= '9') || (Character >= 'A' && Character <= 'Z') || + (Character >= 'a' && Character <= 'z'); +} + +[[nodiscard]] bool isYAMLNumeric(std::string_view Value) noexcept { + if (Value.empty() || Value == "+" || Value == "-") + return false; + if (Value == ".nan" || Value == ".NaN" || Value == ".NAN") + return true; + + std::string_view Tail = Value; + if (Tail.front() == '+' || Tail.front() == '-') + Tail.remove_prefix(1); + if (Tail == ".inf" || Tail == ".Inf" || Tail == ".INF") + return true; + if (Value.starts_with("0o")) { + return Value.size() > 2U && + Value.substr(2).find_first_not_of("01234567") == std::string_view::npos; + } + if (Value.starts_with("0x")) { + return Value.size() > 2U && + Value.substr(2).find_first_not_of("0123456789abcdefABCDEF") == std::string_view::npos; + } + + if (Tail.empty() || Tail.front() == 'e' || Tail.front() == 'E') + return false; + std::size_t Position = 0; + if (Tail.front() == '.') { + ++Position; + if (Position == Tail.size() || Tail[Position] < '0' || Tail[Position] > '9') + return false; + while (Position < Tail.size() && Tail[Position] >= '0' && Tail[Position] <= '9') + ++Position; + } else { + while (Position < Tail.size() && Tail[Position] >= '0' && Tail[Position] <= '9') + ++Position; + if (Position == 0) + return false; + } + + if (Position < Tail.size() && Tail[Position] == '.') { + ++Position; + while (Position < Tail.size() && Tail[Position] >= '0' && Tail[Position] <= '9') + ++Position; + } + if (Position == Tail.size()) + return true; + if (Tail[Position] != 'e' && Tail[Position] != 'E') + return false; + ++Position; + if (Position < Tail.size() && (Tail[Position] == '+' || Tail[Position] == '-')) + ++Position; + const std::size_t ExponentBegin = Position; + while (Position < Tail.size() && Tail[Position] >= '0' && Tail[Position] <= '9') + ++Position; + return Position == Tail.size() && Position != ExponentBegin; +} + +[[nodiscard]] YAMLQuoting yamlQuoting(std::string_view Value) noexcept { + if (Value.empty()) + return YAMLQuoting::Single; + + YAMLQuoting Result = YAMLQuoting::None; + const auto IsSpace = [](unsigned char Character) { + return Character == ' ' || Character == '\t' || Character == '\r' || Character == '\n' || + Character == '\f' || Character == '\v'; + }; + if (IsSpace(static_cast<unsigned char>(Value.front())) || + IsSpace(static_cast<unsigned char>(Value.back())) || Value == "null" || Value == "Null" || + Value == "NULL" || Value == "~" || Value == "true" || Value == "True" || Value == "TRUE" || + Value == "false" || Value == "False" || Value == "FALSE" || isYAMLNumeric(Value)) { + Result = YAMLQuoting::Single; + } + + switch (Value.front()) { + case '-': + case '?': + case ':': + case '\\': + case ',': + case '[': + case ']': + case '{': + case '}': + case '#': + case '&': + case '*': + case '!': + case '|': + case '>': + case '\'': + case '"': + case '%': + case '@': + case '`': + Result = YAMLQuoting::Single; + break; + default: + break; + } + + for (const char RawCharacter : Value) { + const auto Character = static_cast<unsigned char>(RawCharacter); + if (isASCIIAlphanumeric(Character)) + continue; + switch (Character) { + case '_': + case '-': + case '^': + case '.': + case ',': + case ' ': + case '\t': + continue; + case '\n': + case '\r': + case 0x7f: + return YAMLQuoting::Double; + default: + if (Character <= 0x1fU || (Character & 0x80U) != 0) + return YAMLQuoting::Double; + Result = YAMLQuoting::Single; + break; + } + } + return Result; +} + +[[nodiscard]] std::string yamlScalar(std::string_view Value) { + const YAMLQuoting Quoting = yamlQuoting(Value); + if (Quoting == YAMLQuoting::None) + return std::string(Value); + + std::string Result; + Result.reserve(Value.size() + 2U); + const char Quote = Quoting == YAMLQuoting::Single ? '\'' : '"'; + Result.push_back(Quote); + for (const char Character : Value) { + if (Quoting == YAMLQuoting::Single) { + Result.push_back(Character); + if (Character == '\'') + Result.push_back('\''); + continue; + } + switch (Character) { + case '\\': + Result.append("\\\\"); + break; + case '"': + Result.append("\\\""); + break; + case '\0': + Result.append("\\0"); + break; + case '\a': + Result.append("\\a"); + break; + case '\b': + Result.append("\\b"); + break; + case '\t': + Result.append("\\t"); + break; + case '\n': + Result.append("\\n"); + break; + case '\v': + Result.append("\\v"); + break; + case '\f': + Result.append("\\f"); + break; + case '\r': + Result.append("\\r"); + break; + case '\x1b': + Result.append("\\e"); + break; + default: { + const auto Byte = static_cast<unsigned char>(Character); + if (Byte < 0x20U || Byte == 0x7fU) { + static constexpr std::string_view Hex = "0123456789ABCDEF"; + Result.append("\\x"); + Result.push_back(Hex[Byte >> 4U]); + Result.push_back(Hex[Byte & 0xfU]); + } else { + Result.push_back(Character); + } + break; + } + } + } + Result.push_back(Quote); + return Result; +} + +void appendBlockScalar(std::string &Output, std::string_view Text, std::size_t Indent) { + std::size_t Position = 0; + do { + Output.append(Indent, ' '); + const std::size_t End = Text.find('\n', Position); + if (End == std::string_view::npos) { + Output.append(Text.substr(Position)); + Output.push_back('\n'); + return; + } + Output.append(Text.substr(Position, End - Position)); + Output.push_back('\n'); + Position = End + 1U; + } while (Position < Text.size()); +} + +[[nodiscard]] std::optional<std::uint32_t> parseDecimal32(std::string_view Text) noexcept { + std::uint32_t Value = 0; + if (Text.empty()) + return std::nullopt; + const auto [End, Error] = std::from_chars(Text.data(), Text.data() + Text.size(), Value, 10); + if (Error != std::errc{} || End != Text.data() + Text.size()) + return std::nullopt; + return Value; +} + +[[nodiscard]] bool isDefaultFrameState(const MachineFrameState &State) { + return State == MachineFrameState{}; +} + +class MIRDocumentPrinter final { +public: + MIRDocumentPrinter(const MIRDocument &Document, const TargetDescription &Target, + MIRDocumentPrinterOptions Options) + : Document(Document), Target(Target), Options(std::move(Options)) {} + + [[nodiscard]] Expected<std::string> run(); + +private: + struct InstructionLocation final { + std::uint32_t Block = 0; + std::uint32_t Offset = 0; + }; + + [[nodiscard]] Expected<void> printFunction(const MachineFunction &Function, + const MIRFunctionDocumentInfo *DocumentInfo); + [[nodiscard]] Expected<MIParserContext> createBodyContext(const MachineFunction &Function) const; + [[nodiscard]] Expected<void> printRegisters(const MachineFunction &Function, + const MIParserContext &Context); + [[nodiscard]] Expected<void> printFrameInfo(const MachineFunction &Function, + const MIParserContext &Context); + [[nodiscard]] Expected<void> printStackObjects(const MachineFunction &Function, + const MIParserContext &Context, bool Fixed); + [[nodiscard]] Expected<void> printConstants(const MachineFunction &Function, + const MIParserContext &Context); + [[nodiscard]] Expected<void> printJumpTable(const MachineFunction &Function, + const MIParserContext &Context); + [[nodiscard]] Expected<void> printMetadata(const MachineFunction &Function, + const MIParserContext &Context); + [[nodiscard]] Expected<void> printTargetState(const MachineFunction &Function); + [[nodiscard]] Expected<void> printFunctionRecords(const MachineFunction &Function, + const MIParserContext &Context); + [[nodiscard]] Expected<void> verifyOmittedEmbeddedIRLayout(const MachineFunction &Function) const; + + [[nodiscard]] Expected<std::string> physicalRegister(PhysicalRegister Register) const; + [[nodiscard]] Expected<std::string> frameReference(FrameIndex Index, + const MIParserContext &Context) const; + [[nodiscard]] Expected<std::string> metadataReference(MachineMetadataReference Reference, + const MIParserContext &Context) const; + [[nodiscard]] Expected<std::string> metadataValue(const MachineMetadataValue &Value, + const MIParserContext &Context) const; + [[nodiscard]] Expected<std::string> debugLocation(const MachineDebugLocation &Location, + const MIParserContext &Context) const; + [[nodiscard]] Expected<std::string> diLocation(const MachineDILocation &Location, + std::size_t FrameIndex, + const MIParserContext &Context) const; + [[nodiscard]] Expected<InstructionLocation> instructionLocation(const MachineFunction &Function, + MachineInstrStableID ID) const; + [[nodiscard]] Expected<std::string> blockReference(const MachineFunction &Function, + MachineBlockStableID ID) const; + [[nodiscard]] Expected<std::string> targetNode(const MIRTargetNode &Node) const; + [[nodiscard]] const MIRFunctionDocumentInfo * + documentInfo(const MachineFunction &Function) const noexcept; + + void printBoolean(std::string_view Key, bool Value, bool Always = false); + void printUnsigned(std::string_view Key, std::uint64_t Value, bool Always = false); + void printSigned(std::string_view Key, std::int64_t Value, bool Always = false); + + const MIRDocument &Document; + const TargetDescription &Target; + MIRDocumentPrinterOptions Options; + std::string Output; +}; + +void MIRDocumentPrinter::printBoolean(std::string_view Key, bool Value, bool Always) { + if (!Always && Options.Mode == MIPrintMode::Readable && !Value) + return; + Output.append(Key); + Output.append(Value ? ": true\n" : ": false\n"); +} + +void MIRDocumentPrinter::printUnsigned(std::string_view Key, std::uint64_t Value, bool Always) { + if (!Always && Options.Mode == MIPrintMode::Readable && Value == 0) + return; + Output.append(Key); + Output.append(": "); + Output.append(std::to_string(Value)); + Output.push_back('\n'); +} + +void MIRDocumentPrinter::printSigned(std::string_view Key, std::int64_t Value, bool Always) { + if (!Always && Options.Mode == MIPrintMode::Readable && Value == 0) + return; + Output.append(Key); + Output.append(": "); + Output.append(std::to_string(Value)); + Output.push_back('\n'); +} + +Expected<std::string> MIRDocumentPrinter::physicalRegister(PhysicalRegister Register) const { + const auto Name = Target.registers().physicalRegisterMIRName(Register); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, + "physical register has no canonical MIR name"); + } + return "$" + std::string(*Name); +} + +const MIRFunctionDocumentInfo * +MIRDocumentPrinter::documentInfo(const MachineFunction &Function) const noexcept { + const auto Infos = Document.functionDocumentInfo(); + const auto Found = std::ranges::find_if(Infos, [&Function](const MIRFunctionDocumentInfo &Info) { + return Info.Function == Function.stableID(); + }); + return Found == Infos.end() ? nullptr : &*Found; +} + +Expected<void> +MIRDocumentPrinter::verifyOmittedEmbeddedIRLayout(const MachineFunction &Function) const { + if (Options.EmitEmbeddedIR) + return {}; + + const EmbeddedIRReferenceIndex *Embedded = Document.embeddedIR(); + if (Embedded == nullptr) + return {}; + + bool HasLayoutDependentContent = false; + for (const VirtualRegisterInfo &RegisterInfo : Function.registerInfo().virtualRegisters()) { + if (!RegisterInfo.IsTombstone && RegisterInfo.Type && + RegisterInfo.Type->isPointerOrPointerVector()) { + HasLayoutDependentContent = true; + break; + } + } + if (!HasLayoutDependentContent) { + for (const MachineConstantPoolEntry &Entry : Function.constantPool().entries()) { + const MachineConstantPoolEntryDesc *Description = Entry.description(); + if (Description != nullptr && + std::holds_alternative<GenericConstantData>(Description->Value)) { + HasLayoutDependentContent = true; + break; + } + } + } + if (!HasLayoutDependentContent) + return {}; + + if (Options.DataLayout.empty()) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "omitting embedded LLVM IR with layout-dependent MIR requires an explicit printer " + "DataLayout"); + } + + auto Layout = DataLayout::parse(Options.DataLayout); + if (!Layout) + return std::unexpected(Layout.error()); + for (const VirtualRegisterInfo &RegisterInfo : Function.registerInfo().virtualRegisters()) { + if (RegisterInfo.IsTombstone || !RegisterInfo.Type || + !RegisterInfo.Type->isPointerOrPointerVector()) { + continue; + } + const auto AddressSpace = RegisterInfo.Type->addressSpace(); + if (!AddressSpace) { + return makeUnexpected(ErrorCode::VerificationError, + "pointer virtual register has no address space"); + } + auto PointerSize = Layout->getPointerSizeInBits(*AddressSpace); + if (!PointerSize) { + return makeUnexpected(ErrorCode::InvalidArgument, + "printer DataLayout cannot determine pointer width: " + + std::string(PointerSize.error().message())); + } + if (*PointerSize != RegisterInfo.Type->scalarBitWidth()) { + return makeUnexpected(ErrorCode::VerificationError, + "printer DataLayout does not match pointer virtual-register width"); + } + } + + for (const MachineConstantPoolEntry &Entry : Function.constantPool().entries()) { + const MachineConstantPoolEntryDesc *Description = Entry.description(); + if (Description == nullptr) + continue; + const auto *Generic = std::get_if<GenericConstantData>(&Description->Value); + if (Generic == nullptr) + continue; + auto StorageSize = Layout->getTypeAllocSize(Generic->Type); + if (!StorageSize) { + return makeUnexpected(ErrorCode::InvalidArgument, + "printer DataLayout cannot lay out generic constant: " + + std::string(StorageSize.error().message())); + } + if (*StorageSize != Description->Size) { + return makeUnexpected(ErrorCode::VerificationError, + "printer DataLayout does not match generic constant storage size"); + } + } + return {}; +} + +Expected<MIParserContext> +MIRDocumentPrinter::createBodyContext(const MachineFunction &Function) const { + auto Created = createMachineFunctionPrintContext(Function); + if (!Created) + return std::unexpected(Created.error()); + MIParserContext Context = std::move(*Created); + Context.SyncScopes.assign(Document.syncScopes().begin(), Document.syncScopes().end()); + Context.Metadata.clear(); + std::unordered_set<std::uint32_t> EmbeddedMetadataSlots; + if (const EmbeddedIRReferenceIndex *Embedded = Document.embeddedIR()) { + for (const EmbeddedIRMetadataDefinition &Definition : Embedded->metadataDefinitions()) { + const auto Slot = parseDecimal32(Definition.Name); + if (Slot) { + if (!EmbeddedMetadataSlots.insert(*Slot).second) { + return makeUnexpected(ErrorCode::VerificationError, + "embedded LLVM IR metadata slots are not unique"); + } + Context.Metadata.push_back( + {.TextID = *Slot, .Reference = MachineMetadataReference::embeddedIR(*Slot)}); + } + } + } + + std::unordered_set<std::uint32_t> UsedMetadataSlots = EmbeddedMetadataSlots; + for (const MachineMetadataNode &Node : Function.metadata().nodes()) { + if (!EmbeddedMetadataSlots.contains(Node.Slot)) + UsedMetadataSlots.insert(Node.Slot); + } + std::uint64_t NextMetadataSlot = 0; + for (const MachineMetadataNode &Node : Function.metadata().nodes()) { + std::uint32_t TextID = Node.Slot; + if (EmbeddedMetadataSlots.contains(TextID)) { + while (NextMetadataSlot <= std::numeric_limits<std::uint32_t>::max() && + UsedMetadataSlots.contains(static_cast<std::uint32_t>(NextMetadataSlot))) { + ++NextMetadataSlot; + } + if (NextMetadataSlot > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::Unsupported, "MIR metadata text slot space is exhausted"); + } + TextID = static_cast<std::uint32_t>(NextMetadataSlot); + UsedMetadataSlots.insert(TextID); + ++NextMetadataSlot; + } + Context.Metadata.push_back( + {.TextID = TextID, .Reference = MachineMetadataReference::machineFunction(Node.Slot)}); + } + return Context; +} + +Expected<std::string> MIRDocumentPrinter::frameReference(FrameIndex Index, + const MIParserContext &Context) const { + const auto Stack = std::ranges::find_if( + Context.StackObjects, [Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; }); + if (Stack != Context.StackObjects.end()) + return "%stack." + std::to_string(Stack->TextID); + const auto Fixed = + std::ranges::find_if(Context.FixedStackObjects, + [Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; }); + if (Fixed != Context.FixedStackObjects.end()) + return "%fixed-stack." + std::to_string(Fixed->TextID); + return makeUnexpected(ErrorCode::VerificationError, + "frame state references an unknown stack object"); +} + +Expected<std::string> MIRDocumentPrinter::metadataReference(MachineMetadataReference Reference, + const MIParserContext &Context) const { + if (!Reference.isValid()) + return makeUnexpected(ErrorCode::VerificationError, "invalid metadata reference"); + const auto Slot = + std::ranges::find_if(Context.Metadata, [Reference](const MIMetadataSlot &Candidate) { + return Candidate.Reference == Reference; + }); + if (Slot == Context.Metadata.end()) { + return makeUnexpected(ErrorCode::VerificationError, "metadata reference has no MIR text slot"); + } + return "!" + std::to_string(Slot->TextID); +} + +Expected<std::string> MIRDocumentPrinter::metadataValue(const MachineMetadataValue &Value, + const MIParserContext &Context) const { + if (const MachineMetadataReference *Reference = Value.reference()) + return metadataReference(*Reference, Context); + const MachineDIExpression *Expression = Value.expression(); + if (Expression == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "metadata value is not printable"); + return formatDIExpression(Expression->elements()); +} + +Expected<std::string> MIRDocumentPrinter::diLocation(const MachineDILocation &Location, + std::size_t FrameIndex, + const MIParserContext &Context) const { + if (FrameIndex >= Location.inlineChain().size()) { + return makeUnexpected(ErrorCode::VerificationError, + "inline DILocation has an invalid frame chain"); + } + const MachineDILocationFrame &Frame = Location.inlineChain()[FrameIndex]; + auto Scope = metadataReference(Frame.Scope, Context); + if (!Scope) + return std::unexpected(Scope.error()); + std::string Text = "!DILocation(line: " + std::to_string(Frame.Line); + if (Options.Mode == MIPrintMode::Complete || Frame.Column != 0) + Text.append(", column: " + std::to_string(Frame.Column)); + Text.append(", scope: " + *Scope); + if (FrameIndex + 1U < Location.inlineChain().size()) { + auto Nested = diLocation(Location, FrameIndex + 1U, Context); + if (!Nested) + return std::unexpected(Nested.error()); + Text.append(", inlinedAt: " + *Nested); + } else if (Location.terminalInlinedAt()) { + auto Terminal = metadataReference(*Location.terminalInlinedAt(), Context); + if (!Terminal) + return std::unexpected(Terminal.error()); + Text.append(", inlinedAt: " + *Terminal); + } + if (Options.Mode == MIPrintMode::Complete || Frame.IsImplicitCode) + Text.append(Frame.IsImplicitCode ? ", isImplicitCode: true" : ", isImplicitCode: false"); + if (Options.Mode == MIPrintMode::Complete || Frame.AtomGroup != 0) + Text.append(", atomGroup: " + std::to_string(Frame.AtomGroup)); + if (Options.Mode == MIPrintMode::Complete || Frame.AtomRank != 0) + Text.append(", atomRank: " + std::to_string(Frame.AtomRank)); + Text.push_back(')'); + return Text; +} + +Expected<std::string> MIRDocumentPrinter::debugLocation(const MachineDebugLocation &Location, + const MIParserContext &Context) const { + if (const MachineMetadataReference *Reference = Location.reference()) + return metadataReference(*Reference, Context); + if (const MachineDILocation *Inline = Location.inlineLocation()) + return diLocation(*Inline, 0, Context); + return makeUnexpected(ErrorCode::VerificationError, "debug location is not printable"); +} + +Expected<MIRDocumentPrinter::InstructionLocation> +MIRDocumentPrinter::instructionLocation(const MachineFunction &Function, + MachineInstrStableID ID) const { + std::uint64_t BlockNumber = 0; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next(), ++BlockNumber) { + if (BlockNumber > std::numeric_limits<std::uint32_t>::max()) + break; + std::uint64_t Offset = 0; + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next(), ++Offset) { + if (Instruction->stableID() != ID) + continue; + if (Offset > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::Unsupported, "MIR instruction offset exceeds 32 bits"); + } + return InstructionLocation{static_cast<std::uint32_t>(BlockNumber), + static_cast<std::uint32_t>(Offset)}; + } + } + return makeUnexpected(ErrorCode::VerificationError, + "function record references an unlinked instruction"); +} + +Expected<std::string> MIRDocumentPrinter::blockReference(const MachineFunction &Function, + MachineBlockStableID ID) const { + std::uint64_t Number = 0; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next(), ++Number) { + if (Block->stableID() == ID) { + if (Number > std::numeric_limits<std::uint32_t>::max()) + break; + return "%bb." + std::to_string(Number); + } + } + return makeUnexpected(ErrorCode::VerificationError, + "frame record references an unlinked basic block"); +} + +Expected<std::string> MIRDocumentPrinter::targetNode(const MIRTargetNode &Node) const { + switch (Node.Kind) { + case MIRTargetNodeKind::Null: + return std::string("null"); + case MIRTargetNodeKind::Scalar: + return yamlScalar(Node.Scalar); + case MIRTargetNodeKind::Sequence: { + std::string Text = "[ "; + for (std::size_t Index = 0; Index < Node.Sequence.size(); ++Index) { + if (Index != 0) + Text.append(", "); + auto Element = targetNode(Node.Sequence[Index]); + if (!Element) + return std::unexpected(Element.error()); + Text.append(*Element); + } + Text.append(" ]"); + return Text; + } + case MIRTargetNodeKind::Mapping: { + std::string Text = "{ "; + for (std::size_t Index = 0; Index < Node.Mapping.size(); ++Index) { + if (Index != 0) + Text.append(", "); + Text.append(yamlScalar(Node.Mapping[Index].Key)); + Text.append(": "); + auto Value = targetNode(Node.Mapping[Index].Value); + if (!Value) + return std::unexpected(Value.error()); + Text.append(*Value); + } + Text.append(" }"); + return Text; + } + case MIRTargetNodeKind::Count: + break; + } + return makeUnexpected(ErrorCode::VerificationError, "target MIR node has an invalid kind"); +} + +Expected<void> MIRDocumentPrinter::printRegisters(const MachineFunction &Function, + const MIParserContext &Context) { + if (Context.VirtualRegisters.empty()) { + if (Options.Mode == MIPrintMode::Complete) + Output.append("registers: []\n"); + } else { + Output.append("registers:\n"); + for (const MIVirtualRegisterSlot &Slot : Context.VirtualRegisters) { + auto InfoResult = Function.registerInfo().virtualRegisterInfo(Slot.Reg); + if (!InfoResult) + return std::unexpected(InfoResult.error()); + const VirtualRegisterInfo &Info = InfoResult->get(); + std::string Class = "_"; + if (Info.RegClass) { + const auto Name = Target.registers().registerClassMIRName(*Info.RegClass); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, + "virtual register class has no canonical MIR name"); + Class = *Name; + } else if (Info.Bank) { + const auto Name = Target.registers().registerBankMIRName(*Info.Bank); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, + "virtual register bank has no canonical MIR name"); + Class = *Name; + } + if (Info.AllocationHints.size() > 1) { + return makeUnexpected( + ErrorCode::Unsupported, + "fixed LLVM MIR can serialize only one simple preferred-register hint"); + } + + Output.append(" - { id: "); + Output.append(std::to_string(Slot.TextID)); + Output.append(", class: "); + Output.append(yamlScalar(Class)); + if (!Info.AllocationHints.empty()) { + auto Preferred = physicalRegister(Info.AllocationHints.front()); + if (!Preferred) + return std::unexpected(Preferred.error()); + Output.append(", preferred-register: "); + Output.append(yamlScalar(*Preferred)); + } + if (Info.TargetFlags != 0) { + auto Names = Target.mirText().virtualRegisterFlagNames(Info.TargetFlags); + if (!Names) + return std::unexpected(Names.error()); + Output.append(", flags: [ "); + for (std::size_t Index = 0; Index < Names->size(); ++Index) { + if (Index != 0) + Output.append(", "); + Output.append(yamlScalar((*Names)[Index])); + } + Output.append(" ]"); + } + Output.append(" }\n"); + } + } + + if (Function.registerInfo().liveIns().empty()) { + if (Options.Mode == MIPrintMode::Complete) + Output.append("liveins: []\n"); + } else { + Output.append("liveins:\n"); + for (const PhysicalRegisterLiveIn &LiveIn : Function.registerInfo().liveIns()) { + auto Name = physicalRegister(LiveIn.PhysReg); + if (!Name) + return std::unexpected(Name.error()); + Output.append(" - { reg: "); + Output.append(yamlScalar(*Name)); + if (LiveIn.InitialVirtualRegister) { + const auto Slot = std::ranges::find_if( + Context.VirtualRegisters, [&LiveIn](const MIVirtualRegisterSlot &Candidate) { + return Candidate.Reg == *LiveIn.InitialVirtualRegister; + }); + if (Slot == Context.VirtualRegisters.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "live-in references an unknown virtual register"); + } + Output.append(", virtual-reg: "); + Output.append(yamlScalar("%" + std::to_string(Slot->TextID))); + } + Output.append(" }\n"); + } + } + + if (Function.records().hasCalleeSavedRegisters()) { + Output.append("calleeSavedRegisters: [ "); + const auto Registers = Function.records().calleeSavedRegisters(); + for (std::size_t Index = 0; Index < Registers.size(); ++Index) { + if (Index != 0) + Output.append(", "); + auto Name = physicalRegister(Registers[Index]); + if (!Name) + return std::unexpected(Name.error()); + Output.append(yamlScalar(*Name)); + } + Output.append(" ]\n"); + } + return {}; +} + +Expected<void> MIRDocumentPrinter::printFrameInfo(const MachineFunction &Function, + const MIParserContext &Context) { + const MachineFrameState &State = Function.frameInfo().state(); + if (Options.Mode == MIPrintMode::Readable && isDefaultFrameState(State)) + return {}; + if (State.MaxCallFrameSize && + *State.MaxCallFrameSize > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::Unsupported, + "fixed LLVM MIR maxCallFrameSize is limited to 32 bits"); + } + Output.append("frameInfo:\n"); + const auto Bool = [this](std::string_view Key, bool Value) { + if (Options.Mode == MIPrintMode::Readable && !Value) + return; + Output.append(" "); + Output.append(Key); + Output.append(Value ? ": true\n" : ": false\n"); + }; + const auto Unsigned = [this](std::string_view Key, std::uint64_t Value) { + if (Options.Mode == MIPrintMode::Readable && Value == 0) + return; + Output.append(" "); + Output.append(Key); + Output.append(": "); + Output.append(std::to_string(Value)); + Output.push_back('\n'); + }; + + Bool("isFrameAddressTaken", State.IsFrameAddressTaken); + Bool("isReturnAddressTaken", State.IsReturnAddressTaken); + Bool("hasStackMap", State.HasStackMap); + Bool("hasPatchPoint", State.HasPatchPoint); + Unsigned("stackSize", State.StackSize); + if (Options.Mode == MIPrintMode::Complete || State.OffsetAdjustment != 0) { + Output.append(" offsetAdjustment: "); + Output.append(std::to_string(State.OffsetAdjustment)); + Output.push_back('\n'); + } + Unsigned("maxAlignment", State.MaxAlignment ? State.MaxAlignment->bytes() : 0); + Bool("adjustsStack", State.AdjustsStack); + Bool("hasCalls", State.HasCalls); + if (Options.Mode == MIPrintMode::Complete || + State.FramePointerPolicy != MachineFramePointerPolicy::None) { + Output.append(" framePointerPolicy: "); + Output.append(getMachineFramePointerPolicyName(State.FramePointerPolicy)); + Output.push_back('\n'); + } + if (Options.Mode == MIPrintMode::Complete || State.StackProtector) { + std::string Reference; + if (State.StackProtector) { + auto PrintedReference = frameReference(*State.StackProtector, Context); + if (!PrintedReference) + return std::unexpected(PrintedReference.error()); + Reference = std::move(*PrintedReference); + } + Output.append(" stackProtector: "); + Output.append(yamlScalar(Reference)); + Output.push_back('\n'); + } + if (Options.Mode == MIPrintMode::Complete || State.FunctionContext) { + std::string Reference; + if (State.FunctionContext) { + auto PrintedReference = frameReference(*State.FunctionContext, Context); + if (!PrintedReference) + return std::unexpected(PrintedReference.error()); + Reference = std::move(*PrintedReference); + } + Output.append(" functionContext: "); + Output.append(yamlScalar(Reference)); + Output.push_back('\n'); + } + if (Options.Mode == MIPrintMode::Complete || State.MaxCallFrameSize) { + Unsigned("maxCallFrameSize", + State.MaxCallFrameSize.value_or(std::numeric_limits<std::uint32_t>::max())); + } + Unsigned("cvBytesOfCalleeSavedRegisters", State.CVBytesOfCalleeSavedRegisters); + Bool("hasOpaqueSPAdjustment", State.HasOpaqueSPAdjustment); + Bool("hasVAStart", State.HasVAStart); + Bool("hasMustTailInVarArgFunc", State.HasMustTailInVarArgFunc); + Bool("hasTailCall", State.HasTailCall); + Bool("isCalleeSavedInfoValid", State.IsCalleeSavedInfoValid); + Unsigned("localFrameSize", State.LocalFrameSize); + + const auto PrintPoints = + [this, &Function](std::string_view Key, + std::span<const MachineFrameSaveRestorePoint> Points) -> Expected<void> { + if (Points.empty()) + return {}; + Output.append(" "); + Output.append(Key); + Output.append(":\n"); + for (const MachineFrameSaveRestorePoint &Point : Points) { + auto Block = blockReference(Function, Point.Block); + if (!Block) + return std::unexpected(Block.error()); + Output.append(" - { point: "); + Output.append(yamlScalar(*Block)); + if (!Point.Registers.empty()) { + Output.append(", registers: [ "); + for (std::size_t Index = 0; Index < Point.Registers.size(); ++Index) { + if (Index != 0) + Output.append(", "); + auto Name = physicalRegister(Point.Registers[Index]); + if (!Name) + return std::unexpected(Name.error()); + Output.append(yamlScalar(*Name)); + } + Output.append(" ]"); + } + Output.append(" }\n"); + } + return {}; + }; + if (auto Printed = PrintPoints("savePoint", State.SavePoints); !Printed) + return Printed; + return PrintPoints("restorePoint", State.RestorePoints); +} + +Expected<void> MIRDocumentPrinter::printStackObjects(const MachineFunction &Function, + const MIParserContext &Context, bool Fixed) { + const auto &Slots = Fixed ? Context.FixedStackObjects : Context.StackObjects; + if (Slots.empty()) { + if (Options.Mode == MIPrintMode::Complete) + Output.append(Fixed ? "fixedStack: []\n" : "stack: []\n"); + return {}; + } + Output.append(Fixed ? "fixedStack:\n" : "stack:\n"); + for (const MIFrameObjectSlot &Slot : Slots) { + const StackObject *Object = Function.frameInfo().object(Slot.Index); + const StackObjectDesc *Description = Object == nullptr ? nullptr : Object->description(); + if (Description == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "MIR stack slot is a tombstone"); + if (Description->IsDead || Description->AddressTaken || !Description->Lifetime.empty() || + Description->TargetFlags != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "stack object contains state not represented by fixed LLVM MIR"); + } + + const std::string_view Type = Object->isSpill() ? "spill-slot" + : Object->isVariableSized() ? "variable-sized" + : "default"; + + Output.append(" - { id: "); + Output.append(std::to_string(Slot.TextID)); + if (!Fixed && Description->AllocaName) { + Output.append(", name: "); + Output.append(yamlScalar(*Description->AllocaName)); + } + if (Options.Mode == MIPrintMode::Complete || Type != "default") { + Output.append(", type: "); + Output.append(Type); + } + if (Options.Mode == MIPrintMode::Complete || Description->Offset.value_or(0) != 0) { + Output.append(", offset: "); + Output.append(std::to_string(Description->Offset.value_or(0))); + } + if (Description->Kind != StackObjectKind::VariableSized) { + Output.append(", size: "); + Output.append(std::to_string(Description->Size.value_or(0))); + } + if (Options.Mode == MIPrintMode::Complete || Description->Align.bytes() != 1) { + Output.append(", alignment: "); + Output.append(std::to_string(Description->Align.bytes())); + } + if (Options.Mode == MIPrintMode::Complete || Description->Stack != StackID::defaultStack()) { + Output.append(", stack-id: "); + Output.append(getStackIDName(Description->Stack)); + } + if (Fixed && Description->Kind != StackObjectKind::FixedSpillSlot) { + if (Options.Mode == MIPrintMode::Complete || Description->IsImmutable) + Output.append(Description->IsImmutable ? ", isImmutable: true" : ", isImmutable: false"); + if (Options.Mode == MIPrintMode::Complete || Description->IsAliased) + Output.append(Description->IsAliased ? ", isAliased: true" : ", isAliased: false"); + } else if (!Fixed && (Description->IsImmutable || Description->IsAliased)) { + return makeUnexpected(ErrorCode::Unsupported, + "ordinary stack object immutable/aliased state is not serialized"); + } + if (Description->CalleeSavedRegister) { + auto Name = physicalRegister(*Description->CalleeSavedRegister); + if (!Name) + return std::unexpected(Name.error()); + Output.append(", callee-saved-register: "); + Output.append(yamlScalar(*Name)); + } + if (Options.Mode == MIPrintMode::Complete || !Description->CalleeSavedRestored) { + Output.append(Description->CalleeSavedRestored ? ", callee-saved-restored: true" + : ", callee-saved-restored: false"); + } + if (!Fixed && Description->LocalOffset) { + Output.append(", local-offset: "); + Output.append(std::to_string(*Description->LocalOffset)); + } + if (Description->DebugInfo) { + const MachineVariableDebugInfo &Debug = *Description->DebugInfo; + if (Debug.Variable) { + auto Value = metadataReference(*Debug.Variable, Context); + if (!Value) + return std::unexpected(Value.error()); + Output.append(", debug-info-variable: "); + Output.append(yamlScalar(*Value)); + } + if (Debug.Expression) { + auto Value = metadataValue(*Debug.Expression, Context); + if (!Value) + return std::unexpected(Value.error()); + Output.append(", debug-info-expression: "); + Output.append(yamlScalar(*Value)); + } + if (Debug.Location) { + auto Value = debugLocation(*Debug.Location, Context); + if (!Value) + return std::unexpected(Value.error()); + Output.append(", debug-info-location: "); + Output.append(yamlScalar(*Value)); + } + } + Output.append(" }\n"); + } + return {}; +} + +Expected<void> MIRDocumentPrinter::printConstants(const MachineFunction &Function, + const MIParserContext &Context) { + if (Context.ConstantPoolItems.empty()) { + if (Options.Mode == MIPrintMode::Complete) + Output.append("constants: []\n"); + return {}; + } + Output.append("constants:\n"); + for (const MIConstantPoolSlot &Slot : Context.ConstantPoolItems) { + const MachineConstantPoolEntry *Entry = Function.constantPool().entry(Slot.Index); + const MachineConstantPoolEntryDesc *Description = + Entry == nullptr ? nullptr : Entry->description(); + if (Description == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "constant-pool slot is a tombstone"); + if (Description->Merge != ConstantPoolMergeKind::Never || + Description->Relocation != ConstantPoolRelocationMode::None || + Description->Placement != ConstantPoolPlacement{}) { + return makeUnexpected(ErrorCode::Unsupported, + "constant-pool placement state is not represented by fixed LLVM MIR"); + } + + bool IsTargetSpecific = false; + Expected<std::string> Value = + makeUnexpected(ErrorCode::Unsupported, "constant-pool kind is not MIR serializable"); + if (const auto *Generic = std::get_if<GenericConstantData>(&Description->Value)) { + Value = Generic->toLLVMString(); + } else if (const auto *TargetConstantValue = std::get_if<TargetConstant>(&Description->Value)) { + IsTargetSpecific = true; + Value = Target.mirText().printTargetConstant(*TargetConstantValue); + } + if (!Value) + return std::unexpected(Value.error()); + + Output.append(" - id: "); + Output.append(std::to_string(Slot.TextID)); + Output.append("\n value: "); + Output.append(yamlScalar(*Value)); + Output.append("\n alignment: "); + Output.append(std::to_string(Description->Align.bytes())); + Output.push_back('\n'); + if (IsTargetSpecific) + Output.append(" isTargetSpecific: true\n"); + } + return {}; +} + +Expected<void> MIRDocumentPrinter::printJumpTable(const MachineFunction &Function, + const MIParserContext &Context) { + if (Context.JumpTables.empty()) + return {}; + const MachineJumpTableEntry *FirstEntry = + Function.jumpTableInfo().table(Context.JumpTables.front().Index); + const MachineJumpTableDesc *First = FirstEntry == nullptr ? nullptr : FirstEntry->description(); + if (First == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "jump-table slot is not defined"); + + std::string_view Kind; + std::uint8_t ExpectedSize = 8; + switch (First->Encoding) { + case JumpTableEntryEncoding::BlockAddress: + Kind = "block-address"; + break; + case JumpTableEntryEncoding::GPRelative64BlockAddress: + Kind = "gp-rel64-block-address"; + break; + case JumpTableEntryEncoding::GPRelative32BlockAddress: + Kind = "gp-rel32-block-address"; + ExpectedSize = 4; + break; + case JumpTableEntryEncoding::LabelDifference32: + Kind = "label-difference32"; + ExpectedSize = 4; + break; + case JumpTableEntryEncoding::LabelDifference64: + Kind = "label-difference64"; + break; + case JumpTableEntryEncoding::Inline: + Kind = "inline"; + ExpectedSize = 4; + break; + case JumpTableEntryEncoding::Custom32: + Kind = "custom32"; + ExpectedSize = 4; + break; + case JumpTableEntryEncoding::Target: + case JumpTableEntryEncoding::Count: + return makeUnexpected(ErrorCode::Unsupported, + "target jump-table encoding has no fixed LLVM MIR spelling"); + } + + Output.append("jumpTable:\n kind: "); + Output.append(Kind); + Output.append("\n entries:\n"); + for (const MIJumpTableSlot &Slot : Context.JumpTables) { + const MachineJumpTableEntry *Entry = Function.jumpTableInfo().table(Slot.Index); + const MachineJumpTableDesc *Description = Entry == nullptr ? nullptr : Entry->description(); + if (Description == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "jump-table slot is not defined"); + if (Description->Encoding != First->Encoding || Description->EntrySize != ExpectedSize || + Description->Align.bytes() != ExpectedSize || + Description->Placement != JumpTablePlacement{} || Description->TargetEncoding != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "jump-table entries cannot share one fixed LLVM MIR kind"); + } + Output.append(" - { id: "); + Output.append(std::to_string(Slot.TextID)); + Output.append(", blocks: [ "); + for (std::size_t Index = 0; Index < Description->Targets.size(); ++Index) { + if (Index != 0) + Output.append(", "); + if (Description->Targets[Index] == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "jump-table entry contains a null block"); + auto Block = blockReference(Function, Description->Targets[Index]->stableID()); + if (!Block) + return std::unexpected(Block.error()); + Output.append(yamlScalar(*Block)); + } + Output.append(" ] }\n"); + } + return {}; +} + +Expected<void> MIRDocumentPrinter::printMetadata(const MachineFunction &Function, + const MIParserContext &Context) { + if (Function.metadata().nodes().empty()) + return {}; + Output.append("machineMetadataNodes:\n"); + for (const MachineMetadataNode &Node : Function.metadata().nodes()) { + auto NodeReference = + metadataReference(MachineMetadataReference::machineFunction(Node.Slot), Context); + if (!NodeReference) + return std::unexpected(NodeReference.error()); + std::string Text = *NodeReference + " = "; + if (Node.IsDistinct) + Text.append("distinct "); + Text.append("!{"); + for (std::size_t Index = 0; Index < Node.Elements.size(); ++Index) { + if (Index != 0) + Text.append(", "); + const MachineMetadataElement &Element = Node.Elements[Index]; + if (Element.isNull()) { + Text.append("null"); + } else if (const MachineMetadataReference *Reference = Element.reference()) { + auto Printed = metadataReference(*Reference, Context); + if (!Printed) + return std::unexpected(Printed.error()); + Text.append(*Printed); + } else if (const std::string *String = Element.string()) { + Text.append("!\""); + for (const char Character : *String) { + if (Character == '\\' || Character == '"') + Text.push_back('\\'); + Text.push_back(Character); + } + Text.push_back('"'); + } else { + return makeUnexpected(ErrorCode::VerificationError, + "machine metadata tuple contains an invalid element"); + } + } + Text.push_back('}'); + Output.append(" - "); + Output.append(yamlScalar(Text)); + Output.push_back('\n'); + } + return {}; +} + +Expected<void> MIRDocumentPrinter::printTargetState(const MachineFunction &Function) { + if (!Function.targetState().hasValue()) + return {}; + auto Node = Target.mirText().printFunctionState(Function.targetState()); + if (!Node) + return std::unexpected(Node.error()); + auto Text = targetNode(*Node); + if (!Text) + return std::unexpected(Text.error()); + Output.append("machineFunctionInfo: "); + Output.append(*Text); + Output.push_back('\n'); + return {}; +} + +Expected<void> MIRDocumentPrinter::printFunctionRecords(const MachineFunction &Function, + const MIParserContext &Context) { + const MachineFunctionRecords &Records = Function.records(); + if (!Records.entryValues().empty()) { + Output.append("entry_values:\n"); + for (const MachineEntryValueInfo &Entry : Records.entryValues()) { + auto RegisterName = physicalRegister(Entry.Reg); + if (!RegisterName) + return std::unexpected(RegisterName.error()); + if (!Entry.DebugInfo.Variable || !Entry.DebugInfo.Expression || !Entry.DebugInfo.Location) { + return makeUnexpected(ErrorCode::Unsupported, + "MIR entry value requires complete debug metadata"); + } + auto Variable = metadataReference(*Entry.DebugInfo.Variable, Context); + auto Expression = metadataValue(*Entry.DebugInfo.Expression, Context); + auto Location = debugLocation(*Entry.DebugInfo.Location, Context); + if (!Variable) + return std::unexpected(Variable.error()); + if (!Expression) + return std::unexpected(Expression.error()); + if (!Location) + return std::unexpected(Location.error()); + Output.append(" - { entry-value-register: "); + Output.append(yamlScalar(*RegisterName)); + Output.append(", debug-info-variable: "); + Output.append(yamlScalar(*Variable)); + Output.append(", debug-info-expression: "); + Output.append(yamlScalar(*Expression)); + Output.append(", debug-info-location: "); + Output.append(yamlScalar(*Location)); + Output.append(" }\n"); + } + } else if (Options.Mode == MIPrintMode::Complete) { + Output.append("entry_values: []\n"); + } + + if (!Records.callSites().empty()) { + Output.append("callSites:\n"); + for (const MachineCallSiteInfo &CallSite : Records.callSites()) { + auto Location = instructionLocation(Function, CallSite.CallInstruction); + if (!Location) + return std::unexpected(Location.error()); + Output.append(" - { bb: "); + Output.append(std::to_string(Location->Block)); + Output.append(", offset: "); + Output.append(std::to_string(Location->Offset)); + if (!CallSite.ForwardedArguments.empty()) { + Output.append(", fwdArgRegs: [ "); + for (std::size_t Index = 0; Index < CallSite.ForwardedArguments.size(); ++Index) { + if (Index != 0) + Output.append(", "); + const MachineCallSiteArgument &Argument = CallSite.ForwardedArguments[Index]; + auto RegisterName = physicalRegister(Argument.Reg); + if (!RegisterName) + return std::unexpected(RegisterName.error()); + Output.append("{ arg: "); + Output.append(std::to_string(Argument.ArgumentNumber)); + Output.append(", reg: "); + Output.append(yamlScalar(*RegisterName)); + Output.append(" }"); + } + Output.append(" ]"); + } + if (!CallSite.CalleeTypeIDs.empty()) { + Output.append(", calleeTypeIds: [ "); + for (std::size_t Index = 0; Index < CallSite.CalleeTypeIDs.size(); ++Index) { + if (Index != 0) + Output.append(", "); + Output.append(std::to_string(CallSite.CalleeTypeIDs[Index])); + } + Output.append(" ]"); + } + Output.append(" }\n"); + } + } else if (Options.Mode == MIPrintMode::Complete) { + Output.append("callSites: []\n"); + } + + if (!Records.debugValueSubstitutions().empty()) { + Output.append("debugValueSubstitutions:\n"); + for (const MachineDebugValueSubstitution &Substitution : Records.debugValueSubstitutions()) { + Output.append(" - { srcinst: "); + Output.append(std::to_string(Substitution.Source.InstructionNumber)); + Output.append(", srcop: "); + Output.append(std::to_string(Substitution.Source.OperandNumber)); + Output.append(", dstinst: "); + Output.append(std::to_string(Substitution.Destination.InstructionNumber)); + Output.append(", dstop: "); + Output.append(std::to_string(Substitution.Destination.OperandNumber)); + Output.append(", subreg: "); + Output.append(std::to_string(Substitution.Subregister.value())); + Output.append(" }\n"); + } + } else if (Options.Mode == MIPrintMode::Complete) { + Output.append("debugValueSubstitutions: []\n"); + } + + if (!Records.calledGlobals().empty()) { + Output.append("calledGlobals:\n"); + for (const MachineCalledGlobal &Called : Records.calledGlobals()) { + auto Location = instructionLocation(Function, Called.CallInstruction); + if (!Location) + return std::unexpected(Location.error()); + Output.append(" - { bb: "); + Output.append(std::to_string(Location->Block)); + Output.append(", offset: "); + Output.append(std::to_string(Location->Offset)); + Output.append(", callee: "); + Output.append(yamlScalar(Called.Callee)); + Output.append(", flags: "); + Output.append(std::to_string(Called.Flags)); + Output.append(" }\n"); + } + } + + if (!Records.prefetchTargets().empty()) { + Output.append("prefetch-targets: [ "); + const auto Targets = Records.prefetchTargets(); + for (std::size_t Index = 0; Index < Targets.size(); ++Index) { + if (Index != 0) + Output.append(", "); + const MachinePrefetchTarget &TargetValue = Targets[Index]; + std::string Text = "bb_id " + std::to_string(TargetValue.Block.BaseID) + ", " + + std::to_string(TargetValue.Block.CloneID) + ", " + + std::to_string(TargetValue.CallsiteIndex); + Output.append(yamlScalar(Text)); + } + Output.append(" ]\n"); + } + return {}; +} + +Expected<void> MIRDocumentPrinter::printFunction(const MachineFunction &Function, + const MIRFunctionDocumentInfo *) { + if (Function.compilationStatus().state() != MachineCompilationState::Success) { + return makeUnexpected(ErrorCode::Unsupported, + "fixed LLVM MIR cannot serialize this compilation failure state"); + } + auto Context = createBodyContext(Function); + if (!Context) + return std::unexpected(Context.error()); + + Output.append("---\nname: "); + Output.append(yamlScalar(Function.name())); + Output.push_back('\n'); + const auto Alignment = Function.attributes().alignment(); + if (Options.Mode == MIPrintMode::Complete || Alignment) { + Output.append("alignment: "); + Output.append(std::to_string(Alignment ? Alignment->bytes() : 1U)); + Output.push_back('\n'); + } + + const MachineFunctionProperties &Properties = Function.properties(); + const MachineFunctionAttributes &Attributes = Function.attributes(); + printBoolean("exposesReturnsTwice", + Attributes.has(MachineFunctionAttribute::ExposesReturnsTwice)); + printBoolean("legalized", Properties.has(MachineFunctionProperty::Legalized)); + printBoolean("regBankSelected", Properties.has(MachineFunctionProperty::RegBankSelected)); + printBoolean("selected", Properties.has(MachineFunctionProperty::Selected)); + printBoolean("failedISel", Properties.has(MachineFunctionProperty::FailedISel)); + printBoolean("tracksRegLiveness", Properties.has(MachineFunctionProperty::TracksLiveness)); + printBoolean("hasWinCFI", Attributes.has(MachineFunctionAttribute::HasWinCFI)); + printBoolean("noPhis", Properties.has(MachineFunctionProperty::NoPHIs), true); + printBoolean("isSSA", Properties.has(MachineFunctionProperty::IsSSA), true); + printBoolean("noVRegs", Properties.has(MachineFunctionProperty::NoVRegs), true); + printBoolean("hasFakeUses", Attributes.has(MachineFunctionAttribute::HasFakeUses), true); + printBoolean("callsEHReturn", Attributes.has(MachineFunctionAttribute::CallsEHReturn)); + printBoolean("callsUnwindInit", Attributes.has(MachineFunctionAttribute::CallsUnwindInit)); + printBoolean("hasEHContTarget", Attributes.has(MachineFunctionAttribute::HasEHContTarget)); + printBoolean("hasEHScopes", Attributes.has(MachineFunctionAttribute::HasEHScopes)); + printBoolean("hasEHFunclets", Attributes.has(MachineFunctionAttribute::HasEHFunclets)); + printBoolean("isOutlined", Attributes.has(MachineFunctionAttribute::IsOutlined)); + printBoolean("debugInstrRef", Attributes.has(MachineFunctionAttribute::DebugInstrRef)); + printBoolean("failsVerification", Properties.has(MachineFunctionProperty::FailsVerification)); + printBoolean("tracksDebugUserValues", + Attributes.has(MachineFunctionAttribute::TracksDebugUserValues)); + + if (auto Printed = printRegisters(Function, *Context); !Printed) + return Printed; + if (auto Printed = printFrameInfo(Function, *Context); !Printed) + return Printed; + if (auto Printed = printStackObjects(Function, *Context, true); !Printed) + return Printed; + if (auto Printed = printStackObjects(Function, *Context, false); !Printed) + return Printed; + if (auto Printed = printFunctionRecords(Function, *Context); !Printed) + return Printed; + if (auto Printed = printConstants(Function, *Context); !Printed) + return Printed; + if (auto Printed = printTargetState(Function); !Printed) + return Printed; + if (auto Printed = printJumpTable(Function, *Context); !Printed) + return Printed; + if (auto Printed = printMetadata(Function, *Context); !Printed) + return Printed; + + auto Body = + printMachineFunctionBody(Function, Target, *Context, MIPrinterOptions{.Mode = Options.Mode}); + if (!Body) + return std::unexpected(Body.error()); + Output.append("body: |\n"); + appendBlockScalar(Output, *Body, 2); + Output.append("...\n"); + return {}; +} + +Expected<std::string> MIRDocumentPrinter::run() { + auto UnitVerification = Document.unit().verify(); + if (!UnitVerification) + return std::unexpected(UnitVerification.error()); + if (Document.unit().functionCount() == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "MIR document must contain at least one machine function"); + } + + for (const MIRFunctionDocumentInfo &Info : Document.functionDocumentInfo()) { + if (Document.unit().function(Info.Function) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "MIR document info references an unknown machine function"); + } + } + + for (std::size_t Index = 0; Index < Document.unit().functionCount(); ++Index) { + const MachineFunction *Function = Document.unit().functionAt(Index); + if (Function == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit contains a null machine function"); + } + auto ReferenceVerification = + verifyMIRDocumentReferences(Document, *Function, Options.EmitEmbeddedIR); + if (!ReferenceVerification) + return std::unexpected(ReferenceVerification.error()); + auto LayoutVerification = verifyOmittedEmbeddedIRLayout(*Function); + if (!LayoutVerification) + return std::unexpected(LayoutVerification.error()); + } + + if (Options.EmitEmbeddedIR && Document.embeddedIR() != nullptr) { + Output.append("--- |\n"); + appendBlockScalar(Output, Document.embeddedIR()->source().text(), 2); + Output.append("...\n"); + } + for (std::size_t Index = 0; Index < Document.unit().functionCount(); ++Index) { + const MachineFunction *Function = Document.unit().functionAt(Index); + if (Function == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "compilation unit contains a null machine function"); + if (auto Printed = printFunction(*Function, documentInfo(*Function)); !Printed) + return std::unexpected(Printed.error()); + } + return std::move(Output); +} + +} // namespace + +Expected<std::string> printMIRDocument(const MIRDocument &Document, const TargetDescription &Target, + MIRDocumentPrinterOptions Options) { + return MIRDocumentPrinter(Document, Target, std::move(Options)).run(); +} + +} // namespace rcc::mir + +namespace rcc::mir { +namespace { + +[[nodiscard]] bool isBareNameCharacter(char Character) noexcept { + return (Character >= 'a' && Character <= 'z') || (Character >= 'A' && Character <= 'Z') || + (Character >= '0' && Character <= '9') || Character == '_' || Character == '-' || + Character == '.' || Character == '$'; +} + +[[nodiscard]] std::string quoteName(std::string_view Name) { + if (!Name.empty() && std::ranges::all_of(Name, isBareNameCharacter)) + return std::string(Name); + + constexpr std::string_view Hex = "0123456789ABCDEF"; + std::string Result; + Result.reserve(Name.size() + 2U); + Result.push_back('"'); + for (const char RawCharacter : Name) { + const auto Character = static_cast<unsigned char>(RawCharacter); + if (Character >= 0x20U && Character <= 0x7EU && Character != '"' && Character != '\\') { + Result.push_back(static_cast<char>(Character)); + continue; + } + Result.push_back('\\'); + Result.push_back(Hex[Character >> 4U]); + Result.push_back(Hex[Character & 0xFU]); + } + Result.push_back('"'); + return Result; +} + +[[nodiscard]] std::string quoteString(std::string_view Value) { + std::string Result = quoteName(Value); + if (!Result.empty() && Result.front() == '"') + return Result; + Result.insert(Result.begin(), '"'); + Result.push_back('"'); + return Result; +} + +[[nodiscard]] std::string hexadecimalWords(std::span<const std::uint64_t> Words, + std::uint32_t BitWidth) { + constexpr std::string_view Hex = "0123456789abcdef"; + const std::size_t DigitCount = (static_cast<std::size_t>(BitWidth) + 3U) / 4U; + std::string Result(DigitCount, '0'); + for (std::size_t Digit = 0; Digit < DigitCount; ++Digit) { + const std::size_t BitOffset = Digit * 4U; + const std::uint64_t Nibble = (Words[BitOffset / 64U] >> (BitOffset % 64U)) & 0xFU; + Result[DigitCount - Digit - 1U] = Hex[Nibble]; + } + return Result; +} + +[[nodiscard]] std::string_view integerPredicateName(IntegerPredicate Predicate) noexcept { + switch (Predicate) { + case IntegerPredicate::Equal: + return "eq"; + case IntegerPredicate::NotEqual: + return "ne"; + case IntegerPredicate::SignedGreaterThan: + return "sgt"; + case IntegerPredicate::SignedGreaterEqual: + return "sge"; + case IntegerPredicate::SignedLessThan: + return "slt"; + case IntegerPredicate::SignedLessEqual: + return "sle"; + case IntegerPredicate::UnsignedGreaterThan: + return "ugt"; + case IntegerPredicate::UnsignedGreaterEqual: + return "uge"; + case IntegerPredicate::UnsignedLessThan: + return "ult"; + case IntegerPredicate::UnsignedLessEqual: + return "ule"; + case IntegerPredicate::Count: + return {}; + } + return {}; +} + +[[nodiscard]] std::string_view floatingPredicateName(FloatingPredicate Predicate) noexcept { + switch (Predicate) { + case FloatingPredicate::AlwaysFalse: + return "false"; + case FloatingPredicate::OrderedEqual: + return "oeq"; + case FloatingPredicate::OrderedGreaterThan: + return "ogt"; + case FloatingPredicate::OrderedGreaterEqual: + return "oge"; + case FloatingPredicate::OrderedLessThan: + return "olt"; + case FloatingPredicate::OrderedLessEqual: + return "ole"; + case FloatingPredicate::OrderedNotEqual: + return "one"; + case FloatingPredicate::Ordered: + return "ord"; + case FloatingPredicate::Unordered: + return "uno"; + case FloatingPredicate::UnorderedEqual: + return "ueq"; + case FloatingPredicate::UnorderedGreaterThan: + return "ugt"; + case FloatingPredicate::UnorderedGreaterEqual: + return "uge"; + case FloatingPredicate::UnorderedLessThan: + return "ult"; + case FloatingPredicate::UnorderedLessEqual: + return "ule"; + case FloatingPredicate::UnorderedNotEqual: + return "une"; + case FloatingPredicate::AlwaysTrue: + return "true"; + case FloatingPredicate::Count: + return {}; + } + return {}; +} + +[[nodiscard]] std::string_view atomicOrderingName(AtomicOrdering Ordering) noexcept { + switch (Ordering) { + case AtomicOrdering::Unordered: + return "unordered"; + case AtomicOrdering::Monotonic: + return "monotonic"; + case AtomicOrdering::Acquire: + return "acquire"; + case AtomicOrdering::Release: + return "release"; + case AtomicOrdering::AcquireRelease: + return "acq_rel"; + case AtomicOrdering::SequentiallyConsistent: + return "seq_cst"; + case AtomicOrdering::NotAtomic: + return {}; + } + return {}; +} + +[[nodiscard]] Expected<std::string> llvmMIRScalarLowLevelType(LowLevelType Type) { + if (Type.isToken()) + return "s0"; + if (Type.isCategoryNeutralScalar()) + return "s" + std::to_string(Type.scalarBitWidth()); + if (Type.isPointer()) { + const auto AddressSpace = Type.addressSpace(); + if (!AddressSpace) { + return makeUnexpected(ErrorCode::VerificationError, + "pointer low-level type has no address space"); + } + return "p" + std::to_string(*AddressSpace); + } + return makeUnexpected(ErrorCode::VerificationError, + "invalid scalar low-level type cannot be printed as LLVM MIR"); +} + +[[nodiscard]] Expected<std::string> llvmMIRLowLevelType(LowLevelType Type) { + if (!Type.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "invalid low-level type cannot be printed as LLVM MIR"); + } + if (!Type.isVector()) + return llvmMIRScalarLowLevelType(Type); + + const auto ElementType = Type.scalarElementType(); + if (!ElementType) { + return makeUnexpected(ErrorCode::VerificationError, + "vector low-level type has no scalar element type"); + } + auto Scalar = llvmMIRScalarLowLevelType(*ElementType); + if (!Scalar) + return std::unexpected(Scalar.error()); + + std::string Result = "<"; + if (Type.isScalableVector()) + Result.append("vscale x "); + Result.append(std::to_string(Type.minimumElementCount())); + Result.append(" x "); + Result.append(*Scalar); + Result.push_back('>'); + return Result; +} + +class BodyPrinter final { +public: + BodyPrinter(const MachineFunction &Function, const TargetDescription &Target, + const MIParserContext &Context, MIPrinterOptions Options) + : Function(Function), Target(Target), Context(Context), Options(Options) {} + + [[nodiscard]] Expected<std::string> run(); + [[nodiscard]] Expected<std::string> runInstruction(const MachineInstr &Instruction); + [[nodiscard]] Expected<std::string> runOperand(const MachineOperand &Operand); + +private: + [[nodiscard]] Expected<void> initialize(); + [[nodiscard]] Expected<void> printBlock(const MachineBasicBlock &Block); + [[nodiscard]] Expected<void> printInstruction(const MachineInstr &Instruction); + [[nodiscard]] Expected<void> printOperand(const MachineOperand &Operand, bool IsExplicitDef, + bool PrintVirtualType = true); + [[nodiscard]] Expected<void> printInlineAsmExtraInfo(const MachineOperand &Operand); + [[nodiscard]] Expected<std::size_t> printInlineAsmOperandFlag(const MachineOperand &Operand); + [[nodiscard]] Expected<void> printRegisterOperand(const MachineOperand &Operand, + bool IsExplicitDef, bool PrintVirtualType); + [[nodiscard]] Expected<void> printTargetFlags(std::uint32_t Flags); + [[nodiscard]] Expected<void> printMetadataReference(MachineMetadataReference Reference); + [[nodiscard]] Expected<void> printDILocation(const MachineDILocation &Location, + std::size_t FrameIndex); + [[nodiscard]] Expected<void> printCFIInstruction(CFIInstructionIndex Index); + [[nodiscard]] Expected<void> printMemoryOperand(const MachineMemOperand &Operand); + [[nodiscard]] Expected<void> printMemoryBase(const MemoryBase &Base, std::int64_t Offset); + + [[nodiscard]] Expected<std::uint32_t> blockID(const MachineBasicBlock &Block) const; + [[nodiscard]] Expected<std::uint32_t> virtualRegisterID(Register Reg) const; + [[nodiscard]] Expected<std::uint32_t> frameID(FrameIndex Index) const; + [[nodiscard]] Expected<std::uint32_t> constantPoolID(ConstantPoolIndex Index) const; + [[nodiscard]] Expected<std::uint32_t> jumpTableID(JumpTableIndex Index) const; + [[nodiscard]] Expected<std::uint32_t> metadataID(MachineMetadataReference Reference) const; + [[nodiscard]] Expected<void> appendPhysicalRegister(PhysicalRegister Register); + void appendBlockReference(const MachineBasicBlock &Block); + void appendOffset(std::int64_t Offset); + + const MachineFunction &Function; + const TargetDescription &Target; + const MIParserContext &Context; + MIPrinterOptions Options; + std::string Output; + std::unordered_map<const MachineBasicBlock *, std::uint32_t> BlockIDs; +}; + +Expected<void> BodyPrinter::initialize() { + std::uint64_t NextID = 0; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + if (NextID > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine function has too many blocks to print"); + } + BlockIDs.emplace(Block, static_cast<std::uint32_t>(NextID)); + ++NextID; + } + return {}; +} + +Expected<std::uint32_t> BodyPrinter::blockID(const MachineBasicBlock &Block) const { + const auto Found = BlockIDs.find(&Block); + if (Found == BlockIDs.end()) + return makeUnexpected(ErrorCode::VerificationError, + "machine operand references an unlinked basic block"); + return Found->second; +} + +Expected<std::uint32_t> BodyPrinter::virtualRegisterID(Register Reg) const { + if (!Context.VirtualRegisters.empty()) { + const auto Found = + std::ranges::find_if(Context.VirtualRegisters, + [Reg](const MIVirtualRegisterSlot &Slot) { return Slot.Reg == Reg; }); + if (Found == Context.VirtualRegisters.end()) { + return makeUnexpected(ErrorCode::VerificationError, "virtual register has no MIR text slot"); + } + return Found->TextID; + } + const auto VirtualIndex = Reg.virtualIndex(); + if (!VirtualIndex) + return makeUnexpected(ErrorCode::VerificationError, "virtual register has no numeric index"); + return *VirtualIndex; +} + +Expected<std::uint32_t> BodyPrinter::frameID(FrameIndex Index) const { + const auto InStack = std::ranges::find_if( + Context.StackObjects, [Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; }); + if (InStack != Context.StackObjects.end()) + return InStack->TextID; + const auto InFixed = + std::ranges::find_if(Context.FixedStackObjects, + [Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; }); + if (InFixed != Context.FixedStackObjects.end()) + return InFixed->TextID; + return makeUnexpected(ErrorCode::VerificationError, "frame index has no MIR text slot"); +} + +Expected<std::uint32_t> BodyPrinter::constantPoolID(ConstantPoolIndex Index) const { + const auto Found = + std::ranges::find_if(Context.ConstantPoolItems, + [Index](const MIConstantPoolSlot &Slot) { return Slot.Index == Index; }); + if (Found == Context.ConstantPoolItems.end()) { + return makeUnexpected(ErrorCode::VerificationError, "constant-pool index has no MIR text slot"); + } + return Found->TextID; +} + +Expected<std::uint32_t> BodyPrinter::jumpTableID(JumpTableIndex Index) const { + const auto Found = std::ranges::find_if( + Context.JumpTables, [Index](const MIJumpTableSlot &Slot) { return Slot.Index == Index; }); + if (Found == Context.JumpTables.end()) + return makeUnexpected(ErrorCode::VerificationError, "jump-table index has no MIR text slot"); + return Found->TextID; +} + +Expected<std::uint32_t> BodyPrinter::metadataID(MachineMetadataReference Reference) const { + const auto Found = + std::ranges::find_if(Context.Metadata, [Reference](const MIMetadataSlot &Slot) { + return Slot.Reference == Reference; + }); + if (Found != Context.Metadata.end()) + return Found->TextID; + if (Reference.scope() == MachineMetadataScope::MachineFunction) + return Reference.slot(); + return makeUnexpected(ErrorCode::VerificationError, + "embedded metadata reference has no MIR text slot"); +} + +Expected<void> BodyPrinter::appendPhysicalRegister(PhysicalRegister Register) { + const auto Name = Target.registers().physicalRegisterMIRName(Register); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, "physical register has no MIR name"); + Output.push_back('$'); + Output.append(*Name); + return {}; +} + +void BodyPrinter::appendBlockReference(const MachineBasicBlock &Block) { + Output.append("%bb."); + Output.append(std::to_string(BlockIDs.at(&Block))); +} + +void BodyPrinter::appendOffset(std::int64_t Offset) { + if (Offset == 0) + return; + if (Offset > 0) { + Output.append(" + "); + Output.append(std::to_string(Offset)); + return; + } + Output.append(" - "); + const std::uint64_t Magnitude = Offset == std::numeric_limits<std::int64_t>::min() + ? std::uint64_t{1} << 63U + : static_cast<std::uint64_t>(-Offset); + Output.append(std::to_string(Magnitude)); +} + +Expected<void> BodyPrinter::printTargetFlags(std::uint32_t Flags) { + if (Flags == 0) + return {}; + auto Names = Target.mirText().operandFlagNames(Flags); + if (!Names) + return std::unexpected(Names.error()); + Output.append("target-flags("); + for (std::size_t Index = 0; Index < Names->size(); ++Index) { + if (Index != 0) + Output.append(", "); + Output.append((*Names)[Index]); + } + Output.append(") "); + return {}; +} + +Expected<void> BodyPrinter::printMetadataReference(MachineMetadataReference Reference) { + auto ID = metadataID(Reference); + if (!ID) + return std::unexpected(ID.error()); + Output.push_back('!'); + Output.append(std::to_string(*ID)); + return {}; +} + +Expected<void> BodyPrinter::printDILocation(const MachineDILocation &Location, + std::size_t FrameIndex) { + const auto Frames = Location.inlineChain(); + if (FrameIndex >= Frames.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "inline DILocation has an invalid frame index"); + } + const MachineDILocationFrame &Frame = Frames[FrameIndex]; + Output.append("!DILocation(line: "); + Output.append(std::to_string(Frame.Line)); + if (Options.Mode == MIPrintMode::Complete || Frame.Column != 0) { + Output.append(", column: "); + Output.append(std::to_string(Frame.Column)); + } + Output.append(", scope: "); + if (auto Printed = printMetadataReference(Frame.Scope); !Printed) + return Printed; + if (FrameIndex + 1U < Frames.size()) { + Output.append(", inlinedAt: "); + if (auto Printed = printDILocation(Location, FrameIndex + 1U); !Printed) + return Printed; + } else if (Location.terminalInlinedAt()) { + Output.append(", inlinedAt: "); + if (auto Printed = printMetadataReference(*Location.terminalInlinedAt()); !Printed) { + return Printed; + } + } + if (Options.Mode == MIPrintMode::Complete || Frame.IsImplicitCode) { + Output.append(", isImplicitCode: "); + Output.append(Frame.IsImplicitCode ? "true" : "false"); + } + if (Options.Mode == MIPrintMode::Complete || Frame.AtomGroup != 0) { + Output.append(", atomGroup: "); + Output.append(std::to_string(Frame.AtomGroup)); + } + if (Options.Mode == MIPrintMode::Complete || Frame.AtomRank != 0) { + Output.append(", atomRank: "); + Output.append(std::to_string(Frame.AtomRank)); + } + Output.push_back(')'); + return {}; +} + +Expected<void> BodyPrinter::printRegisterOperand(const MachineOperand &Operand, bool IsExplicitDef, + bool PrintVirtualType) { + const RegisterOperandFlags Flags = Operand.registerFlags(); + if (Flags.has(RegisterOperandFlag::Implicit) && Flags.has(RegisterOperandFlag::Define)) { + Output.append("implicit-def "); + } else { + if (!IsExplicitDef && Flags.has(RegisterOperandFlag::Define)) + Output.append("def "); + if (Flags.has(RegisterOperandFlag::Implicit)) + Output.append("implicit "); + } + if (Flags.has(RegisterOperandFlag::Dead)) + Output.append("dead "); + if (Flags.has(RegisterOperandFlag::Kill)) + Output.append("killed "); + if (Flags.has(RegisterOperandFlag::Undef)) + Output.append("undef "); + if (Flags.has(RegisterOperandFlag::InternalRead)) + Output.append("internal "); + if (Flags.has(RegisterOperandFlag::EarlyClobber)) + Output.append("early-clobber "); + if (Flags.has(RegisterOperandFlag::Debug) && + (Operand.parent() == nullptr || !Operand.parent()->isDebugInstr())) + Output.append("debug-use "); + if (Flags.has(RegisterOperandFlag::Renamable)) + Output.append("renamable "); + + const auto RegisterValue = Operand.registerValue(); + if (!RegisterValue) { + return makeUnexpected(ErrorCode::VerificationError, "register operand has no register value"); + } + const Register Reg = *RegisterValue; + if (Reg.isNoRegister()) { + Output.append("$noreg"); + } else if (Reg.isPhysical()) { + const auto Physical = Reg.physicalRegister(); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "physical register has no physical-register value"); + } + auto Printed = appendPhysicalRegister(*Physical); + if (!Printed) + return Printed; + } else { + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + const VirtualRegisterInfo &RegisterInfo = Info->get(); + Output.push_back('%'); + if (RegisterInfo.Name) { + Output.append(*RegisterInfo.Name); + } else { + auto VirtualIndex = virtualRegisterID(Reg); + if (!VirtualIndex) + return std::unexpected(VirtualIndex.error()); + Output.append(std::to_string(*VirtualIndex)); + } + if (Operand.subReg().value() != 0) { + const SubRegIndexDesc *SubReg = Target.registers().subRegIndex(Operand.subReg()); + if (SubReg == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "virtual register has an unknown subregister"); + Output.push_back('.'); + Output.append(SubReg->Name); + } + + const bool PrintRegisterConstraint = + IsExplicitDef || Function.registerInfo().getUniqueVRegDef(Reg) == nullptr; + if (PrintRegisterConstraint && RegisterInfo.RegClass) { + const auto Name = Target.registers().registerClassMIRName(*RegisterInfo.RegClass); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, + "virtual register class has no MIR name"); + Output.push_back(':'); + Output.append(*Name); + } else if (PrintRegisterConstraint && RegisterInfo.Bank) { + const auto Name = Target.registers().registerBankMIRName(*RegisterInfo.Bank); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, + "virtual register bank has no MIR name"); + Output.push_back(':'); + Output.append(*Name); + } else if (PrintRegisterConstraint && RegisterInfo.Type) { + Output.append(":_"); + } + if (PrintVirtualType && RegisterInfo.Type) { + auto Type = llvmMIRLowLevelType(*RegisterInfo.Type); + if (!Type) + return std::unexpected(Type.error()); + Output.push_back('('); + Output.append(*Type); + Output.push_back(')'); + } + } + + if (Operand.tiedOperandIndex() && Operand.isUse()) { + Output.append("(tied-def "); + Output.append(std::to_string(*Operand.tiedOperandIndex())); + Output.push_back(')'); + } + return {}; +} + +Expected<void> BodyPrinter::printCFIInstruction(CFIInstructionIndex Index) { + const MachineCFIInstruction *Instruction = Function.cfiInfo().instruction(Index); + if (Instruction == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "CFI operand references an unknown instruction"); + const auto PrintRegister = [this](DwarfRegisterNumber Dwarf) -> Expected<void> { + const auto Register = Target.mirText().physicalRegister(Dwarf.value()); + if (!Register) { + return makeUnexpected(ErrorCode::VerificationError, + "CFI instruction has no canonical physical register"); + } + return appendPhysicalRegister(*Register); + }; + const auto PrintRegisterOffset = + [this, &PrintRegister](std::string_view Name, + const MachineCFIRegisterOffsetRule &Rule) -> Expected<void> { + Output.append(Name); + Output.push_back(' '); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.Offset)); + return {}; + }; + + switch (Instruction->operation()) { + case MachineCFIOperation::SameValue: { + Output.append("same_value "); + return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register); + } + case MachineCFIOperation::RememberState: + Output.append("remember_state"); + return {}; + case MachineCFIOperation::RestoreState: + Output.append("restore_state"); + return {}; + case MachineCFIOperation::Offset: + return PrintRegisterOffset("offset", *Instruction->getIf<MachineCFIRegisterOffsetRule>()); + case MachineCFIOperation::LLVMDefAddressSpaceCFA: { + const auto &Rule = *Instruction->getIf<MachineCFIAddressSpaceCFARule>(); + Output.append("llvm_def_aspace_cfa "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.Offset)); + Output.append(", "); + Output.append(std::to_string(Rule.AddressSpace)); + return {}; + } + case MachineCFIOperation::DefCFARegister: + Output.append("def_cfa_register "); + return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register); + case MachineCFIOperation::DefCFAOffset: + Output.append("def_cfa_offset "); + Output.append(std::to_string(Instruction->getIf<MachineCFIOffsetRule>()->Offset)); + return {}; + case MachineCFIOperation::DefCFA: + return PrintRegisterOffset("def_cfa", *Instruction->getIf<MachineCFIRegisterOffsetRule>()); + case MachineCFIOperation::RelativeOffset: + return PrintRegisterOffset("rel_offset", *Instruction->getIf<MachineCFIRegisterOffsetRule>()); + case MachineCFIOperation::AdjustCFAOffset: + Output.append("adjust_cfa_offset "); + Output.append(std::to_string(Instruction->getIf<MachineCFIOffsetRule>()->Offset)); + return {}; + case MachineCFIOperation::Escape: { + Output.append("escape "); + constexpr std::string_view Hex = "0123456789ABCDEF"; + const auto &Bytes = Instruction->getIf<MachineCFIEscapeRule>()->Bytes; + for (std::size_t ByteIndex = 0; ByteIndex < Bytes.size(); ++ByteIndex) { + if (ByteIndex != 0) + Output.append(", "); + Output.append("0x"); + Output.push_back(Hex[Bytes[ByteIndex] >> 4U]); + Output.push_back(Hex[Bytes[ByteIndex] & 0xFU]); + } + return {}; + } + case MachineCFIOperation::Restore: + Output.append("restore "); + return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register); + case MachineCFIOperation::Undefined: + Output.append("undefined "); + return PrintRegister(Instruction->getIf<MachineCFIRegisterRule>()->Register); + case MachineCFIOperation::Register: { + const auto &Rule = *Instruction->getIf<MachineCFIRegisterRegisterRule>(); + Output.append("register "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + return PrintRegister(Rule.SourceRegister); + } + case MachineCFIOperation::WindowSave: + Output.append("window_save"); + return {}; + case MachineCFIOperation::NegateReturnAddressState: + Output.append("negate_ra_sign_state"); + return {}; + case MachineCFIOperation::NegateReturnAddressStateWithPC: + Output.append("negate_ra_sign_state_with_pc"); + return {}; + case MachineCFIOperation::GNUArgsSize: + case MachineCFIOperation::Label: + case MachineCFIOperation::ValueOffset: + return makeUnexpected( + ErrorCode::Unsupported, + "CFI operation has no textual representation in the fixed LLVM MIR baseline"); + case MachineCFIOperation::LLVMRegisterPair: { + const auto &Rule = *Instruction->getIf<MachineCFIRegisterPairRule>(); + Output.append("llvm_register_pair "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + if (auto Printed = PrintRegister(Rule.FirstRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.FirstSizeInBits)); + Output.append(", "); + if (auto Printed = PrintRegister(Rule.SecondRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.SecondSizeInBits)); + return {}; + } + case MachineCFIOperation::LLVMVectorRegisters: { + const auto &Rule = *Instruction->getIf<MachineCFIVectorRegistersRule>(); + Output.append("llvm_vector_registers "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + for (const MachineCFIVectorRegisterLane &Lane : Rule.VectorRegisters) { + Output.append(", "); + if (auto Printed = PrintRegister(Lane.Register); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Lane.Lane)); + Output.append(", "); + Output.append(std::to_string(Lane.SizeInBits)); + } + return {}; + } + case MachineCFIOperation::LLVMVectorOffset: { + const auto &Rule = *Instruction->getIf<MachineCFIVectorOffsetRule>(); + Output.append("llvm_vector_offset "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.RegisterSizeInBits)); + Output.append(", "); + if (auto Printed = PrintRegister(Rule.MaskRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.MaskRegisterSizeInBits)); + Output.append(", "); + Output.append(std::to_string(Rule.Offset)); + return {}; + } + case MachineCFIOperation::LLVMVectorRegisterMask: { + const auto &Rule = *Instruction->getIf<MachineCFIVectorRegisterMaskRule>(); + Output.append("llvm_vector_register_mask "); + if (auto Printed = PrintRegister(Rule.Register); !Printed) + return Printed; + Output.append(", "); + if (auto Printed = PrintRegister(Rule.SpillRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.SpillRegisterLaneSizeInBits)); + Output.append(", "); + if (auto Printed = PrintRegister(Rule.MaskRegister); !Printed) + return Printed; + Output.append(", "); + Output.append(std::to_string(Rule.MaskRegisterSizeInBits)); + return {}; + } + case MachineCFIOperation::Count: + return makeUnexpected(ErrorCode::VerificationError, "CFI instruction has an invalid operation"); + } + return makeUnexpected(ErrorCode::InternalError, "CFI printer did not handle an operation"); +} + +Expected<void> BodyPrinter::printOperand(const MachineOperand &Operand, bool IsExplicitDef, + bool PrintVirtualType) { + if (Operand.isRegister()) + return printRegisterOperand(Operand, IsExplicitDef, PrintVirtualType); + if (auto Flags = printTargetFlags(Operand.targetFlags()); !Flags) + return Flags; + + switch (Operand.kind()) { + case MachineOperand::Kind::Register: + return makeUnexpected(ErrorCode::InternalError, + "register operand reached the non-register printer"); + case MachineOperand::Kind::Immediate: + Output.append(std::to_string(*Operand.getIf<std::int64_t>())); + return {}; + case MachineOperand::Kind::IntegerImmediate: { + const APInt &Value = Operand.getIf<IntegerImmediateOperand>()->Value; + Output.push_back('i'); + Output.append(std::to_string(Value.bitWidth())); + Output.push_back(' '); + if (Value.bitWidth() == 1U) + Output.append(Value.isZero() ? "false" : "true"); + else + Output.append(Value.toSignedDecimal()); + return {}; + } + case MachineOperand::Kind::FloatingImmediate: { + const APFloat &Bits = Operand.getIf<FloatingImmediateOperand>()->Bits; + auto Constant = GenericConstantData::floating(Bits); + if (!Constant) + return std::unexpected(Constant.error()); + auto Text = Constant->toLLVMString(); + if (!Text) + return std::unexpected(Text.error()); + Output.append(*Text); + return {}; + } + case MachineOperand::Kind::BasicBlock: { + const MachineBasicBlock *Block = Operand.getIf<BasicBlockOperand>()->Block; + if (Block == nullptr || !BlockIDs.contains(Block)) { + return makeUnexpected(ErrorCode::VerificationError, + "basic-block operand references an unlinked block"); + } + appendBlockReference(*Block); + return {}; + } + case MachineOperand::Kind::FrameIndex: { + const auto &Frame = *Operand.getIf<FrameIndexOperand>(); + auto ID = frameID(Frame.Index); + if (!ID) + return std::unexpected(ID.error()); + const auto Stack = std::ranges::find_if( + Context.StackObjects, + [Index = Frame.Index](const MIFrameObjectSlot &Slot) { return Slot.Index == Index; }); + if (Stack != Context.StackObjects.end()) { + Output.append("%stack."); + Output.append(std::to_string(*ID)); + if (Stack->Name) { + Output.push_back('.'); + Output.append(quoteName(*Stack->Name)); + } + } else { + Output.append("%fixed-stack."); + Output.append(std::to_string(*ID)); + } + appendOffset(Frame.Offset); + return {}; + } + case MachineOperand::Kind::ConstantPoolIndex: { + const auto &Constant = *Operand.getIf<ConstantPoolOperand>(); + auto ID = constantPoolID(Constant.Index); + if (!ID) + return std::unexpected(ID.error()); + Output.append("%const."); + Output.append(std::to_string(*ID)); + appendOffset(Constant.Offset); + return {}; + } + case MachineOperand::Kind::JumpTableIndex: { + auto ID = jumpTableID(Operand.getIf<JumpTableOperand>()->Index); + if (!ID) + return std::unexpected(ID.error()); + Output.append("%jump-table."); + Output.append(std::to_string(*ID)); + return {}; + } + case MachineOperand::Kind::GlobalSymbol: { + const auto &Global = *Operand.getIf<GlobalSymbolOperand>(); + Output.push_back('@'); + Output.append(quoteName(Global.Name)); + appendOffset(Global.Offset); + return {}; + } + case MachineOperand::Kind::ExternalSymbol: { + const auto &External = *Operand.getIf<ExternalSymbolOperand>(); + Output.push_back('&'); + Output.append(quoteName(External.Name)); + appendOffset(External.Offset); + return {}; + } + case MachineOperand::Kind::BlockAddress: { + const auto &Address = *Operand.getIf<BlockAddressOperand>(); + Output.append("blockaddress(@"); + Output.append(quoteName(Address.FunctionName)); + Output.append(", %ir-block."); + Output.append(quoteName(Address.BlockName)); + Output.push_back(')'); + appendOffset(Address.Offset); + return {}; + } + case MachineOperand::Kind::RegisterMask: { + const RegisterMask &Mask = *Operand.getIf<RegisterMaskOperand>()->Mask; + if (const auto Name = Target.mirText().registerMaskName(Mask)) { + Output.append(*Name); + return {}; + } + Output.append("CustomRegMask("); + bool First = true; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask.preserves(Register.Reg)) + continue; + if (!First) + Output.append(", "); + First = false; + if (auto Printed = appendPhysicalRegister(Register.Reg); !Printed) + return Printed; + } + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::LiveOutRegisterMask: { + const RegisterMask &Mask = *Operand.getIf<LiveOutRegisterMaskOperand>()->Mask; + Output.append("liveout("); + bool First = true; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask.preserves(Register.Reg)) + continue; + if (!First) + Output.append(", "); + First = false; + if (auto Printed = appendPhysicalRegister(Register.Reg); !Printed) + return Printed; + } + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::Metadata: { + const MachineMetadataValue &Value = Operand.getIf<MetadataOperand>()->Value; + if (const auto *Reference = Value.reference()) + return printMetadataReference(*Reference); + const MachineDIExpression *Expression = Value.expression(); + if (Expression == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "metadata operand has no printable value"); + } + Output.append(formatDIExpression(Expression->elements())); + return {}; + } + case MachineOperand::Kind::TargetIndex: { + const auto &TargetIndexValue = *Operand.getIf<TargetIndexOperand>(); + const auto Name = Target.mirText().targetIndexName(TargetIndexValue.Index); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, "target index has no MIR name"); + Output.append("target-index("); + Output.append(*Name); + Output.push_back(')'); + appendOffset(TargetIndexValue.Offset); + return {}; + } + case MachineOperand::Kind::MCSymbol: { + const auto &Symbol = *Operand.getIf<MCSymbolOperand>(); + Output.append("<mcsymbol "); + Output.append(quoteName(Symbol.Name)); + Output.push_back('>'); + appendOffset(Symbol.Offset); + return {}; + } + case MachineOperand::Kind::CFIIndex: + return printCFIInstruction(Operand.getIf<CFIIndexOperand>()->Index); + case MachineOperand::Kind::Intrinsic: { + const auto Name = Target.mirText().intrinsicName(Operand.getIf<IntrinsicOperand>()->ID); + if (!Name) + return makeUnexpected(ErrorCode::VerificationError, "intrinsic has no MIR name"); + Output.append("intrinsic(@"); + Output.append(*Name); + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::Predicate: { + const PredicateOperand &Predicate = *Operand.getIf<PredicateOperand>(); + if (const auto *Integer = Predicate.integerPredicate()) { + Output.append("intpred("); + Output.append(integerPredicateName(*Integer)); + } else { + Output.append("floatpred("); + Output.append(floatingPredicateName(*Predicate.floatingPredicate())); + } + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::ShuffleMask: { + Output.append("shufflemask("); + const auto &Elements = Operand.getIf<ShuffleMaskOperand>()->Elements; + for (std::size_t Index = 0; Index < Elements.size(); ++Index) { + if (Index != 0) + Output.append(", "); + if (Elements[Index] == ShuffleMaskOperand::Undef) + Output.append("undef"); + else + Output.append(std::to_string(Elements[Index])); + } + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::DebugInstrRef: { + const auto &Reference = *Operand.getIf<DebugInstrRefOperand>(); + Output.append("dbg-instr-ref("); + Output.append(std::to_string(Reference.InstructionNumber)); + Output.append(", "); + Output.append(std::to_string(Reference.OperandNumber)); + Output.push_back(')'); + return {}; + } + case MachineOperand::Kind::LaneMask: + Output.append("lanemask(0x"); + Output.append( + hexadecimalWords(std::array{Operand.getIf<LaneMaskOperand>()->Mask.rawValue()}, 64)); + Output.push_back(')'); + return {}; + case MachineOperand::Kind::Count: + return makeUnexpected(ErrorCode::VerificationError, "machine operand has an invalid kind"); + } + return makeUnexpected(ErrorCode::InternalError, "machine operand printer did not handle a kind"); +} + +Expected<void> BodyPrinter::printInlineAsmExtraInfo(const MachineOperand &Operand) { + const auto *Value = Operand.getIf<std::int64_t>(); + if (Value == nullptr || Operand.targetFlags() != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "inline assembly extra info must be an unflagged immediate"); + } + const InlineAsmExtraInfo ExtraInfo(static_cast<std::uint32_t>(*Value)); + const std::vector<std::string_view> Names = ExtraInfo.names(); + for (std::size_t Index = 0; Index < Names.size(); ++Index) { + if (Index != 0) + Output.push_back(' '); + Output.append(Names[Index]); + } + return {}; +} + +Expected<std::size_t> BodyPrinter::printInlineAsmOperandFlag(const MachineOperand &Operand) { + const auto *Value = Operand.getIf<std::int64_t>(); + if (Value == nullptr || *Value < 0 || + static_cast<std::uint64_t>(*Value) > std::numeric_limits<std::uint32_t>::max() || + Operand.targetFlags() != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "inline assembly operand flag must be an unflagged 32-bit immediate"); + } + + const InlineAsmOperandFlag Flag(static_cast<std::uint32_t>(*Value)); + const std::string_view KindName = InlineAsmOperandFlag::kindName(Flag.kind()); + if (KindName.empty() || Flag.operandCount() == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "inline assembly operand flag has an invalid kind or operand count"); + } + Output.append(KindName); + + if (const auto MatchingOperand = Flag.matchingOperand()) { + Output.append(" tiedto:$"); + Output.append(std::to_string(*MatchingOperand)); + return Flag.operandCount(); + } + if (Flag.kind() == InlineAsmOperandKind::Memory || + Flag.kind() == InlineAsmOperandKind::Function) { + const auto Constraint = InlineAsmOperandFlag::memoryConstraintName(Flag.memoryConstraint()); + if (Constraint) { + Output.push_back(':'); + Output.append(*Constraint); + } + return Flag.operandCount(); + } + if (Flag.isRegisterKind()) { + if (const auto RegisterClass = Flag.registerClass()) { + const auto Name = + Target.registers().registerClassMIRName(RegisterClassID::fromValue(*RegisterClass)); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, + "inline assembly operand flag references an unknown register class"); + } + Output.push_back(':'); + Output.append(*Name); + } + } + return Flag.operandCount(); +} + +Expected<void> BodyPrinter::printMemoryBase(const MemoryBase &Base, std::int64_t Offset) { + switch (Base.kind()) { + case MemoryBase::Kind::Unknown: + Output.append("unknown-address"); + break; + case MemoryBase::Kind::FrameIndex: { + const auto Frame = Base.frameIndex(); + if (!Frame) { + return makeUnexpected(ErrorCode::VerificationError, + "frame-index memory base has no frame index"); + } + const FrameIndex Index = *Frame; + auto ID = frameID(Index); + if (!ID) + return std::unexpected(ID.error()); + const auto Stack = + std::ranges::find_if(Context.StackObjects, [Index](const MIFrameObjectSlot &Slot) { + return Slot.Index == Index; + }); + if (Stack != Context.StackObjects.end()) { + Output.append("%stack."); + Output.append(std::to_string(*ID)); + if (Stack->Name) { + Output.push_back('.'); + Output.append(quoteName(*Stack->Name)); + } + } else { + Output.append("%fixed-stack."); + Output.append(std::to_string(*ID)); + } + break; + } + case MemoryBase::Kind::ConstantPool: { + const auto Constant = Base.constantPoolIndex(); + if (!Constant) { + return makeUnexpected(ErrorCode::VerificationError, "constant-pool memory base has no index"); + } + auto ID = constantPoolID(*Constant); + if (!ID) + return std::unexpected(ID.error()); + Output.append("%const."); + Output.append(std::to_string(*ID)); + break; + } + case MemoryBase::Kind::JumpTable: { + const auto JumpTable = Base.jumpTableIndex(); + if (!JumpTable) { + return makeUnexpected(ErrorCode::VerificationError, "jump-table memory base has no index"); + } + auto ID = jumpTableID(*JumpTable); + if (!ID) + return std::unexpected(ID.error()); + Output.append("%jump-table."); + Output.append(std::to_string(*ID)); + break; + } + case MemoryBase::Kind::GlobalSymbol: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, "global memory base has no symbol name"); + } + Output.push_back('@'); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::ExternalSymbol: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, + "external memory base has no symbol name"); + } + Output.push_back('&'); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::IRValue: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, "IR-value memory base has no value name"); + } + Output.append("%ir."); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::IRConstant: { + const GenericConstantData *Constant = Base.constantValue(); + if (Constant == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "IR-constant memory base has no constant value"); + } + auto Printed = Constant->toLLVMString(); + if (!Printed) + return std::unexpected(Printed.error()); + Output.push_back('`'); + Output.append(*Printed); + Output.push_back('`'); + break; + } + case MemoryBase::Kind::Stack: + Output.append("stack"); + break; + case MemoryBase::Kind::GOT: + Output.append("got"); + break; + case MemoryBase::Kind::ConstantPoolPseudo: + Output.append("constant-pool"); + break; + case MemoryBase::Kind::JumpTablePseudo: + Output.append("jump-table"); + break; + case MemoryBase::Kind::CallEntryGlobal: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, + "global call-entry memory base has no symbol name"); + } + Output.append("call-entry @"); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::CallEntryExternal: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, + "external call-entry memory base has no symbol name"); + } + Output.append("call-entry &"); + Output.append(quoteName(*Name)); + break; + } + case MemoryBase::Kind::Custom: { + const auto &Name = Base.symbolName(); + if (!Name) { + return makeUnexpected(ErrorCode::VerificationError, "custom memory base has no name"); + } + Output.append("custom "); + Output.append(quoteName(*Name)); + break; + } + } + appendOffset(Offset); + return {}; +} + +Expected<void> BodyPrinter::printMemoryOperand(const MachineMemOperand &Operand) { + const MachineMemOperandDesc &Description = Operand.description(); + Output.push_back('('); + if (Description.Accesses.has(MemoryAccessFlag::Volatile)) + Output.append("volatile "); + if (Description.Accesses.has(MemoryAccessFlag::NonTemporal)) + Output.append("non-temporal "); + if (Description.Accesses.has(MemoryAccessFlag::Dereferenceable)) + Output.append("dereferenceable "); + if (Description.Accesses.has(MemoryAccessFlag::Invariant)) + Output.append("invariant "); + if (Description.TargetFlags != 0) { + auto Names = Target.mirText().machineMemOperandFlagNames(Description.TargetFlags); + if (!Names) + return std::unexpected(Names.error()); + for (std::string_view Name : *Names) { + Output.append(quoteName(Name)); + Output.push_back(' '); + } + } + + if (Operand.isLoad()) + Output.append("load"); + else + Output.append("store"); + if (Operand.isLoad() && Operand.isStore()) + Output.append(" store"); + + if (Description.SyncScope.value() != 0) { + const auto Scope = + std::ranges::find_if(Context.SyncScopes, [&Description](const MISyncScopeSlot &Slot) { + return Slot.ID == Description.SyncScope; + }); + if (Scope == Context.SyncScopes.end()) { + return makeUnexpected(ErrorCode::VerificationError, "synchronization scope has no MIR name"); + } + Output.append(" syncscope("); + Output.append(quoteString(Scope->Name)); + Output.push_back(')'); + } + if (Description.SuccessOrdering != AtomicOrdering::NotAtomic) { + Output.push_back(' '); + Output.append(atomicOrderingName(Description.SuccessOrdering)); + } + if (Description.FailureOrdering != AtomicOrdering::NotAtomic) { + Output.push_back(' '); + Output.append(atomicOrderingName(Description.FailureOrdering)); + } + + Output.push_back(' '); + if (Description.MemoryType) { + auto Type = llvmMIRLowLevelType(*Description.MemoryType); + if (!Type) + return std::unexpected(Type.error()); + Output.push_back('('); + Output.append(*Type); + Output.push_back(')'); + } else { + switch (Description.Size.kind()) { + case MemorySize::Kind::Unknown: + Output.append("unknown-size"); + break; + case MemorySize::Kind::Fixed: + Output.append(std::to_string(Description.Size.minimumBytes())); + break; + case MemorySize::Kind::Scalable: + Output.append("(<vscale x "); + Output.append(std::to_string(Description.Size.minimumBytes())); + Output.append(" x s8>)"); + break; + } + } + + const std::string_view Preposition = + Operand.isLoad() && Operand.isStore() ? " on " : (Operand.isLoad() ? " from " : " into "); + if (Description.Base.kind() != MemoryBase::Kind::Unknown || Description.Offset != 0) { + Output.append(Preposition); + if (auto Printed = printMemoryBase(Description.Base, Description.Offset); !Printed) { + return Printed; + } + } + + if (!Description.Size.isKnown() || + (Description.Size.minimumBytes() != 0 && + Description.Align.bytes() != Description.Size.minimumBytes())) { + Output.append(", align "); + Output.append(std::to_string(Description.Align.bytes())); + } + if (Description.BaseAlign && *Description.BaseAlign != Description.Align) { + Output.append(", basealign "); + Output.append(std::to_string(Description.BaseAlign->bytes())); + } + if (Description.AddressSpace != 0) { + Output.append(", addrspace "); + Output.append(std::to_string(Description.AddressSpace)); + } + + const auto PrintMetadata = + [this](std::string_view Name, + const std::optional<MachineMetadataReference> &Reference) -> Expected<void> { + if (!Reference) + return {}; + Output.append(", "); + Output.append(Name); + Output.push_back(' '); + return printMetadataReference(*Reference); + }; + if (auto Printed = PrintMetadata("!tbaa", Description.TBAA); !Printed) + return Printed; + if (auto Printed = PrintMetadata("!alias.scope", Description.AliasScope); !Printed) { + return Printed; + } + if (auto Printed = PrintMetadata("!noalias", Description.NoAlias); !Printed) + return Printed; + if (auto Printed = PrintMetadata("!noalias.addrspace", Description.NoAliasAddressSpace); + !Printed) { + return Printed; + } + if (auto Printed = PrintMetadata("!range", Description.RangeMetadata); !Printed) { + return Printed; + } + Output.push_back(')'); + return {}; +} + +Expected<void> BodyPrinter::printInstruction(const MachineInstr &Instruction) { + const InstructionDescriptor *Descriptor = Target.instructions().descriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine instruction has no target descriptor"); + } + std::size_t ExplicitDefs = 0; + while (ExplicitDefs < Instruction.operandCount()) { + const MachineOperand *Operand = Instruction.operandAt(ExplicitDefs); + if (Operand == nullptr || !Operand->isRegister() || Operand->isImplicit() || + !Operand->isDef() || Operand->isUse()) + break; + ++ExplicitDefs; + } + + Output.append(" "); + for (std::size_t Index = 0; Index < ExplicitDefs; ++Index) { + if (Index != 0) + Output.append(", "); + if (auto Printed = printOperand(*Instruction.operandAt(Index), true); !Printed) { + return Printed; + } + } + if (ExplicitDefs != 0) + Output.append(" = "); + + const MachineInstrFlags Flags = Instruction.flags(); + constexpr std::array FlagSpellings{ + std::pair{MachineInstrFlag::FrameSetup, std::string_view{"frame-setup"}}, + std::pair{MachineInstrFlag::FrameDestroy, std::string_view{"frame-destroy"}}, + std::pair{MachineInstrFlag::FastMathNoNaNs, std::string_view{"nnan"}}, + std::pair{MachineInstrFlag::FastMathNoInfs, std::string_view{"ninf"}}, + std::pair{MachineInstrFlag::FastMathNoSignedZeros, std::string_view{"nsz"}}, + std::pair{MachineInstrFlag::FastMathAllowReciprocal, std::string_view{"arcp"}}, + std::pair{MachineInstrFlag::FastMathAllowContract, std::string_view{"contract"}}, + std::pair{MachineInstrFlag::FastMathApproxFunc, std::string_view{"afn"}}, + std::pair{MachineInstrFlag::FastMathAllowReassoc, std::string_view{"reassoc"}}, + std::pair{MachineInstrFlag::NoUnsignedWrap, std::string_view{"nuw"}}, + std::pair{MachineInstrFlag::NoSignedWrap, std::string_view{"nsw"}}, + std::pair{MachineInstrFlag::NoUSWrap, std::string_view{"nusw"}}, + std::pair{MachineInstrFlag::Exact, std::string_view{"exact"}}, + std::pair{MachineInstrFlag::NoFPExcept, std::string_view{"nofpexcept"}}, + std::pair{MachineInstrFlag::Unpredictable, std::string_view{"unpredictable"}}, + std::pair{MachineInstrFlag::NoConvergent, std::string_view{"noconvergent"}}, + std::pair{MachineInstrFlag::NonNeg, std::string_view{"nneg"}}, + std::pair{MachineInstrFlag::Disjoint, std::string_view{"disjoint"}}, + std::pair{MachineInstrFlag::SameSign, std::string_view{"samesign"}}, + std::pair{MachineInstrFlag::InBounds, std::string_view{"inbounds"}}, + }; + for (const auto &[Flag, Spelling] : FlagSpellings) { + if (Flags.has(Flag)) { + Output.append(Spelling); + Output.push_back(' '); + } + } + Output.append(Descriptor->Name); + + bool HasPrintedTail = false; + const bool IsInlineAsm = Descriptor->Name == "INLINEASM" || Descriptor->Name == "INLINEASM_BR"; + std::size_t InlineAsmPayloadRemaining = 0; + for (std::size_t Index = ExplicitDefs; Index < Instruction.operandCount(); ++Index) { + Output.append(HasPrintedTail ? ", " : " "); + HasPrintedTail = true; + const MachineOperand &Operand = *Instruction.operandAt(Index); + if (IsInlineAsm && Index == InlineAsmExtraInfoOperandIndex) { + auto Printed = printInlineAsmExtraInfo(Operand); + if (!Printed) + return Printed; + continue; + } + if (IsInlineAsm && Index >= InlineAsmFirstConstraintOperandIndex && + InlineAsmPayloadRemaining == 0) { + auto OperandCount = printInlineAsmOperandFlag(Operand); + if (!OperandCount) + return std::unexpected(OperandCount.error()); + InlineAsmPayloadRemaining = *OperandCount; + continue; + } + bool PrintVirtualType = true; + if (Descriptor->Flags.has(InstructionFlag::Generic) && + !Descriptor->Flags.has(InstructionFlag::Phi) && Index < Descriptor->Operands.size()) { + const std::optional<OperandTypeConstraint> &UseConstraint = + Descriptor->Operands[Index].TypeConstraint; + if (UseConstraint) { + for (std::size_t DefIndex = 0; DefIndex < ExplicitDefs; ++DefIndex) { + if (DefIndex >= Descriptor->Operands.size()) + break; + const std::optional<OperandTypeConstraint> &DefConstraint = + Descriptor->Operands[DefIndex].TypeConstraint; + if (DefConstraint && DefConstraint->EqualityGroup == UseConstraint->EqualityGroup) { + PrintVirtualType = false; + break; + } + } + } else if (const Register *UseRegister = Operand.getIf<Register>(); + UseRegister != nullptr && UseRegister->isVirtual()) { + const auto UseInfo = Function.registerInfo().virtualRegisterInfo(*UseRegister); + for (std::size_t DefIndex = 0; DefIndex < ExplicitDefs && UseInfo && UseInfo->get().Type; + ++DefIndex) { + if (DefIndex >= Descriptor->Operands.size() || + Descriptor->Operands[DefIndex].TypeConstraint) + continue; + const MachineOperand *DefOperand = Instruction.operandAt(DefIndex); + const Register *DefRegister = + DefOperand == nullptr ? nullptr : DefOperand->getIf<Register>(); + if (DefRegister == nullptr || !DefRegister->isVirtual()) + continue; + const auto DefInfo = Function.registerInfo().virtualRegisterInfo(*DefRegister); + if (DefInfo && DefInfo->get().Type == UseInfo->get().Type) { + PrintVirtualType = false; + break; + } + } + } + } + auto Printed = printOperand(Operand, false, PrintVirtualType); + if (!Printed) + return Printed; + if (IsInlineAsm && InlineAsmPayloadRemaining != 0) + --InlineAsmPayloadRemaining; + } + if (InlineAsmPayloadRemaining != 0) + return makeUnexpected(ErrorCode::VerificationError, + "inline assembly operand flag has missing payload operands"); + + const auto BeginAttachment = [this, &HasPrintedTail](std::string_view Name) { + Output.append(HasPrintedTail ? ", " : " "); + HasPrintedTail = true; + Output.append(Name); + Output.push_back(' '); + }; + if (Instruction.preInstrSymbol()) { + BeginAttachment("pre-instr-symbol"); + Output.append("<mcsymbol "); + Output.append(quoteName(*Instruction.preInstrSymbol())); + Output.push_back('>'); + } + if (Instruction.postInstrSymbol()) { + BeginAttachment("post-instr-symbol"); + Output.append("<mcsymbol "); + Output.append(quoteName(*Instruction.postInstrSymbol())); + Output.push_back('>'); + } + if (Instruction.heapAllocMarker()) { + BeginAttachment("heap-alloc-marker"); + if (auto Printed = printMetadataReference(*Instruction.heapAllocMarker()); !Printed) { + return Printed; + } + } + if (Instruction.pcSections()) { + BeginAttachment("pcsections"); + if (auto Printed = printMetadataReference(*Instruction.pcSections()); !Printed) { + return Printed; + } + } + if (Instruction.mmra()) { + BeginAttachment("mmra"); + if (auto Printed = printMetadataReference(*Instruction.mmra()); !Printed) + return Printed; + } + if (Instruction.cfiType()) { + BeginAttachment("cfi-type"); + Output.append(std::to_string(*Instruction.cfiType())); + } + if (Instruction.deactivationSymbol()) { + BeginAttachment("deactivation-symbol"); + Output.push_back('@'); + Output.append(quoteName(*Instruction.deactivationSymbol())); + } + if (Instruction.debugInstrNumber() != 0) { + BeginAttachment("debug-instr-number"); + Output.append(std::to_string(Instruction.debugInstrNumber())); + } + if (Instruction.debugLocation()) { + BeginAttachment("debug-location"); + if (const MachineMetadataReference *Reference = Instruction.debugLocation()->reference()) { + if (auto Printed = printMetadataReference(*Reference); !Printed) + return Printed; + } else if (const MachineDILocation *Location = Instruction.debugLocation()->inlineLocation()) { + if (auto Printed = printDILocation(*Location, 0); !Printed) + return Printed; + } else { + return makeUnexpected(ErrorCode::VerificationError, + "machine debug location has no printable value"); + } + } + + if (!Instruction.memOperands().empty()) { + Output.append(" :: "); + for (std::size_t Index = 0; Index < Instruction.memOperands().size(); ++Index) { + if (Index != 0) + Output.append(", "); + if (auto Printed = printMemoryOperand(*Instruction.memOperands()[Index]); !Printed) { + return Printed; + } + } + } + return {}; +} + +Expected<void> BodyPrinter::printBlock(const MachineBasicBlock &Block) { + auto ID = blockID(Block); + if (!ID) + return std::unexpected(ID.error()); + Output.append("bb."); + Output.append(std::to_string(*ID)); + const auto &Name = Block.attributes().name(); + if (Name) { + Output.push_back('.'); + Output.append(quoteName(*Name)); + } + + std::vector<std::string> Attributes; + const auto AddFlag = [&Block, &Attributes](MachineBasicBlockAttribute Attribute, + std::string_view Spelling) { + if (Block.attributes().has(Attribute)) + Attributes.emplace_back(Spelling); + }; + AddFlag(MachineBasicBlockAttribute::MachineBlockAddressTaken, "machine-block-address-taken"); + if (Block.attributes().has(MachineBasicBlockAttribute::IRBlockAddressTaken)) { + std::string Attribute = "ir-block-address-taken"; + const auto &IRBlockName = Block.attributes().irBlockName(); + if (IRBlockName) { + Attribute.append(" %ir-block."); + Attribute.append(quoteName(*IRBlockName)); + } + Attributes.push_back(std::move(Attribute)); + } else { + const auto &IRBlockName = Block.attributes().irBlockName(); + if (IRBlockName) + Attributes.push_back("%ir-block." + quoteName(*IRBlockName)); + } + AddFlag(MachineBasicBlockAttribute::LandingPad, "landing-pad"); + AddFlag(MachineBasicBlockAttribute::InlineAsmBrIndirectTarget, "inlineasm-br-indirect-target"); + AddFlag(MachineBasicBlockAttribute::EHFuncletEntry, "ehfunclet-entry"); + AddFlag(MachineBasicBlockAttribute::EHScopeEntry, "ehscope-entry"); + if (Block.attributes().alignment().bytes() != 1) { + Attributes.push_back("align " + std::to_string(Block.attributes().alignment().bytes())); + } + const MachineBasicBlockSectionID Section = Block.attributes().sectionID(); + if (!Section.isDefault()) { + std::string Value; + if (Section.kind() == MachineBasicBlockSectionKind::Exception) + Value = "Exception"; + else if (Section.kind() == MachineBasicBlockSectionKind::Cold) + Value = "Cold"; + else + Value = std::to_string(Section.number()); + Attributes.push_back("bbsections " + Value); + } + const auto &ProfileID = Block.attributes().profileID(); + if (ProfileID) { + const MachineBasicBlockProfileID Profile = *ProfileID; + Attributes.push_back("bb_id " + std::to_string(Profile.BaseID) + "." + + std::to_string(Profile.CloneID)); + } + if (Block.attributes().callFrameSize() != 0) { + Attributes.push_back("call-frame-size " + std::to_string(Block.attributes().callFrameSize())); + } + if (!Attributes.empty()) { + Output.append(" ("); + for (std::size_t Index = 0; Index < Attributes.size(); ++Index) { + if (Index != 0) + Output.append(", "); + Output.append(Attributes[Index]); + } + Output.push_back(')'); + } + Output.append(":\n"); + + bool CanPredictProbabilities = true; + if (Block.successors().size() > 1) { + const BranchProbability FirstProbability = Block.successors().front().Probability; + CanPredictProbabilities = std::ranges::all_of( + Block.successors(), [FirstProbability](const MachineSuccessor &Successor) { + return Successor.Probability == FirstProbability; + }); + } + std::vector<MachineBasicBlock *> GuessedSuccessors; + bool IsFallthrough = false; + auto Guessed = guessSuccessors(Block, Target, GuessedSuccessors, IsFallthrough); + if (!Guessed) + return std::unexpected(Guessed.error()); + MachineBasicBlock *NextBlock = const_cast<MachineBasicBlock *>(Block.next()); + if (IsFallthrough && NextBlock != nullptr && + std::ranges::find(GuessedSuccessors, NextBlock) == GuessedSuccessors.end()) { + GuessedSuccessors.push_back(NextBlock); + } + const bool CanPredictSuccessors = + GuessedSuccessors.size() == Block.successors().size() && + std::ranges::equal(GuessedSuccessors, Block.successors(), {}, {}, + [](const MachineSuccessor &Successor) { return Successor.Block; }); + + bool HasLineAttributes = false; + const bool Simplify = Options.Mode == MIPrintMode::Readable; + if ((!Block.successors().empty() && !Simplify) || !CanPredictProbabilities || + !CanPredictSuccessors) { + Output.append(" successors:"); + if (!Block.successors().empty()) + Output.push_back(' '); + for (std::size_t Index = 0; Index < Block.successors().size(); ++Index) { + if (Index != 0) + Output.append(", "); + const MachineSuccessor &Successor = Block.successors()[Index]; + if (Successor.Block == nullptr || !BlockIDs.contains(Successor.Block)) { + return makeUnexpected(ErrorCode::VerificationError, + "successor references an unlinked block"); + } + appendBlockReference(*Successor.Block); + if (!Simplify || !CanPredictProbabilities) { + Output.append("(0x"); + Output.append(hexadecimalWords( + std::array{static_cast<std::uint64_t>(Successor.Probability.rawValue())}, 32)); + Output.push_back(')'); + } + } + Output.push_back('\n'); + HasLineAttributes = true; + } + if (!Block.liveIns().empty()) { + Output.append(" liveins: "); + for (std::size_t Index = 0; Index < Block.liveIns().size(); ++Index) { + if (Index != 0) + Output.append(", "); + const MachineBasicBlockLiveIn &LiveIn = Block.liveIns()[Index]; + if (auto Printed = appendPhysicalRegister(LiveIn.PhysReg); !Printed) + return Printed; + if (LiveIn.Lanes != LaneMask::all()) { + Output.append(": 0x"); + Output.append(hexadecimalWords(std::array{LiveIn.Lanes.rawValue()}, 64)); + } + } + Output.push_back('\n'); + HasLineAttributes = true; + } + + if (HasLineAttributes && Block.firstInstruction() != nullptr) + Output.push_back('\n'); + + bool InBundle = false; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (auto Printed = printInstruction(*Instruction); !Printed) + return Printed; + const bool HasPredecessor = Instruction->flags().has(MachineInstrFlag::BundledWithPred); + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if (!InBundle && HasSuccessor && !HasPredecessor) { + Output.append(" {"); + InBundle = true; + } + Output.push_back('\n'); + if (InBundle && !HasSuccessor) { + Output.append(" }\n"); + InBundle = false; + } + } + if (InBundle) { + return makeUnexpected(ErrorCode::VerificationError, + "unterminated instruction bundle cannot be printed"); + } + return {}; +} + +Expected<std::string> BodyPrinter::run() { + auto Verification = MachineVerifier::verify( + Function, Target, + MachineVerifierOptions{ + .Mode = MachineVerifierMode::Generic, .Stage = std::nullopt, .VerifyProperties = false}); + if (!Verification) + return std::unexpected(Verification.error()); + if (auto Initialized = initialize(); !Initialized) + return std::unexpected(Initialized.error()); + + bool First = true; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + if (!First) + Output.push_back('\n'); + First = false; + if (auto Printed = printBlock(*Block); !Printed) + return std::unexpected(Printed.error()); + } + return std::move(Output); +} + +Expected<std::string> BodyPrinter::runInstruction(const MachineInstr &Instruction) { + if (&Instruction.function() != &Function) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine instruction belongs to a different function"); + } + if (auto Initialized = initialize(); !Initialized) + return std::unexpected(Initialized.error()); + if (auto Printed = printInstruction(Instruction); !Printed) + return std::unexpected(Printed.error()); + if (!Output.starts_with(" ")) + return makeUnexpected(ErrorCode::InternalError, "machine instruction printer lost indentation"); + Output.erase(0, 2); + return std::move(Output); +} + +Expected<std::string> BodyPrinter::runOperand(const MachineOperand &Operand) { + if (Operand.parent() != nullptr && &Operand.parent()->function() != &Function) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine operand belongs to a different function"); + } + if (auto Initialized = initialize(); !Initialized) + return std::unexpected(Initialized.error()); + if (auto Printed = printOperand(Operand, false); !Printed) + return std::unexpected(Printed.error()); + return std::move(Output); +} + +} // namespace + +Expected<MIParserContext> createMachineFunctionPrintContext(const MachineFunction &Function) { + MIParserContext Context; + const auto VirtualRegisters = Function.registerInfo().virtualRegisters(); + for (std::size_t Index = 0; Index < VirtualRegisters.size(); ++Index) { + if (VirtualRegisters[Index].IsTombstone) + continue; + if (Index > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::Unsupported, + "MIR cannot represent more than 2^32 virtual registers"); + } + auto Reg = Register::makeVirtual(Index); + if (!Reg) + return std::unexpected(Reg.error()); + Context.VirtualRegisters.push_back({.TextID = static_cast<std::uint32_t>(Index), .Reg = *Reg}); + } + + std::uint32_t StackID = 0; + std::uint32_t FixedID = 0; + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + if (Description == nullptr || Description->IsDead) + continue; + MIFrameObjectSlot Slot{ + .TextID = Object.isFixed() ? FixedID++ : StackID++, + .Index = Object.index(), + .Name = Description->AllocaName, + }; + (Object.isFixed() ? Context.FixedStackObjects : Context.StackObjects) + .push_back(std::move(Slot)); + } + + std::uint32_t ConstantID = 0; + for (const MachineConstantPoolEntry &Entry : Function.constantPool().entries()) { + if (Entry.description() != nullptr) + Context.ConstantPoolItems.push_back({.TextID = ConstantID++, .Index = Entry.index()}); + } + std::uint32_t JumpTableID = 0; + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + if (Entry.description() != nullptr) + Context.JumpTables.push_back({.TextID = JumpTableID++, .Index = Entry.index()}); + } + for (const MachineMetadataNode &Node : Function.metadata().nodes()) { + Context.Metadata.push_back( + {.TextID = Node.Slot, .Reference = MachineMetadataReference::machineFunction(Node.Slot)}); + } + return Context; +} + +Expected<std::string> printMachineInstruction(const MachineInstr &Instruction, + const TargetDescription &Target, + const MIParserContext &Context, + MIPrinterOptions Options) { + return BodyPrinter(Instruction.function(), Target, Context, Options).runInstruction(Instruction); +} + +Expected<std::string> printMachineOperand(const MachineOperand &Operand, + const TargetDescription &Target, + const MIParserContext &Context, + MIPrinterOptions Options) { + if (Operand.parent() == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "standalone machine operand has no owning function"); + } + return BodyPrinter(Operand.parent()->function(), Target, Context, Options).runOperand(Operand); +} + +Expected<void> guessSuccessors(const MachineBasicBlock &Block, const TargetDescription &Target, + std::vector<MachineBasicBlock *> &Result, bool &IsFallthrough) { + Result.clear(); + const InstructionDescriptor *LastCodeDescriptor = nullptr; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = + Target.instructions().descriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction has no target descriptor while inferring successors"); + } + if (!Descriptor->Flags.has(InstructionFlag::Phi)) { + for (const MachineOperand &Operand : Instruction->operands()) { + const BasicBlockOperand *TargetBlock = Operand.getIf<BasicBlockOperand>(); + if (TargetBlock != nullptr && TargetBlock->Block != nullptr && + std::ranges::find(Result, TargetBlock->Block) == Result.end()) { + Result.push_back(TargetBlock->Block); + } + } + } + if (!Descriptor->Flags.has(InstructionFlag::Meta)) + LastCodeDescriptor = Descriptor; + } + IsFallthrough = + LastCodeDescriptor == nullptr || !LastCodeDescriptor->Flags.has(InstructionFlag::Barrier); + return {}; +} + +Expected<std::string> printMachineFunctionBody(const MachineFunction &Function, + const TargetDescription &Target, + const MIParserContext &Context, + MIPrinterOptions Options) { + return BodyPrinter(Function, Target, Context, Options).run(); +} + +} // namespace rcc::mir diff --git a/backend/lib/CodeGen/MIRYamlMapping.cpp b/backend/lib/CodeGen/MIRYamlMapping.cpp new file mode 100644 index 0000000..c36f006 --- /dev/null +++ b/backend/lib/CodeGen/MIRYamlMapping.cpp @@ -0,0 +1,2935 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MIRYamlMapping.h" + +#include "MIRParser/MILexer.h" +#include "rcc/AsmParser/LLParser.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineConstantPool.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionAttributes.h" +#include "rcc/CodeGen/MachineFunctionProperties.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineMetadata.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMIRTextInfo.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <array> +#include <charconv> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <initializer_list> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <string_view> +#include <system_error> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <variant> +#include <vector> + +namespace rcc::mir { + +namespace parser_detail { + +[[nodiscard]] inline std::string_view trim(std::string_view Text) noexcept { + while (!Text.empty() && (Text.front() == ' ' || Text.front() == '\t' || Text.front() == '\r' || + Text.front() == '\n')) { + Text.remove_prefix(1); + } + while (!Text.empty() && (Text.back() == ' ' || Text.back() == '\t' || Text.back() == '\r' || + Text.back() == '\n')) { + Text.remove_suffix(1); + } + return Text; +} + +[[nodiscard]] inline bool parseUnsigned(std::string_view Text, std::uint64_t &Value) noexcept { + Text = trim(Text); + int Base = 10; + if (Text.starts_with("0x") || Text.starts_with("0X")) { + Text.remove_prefix(2); + Base = 16; + } + if (Text.empty()) + return false; + const auto [End, Error] = std::from_chars(Text.data(), Text.data() + Text.size(), Value, Base); + return Error == std::errc{} && End == Text.data() + Text.size(); +} + +[[nodiscard]] inline bool parseSigned(std::string_view Text, std::int64_t &Value) noexcept { + Text = trim(Text); + if (Text.empty()) + return false; + const auto [End, Error] = std::from_chars(Text.data(), Text.data() + Text.size(), Value, 10); + return Error == std::errc{} && End == Text.data() + Text.size(); +} + +[[nodiscard]] inline bool parseUnsigned32(std::string_view Text, std::uint32_t &Value) noexcept { + std::uint64_t Wide = 0; + if (!parseUnsigned(Text, Wide) || Wide > std::numeric_limits<std::uint32_t>::max()) + return false; + Value = static_cast<std::uint32_t>(Wide); + return true; +} + +[[nodiscard]] inline bool parseBool(std::string_view Text, bool &Value) noexcept { + if (Text == "true") { + Value = true; + return true; + } + if (Text == "false") { + Value = false; + return true; + } + return false; +} + +[[nodiscard]] inline std::optional<std::uint32_t> +parsePrefixedID(std::string_view Text, std::string_view Prefix) noexcept { + Text = trim(Text); + if (!Text.starts_with(Prefix)) + return std::nullopt; + Text.remove_prefix(Prefix.size()); + const std::size_t Suffix = Text.find_first_of(". \t"); + if (Suffix != std::string_view::npos) + Text = Text.substr(0, Suffix); + std::uint32_t ID = 0; + if (!parseUnsigned32(Text, ID)) + return std::nullopt; + return ID; +} + +} // namespace parser_detail + +class MetadataTextParser final { +public: + using Resolver = std::function<Expected<MachineMetadataReference>(std::uint32_t Slot)>; + + MetadataTextParser(std::string Text, Resolver Resolve) + : Lexer("MIR metadata scalar", std::move(Text)), Resolve(std::move(Resolve)) { + do { + Tokens.push_back(Lexer.next()); + } while (!Tokens.back().isErrorOrEnd()); + } + + [[nodiscard]] Expected<MachineMetadataReference> parseReferenceOnly(); + [[nodiscard]] Expected<MachineMetadataValue> parseValue(); + [[nodiscard]] Expected<MachineDebugLocation> parseLocation(); + + [[nodiscard]] const TextRange &diagnosticRange() const noexcept { + return token().range(); + } + +private: + [[nodiscard]] const MIToken &token() const noexcept { + return Tokens[Position]; + } + [[nodiscard]] bool at(MITokenKind Kind) const noexcept { + return token().is(Kind); + } + void advance() noexcept { + if (Position + 1U < Tokens.size()) + ++Position; + } + [[nodiscard]] bool consume(MITokenKind Kind) noexcept { + if (!at(Kind)) + return false; + advance(); + return true; + } + [[nodiscard]] Expected<void> expect(MITokenKind Kind, std::string_view Description); + [[nodiscard]] Expected<void> finish(); + [[nodiscard]] Expected<MachineMetadataReference> parseReference(); + [[nodiscard]] Expected<MachineDIExpression> parseExpression(); + [[nodiscard]] Expected<void> parseDILocation(std::vector<MachineDILocationFrame> &Frames, + std::optional<MachineMetadataReference> &Terminal); + + MILexer Lexer; + Resolver Resolve; + std::vector<MIToken> Tokens; + std::size_t Position = 0; +}; + +using parser_detail::parseBool; +using parser_detail::parsePrefixedID; +using parser_detail::parseSigned; +using parser_detail::parseUnsigned; +using parser_detail::parseUnsigned32; +using parser_detail::trim; + +bool MIRDocumentParser::boolean(const yaml::Node &Node, bool &Value) { + std::string_view Text; + if (scalar(Node, Text, "boolean") || !parseBool(Text, Value)) + return fail(Node.range(), "expected 'true' or 'false'"); + return false; +} + +bool MIRDocumentParser::unsignedValue(const yaml::Node &Node, std::uint64_t &Value, + std::string_view Description) { + std::string_view Text; + if (scalar(Node, Text, Description) || !parseUnsigned(Text, Value)) + return fail(Node.range(), "expected " + std::string(Description)); + return false; +} + +bool MIRDocumentParser::signedValue(const yaml::Node &Node, std::int64_t &Value, + std::string_view Description) { + std::string_view Text; + if (scalar(Node, Text, Description) || !parseSigned(Text, Value)) + return fail(Node.range(), "expected " + std::string(Description)); + return false; +} + +bool MIRDocumentParser::alignment(const yaml::Node &Node, Alignment &Value, bool AllowZero) { + std::uint64_t Bytes = 0; + if (unsignedValue(Node, Bytes, "an unsigned alignment")) + return true; + if (Bytes == 0 && AllowZero) { + Value = Alignment{}; + return false; + } + auto Parsed = Alignment::create(Bytes); + if (!Parsed) + return importError(Node.range(), Parsed.error()); + Value = *Parsed; + return false; +} + +bool MIRDocumentParser::physicalRegister(std::string_view Text, const TextRange &Range, + PhysicalRegister &Reg) { + Text = trim(Text); + if (!Text.starts_with('$') || Text.size() == 1) + return fail(Range, "physical register must use the '$name' MIR spelling"); + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(Text.substr(1)); + if (Description == nullptr) + return fail(Range, "unknown physical register '" + std::string(Text) + "'"); + Reg = Description->Reg; + return false; +} + +bool MIRDocumentParser::physicalRegister(const yaml::Node &Node, PhysicalRegister &Reg) { + std::string_view Text; + return scalar(Node, Text, "physical register") || physicalRegister(Text, Node.range(), Reg); +} + +bool MIRDocumentParser::virtualRegister(std::string_view Text, const TextRange &Range, + MachineFunction &Function, MIParserContext &Context, + Register &Reg) { + const auto ID = parsePrefixedID(Text, "%"); + if (!ID) + return fail(Range, "virtual register must use the '%N' MIR spelling"); + auto Found = + std::ranges::find_if(Context.VirtualRegisters, + [ID](const MIVirtualRegisterSlot &Slot) { return Slot.TextID == *ID; }); + if (Found == Context.VirtualRegisters.end()) { + auto Created = Function.registerInfo().createVirtualRegister(); + if (!Created) + return importError(Range, Created.error()); + Context.VirtualRegisters.push_back({*ID, *Created}); + Reg = *Created; + return false; + } + Reg = Found->Reg; + return false; +} + +bool MIRDocumentParser::parseConstants(const yaml::Node &Root, MachineFunction &Function, + MIParserContext &Context) { + const yaml::Node *Constants = mappingValue(Root, "constants"); + if (Constants == nullptr) + return false; + if (requireKind(*Constants, yaml::NodeKind::Sequence, "constant pool")) + return true; + constexpr std::array Keys{ + std::string_view{"id"}, + std::string_view{"value"}, + std::string_view{"alignment"}, + std::string_view{"isTargetSpecific"}, + }; + std::vector<LLNamedTypeDefinition> NamedTypes; + const std::string_view DataLayoutText = effectiveDataLayout(); + if (ActiveDocument->EmbeddedIR) { + const auto Definitions = ActiveDocument->EmbeddedIR->typeDefinitions(); + NamedTypes.reserve(Definitions.size()); + for (const EmbeddedIRTypeDefinition &Definition : Definitions) { + NamedTypes.push_back({.Name = Definition.Name, + .Definition = Definition.Definition, + .NumericName = Definition.NumericName}); + } + } + auto Layout = DataLayout::parse(DataLayoutText); + if (!Layout) + return importError(Constants->range(), Layout.error(), "mir-data-layout"); + std::unordered_set<std::uint32_t> IDs; + for (const yaml::NodeID ConstantID : Constants->sequence()) { + const yaml::Node *Constant = node(ConstantID); + if (Constant == nullptr || + requireKind(*Constant, yaml::NodeKind::Mapping, "constant pool entry") || + validateKeys(*Constant, Keys, "constant pool entry")) { + return true; + } + const yaml::Node *IDNode = mappingValue(*Constant, "id"); + const yaml::Node *ValueNode = mappingValue(*Constant, "value"); + if (IDNode == nullptr || ValueNode == nullptr) + return fail(Constant->range(), "constant pool entry requires 'id' and 'value'"); + std::uint64_t WideID = 0; + if (unsignedValue(*IDNode, WideID, "a 32-bit constant pool ID") || + WideID > std::numeric_limits<std::uint32_t>::max()) { + return true; + } + const auto TextID = static_cast<std::uint32_t>(WideID); + if (!IDs.insert(TextID).second) + return fail(IDNode->range(), "duplicate constant pool ID"); + + bool IsTargetSpecific = false; + if (const yaml::Node *TargetNode = mappingValue(*Constant, "isTargetSpecific")) { + if (boolean(*TargetNode, IsTargetSpecific)) + return true; + } + if (IsTargetSpecific) { + std::string_view ValueText; + if (scalar(*ValueNode, ValueText, "target constant")) + return true; + auto TargetConstantValue = Target.mirText().parseTargetConstant(ValueText); + if (!TargetConstantValue) + return importError(ValueNode->range(), TargetConstantValue.error(), "mir-target-constant"); + auto Emission = TargetConstantValue->emit(); + if (!Emission) + return importError(ValueNode->range(), Emission.error(), "mir-target-constant"); + auto RequiredAlignment = TargetConstantValue->requiredAlignment(); + if (!RequiredAlignment) + return importError(ValueNode->range(), RequiredAlignment.error(), "mir-target-constant"); + + MachineConstantPoolEntryDesc Description; + Description.Size = Emission->Bytes.size(); + Description.Align = *RequiredAlignment; + Description.Value = std::move(*TargetConstantValue); + if (const yaml::Node *AlignmentNode = mappingValue(*Constant, "alignment")) { + if (alignment(*AlignmentNode, Description.Align, true)) + return true; + } + auto Added = Function.constantPool().addEntry(std::move(Description)); + if (!Added) + return importError(Constant->range(), Added.error()); + Context.ConstantPoolItems.push_back({.TextID = TextID, .Index = *Added}); + continue; + } + + std::string_view ValueText; + if (scalar(*ValueNode, ValueText, "LLVM constant")) + return true; + const LLParserOptions ConstantOptions{ + .NamedTypes = NamedTypes, + }; + auto Data = parseLLVMConstant(trim(ValueText), ConstantOptions); + if (!Data) + return importError(ValueNode->range(), Data.error(), "mir-llvm-constant"); + auto Size = Layout->getTypeAllocSize(Data->Type); + if (!Size) + return importError(ValueNode->range(), Size.error(), "mir-llvm-constant"); + + MachineConstantPoolEntryDesc Description; + Description.Size = *Size; + Description.Value = std::move(*Data); + if (const yaml::Node *AlignmentNode = mappingValue(*Constant, "alignment")) { + if (alignment(*AlignmentNode, Description.Align, true)) + return true; + } else { + auto Preferred = + Layout->getPrefTypeAlign(std::get<GenericConstantData>(Description.Value).Type); + if (!Preferred) + return importError(ValueNode->range(), Preferred.error(), "mir-llvm-constant"); + auto PreferredAlignment = Alignment::create(*Preferred); + if (!PreferredAlignment) + return importError(ValueNode->range(), PreferredAlignment.error(), "mir-llvm-constant"); + Description.Align = *PreferredAlignment; + } + auto Added = Function.constantPool().addEntry(std::move(Description)); + if (!Added) + return importError(Constant->range(), Added.error()); + Context.ConstantPoolItems.push_back({.TextID = TextID, .Index = *Added}); + } + return false; +} + +bool MIRDocumentParser::reserveJumpTables(const yaml::Node &Root, MachineFunction &Function, + MIParserContext &Context, + std::vector<PendingJumpTable> &Pending) { + const yaml::Node *JumpTable = mappingValue(Root, "jumpTable"); + if (JumpTable == nullptr) + return false; + if (requireKind(*JumpTable, yaml::NodeKind::Mapping, "jump table")) + return true; + constexpr std::array JumpTableKeys{ + std::string_view{"kind"}, + std::string_view{"entries"}, + }; + if (validateKeys(*JumpTable, JumpTableKeys, "jump table")) + return true; + const yaml::Node *KindNode = mappingValue(*JumpTable, "kind"); + if (KindNode == nullptr) + return fail(JumpTable->range(), "jump table requires 'kind'"); + std::string_view KindName; + if (scalar(*KindNode, KindName, "jump table kind")) + return true; + + JumpTableEntryEncoding Encoding; + std::uint8_t EntrySize = 8; + if (KindName == "block-address") + Encoding = JumpTableEntryEncoding::BlockAddress; + else if (KindName == "gp-rel64-block-address") + Encoding = JumpTableEntryEncoding::GPRelative64BlockAddress; + else if (KindName == "gp-rel32-block-address") { + Encoding = JumpTableEntryEncoding::GPRelative32BlockAddress; + EntrySize = 4; + } else if (KindName == "label-difference32") { + Encoding = JumpTableEntryEncoding::LabelDifference32; + EntrySize = 4; + } else if (KindName == "label-difference64") + Encoding = JumpTableEntryEncoding::LabelDifference64; + else if (KindName == "inline") { + Encoding = JumpTableEntryEncoding::Inline; + EntrySize = 4; + } else if (KindName == "custom32") { + Encoding = JumpTableEntryEncoding::Custom32; + EntrySize = 4; + } else { + return fail(KindNode->range(), "unknown enumerated scalar"); + } + + const yaml::Node *Entries = mappingValue(*JumpTable, "entries"); + if (Entries == nullptr) + return false; + if (requireKind(*Entries, yaml::NodeKind::Sequence, "jump table entries")) + return true; + constexpr std::array EntryKeys{ + std::string_view{"id"}, + std::string_view{"blocks"}, + }; + std::unordered_set<std::uint32_t> IDs; + for (const yaml::NodeID EntryID : Entries->sequence()) { + const yaml::Node *Entry = node(EntryID); + if (Entry == nullptr || requireKind(*Entry, yaml::NodeKind::Mapping, "jump table entry") || + validateKeys(*Entry, EntryKeys, "jump table entry")) { + return true; + } + const yaml::Node *IDNode = mappingValue(*Entry, "id"); + if (IDNode == nullptr) + return fail(Entry->range(), "jump table entry requires 'id'"); + std::uint64_t WideID = 0; + if (unsignedValue(*IDNode, WideID, "a 32-bit jump table ID") || + WideID > std::numeric_limits<std::uint32_t>::max()) { + return true; + } + const auto TextID = static_cast<std::uint32_t>(WideID); + if (!IDs.insert(TextID).second) + return fail(IDNode->range(), "duplicate jump table ID"); + auto Reserved = Function.jumpTableInfo().reserveTable(); + if (!Reserved) + return importError(Entry->range(), Reserved.error()); + PendingJumpTable NewPending{ + .Index = *Reserved, + .TextID = TextID, + .Encoding = Encoding, + .EntrySize = EntrySize, + .Blocks = {}, + }; + if (const yaml::Node *Blocks = mappingValue(*Entry, "blocks")) { + if (requireKind(*Blocks, yaml::NodeKind::Sequence, "jump table blocks")) + return true; + for (const yaml::NodeID BlockID : Blocks->sequence()) { + const yaml::Node *BlockNode = node(BlockID); + std::string_view Text; + if (BlockNode == nullptr || scalar(*BlockNode, Text, "basic block reference")) + return true; + NewPending.Blocks.emplace_back(Text); + } + } + if (NewPending.Blocks.empty()) + return fail(Entry->range(), "jump table entry must contain at least one block"); + Context.JumpTables.push_back({.TextID = TextID, .Index = *Reserved}); + Pending.push_back(std::move(NewPending)); + } + return false; +} + +bool MIRDocumentParser::defineJumpTables(MachineFunction &Function, + std::span<const PendingJumpTable> Pending) { + for (const PendingJumpTable &Table : Pending) { + MachineJumpTableDesc Description; + Description.Encoding = Table.Encoding; + Description.EntrySize = Table.EntrySize; + Description.Align = *Alignment::create(Table.EntrySize); + for (const std::string &Text : Table.Blocks) { + MachineBasicBlock *Block = nullptr; + if (parseBlockReference(Text, Result.Source.range(0, 0), Function, Block)) + return true; + if (Block == nullptr) + return fail(Result.Source.range(0, 0), "reference to undefined machine basic block"); + Description.Targets.push_back(Block); + } + auto DefineResult = + Function.jumpTableInfo().defineReservedTable(Table.Index, std::move(Description)); + if (!DefineResult) + return importError(Result.Source.range(0, 0), DefineResult.error()); + } + return false; +} + +} // namespace rcc::mir + +namespace rcc::mir { +namespace { + +[[nodiscard]] bool parseIndex(std::string_view Text, std::uint64_t &Value) noexcept { + if (Text.empty()) + return false; + const auto [End, Error] = std::from_chars(Text.data(), Text.data() + Text.size(), Value, 10); + return Error == std::errc{} && End == Text.data() + Text.size(); +} + +[[nodiscard]] const auto &namedOpcodes() { + static constexpr auto Opcodes = std::to_array<std::pair<std::string_view, std::uint64_t>>({ + {"DW_OP_addr", 0x03}, + {"DW_OP_deref", 0x06}, + {"DW_OP_const1u", 0x08}, + {"DW_OP_const1s", 0x09}, + {"DW_OP_const2u", 0x0A}, + {"DW_OP_const2s", 0x0B}, + {"DW_OP_const4u", 0x0C}, + {"DW_OP_const4s", 0x0D}, + {"DW_OP_const8u", 0x0E}, + {"DW_OP_const8s", 0x0F}, + {"DW_OP_constu", 0x10}, + {"DW_OP_consts", 0x11}, + {"DW_OP_dup", 0x12}, + {"DW_OP_drop", 0x13}, + {"DW_OP_over", 0x14}, + {"DW_OP_pick", 0x15}, + {"DW_OP_swap", 0x16}, + {"DW_OP_rot", 0x17}, + {"DW_OP_xderef", 0x18}, + {"DW_OP_abs", 0x19}, + {"DW_OP_and", 0x1A}, + {"DW_OP_div", 0x1B}, + {"DW_OP_minus", 0x1C}, + {"DW_OP_mod", 0x1D}, + {"DW_OP_mul", 0x1E}, + {"DW_OP_neg", 0x1F}, + {"DW_OP_not", 0x20}, + {"DW_OP_or", 0x21}, + {"DW_OP_plus", 0x22}, + {"DW_OP_plus_uconst", 0x23}, + {"DW_OP_shl", 0x24}, + {"DW_OP_shr", 0x25}, + {"DW_OP_shra", 0x26}, + {"DW_OP_xor", 0x27}, + {"DW_OP_bra", 0x28}, + {"DW_OP_eq", 0x29}, + {"DW_OP_ge", 0x2A}, + {"DW_OP_gt", 0x2B}, + {"DW_OP_le", 0x2C}, + {"DW_OP_lt", 0x2D}, + {"DW_OP_ne", 0x2E}, + {"DW_OP_skip", 0x2F}, + {"DW_OP_regx", 0x90}, + {"DW_OP_fbreg", 0x91}, + {"DW_OP_bregx", 0x92}, + {"DW_OP_piece", 0x93}, + {"DW_OP_deref_size", 0x94}, + {"DW_OP_xderef_size", 0x95}, + {"DW_OP_nop", 0x96}, + {"DW_OP_push_object_address", 0x97}, + {"DW_OP_call2", 0x98}, + {"DW_OP_call4", 0x99}, + {"DW_OP_call_ref", 0x9A}, + {"DW_OP_form_tls_address", 0x9B}, + {"DW_OP_call_frame_cfa", 0x9C}, + {"DW_OP_bit_piece", 0x9D}, + {"DW_OP_implicit_value", 0x9E}, + {"DW_OP_stack_value", 0x9F}, + {"DW_OP_implicit_pointer", 0xA0}, + {"DW_OP_addrx", 0xA1}, + {"DW_OP_constx", 0xA2}, + {"DW_OP_entry_value", 0xA3}, + {"DW_OP_const_type", 0xA4}, + {"DW_OP_regval_type", 0xA5}, + {"DW_OP_deref_type", 0xA6}, + {"DW_OP_xderef_type", 0xA7}, + {"DW_OP_convert", 0xA8}, + {"DW_OP_reinterpret", 0xA9}, + {"DW_OP_LLVM_fragment", 0x1000}, + {"DW_OP_LLVM_convert", 0x1001}, + {"DW_OP_LLVM_tag_offset", 0x1002}, + {"DW_OP_LLVM_entry_value", 0x1003}, + {"DW_OP_LLVM_implicit_pointer", 0x1004}, + {"DW_OP_LLVM_arg", 0x1005}, + {"DW_OP_LLVM_extract_bits_sext", 0x1006}, + {"DW_OP_LLVM_extract_bits_zext", 0x1007}, + }); + return Opcodes; +} + +[[nodiscard]] const auto &attributeEncodings() { + static constexpr auto Encodings = std::to_array<std::pair<std::string_view, std::uint64_t>>({ + {"DW_ATE_address", 0x01}, + {"DW_ATE_boolean", 0x02}, + {"DW_ATE_complex_float", 0x03}, + {"DW_ATE_float", 0x04}, + {"DW_ATE_signed", 0x05}, + {"DW_ATE_signed_char", 0x06}, + {"DW_ATE_unsigned", 0x07}, + {"DW_ATE_unsigned_char", 0x08}, + {"DW_ATE_imaginary_float", 0x09}, + {"DW_ATE_packed_decimal", 0x0A}, + {"DW_ATE_numeric_string", 0x0B}, + {"DW_ATE_edited", 0x0C}, + {"DW_ATE_signed_fixed", 0x0D}, + {"DW_ATE_unsigned_fixed", 0x0E}, + {"DW_ATE_decimal_float", 0x0F}, + {"DW_ATE_UTF", 0x10}, + {"DW_ATE_UCS", 0x11}, + {"DW_ATE_ASCII", 0x12}, + }); + return Encodings; +} + +[[nodiscard]] std::optional<std::string> opcodeName(std::uint64_t Opcode) { + if (Opcode >= 0x30U && Opcode <= 0x4FU) + return "DW_OP_lit" + std::to_string(Opcode - 0x30U); + if (Opcode >= 0x50U && Opcode <= 0x6FU) + return "DW_OP_reg" + std::to_string(Opcode - 0x50U); + if (Opcode >= 0x70U && Opcode <= 0x8FU) + return "DW_OP_breg" + std::to_string(Opcode - 0x70U); + const auto &Opcodes = namedOpcodes(); + const auto *const Found = + std::ranges::find_if(Opcodes, [Opcode](const auto &Entry) { return Entry.second == Opcode; }); + if (Found == Opcodes.end()) + return std::nullopt; + return std::string(Found->first); +} + +[[nodiscard]] std::size_t opcodeSize(std::uint64_t Opcode) noexcept { + if (Opcode >= 0x70U && Opcode <= 0x8FU) + return 2; + switch (Opcode) { + case 0x92: + case 0x1000: + case 0x1001: + case 0x1006: + case 0x1007: + return 3; + case 0x10: + case 0x11: + case 0x23: + case 0x90: + case 0x94: + case 0x1002: + case 0x1003: + case 0x1005: + return 2; + default: + return 1; + } +} + +[[nodiscard]] std::optional<std::string_view> attributeName(std::uint64_t Encoding) noexcept { + const auto &Encodings = attributeEncodings(); + const auto *const Found = std::ranges::find_if( + Encodings, [Encoding](const auto &Entry) { return Entry.second == Encoding; }); + if (Found == Encodings.end()) + return std::nullopt; + return Found->first; +} + +} // namespace + +std::optional<std::uint64_t> parseDIExpressionOpcodeName(std::string_view Name) noexcept { + const auto NumberedOpcode = [Name](std::string_view Prefix, + std::uint64_t Base) -> std::optional<std::uint64_t> { + if (!Name.starts_with(Prefix)) + return std::nullopt; + std::uint64_t Index = 0; + if (!parseIndex(Name.substr(Prefix.size()), Index) || Index > 31) + return std::nullopt; + return Base + Index; + }; + if (auto Opcode = NumberedOpcode("DW_OP_lit", 0x30U)) + return Opcode; + if (auto Opcode = NumberedOpcode("DW_OP_reg", 0x50U)) + return Opcode; + if (auto Opcode = NumberedOpcode("DW_OP_breg", 0x70U)) + return Opcode; + + const auto &NamedOpcodes = namedOpcodes(); + const auto *const Found = + std::ranges::find_if(NamedOpcodes, [Name](const auto &Entry) { return Entry.first == Name; }); + if (Found == NamedOpcodes.end()) + return std::nullopt; + return Found->second; +} + +std::optional<std::uint64_t> +parseDIExpressionAttributeEncodingName(std::string_view Name) noexcept { + const auto &Encodings = attributeEncodings(); + const auto *const Found = + std::ranges::find_if(Encodings, [Name](const auto &Entry) { return Entry.first == Name; }); + if (Found == Encodings.end()) + return std::nullopt; + return Found->second; +} + +std::string invalidDIExpressionElementMessage(std::string_view Name) { + if (Name.starts_with("DW_OP_")) + return "invalid DWARF op '" + std::string(Name) + "'"; + if (Name.starts_with("DW_ATE_")) + return "invalid DWARF attribute encoding '" + std::string(Name) + "'"; + return "expected unsigned integer"; +} + +std::string formatDIExpression(std::span<const std::uint64_t> Elements) { + std::string Result = "!DIExpression("; + std::size_t Position = 0; + bool First = true; + while (Position < Elements.size()) { + const std::uint64_t Opcode = Elements[Position]; + const auto Name = opcodeName(Opcode); + const std::size_t Size = opcodeSize(Opcode); + if (!Name || Position + Size > Elements.size()) { + Result = "!DIExpression("; + for (std::size_t Index = 0; Index < Elements.size(); ++Index) { + if (Index != 0) + Result.append(", "); + Result.append(std::to_string(Elements[Index])); + } + Result.push_back(')'); + return Result; + } + + if (!First) + Result.append(", "); + First = false; + Result.append(*Name); + for (std::size_t Argument = 1; Argument < Size; ++Argument) { + Result.append(", "); + if (Opcode == 0x1001U && Argument == 2U) { + if (const auto Encoding = attributeName(Elements[Position + Argument])) { + Result.append(*Encoding); + continue; + } + } + Result.append(std::to_string(Elements[Position + Argument])); + } + Position += Size; + } + Result.push_back(')'); + return Result; +} + +} // namespace rcc::mir + +namespace rcc::mir { + +using parser_detail::parseSigned; +using parser_detail::parseUnsigned; +using parser_detail::parseUnsigned32; + +Expected<void> MetadataTextParser::expect(MITokenKind Kind, std::string_view Description) { + if (consume(Kind)) + return {}; + return makeUnexpected(ErrorCode::ParseError, + "expected " + std::string(Description) + " in MIR metadata scalar"); +} + +Expected<void> MetadataTextParser::finish() { + while (consume(MITokenKind::Newline)) { + } + if (Lexer.failed()) { + if (!Lexer.diagnostics().empty()) { + return makeUnexpected(ErrorCode::ParseError, + std::string(Lexer.diagnostics().front().Message)); + } + return makeUnexpected(ErrorCode::ParseError, "invalid MIR metadata scalar"); + } + if (!at(MITokenKind::EndOfFile)) + return makeUnexpected(ErrorCode::ParseError, "unexpected trailing MIR metadata tokens"); + return {}; +} + +Expected<MachineMetadataReference> MetadataTextParser::parseReference() { + if (auto Exclaim = expect(MITokenKind::Exclaim, "'!' before a metadata slot"); !Exclaim) + return std::unexpected(Exclaim.error()); + if (!at(MITokenKind::IntegerLiteral)) + return makeUnexpected(ErrorCode::ParseError, "expected a 32-bit metadata slot"); + std::uint32_t Slot = 0; + if (!parseUnsigned32(token().integerValue(), Slot)) + return makeUnexpected(ErrorCode::ParseError, "metadata slot is outside the 32-bit range"); + advance(); + return Resolve(Slot); +} + +Expected<MachineMetadataReference> MetadataTextParser::parseReferenceOnly() { + auto Reference = parseReference(); + if (!Reference) + return std::unexpected(Reference.error()); + if (auto Finished = finish(); !Finished) + return std::unexpected(Finished.error()); + return Reference; +} + +Expected<MachineDIExpression> MetadataTextParser::parseExpression() { + if (!at(MITokenKind::MetadataKeyword) || !token().isMetadataKeyword("!DIExpression")) { + return makeUnexpected(ErrorCode::ParseError, "expected an inline !DIExpression"); + } + advance(); + if (auto Left = expect(MITokenKind::LeftParen, "'(' after !DIExpression"); !Left) + return std::unexpected(Left.error()); + std::vector<std::uint64_t> Elements; + while (!at(MITokenKind::RightParen)) { + if (at(MITokenKind::Identifier)) { + const auto Opcode = parseDIExpressionOpcodeName(token().stringValue()); + const auto Encoding = parseDIExpressionAttributeEncodingName(token().stringValue()); + if (!Opcode && !Encoding) + return makeUnexpected(ErrorCode::ParseError, + invalidDIExpressionElementMessage(token().stringValue())); + Elements.push_back(Opcode ? *Opcode : *Encoding); + advance(); + } else if (at(MITokenKind::IntegerLiteral)) { + std::int64_t Signed = 0; + std::uint64_t Unsigned = 0; + if (parseSigned(token().integerValue(), Signed)) + Elements.push_back(static_cast<std::uint64_t>(Signed)); + else if (parseUnsigned(token().integerValue(), Unsigned)) + Elements.push_back(Unsigned); + else + return makeUnexpected(ErrorCode::ParseError, "DIExpression integer exceeds 64 bits"); + advance(); + } else { + return makeUnexpected(ErrorCode::ParseError, "expected a DIExpression opcode or integer"); + } + if (!consume(MITokenKind::Comma)) + break; + } + if (auto Right = expect(MITokenKind::RightParen, "')' after !DIExpression"); !Right) + return std::unexpected(Right.error()); + return MachineDIExpression(std::move(Elements)); +} + +Expected<MachineMetadataValue> MetadataTextParser::parseValue() { + MachineMetadataValue Value; + if (at(MITokenKind::Exclaim)) { + auto Reference = parseReference(); + if (!Reference) + return std::unexpected(Reference.error()); + auto Created = MachineMetadataValue::reference(*Reference); + if (!Created) + return std::unexpected(Created.error()); + Value = std::move(*Created); + } else { + auto Expression = parseExpression(); + if (!Expression) + return std::unexpected(Expression.error()); + Value = MachineMetadataValue::expression(std::move(*Expression)); + } + if (auto Finished = finish(); !Finished) + return std::unexpected(Finished.error()); + return Value; +} + +Expected<void> +MetadataTextParser::parseDILocation(std::vector<MachineDILocationFrame> &Frames, + std::optional<MachineMetadataReference> &Terminal) { + if (!at(MITokenKind::MetadataKeyword) || !token().isMetadataKeyword("!DILocation")) { + return makeUnexpected(ErrorCode::ParseError, "expected an inline !DILocation"); + } + advance(); + if (auto Left = expect(MITokenKind::LeftParen, "'(' after !DILocation"); !Left) + return Left; + + MachineDILocationFrame Frame; + bool HasLine = false; + bool HasScope = false; + std::vector<MachineDILocationFrame> NestedFrames; + std::optional<MachineMetadataReference> NestedTerminal; + std::unordered_set<std::string> Seen; + while (!at(MITokenKind::RightParen)) { + if (!at(MITokenKind::Identifier)) + return makeUnexpected(ErrorCode::ParseError, "expected a DILocation field name"); + const std::string Field(token().stringValue()); + if (!Seen.insert(Field).second) { + return makeUnexpected(ErrorCode::ParseError, "duplicate DILocation field '" + Field + "'"); + } + advance(); + if (auto Colon = expect(MITokenKind::Colon, "':' after a DILocation field"); !Colon) + return Colon; + + if (Field == "line") { + std::uint32_t Value = 0; + if (!at(MITokenKind::IntegerLiteral) || !parseUnsigned32(token().integerValue(), Value)) { + return makeUnexpected(ErrorCode::ParseError, + "DILocation line must be an unsigned 32-bit integer"); + } + Frame.Line = Value; + HasLine = true; + advance(); + } else if (Field == "column") { + std::uint64_t Value = 0; + if (!at(MITokenKind::IntegerLiteral) || !parseUnsigned(token().integerValue(), Value) || + Value > std::numeric_limits<std::uint16_t>::max()) { + return makeUnexpected(ErrorCode::ParseError, + "DILocation column must be an unsigned 16-bit integer"); + } + Frame.Column = static_cast<std::uint16_t>(Value); + advance(); + } else if (Field == "scope") { + auto Scope = parseReference(); + if (!Scope) + return std::unexpected(Scope.error()); + if (Scope->scope() != MachineMetadataScope::EmbeddedIR) { + return makeUnexpected(ErrorCode::ParseError, + "DILocation scope must reference embedded LLVM IR metadata"); + } + Frame.Scope = *Scope; + HasScope = true; + } else if (Field == "inlinedAt") { + if (at(MITokenKind::MetadataKeyword) && token().isMetadataKeyword("!DILocation")) { + if (auto Nested = parseDILocation(NestedFrames, NestedTerminal); !Nested) + return Nested; + } else { + auto Reference = parseReference(); + if (!Reference) + return std::unexpected(Reference.error()); + if (Reference->scope() != MachineMetadataScope::EmbeddedIR) { + return makeUnexpected(ErrorCode::ParseError, + "DILocation inlinedAt must reference embedded LLVM IR metadata"); + } + NestedTerminal = *Reference; + } + } else if (Field == "isImplicitCode") { + if (!at(MITokenKind::Identifier) || + (token().stringValue() != "true" && token().stringValue() != "false")) { + return makeUnexpected(ErrorCode::ParseError, + "DILocation isImplicitCode must be true or false"); + } + Frame.IsImplicitCode = token().stringValue() == "true"; + advance(); + } else if (Field == "atomGroup") { + std::uint64_t Value = 0; + if (!at(MITokenKind::IntegerLiteral) || !parseUnsigned(token().integerValue(), Value) || + Value > MachineDILocation::MaxAtomGroup) { + return makeUnexpected(ErrorCode::ParseError, "DILocation atomGroup exceeds 61 bits"); + } + Frame.AtomGroup = Value; + advance(); + } else if (Field == "atomRank") { + std::uint64_t Value = 0; + if (!at(MITokenKind::IntegerLiteral) || !parseUnsigned(token().integerValue(), Value) || + Value > 7) { + return makeUnexpected(ErrorCode::ParseError, "DILocation atomRank exceeds three bits"); + } + Frame.AtomRank = static_cast<std::uint8_t>(Value); + advance(); + } else { + return makeUnexpected(ErrorCode::ParseError, "unknown DILocation field '" + Field + "'"); + } + if (!consume(MITokenKind::Comma)) + break; + } + if (auto Right = expect(MITokenKind::RightParen, "')' after !DILocation"); !Right) + return Right; + if (!HasLine || !HasScope) + return makeUnexpected(ErrorCode::ParseError, "DILocation requires line and scope fields"); + Frames.push_back(Frame); + Frames.insert(Frames.end(), NestedFrames.begin(), NestedFrames.end()); + Terminal = NestedTerminal; + return {}; +} + +Expected<MachineDebugLocation> MetadataTextParser::parseLocation() { + MachineDebugLocation Location; + if (at(MITokenKind::Exclaim)) { + auto Reference = parseReference(); + if (!Reference) + return std::unexpected(Reference.error()); + auto Created = MachineDebugLocation::reference(*Reference); + if (!Created) + return std::unexpected(Created.error()); + Location = std::move(*Created); + } else { + std::vector<MachineDILocationFrame> Frames; + std::optional<MachineMetadataReference> Terminal; + if (auto Parsed = parseDILocation(Frames, Terminal); !Parsed) + return std::unexpected(Parsed.error()); + auto Created = MachineDILocation::create(std::move(Frames), Terminal); + if (!Created) + return std::unexpected(Created.error()); + Location = MachineDebugLocation::inlineLocation(std::move(*Created)); + } + if (auto Finished = finish(); !Finished) + return std::unexpected(Finished.error()); + return Location; +} + +} // namespace rcc::mir + +namespace rcc::mir { + +Expected<void> verifyMIRDocumentReferences(const MIRDocument &Document, + const MachineFunction &Function, bool UseEmbeddedIR) { + const EmbeddedIRReferenceIndex *Embedded = UseEmbeddedIR ? Document.embeddedIR() : nullptr; + const EmbeddedIRFunction *IRFunction = nullptr; + if (Embedded != nullptr) { + if (Function.name().empty()) { + std::size_t AnonymousOrdinal = 0; + bool FoundFunction = false; + for (std::size_t Index = 0; Index < Document.unit().functionCount(); ++Index) { + const MachineFunction *Candidate = Document.unit().functionAt(Index); + if (Candidate == &Function) { + FoundFunction = true; + break; + } + if (Candidate != nullptr && Candidate->name().empty()) + ++AnonymousOrdinal; + } + if (FoundFunction) + IRFunction = Embedded->unnamedFunction(AnonymousOrdinal); + } else { + IRFunction = Embedded->function(Function.name()); + } + if (IRFunction == nullptr && !Function.attributes().has(MachineFunctionAttribute::IsOutlined)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine function has no matching embedded LLVM IR function"); + } + } + + const auto ValidateGlobal = + [&Document, Embedded](std::string_view Name, std::string_view Description) -> Expected<void> { + if (Document.unit().function(Name) != nullptr) + return {}; + if (Embedded == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + + " requires embedded LLVM IR to resolve global '" + + std::string(Name) + "'"); + } + if (Embedded->hasGlobalValue(Name)) + return {}; + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + + " references an undefined embedded LLVM IR global '" + + std::string(Name) + "'"); + }; + const auto ValidateBlockAddress = [Embedded](std::string_view FunctionName, + std::string_view BlockName, + std::string_view Description) -> Expected<void> { + if (Embedded == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + + " requires embedded LLVM IR to resolve a block address"); + } + const EmbeddedIRFunction *AddressFunction = Embedded->function(FunctionName); + if (AddressFunction == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + + " references an undefined embedded LLVM IR function '" + + std::string(FunctionName) + "'"); + } + if (AddressFunction->block(BlockName) == nullptr) { + return makeUnexpected( + ErrorCode::VerificationError, + std::string(Description) + " references an undefined embedded LLVM IR block '" + + std::string(BlockName) + "' in function '" + std::string(FunctionName) + "'"); + } + return {}; + }; + const auto ValidateIRValue = [IRFunction](std::string_view Name, + std::string_view Description) -> Expected<void> { + if (IRFunction == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + + " requires embedded LLVM IR to resolve IR value '%" + + std::string(Name) + "'"); + } + if (IRFunction->value(Name) != nullptr) + return {}; + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + + " references an undefined embedded LLVM IR value '%" + + std::string(Name) + "'"); + }; + const auto ValidateMetadata = [Embedded](MachineMetadataReference Reference, + std::initializer_list<std::string_view> ExpectedTypes, + std::string_view Description) -> Expected<void> { + if (!Reference.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + " has an invalid metadata reference"); + } + if (Reference.scope() != MachineMetadataScope::EmbeddedIR) + return {}; + if (Embedded == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + " requires embedded LLVM IR metadata !" + + std::to_string(Reference.slot())); + } + const EmbeddedIRMetadataDefinition *Definition = + Embedded->metadata(std::to_string(Reference.slot())); + if (Definition == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + + " references undefined embedded LLVM IR metadata !" + + std::to_string(Reference.slot())); + } + if (ExpectedTypes.size() == 0 || + std::ranges::find(ExpectedTypes, Definition->DynamicType) != ExpectedTypes.end()) { + return {}; + } + std::string Expected; + for (const std::string_view Type : ExpectedTypes) { + if (!Expected.empty()) + Expected.append(" or "); + Expected.append(Type); + } + return makeUnexpected(ErrorCode::VerificationError, + std::string(Description) + " requires " + Expected + " metadata, but !" + + std::to_string(Reference.slot()) + " is " + + (Definition->DynamicType.empty() + ? std::string("an unclassified metadata node") + : Definition->DynamicType)); + }; + const auto ValidateMetadataValue = + [&ValidateMetadata](const MachineMetadataValue &Value, + std::string_view Description) -> Expected<void> { + const MachineMetadataReference *Reference = Value.reference(); + return Reference == nullptr ? Expected<void>{} : ValidateMetadata(*Reference, {}, Description); + }; + const auto ValidateLocation = + [&ValidateMetadata](const MachineDebugLocation &Location, + std::string_view Description) -> Expected<void> { + if (const MachineMetadataReference *Reference = Location.reference()) + return ValidateMetadata(*Reference, {"DILocation"}, Description); + const MachineDILocation *Inline = Location.inlineLocation(); + if (Inline == nullptr) + return {}; + for (const MachineDILocationFrame &Frame : Inline->inlineChain()) { + auto Verification = + ValidateMetadata(Frame.Scope, {"DISubprogram", "DILexicalBlock", "DILexicalBlockFile"}, + "inline DILocation scope"); + if (!Verification) + return Verification; + } + if (Inline->terminalInlinedAt()) { + return ValidateMetadata(*Inline->terminalInlinedAt(), {"DILocation"}, + "inline DILocation inlinedAt"); + } + return {}; + }; + const auto ValidateVariableDebugInfo = + [&ValidateMetadata, &ValidateMetadataValue, + &ValidateLocation](const MachineVariableDebugInfo &DebugInfo) -> Expected<void> { + if (DebugInfo.Variable) { + auto Verification = + ValidateMetadata(*DebugInfo.Variable, {"DILocalVariable"}, "debug variable"); + if (!Verification) + return Verification; + } + if (DebugInfo.Expression) { + if (const MachineMetadataReference *Reference = DebugInfo.Expression->reference(); + Reference != nullptr) { + auto Verification = ValidateMetadata(*Reference, {"DIExpression"}, "debug expression"); + if (!Verification) + return Verification; + } + auto Verification = ValidateMetadataValue(*DebugInfo.Expression, "debug expression"); + if (!Verification) + return Verification; + } + if (DebugInfo.Location) + return ValidateLocation(*DebugInfo.Location, "debug location"); + return {}; + }; + const auto ValidateGenericType = [Embedded](const auto &Self, + const GenericConstantType &Type) -> Expected<void> { + if (Type.Kind == GenericConstantTypeKind::Named) { + if (Embedded == nullptr) { + return makeUnexpected( + ErrorCode::VerificationError, + "generic constant type requires embedded LLVM IR to resolve named type '%" + Type.Name + + "'"); + } + if (Embedded->typeDefinition(Type.Name) == nullptr) { + return makeUnexpected( + ErrorCode::VerificationError, + "generic constant type references an undefined embedded LLVM IR named type '%" + + Type.Name + "'"); + } + } + for (const GenericConstantType &Element : Type.Elements) { + auto Verification = Self(Self, Element); + if (!Verification) + return Verification; + } + return {}; + }; + const auto ValidateGenericConstant = + [&ValidateBlockAddress, &ValidateGenericType, + &ValidateGlobal](const auto &Self, const GenericConstantData &Constant) -> Expected<void> { + auto TypeVerification = ValidateGenericType(ValidateGenericType, Constant.Type); + if (!TypeVerification) + return TypeVerification; + if (Constant.SourceElementType) { + TypeVerification = ValidateGenericType(ValidateGenericType, *Constant.SourceElementType); + if (!TypeVerification) + return TypeVerification; + } + switch (Constant.Kind) { + case GenericConstantKind::GlobalAddress: + case GenericConstantKind::DSOLocalEquivalent: + case GenericConstantKind::NoCFI: + if (auto Verification = ValidateGlobal(Constant.Symbol, "generic constant"); !Verification) + return Verification; + break; + case GenericConstantKind::BlockAddress: + if (auto Verification = ValidateBlockAddress(Constant.Symbol, Constant.Block, + "generic constant blockaddress"); + !Verification) { + return Verification; + } + break; + default: + break; + } + for (const GenericConstantData &Element : Constant.Elements) { + auto Verification = Self(Self, Element); + if (!Verification) + return Verification; + } + return {}; + }; + + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + if (const auto Name = Block->attributes().irBlockName(); Name) { + if (IRFunction == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine basic block requires embedded LLVM IR to resolve block '" + + std::string(*Name) + "'"); + } + if (IRFunction->block(*Name) == nullptr) { + return makeUnexpected( + ErrorCode::VerificationError, + "machine basic block references an undefined embedded LLVM IR block '" + + std::string(*Name) + "'"); + } + } + } + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + if (Description == nullptr || !Description->AllocaName) + continue; + if (IRFunction == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "stack object requires embedded LLVM IR to resolve alloca '" + + *Description->AllocaName + "'"); + } + if (IRFunction->alloca(*Description->AllocaName) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "stack object references an undefined embedded LLVM IR alloca '" + + *Description->AllocaName + "'"); + } + } + + for (const MachineConstantPoolEntry &Entry : Function.constantPool().entries()) { + const MachineConstantPoolEntryDesc *Description = Entry.description(); + if (Description == nullptr) + continue; + if (const auto *Generic = std::get_if<GenericConstantData>(&Description->Value)) { + auto Verification = ValidateGenericConstant(ValidateGenericConstant, *Generic); + if (!Verification) + return Verification; + } else if (const auto *Address = std::get_if<BlockAddressConstant>(&Description->Value)) { + auto Verification = ValidateBlockAddress(Address->FunctionName, Address->BlockName, + "constant-pool block address"); + if (!Verification) + return Verification; + } + } + + for (const MachineMetadataNode &Node : Function.metadata().nodes()) { + for (const MachineMetadataElement &Element : Node.Elements) { + const MachineMetadataReference *Reference = Element.reference(); + if (Reference == nullptr) + continue; + auto Verification = ValidateMetadata(*Reference, {}, "machine metadata node"); + if (!Verification) + return Verification; + } + } + + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + if (Description == nullptr || !Description->DebugInfo) + continue; + auto Verification = ValidateVariableDebugInfo(*Description->DebugInfo); + if (!Verification) + return Verification; + } + for (const MachineEntryValueInfo &Entry : Function.records().entryValues()) { + auto Verification = ValidateVariableDebugInfo(Entry.DebugInfo); + if (!Verification) + return Verification; + } + for (const MachineCalledGlobal &Called : Function.records().calledGlobals()) { + auto Verification = ValidateGlobal(Called.Callee, "called-global record"); + if (!Verification) + return Verification; + } + + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Instruction->debugLocation()) { + auto Verification = + ValidateLocation(*Instruction->debugLocation(), "instruction debug location"); + if (!Verification) + return Verification; + } + const std::array InstructionMetadata{ + Instruction->heapAllocMarker(), + Instruction->pcSections(), + Instruction->mmra(), + }; + for (const auto &Reference : InstructionMetadata) { + if (!Reference) + continue; + auto Verification = ValidateMetadata(*Reference, {}, "instruction metadata attachment"); + if (!Verification) + return Verification; + } + + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Global = Operand.getIf<GlobalSymbolOperand>(); Global != nullptr) { + auto Verification = ValidateGlobal(Global->Name, "global-symbol operand"); + if (!Verification) + return Verification; + } + if (const auto *Address = Operand.getIf<BlockAddressOperand>(); Address != nullptr) { + auto Verification = ValidateBlockAddress(Address->FunctionName, Address->BlockName, + "block-address operand"); + if (!Verification) + return Verification; + } + if (const auto *Metadata = Operand.getIf<MetadataOperand>(); Metadata != nullptr) { + auto Verification = ValidateMetadataValue(Metadata->Value, "metadata operand"); + if (!Verification) + return Verification; + } + } + + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + const MachineMemOperandDesc &Description = Memory->description(); + const std::optional<std::string_view> BaseName = Description.Base.symbolName(); + switch (Description.Base.kind()) { + case MemoryBase::Kind::GlobalSymbol: + case MemoryBase::Kind::CallEntryGlobal: + if (BaseName) { + auto Verification = ValidateGlobal(*BaseName, "machine memory base"); + if (!Verification) + return Verification; + } + break; + case MemoryBase::Kind::IRValue: + if (BaseName) { + auto Verification = ValidateIRValue(*BaseName, "machine memory base"); + if (!Verification) + return Verification; + } + break; + default: + break; + } + const std::array MetadataReferences{ + Description.TBAA, Description.AliasScope, + Description.NoAlias, Description.NoAliasAddressSpace, + Description.RangeMetadata, + }; + for (const auto &Reference : MetadataReferences) { + if (!Reference) + continue; + auto Verification = ValidateMetadata(*Reference, {}, "machine memory operand metadata"); + if (!Verification) + return Verification; + } + if (Description.SyncScope.value() == 0) + continue; + const auto Scope = + std::ranges::find_if(Document.syncScopes(), [&Description](const auto &Slot) { + return Slot.ID == Description.SyncScope; + }); + if (Scope == Document.syncScopes().end()) { + return makeUnexpected( + ErrorCode::VerificationError, + "machine memory operand synchronization scope has no document name"); + } + } + } + } + return {}; +} + +} // namespace rcc::mir + +namespace rcc::mir { + +using parser_detail::parseBool; +using parser_detail::parsePrefixedID; +using parser_detail::parseSigned; +using parser_detail::parseUnsigned; +using parser_detail::parseUnsigned32; +using parser_detail::trim; + +bool MIRDocumentParser::parseStackObjects(const yaml::Node &Root, MachineFunction &Function, + MIParserContext &Context) { + auto ParseList = [this, &Function, &Context](const yaml::Node &List, bool Fixed) -> bool { + if (requireKind(List, yaml::NodeKind::Sequence, + Fixed ? "fixed stack objects" : "stack objects")) { + return true; + } + constexpr std::array FixedKeys{ + std::string_view{"id"}, + std::string_view{"type"}, + std::string_view{"offset"}, + std::string_view{"size"}, + std::string_view{"alignment"}, + std::string_view{"stack-id"}, + std::string_view{"isImmutable"}, + std::string_view{"isAliased"}, + std::string_view{"callee-saved-register"}, + std::string_view{"callee-saved-restored"}, + std::string_view{"debug-info-variable"}, + std::string_view{"debug-info-expression"}, + std::string_view{"debug-info-location"}, + }; + constexpr std::array StackKeys{ + std::string_view{"id"}, + std::string_view{"name"}, + std::string_view{"type"}, + std::string_view{"offset"}, + std::string_view{"size"}, + std::string_view{"alignment"}, + std::string_view{"stack-id"}, + std::string_view{"callee-saved-register"}, + std::string_view{"callee-saved-restored"}, + std::string_view{"local-offset"}, + std::string_view{"debug-info-variable"}, + std::string_view{"debug-info-expression"}, + std::string_view{"debug-info-location"}, + }; + std::unordered_set<std::uint32_t> IDs; + for (const yaml::NodeID ElementID : List.sequence()) { + const yaml::Node *Element = node(ElementID); + if (Element == nullptr || requireKind(*Element, yaml::NodeKind::Mapping, "stack object") || + (Fixed ? validateKeys(*Element, FixedKeys, "fixed stack object") + : validateKeys(*Element, StackKeys, "stack object"))) { + return true; + } + const yaml::Node *IDNode = mappingValue(*Element, "id"); + if (IDNode == nullptr) + return fail(Element->range(), "stack object requires 'id'"); + std::uint64_t WideID = 0; + if (unsignedValue(*IDNode, WideID, "a 32-bit stack object ID") || + WideID > std::numeric_limits<std::uint32_t>::max()) { + return true; + } + const auto TextID = static_cast<std::uint32_t>(WideID); + if (!IDs.insert(TextID).second) + return fail(IDNode->range(), "duplicate stack object ID"); + + StackObjectDesc Description; + Description.Kind = Fixed ? StackObjectKind::FixedABI : StackObjectKind::Default; + Description.Offset = 0; + if (const yaml::Node *NameNode = mappingValue(*Element, "name")) { + std::string_view Name; + if (scalar(*NameNode, Name, "stack object name")) + return true; + if (!Name.empty()) + Description.AllocaName = std::string(Name); + } + if (const yaml::Node *TypeNode = mappingValue(*Element, "type")) { + std::string_view Type; + if (scalar(*TypeNode, Type, "stack object type")) + return true; + if (Type == "default") + Description.Kind = Fixed ? StackObjectKind::FixedABI : StackObjectKind::Default; + else if (Type == "spill-slot") + Description.Kind = Fixed ? StackObjectKind::FixedSpillSlot : StackObjectKind::SpillSlot; + else if (!Fixed && Type == "variable-sized") + Description.Kind = StackObjectKind::VariableSized; + else + return fail(TypeNode->range(), "unknown stack object type '" + std::string(Type) + "'"); + } + if (const yaml::Node *OffsetNode = mappingValue(*Element, "offset")) { + std::int64_t Offset = 0; + if (signedValue(*OffsetNode, Offset, "a signed stack object offset")) + return true; + Description.Offset = Offset; + } else if (Fixed) { + Description.Offset = 0; + } + if (Description.Kind != StackObjectKind::VariableSized) { + const yaml::Node *SizeNode = mappingValue(*Element, "size"); + if (SizeNode == nullptr) { + if (!Fixed) + return fail(Element->range(), "fixed-size stack object requires 'size'"); + Description.Size = 0; + } else { + std::uint64_t Size = 0; + if (unsignedValue(*SizeNode, Size, "an unsigned stack object size")) + return true; + Description.Size = Size; + } + } + if (const yaml::Node *AlignmentNode = mappingValue(*Element, "alignment")) { + if (alignment(*AlignmentNode, Description.Align, true)) + return true; + } + if (const yaml::Node *StackIDNode = mappingValue(*Element, "stack-id")) { + std::string_view Name; + if (scalar(*StackIDNode, Name, "stack ID")) + return true; + const auto Stack = parseStackIDName(Name); + if (!Stack) + return fail(StackIDNode->range(), "unknown stack ID '" + std::string(Name) + "'"); + Description.Stack = *Stack; + } + if (const yaml::Node *ImmutableNode = mappingValue(*Element, "isImmutable")) { + if (boolean(*ImmutableNode, Description.IsImmutable)) + return true; + } + if (const yaml::Node *AliasedNode = mappingValue(*Element, "isAliased")) { + if (boolean(*AliasedNode, Description.IsAliased)) + return true; + } + if (const yaml::Node *SavedNode = mappingValue(*Element, "callee-saved-register")) { + std::string_view SavedName; + if (scalar(*SavedNode, SavedName, "physical register")) + return true; + if (!trim(SavedName).empty()) { + PhysicalRegister Reg; + if (physicalRegister(SavedName, SavedNode->range(), Reg)) + return true; + Description.CalleeSavedRegister = Reg; + } + } + if (const yaml::Node *RestoredNode = mappingValue(*Element, "callee-saved-restored")) { + if (boolean(*RestoredNode, Description.CalleeSavedRestored)) + return true; + } + if (const yaml::Node *LocalNode = mappingValue(*Element, "local-offset")) { + std::int64_t Offset = 0; + if (signedValue(*LocalNode, Offset, "a signed local frame offset")) + return true; + Description.LocalOffset = Offset; + } + MachineVariableDebugInfo DebugInfo; + if (parseVariableDebugInfo(*Element, Function, DebugInfo)) + return true; + if (DebugInfo.Variable || DebugInfo.Expression || DebugInfo.Location) + Description.DebugInfo = std::move(DebugInfo); + + auto Created = Function.frameInfo().createObject(std::move(Description)); + if (!Created) + return importError(Element->range(), Created.error()); + MIFrameObjectSlot Slot{ + .TextID = TextID, + .Index = *Created, + .Name = std::nullopt, + }; + if (const StackObjectDesc *CreatedDescription = + Function.frameInfo().object(*Created)->description(); + CreatedDescription != nullptr && CreatedDescription->AllocaName) { + Slot.Name = *CreatedDescription->AllocaName; + } + (Fixed ? Context.FixedStackObjects : Context.StackObjects).push_back(std::move(Slot)); + } + return false; + }; + + if (const yaml::Node *Fixed = mappingValue(Root, "fixedStack")) { + if (ParseList(*Fixed, true)) + return true; + } + if (const yaml::Node *Stack = mappingValue(Root, "stack")) { + if (ParseList(*Stack, false)) + return true; + } + return false; +} + +bool MIRDocumentParser::parseFrameReference(const yaml::Node &Node, const MIParserContext &Context, + FrameIndex &Index) { + std::string_view Text; + if (scalar(Node, Text, "frame index")) + return true; + bool Fixed = false; + std::optional<std::uint32_t> ID = parsePrefixedID(Text, "%stack."); + if (!ID) { + ID = parsePrefixedID(Text, "%fixed-stack."); + Fixed = true; + } + if (!ID) + return fail(Node.range(), "frame index must use '%stack.N' or '%fixed-stack.N'"); + const auto &Slots = Fixed ? Context.FixedStackObjects : Context.StackObjects; + const auto Found = std::ranges::find_if( + Slots, [ID](const MIFrameObjectSlot &Slot) { return Slot.TextID == *ID; }); + if (Found == Slots.end()) + return fail(Node.range(), "reference to undefined frame object"); + Index = Found->Index; + return false; +} + +MachineBasicBlock *MIRDocumentParser::blockAt(MachineFunction &Function, + std::uint32_t Number) const noexcept { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (Block->denseNumber() == Number) + return Block; + } + return nullptr; +} + +MachineInstr *MIRDocumentParser::instructionAt(MachineFunction &Function, std::uint32_t BlockNumber, + std::uint32_t Offset) const noexcept { + MachineBasicBlock *Block = blockAt(Function, BlockNumber); + if (Block == nullptr) + return nullptr; + MachineInstr *Instruction = Block->firstInstruction(); + for (std::uint32_t Index = 0; Instruction != nullptr && Index < Offset; ++Index) + Instruction = Instruction->next(); + return Instruction; +} + +bool MIRDocumentParser::parseBlockReference(std::string_view Text, const TextRange &Range, + MachineFunction &Function, MachineBasicBlock *&Block) { + const auto Number = parsePrefixedID(Text, "%bb."); + if (!Number) + return fail(Range, "basic block reference must use the '%bb.N' MIR spelling"); + Block = blockAt(Function, *Number); + if (Block == nullptr) + return fail(Range, "reference to undefined machine basic block"); + return false; +} + +bool MIRDocumentParser::parseSaveRestorePoints(const yaml::Node &Node, MachineFunction &Function, + std::vector<MachineFrameSaveRestorePoint> &Points) { + if (requireKind(Node, yaml::NodeKind::Sequence, "frame save/restore points")) + return true; + constexpr std::array Keys{ + std::string_view{"point"}, + std::string_view{"registers"}, + }; + for (const yaml::NodeID PointID : Node.sequence()) { + const yaml::Node *PointNode = node(PointID); + if (PointNode == nullptr || + requireKind(*PointNode, yaml::NodeKind::Mapping, "frame save/restore point") || + validateKeys(*PointNode, Keys, "frame save/restore point")) { + return true; + } + const yaml::Node *BlockNode = mappingValue(*PointNode, "point"); + if (BlockNode == nullptr) + return fail(PointNode->range(), "frame save/restore point requires 'point'"); + std::string_view BlockText; + MachineBasicBlock *Block = nullptr; + if (scalar(*BlockNode, BlockText, "basic block reference") || + parseBlockReference(BlockText, BlockNode->range(), Function, Block)) { + return true; + } + if (Block == nullptr) + return fail(BlockNode->range(), "reference to undefined machine basic block"); + MachineFrameSaveRestorePoint Point{.Block = Block->stableID(), .Registers = {}}; + if (const yaml::Node *RegistersNode = mappingValue(*PointNode, "registers")) { + if (requireKind(*RegistersNode, yaml::NodeKind::Sequence, "save/restore point registers")) { + return true; + } + for (const yaml::NodeID RegID : RegistersNode->sequence()) { + const yaml::Node *RegNode = node(RegID); + PhysicalRegister Reg; + if (RegNode == nullptr || physicalRegister(*RegNode, Reg)) + return true; + Point.Registers.push_back(Reg); + } + } + Points.push_back(std::move(Point)); + } + return false; +} + +bool MIRDocumentParser::parseFrameState(const yaml::Node &Root, MachineFunction &Function, + const MIParserContext &Context) { + const yaml::Node *Frame = mappingValue(Root, "frameInfo"); + if (Frame == nullptr) + return false; + if (requireKind(*Frame, yaml::NodeKind::Mapping, "frameInfo")) + return true; + constexpr std::array Keys{ + std::string_view{"isFrameAddressTaken"}, + std::string_view{"isReturnAddressTaken"}, + std::string_view{"hasStackMap"}, + std::string_view{"hasPatchPoint"}, + std::string_view{"stackSize"}, + std::string_view{"offsetAdjustment"}, + std::string_view{"maxAlignment"}, + std::string_view{"adjustsStack"}, + std::string_view{"hasCalls"}, + std::string_view{"framePointerPolicy"}, + std::string_view{"stackProtector"}, + std::string_view{"functionContext"}, + std::string_view{"maxCallFrameSize"}, + std::string_view{"cvBytesOfCalleeSavedRegisters"}, + std::string_view{"hasOpaqueSPAdjustment"}, + std::string_view{"hasVAStart"}, + std::string_view{"hasMustTailInVarArgFunc"}, + std::string_view{"hasTailCall"}, + std::string_view{"isCalleeSavedInfoValid"}, + std::string_view{"localFrameSize"}, + std::string_view{"savePoint"}, + std::string_view{"restorePoint"}, + }; + if (validateKeys(*Frame, Keys, "frameInfo")) + return true; + + MachineFrameState State = Function.frameInfo().state(); + const std::array BoolBindings{ + std::pair{"isFrameAddressTaken", &State.IsFrameAddressTaken}, + std::pair{"isReturnAddressTaken", &State.IsReturnAddressTaken}, + std::pair{"hasStackMap", &State.HasStackMap}, + std::pair{"hasPatchPoint", &State.HasPatchPoint}, + std::pair{"adjustsStack", &State.AdjustsStack}, + std::pair{"hasCalls", &State.HasCalls}, + std::pair{"hasOpaqueSPAdjustment", &State.HasOpaqueSPAdjustment}, + std::pair{"hasVAStart", &State.HasVAStart}, + std::pair{"hasMustTailInVarArgFunc", &State.HasMustTailInVarArgFunc}, + std::pair{"hasTailCall", &State.HasTailCall}, + std::pair{"isCalleeSavedInfoValid", &State.IsCalleeSavedInfoValid}, + }; + for (const auto &[Key, Destination] : BoolBindings) { + if (const yaml::Node *Value = mappingValue(*Frame, Key)) { + if (boolean(*Value, *Destination)) + return true; + } + } + + if (const yaml::Node *Value = mappingValue(*Frame, "stackSize")) { + if (unsignedValue(*Value, State.StackSize, "an unsigned stack size")) + return true; + } + if (const yaml::Node *Value = mappingValue(*Frame, "offsetAdjustment")) { + std::int64_t Wide = 0; + if (signedValue(*Value, Wide, "a 32-bit signed offset adjustment") || + Wide < std::numeric_limits<std::int32_t>::min() || + Wide > std::numeric_limits<std::int32_t>::max()) { + return fail(Value->range(), "offset adjustment is outside the 32-bit signed range"); + } + State.OffsetAdjustment = static_cast<std::int32_t>(Wide); + } + if (const yaml::Node *Value = mappingValue(*Frame, "maxAlignment")) { + std::uint64_t Bytes = 0; + if (unsignedValue(*Value, Bytes, "an unsigned maximum alignment")) + return true; + if (Bytes == 0) { + State.MaxAlignment.reset(); + } else { + auto Parsed = Alignment::create(Bytes); + if (!Parsed) + return importError(Value->range(), Parsed.error()); + State.MaxAlignment = *Parsed; + } + } + if (const yaml::Node *Value = mappingValue(*Frame, "framePointerPolicy")) { + std::string_view Name; + if (scalar(*Value, Name, "frame-pointer policy")) + return true; + const auto Policy = parseMachineFramePointerPolicy(Name); + if (!Policy) + return fail(Value->range(), "unknown frame-pointer policy '" + std::string(Name) + "'"); + State.FramePointerPolicy = *Policy; + } + for (const auto &[Key, Destination] : {std::pair{"stackProtector", &State.StackProtector}, + std::pair{"functionContext", &State.FunctionContext}}) { + if (const yaml::Node *Value = mappingValue(*Frame, Key)) { + std::string_view Reference; + if (scalar(*Value, Reference, "frame index")) + return true; + if (Reference.empty()) { + Destination->reset(); + continue; + } + FrameIndex Index; + if (parseFrameReference(*Value, Context, Index)) + return true; + *Destination = Index; + } + } + if (const yaml::Node *Value = mappingValue(*Frame, "maxCallFrameSize")) { + std::uint64_t Size = 0; + if (unsignedValue(*Value, Size, "a 32-bit unsigned maximum call-frame size") || + Size > std::numeric_limits<std::uint32_t>::max()) { + return fail(Value->range(), "maximum call-frame size is outside the 32-bit unsigned range"); + } + if (Size == std::numeric_limits<std::uint32_t>::max()) + State.MaxCallFrameSize.reset(); + else + State.MaxCallFrameSize = Size; + } + for (const auto &[Key, Destination] : + {std::pair{"cvBytesOfCalleeSavedRegisters", &State.CVBytesOfCalleeSavedRegisters}, + std::pair{"localFrameSize", &State.LocalFrameSize}}) { + if (const yaml::Node *Value = mappingValue(*Frame, Key)) { + std::uint64_t Wide = 0; + if (unsignedValue(*Value, Wide, "a 32-bit unsigned frame value") || + Wide > std::numeric_limits<std::uint32_t>::max()) { + return fail(Value->range(), "frame value is outside the 32-bit unsigned range"); + } + *Destination = static_cast<std::uint32_t>(Wide); + } + } + if (const yaml::Node *Value = mappingValue(*Frame, "savePoint")) { + if (parseSaveRestorePoints(*Value, Function, State.SavePoints)) + return true; + } + if (const yaml::Node *Value = mappingValue(*Frame, "restorePoint")) { + if (parseSaveRestorePoints(*Value, Function, State.RestorePoints)) + return true; + } + auto StateResult = Function.frameInfo().setState(std::move(State)); + if (!StateResult) + return importError(Frame->range(), StateResult.error()); + return false; +} + +} // namespace rcc::mir + +namespace rcc::mir { + +using parser_detail::parseBool; +using parser_detail::parsePrefixedID; +using parser_detail::parseSigned; +using parser_detail::parseUnsigned; +using parser_detail::parseUnsigned32; +using parser_detail::trim; + +bool MIRDocumentParser::parseFunctionScalars(const yaml::Node &Root, MachineFunction &Function, + MIRFunctionDocumentInfo &DocumentInfo) { + if (const yaml::Node *AlignNode = mappingValue(Root, "alignment")) { + Alignment Align; + if (alignment(*AlignNode, Align, true)) + return true; + if (Align.bytes() != 1 || AlignNode->scalar() != "0") + Function.attributes().setAlignment(Align); + } + + constexpr std::array PropertyBindings{ + std::pair{"legalized", MachineFunctionProperty::Legalized}, + std::pair{"regBankSelected", MachineFunctionProperty::RegBankSelected}, + std::pair{"selected", MachineFunctionProperty::Selected}, + std::pair{"failedISel", MachineFunctionProperty::FailedISel}, + std::pair{"tracksRegLiveness", MachineFunctionProperty::TracksLiveness}, + std::pair{"noPhis", MachineFunctionProperty::NoPHIs}, + std::pair{"isSSA", MachineFunctionProperty::IsSSA}, + std::pair{"noVRegs", MachineFunctionProperty::NoVRegs}, + std::pair{"failsVerification", MachineFunctionProperty::FailsVerification}, + }; + for (const auto &[Key, Property] : PropertyBindings) { + const yaml::Node *ValueNode = mappingValue(Root, Key); + if (ValueNode == nullptr) + continue; + bool Value = false; + if (boolean(*ValueNode, Value)) + return true; + if (Value) + Function.properties().set(Property); + else + Function.properties().clear(Property); + if (Key == std::string_view("noPhis")) + DocumentInfo.NoPHIs = Value; + else if (Key == std::string_view("isSSA")) + DocumentInfo.IsSSA = Value; + else if (Key == std::string_view("noVRegs")) + DocumentInfo.NoVRegs = Value; + } + + constexpr std::array AttributeBindings{ + std::pair{"exposesReturnsTwice", MachineFunctionAttribute::ExposesReturnsTwice}, + std::pair{"hasWinCFI", MachineFunctionAttribute::HasWinCFI}, + std::pair{"hasFakeUses", MachineFunctionAttribute::HasFakeUses}, + std::pair{"callsEHReturn", MachineFunctionAttribute::CallsEHReturn}, + std::pair{"callsUnwindInit", MachineFunctionAttribute::CallsUnwindInit}, + std::pair{"hasEHContTarget", MachineFunctionAttribute::HasEHContTarget}, + std::pair{"hasEHScopes", MachineFunctionAttribute::HasEHScopes}, + std::pair{"hasEHFunclets", MachineFunctionAttribute::HasEHFunclets}, + std::pair{"isOutlined", MachineFunctionAttribute::IsOutlined}, + std::pair{"debugInstrRef", MachineFunctionAttribute::DebugInstrRef}, + std::pair{"tracksDebugUserValues", MachineFunctionAttribute::TracksDebugUserValues}, + }; + for (const auto &[Key, Attribute] : AttributeBindings) { + const yaml::Node *ValueNode = mappingValue(Root, Key); + if (ValueNode == nullptr) + continue; + bool Value = false; + if (boolean(*ValueNode, Value)) + return true; + if (Value) + Function.attributes().set(Attribute); + else + Function.attributes().clear(Attribute); + if (Key == std::string_view("hasFakeUses")) + DocumentInfo.HasFakeUses = Value; + } + + return false; +} + +bool MIRDocumentParser::recomputeProperties(const yaml::Node &Root, MachineFunction &Function, + const MIRFunctionDocumentInfo &DocumentInfo) { + bool HasPHI = false; + bool HasFakeUse = false; + bool IsSSA = true; + bool HasTiedOperands = false; + bool AllTiedOperandsRewritten = true; + bool HasFrameIndex = false; + bool HasExecutablePseudo = false; + std::unordered_set<Register> Definitions; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return fail(Root.range(), "instruction has no target descriptor"); + HasPHI |= Descriptor->Flags.has(InstructionFlag::Phi); + HasFakeUse |= Descriptor->Flags.has(InstructionFlag::FakeUse); + HasExecutablePseudo |= Descriptor->Flags.has(InstructionFlag::Pseudo) && + !Descriptor->Flags.has(InstructionFlag::Meta) && + !Descriptor->Flags.has(InstructionFlag::TerminalEmission); + const auto Operands = Instruction->operands(); + for (std::size_t Index = 0; Index < Operands.size(); ++Index) { + const MachineOperand &Operand = Operands[Index]; + HasFrameIndex |= Operand.getIf<FrameIndexOperand>() != nullptr; + const auto Reg = Operand.registerValue(); + if (Reg && Reg->isVirtual() && Operand.isDef() && !Definitions.insert(*Reg).second) { + IsSSA = false; + } + if (!Operand.tiedOperandIndex() || !Operand.isUse()) + continue; + HasTiedOperands = true; + const MachineOperand *Definition = Instruction->operandAt(*Operand.tiedOperandIndex()); + if (Definition == nullptr || Definition->registerValue() != Reg) + AllTiedOperandsRewritten = false; + } + for (const MachineMemOperand *Memory : Instruction->memOperands()) + HasFrameIndex |= Memory->description().Base.frameIndex().has_value(); + } + } + const bool NoPHIs = !HasPHI; + const bool NoVRegs = + std::ranges::none_of(Function.registerInfo().virtualRegisters(), + [](const VirtualRegisterInfo &Info) { return !Info.IsTombstone; }); + + const auto ApplyComputed = [this, &Root, &Function](std::optional<bool> Explicit, bool Computed, + MachineFunctionProperty Property, + std::string_view Conflict) -> bool { + if (Explicit && *Explicit && !Computed) { + return fail(Root.range(), std::string(Function.name()) + std::string(Conflict)); + } + if (Explicit.value_or(Computed)) + Function.properties().set(Property); + else + Function.properties().clear(Property); + return false; + }; + if (ApplyComputed(DocumentInfo.NoPHIs, NoPHIs, MachineFunctionProperty::NoPHIs, + " has explicit property NoPhi, but contains at least one PHI") || + ApplyComputed(DocumentInfo.IsSSA, IsSSA, MachineFunctionProperty::IsSSA, + " has explicit property IsSSA, but is not valid SSA") || + ApplyComputed(DocumentInfo.NoVRegs, NoVRegs, MachineFunctionProperty::NoVRegs, + " has explicit property NoVRegs, but contains virtual registers")) { + return true; + } + + if (DocumentInfo.HasFakeUses && !*DocumentInfo.HasFakeUses && HasFakeUse) { + return fail(Root.range(), + std::string(Function.name()) + + " has explicit property hasFakeUses=false, but contains fake uses"); + } + if (DocumentInfo.HasFakeUses.value_or(HasFakeUse)) + Function.attributes().set(MachineFunctionAttribute::HasFakeUses); + else + Function.attributes().clear(MachineFunctionAttribute::HasFakeUses); + + const bool Selected = Function.properties().has(MachineFunctionProperty::Selected); + const bool StructurallyPostSSA = + Selected && NoPHIs && !Function.properties().has(MachineFunctionProperty::IsSSA); + if (AllTiedOperandsRewritten && (HasTiedOperands || StructurallyPostSSA)) + Function.properties().set(MachineFunctionProperty::TiedOpsRewritten); + else + Function.properties().clear(MachineFunctionProperty::TiedOpsRewritten); + + if (HasFrameIndex) + Function.properties().clear(MachineFunctionProperty::NoFrameIndices); + else + Function.properties().set(MachineFunctionProperty::NoFrameIndices); + + const MachineFrameState &FrameState = Function.frameInfo().state(); + const bool StructurallyFinalizedFrame = Selected && NoVRegs && !HasFrameIndex && + FrameState.IsCalleeSavedInfoValid && + FrameState.MaxAlignment.has_value(); + if (StructurallyFinalizedFrame) { + if (!Function.frameInfo().isFrozen()) { + auto Frozen = Function.frameInfo().freeze(FrameState.StackSize, *FrameState.MaxAlignment); + if (!Frozen) + return importError(Root.range(), Frozen.error(), "mir-frame-finalization"); + } + Function.properties().set(MachineFunctionProperty::FrameFinalized); + } else { + Function.properties().clear(MachineFunctionProperty::FrameFinalized); + } + + if (StructurallyFinalizedFrame && !HasExecutablePseudo) + Function.properties().set(MachineFunctionProperty::NoExecutablePseudos); + else + Function.properties().clear(MachineFunctionProperty::NoExecutablePseudos); + return false; +} + +bool MIRDocumentParser::parseRegisters(const yaml::Node &Root, MachineFunction &Function, + MIParserContext &Context) { + const yaml::Node *Registers = mappingValue(Root, "registers"); + if (Registers != nullptr) { + if (requireKind(*Registers, yaml::NodeKind::Sequence, "registers")) + return true; + constexpr std::array RegisterKeys{ + std::string_view{"id"}, + std::string_view{"class"}, + std::string_view{"preferred-register"}, + std::string_view{"flags"}, + }; + std::unordered_set<std::uint32_t> IDs; + for (const yaml::NodeID ElementID : Registers->sequence()) { + const yaml::Node *Element = node(ElementID); + if (Element == nullptr || requireKind(*Element, yaml::NodeKind::Mapping, "register entry") || + validateKeys(*Element, RegisterKeys, "register")) { + return true; + } + const yaml::Node *IDNode = mappingValue(*Element, "id"); + const yaml::Node *ClassNode = mappingValue(*Element, "class"); + if (IDNode == nullptr || ClassNode == nullptr) + return fail(Element->range(), "register entry requires 'id' and 'class'"); + std::uint64_t WideID = 0; + std::string_view ClassName; + if (unsignedValue(*IDNode, WideID, "a 32-bit virtual-register ID") || + WideID > std::numeric_limits<std::uint32_t>::max() || + scalar(*ClassNode, ClassName, "register class or bank")) { + return true; + } + const auto TextID = static_cast<std::uint32_t>(WideID); + if (!IDs.insert(TextID).second) + return fail(IDNode->range(), "duplicate virtual register ID"); + auto Created = Function.registerInfo().createVirtualRegister(); + if (!Created) + return importError(IDNode->range(), Created.error()); + Context.VirtualRegisters.push_back({TextID, *Created}); + + if (ClassName != "_") { + if (const RegisterClassDesc *Class = Target.registers().registerClass(ClassName)) { + auto SetResult = Function.registerInfo().setRegisterClass(*Created, Class->ID); + if (!SetResult) + return importError(ClassNode->range(), SetResult.error()); + SetResult = Function.registerInfo().setRegisterBank(*Created, Class->DefaultBank); + if (!SetResult) + return importError(ClassNode->range(), SetResult.error()); + } else if (const RegisterBankDesc *Bank = Target.registers().registerBank(ClassName)) { + auto SetResult = Function.registerInfo().setRegisterBank(*Created, Bank->ID); + if (!SetResult) + return importError(ClassNode->range(), SetResult.error()); + } else { + return fail(ClassNode->range(), + "unknown register class or bank '" + std::string(ClassName) + "'"); + } + } + + if (const yaml::Node *Preferred = mappingValue(*Element, "preferred-register")) { + std::string_view PreferredName; + PhysicalRegister Reg; + if (scalar(*Preferred, PreferredName, "physical register")) + return true; + if (!trim(PreferredName).empty()) { + if (physicalRegister(PreferredName, Preferred->range(), Reg)) + return true; + auto AddResult = Function.registerInfo().addAllocationHint(*Created, Reg); + if (!AddResult) + return importError(Preferred->range(), AddResult.error()); + } + } + if (const yaml::Node *Flags = mappingValue(*Element, "flags")) { + if (requireKind(*Flags, yaml::NodeKind::Sequence, "virtual-register flags")) + return true; + std::uint8_t Mask = 0; + for (const yaml::NodeID FlagID : Flags->sequence()) { + const yaml::Node *FlagNode = node(FlagID); + std::string_view Name; + if (FlagNode == nullptr || scalar(*FlagNode, Name, "virtual-register flag")) + return true; + const auto Flag = Target.mirText().virtualRegisterFlag(Name); + if (!Flag) + return fail(FlagNode->range(), + "use of undefined register flag '" + std::string(Name) + "'"); + if ((Mask & *Flag) != 0) + return fail(FlagNode->range(), "duplicate virtual-register flag"); + Mask = static_cast<std::uint8_t>(Mask | *Flag); + } + auto SetResult = Function.registerInfo().setTargetRegisterFlags(*Created, Mask); + if (!SetResult) + return importError(Flags->range(), SetResult.error()); + } + } + } + + const yaml::Node *LiveIns = mappingValue(Root, "liveins"); + if (LiveIns != nullptr) { + if (requireKind(*LiveIns, yaml::NodeKind::Sequence, "liveins")) + return true; + constexpr std::array LiveInKeys{ + std::string_view{"reg"}, + std::string_view{"virtual-reg"}, + }; + for (const yaml::NodeID ElementID : LiveIns->sequence()) { + const yaml::Node *Element = node(ElementID); + if (Element == nullptr || requireKind(*Element, yaml::NodeKind::Mapping, "live-in entry") || + validateKeys(*Element, LiveInKeys, "live-in")) { + return true; + } + const yaml::Node *RegNode = mappingValue(*Element, "reg"); + if (RegNode == nullptr) + return fail(Element->range(), "live-in entry requires 'reg'"); + PhysicalRegister Reg; + if (physicalRegister(*RegNode, Reg)) + return true; + std::optional<Register> Initial; + if (const yaml::Node *VirtualNode = mappingValue(*Element, "virtual-reg")) { + std::string_view Text; + if (scalar(*VirtualNode, Text, "virtual register")) + return true; + if (!trim(Text).empty()) { + Register Virtual; + if (virtualRegister(Text, VirtualNode->range(), Function, Context, Virtual)) + return true; + Initial = Virtual; + } + } + auto AddResult = Function.registerInfo().addLiveIn(Reg, Initial); + if (!AddResult) + return importError(Element->range(), AddResult.error()); + } + } + + if (const yaml::Node *CalleeSaved = mappingValue(Root, "calleeSavedRegisters")) { + if (requireKind(*CalleeSaved, yaml::NodeKind::Sequence, "callee-saved registers")) + return true; + std::vector<PhysicalRegister> RegistersValue; + for (const yaml::NodeID RegID : CalleeSaved->sequence()) { + const yaml::Node *RegNode = node(RegID); + PhysicalRegister Reg; + if (RegNode == nullptr || physicalRegister(*RegNode, Reg)) + return true; + RegistersValue.push_back(Reg); + } + auto SetResult = Function.records().setCalleeSavedRegisters(std::move(RegistersValue)); + if (!SetResult) + return importError(CalleeSaved->range(), SetResult.error()); + } + return false; +} + +TextRange MIRDocumentParser::scalarDiagnosticRange(const yaml::Node &Scalar, + const TextRange &ScalarRange) const noexcept { + if (Scalar.scalarStyle() != yaml::ScalarStyle::Literal && + Scalar.scalarStyle() != yaml::ScalarStyle::Folded) { + std::size_t ContentOffset = Scalar.range().Begin.Offset; + if (Scalar.scalarStyle() == yaml::ScalarStyle::SingleQuoted || + Scalar.scalarStyle() == yaml::ScalarStyle::DoubleQuoted) { + ++ContentOffset; + } + return Result.Source.range(ContentOffset + ScalarRange.Begin.Offset, + ContentOffset + ScalarRange.End.Offset); + } + + const std::size_t FirstBodyLine = Scalar.range().Begin.Line + 1U; + const std::size_t LastBodyLine = Scalar.range().End.Line; + if (FirstBodyLine > LastBodyLine || FirstBodyLine == 0) + return Scalar.range(); + + std::size_t Indent = 0; + bool FoundContent = false; + for (std::size_t Line = FirstBodyLine; Line <= LastBodyLine; ++Line) { + const std::string_view Text = Result.Source.line(Line); + std::size_t CurrentIndent = 0; + while (CurrentIndent < Text.size() && Text[CurrentIndent] == ' ') + ++CurrentIndent; + if (CurrentIndent == Text.size()) + continue; + Indent = CurrentIndent; + FoundContent = true; + break; + } + if (!FoundContent) + return Scalar.range(); + + const TextPosition DefaultPosition = Scalar.range().Begin; + const auto MapPosition = [this, FirstBodyLine, LastBodyLine, Indent, + DefaultPosition](TextPosition Position) { + const std::size_t SourceLine = FirstBodyLine + Position.Line - 1U; + if (Position.Line == 0 || SourceLine > LastBodyLine || SourceLine > Result.Source.lineCount()) { + return DefaultPosition; + } + const std::string_view SourceLineText = Result.Source.line(SourceLine); + const std::size_t BodyColumn = Position.Column == 0 ? 0 : Position.Column - 1U; + const std::size_t SourceColumn = std::min(Indent + BodyColumn, SourceLineText.size()); + const std::size_t Offset = Result.Source.lineOffsets()[SourceLine - 1U] + SourceColumn; + return Result.Source.positionAt(Offset); + }; + return {.Begin = MapPosition(ScalarRange.Begin), .End = MapPosition(ScalarRange.End)}; +} + +bool MIRDocumentParser::parseBody(const yaml::Node &Root, MachineFunction &Function, + MIParserContext &Context) { + const yaml::Node *Body = mappingValue(Root, "body"); + std::string BodyText; + TextRange BodyRange = Root.range(); + if (Body != nullptr) { + std::string_view Text; + if (scalar(*Body, Text, "machine function body")) + return true; + BodyText = Text; + BodyRange = Body->range(); + } + + if (!effectiveDataLayout().empty()) { + auto Layout = dataLayout(); + if (!Layout) + return importError(BodyRange, Layout.error(), "mir-data-layout"); + Context.PointerSizeInBits = [Layout = std::move(*Layout)](std::uint32_t AddressSpace) { + return Layout.getPointerSizeInBits(AddressSpace); + }; + } + Context.SyncScopes = ActiveDocument->SyncScopes; + Context.InternSyncScope = [this](std::string_view Name) -> Expected<SyncScopeID> { + if (Name.empty()) + return makeUnexpected(ErrorCode::ParseError, "synchronization scope name cannot be empty"); + const auto Existing = + std::ranges::find_if(ActiveDocument->SyncScopes, + [Name](const MISyncScopeSlot &Slot) { return Slot.Name == Name; }); + if (Existing != ActiveDocument->SyncScopes.end()) + return Existing->ID; + if (ActiveDocument->SyncScopes.size() >= + static_cast<std::size_t>(std::numeric_limits<std::uint32_t>::max())) { + return makeUnexpected(ErrorCode::Unsupported, + "MIR document contains too many synchronization scopes"); + } + const auto ID = + SyncScopeID::fromValue(static_cast<std::uint32_t>(ActiveDocument->SyncScopes.size() + 1U)); + ActiveDocument->SyncScopes.push_back({.Name = std::string(Name), .ID = ID}); + return ID; + }; + MIParserOptions BodyOptions = Options.MachineInstructions; + BodyOptions.VerifyAfterParse = false; + MIParser Parser(Function, Target, + std::string(Result.Source.name()) + ":" + std::string(Function.name()), + std::move(BodyText), Context, BodyOptions); + auto Parsed = Parser.parse(); + if (!Parsed) { + if (!Parser.diagnostics().empty()) { + const TextRange Range = + Body == nullptr ? BodyRange + : scalarDiagnosticRange(*Body, Parser.diagnostics().front().Range); + return fail(Range, Parser.diagnostics().front().Message, "mir-body"); + } + return importError(BodyRange, Parsed.error(), "mir-body"); + } + return false; +} + +bool MIRDocumentParser::convertTargetNode(const yaml::Node &Source, MIRTargetNode &Destination) { + switch (Source.kind()) { + case yaml::NodeKind::Null: + Destination = MIRTargetNode::null(); + return false; + case yaml::NodeKind::Scalar: + Destination = MIRTargetNode::scalar(std::string(Source.scalar())); + return false; + case yaml::NodeKind::Sequence: { + std::vector<MIRTargetNode> Values; + Values.reserve(Source.sequence().size()); + for (const yaml::NodeID ElementID : Source.sequence()) { + const yaml::Node *Element = node(ElementID); + if (Element == nullptr) + return fail(Source.range(), "target MIR sequence contains an invalid node"); + MIRTargetNode Value; + if (convertTargetNode(*Element, Value)) + return true; + Values.push_back(std::move(Value)); + } + Destination = MIRTargetNode::sequence(std::move(Values)); + return false; + } + case yaml::NodeKind::Mapping: { + std::vector<MIRTargetMappingEntry> Entries; + Entries.reserve(Source.mapping().size()); + for (const yaml::MappingEntry &SourceEntry : Source.mapping()) { + const yaml::Node *ValueNode = node(SourceEntry.Value); + if (ValueNode == nullptr) + return fail(SourceEntry.KeyRange, "target MIR mapping contains an invalid node"); + MIRTargetNode Value; + if (convertTargetNode(*ValueNode, Value)) + return true; + Entries.push_back({SourceEntry.Key, std::move(Value)}); + } + Destination = MIRTargetNode::mapping(std::move(Entries)); + return false; + } + } + return fail(Source.range(), "target MIR node has an unknown kind"); +} + +bool MIRDocumentParser::parseTargetState(const yaml::Node &Root, MachineFunction &Function) { + const yaml::Node *StateNode = mappingValue(Root, "machineFunctionInfo"); + if (StateNode == nullptr) + return false; + MIRTargetNode TargetNode; + if (convertTargetNode(*StateNode, TargetNode)) + return true; + auto State = Target.mirText().parseFunctionState(TargetNode); + if (!State) + return importError(StateNode->range(), State.error(), "mir-target-state"); + auto SetResult = Function.setTargetState(std::move(*State)); + if (!SetResult) + return importError(StateNode->range(), SetResult.error(), "mir-target-state"); + return false; +} + +bool MIRDocumentParser::validateDocumentReferences(const MachineFunction &Function) { + auto Verification = verifyMIRDocumentReferences(*ActiveDocument, Function, true); + if (Verification) + return false; + return importError(Result.Source.range(0, 0), Verification.error()); +} + +} // namespace rcc::mir + +namespace rcc::mir { + +using parser_detail::parseBool; +using parser_detail::parsePrefixedID; +using parser_detail::parseSigned; +using parser_detail::parseUnsigned; +using parser_detail::parseUnsigned32; +using parser_detail::trim; + +bool MIRDocumentParser::parseFunctionRecords(const yaml::Node &Root, MachineFunction &Function) { + auto ParseLocation = [this, &Function](const yaml::Node &Entry, + MachineInstr *&Instruction) -> bool { + const yaml::Node *BlockNode = mappingValue(Entry, "bb"); + const yaml::Node *OffsetNode = mappingValue(Entry, "offset"); + if (BlockNode == nullptr || OffsetNode == nullptr) + return fail(Entry.range(), "instruction location requires 'bb' and 'offset'"); + std::uint64_t Block = 0; + std::uint64_t Offset = 0; + if (unsignedValue(*BlockNode, Block, "a 32-bit basic block number") || + unsignedValue(*OffsetNode, Offset, "a 32-bit instruction offset") || + Block > std::numeric_limits<std::uint32_t>::max() || + Offset > std::numeric_limits<std::uint32_t>::max()) { + return true; + } + Instruction = instructionAt(Function, static_cast<std::uint32_t>(Block), + static_cast<std::uint32_t>(Offset)); + if (Instruction == nullptr) + return fail(Entry.range(), "instruction location is outside the machine function body"); + return false; + }; + + if (const yaml::Node *CallSites = mappingValue(Root, "callSites")) { + if (requireKind(*CallSites, yaml::NodeKind::Sequence, "call-site records")) + return true; + constexpr std::array CallSiteKeys{ + std::string_view{"bb"}, + std::string_view{"offset"}, + std::string_view{"fwdArgRegs"}, + std::string_view{"calleeTypeIds"}, + }; + constexpr std::array ArgumentKeys{ + std::string_view{"arg"}, + std::string_view{"reg"}, + }; + for (const yaml::NodeID CallSiteID : CallSites->sequence()) { + const yaml::Node *CallSiteNode = node(CallSiteID); + if (CallSiteNode == nullptr || + requireKind(*CallSiteNode, yaml::NodeKind::Mapping, "call-site record") || + validateKeys(*CallSiteNode, CallSiteKeys, "call-site record")) { + return true; + } + MachineInstr *Call = nullptr; + if (ParseLocation(*CallSiteNode, Call)) + return true; + if (Call == nullptr) + return fail(CallSiteNode->range(), + "instruction location is outside the machine function body"); + MachineCallSiteInfo Info{ + .CallInstruction = Call->stableID(), + .ForwardedArguments = {}, + .CalleeTypeIDs = {}, + }; + if (const yaml::Node *Arguments = mappingValue(*CallSiteNode, "fwdArgRegs")) { + if (requireKind(*Arguments, yaml::NodeKind::Sequence, "forwarded call arguments")) { + return true; + } + for (const yaml::NodeID ArgumentID : Arguments->sequence()) { + const yaml::Node *Argument = node(ArgumentID); + if (Argument == nullptr || + requireKind(*Argument, yaml::NodeKind::Mapping, "forwarded call argument") || + validateKeys(*Argument, ArgumentKeys, "forwarded call argument")) { + return true; + } + const yaml::Node *NumberNode = mappingValue(*Argument, "arg"); + const yaml::Node *RegNode = mappingValue(*Argument, "reg"); + if (NumberNode == nullptr || RegNode == nullptr) + return fail(Argument->range(), "forwarded call argument requires 'arg' and 'reg'"); + std::uint16_t Number = 0; + std::uint64_t WideNumber = 0; + PhysicalRegister Reg; + if (unsignedValue(*NumberNode, WideNumber, "a 16-bit argument number") || + WideNumber > std::numeric_limits<std::uint16_t>::max() || + physicalRegister(*RegNode, Reg)) { + return true; + } + Number = static_cast<std::uint16_t>(WideNumber); + Info.ForwardedArguments.push_back({Reg, Number}); + } + } + if (const yaml::Node *TypeIDs = mappingValue(*CallSiteNode, "calleeTypeIds")) { + if (requireKind(*TypeIDs, yaml::NodeKind::Sequence, "callee type IDs")) + return true; + for (const yaml::NodeID TypeID : TypeIDs->sequence()) { + const yaml::Node *TypeNode = node(TypeID); + std::uint64_t Value = 0; + if (TypeNode == nullptr || + unsignedValue(*TypeNode, Value, "an unsigned callee type ID")) { + return true; + } + Info.CalleeTypeIDs.push_back(Value); + } + } + auto AddResult = Function.records().addCallSite(std::move(Info)); + if (!AddResult) + return importError(CallSiteNode->range(), AddResult.error()); + } + } + + if (const yaml::Node *CalledGlobals = mappingValue(Root, "calledGlobals")) { + if (requireKind(*CalledGlobals, yaml::NodeKind::Sequence, "called-global records")) + return true; + constexpr std::array Keys{ + std::string_view{"bb"}, + std::string_view{"offset"}, + std::string_view{"callee"}, + std::string_view{"flags"}, + }; + for (const yaml::NodeID CalledID : CalledGlobals->sequence()) { + const yaml::Node *CalledNode = node(CalledID); + if (CalledNode == nullptr || + requireKind(*CalledNode, yaml::NodeKind::Mapping, "called-global record") || + validateKeys(*CalledNode, Keys, "called-global record")) { + return true; + } + MachineInstr *Call = nullptr; + if (ParseLocation(*CalledNode, Call)) + return true; + if (Call == nullptr) + return fail(CalledNode->range(), + "instruction location is outside the machine function body"); + const yaml::Node *CalleeNode = mappingValue(*CalledNode, "callee"); + const yaml::Node *FlagsNode = mappingValue(*CalledNode, "flags"); + if (CalleeNode == nullptr || FlagsNode == nullptr) + return fail(CalledNode->range(), "called-global record requires 'callee' and 'flags'"); + std::string_view Callee; + std::uint64_t WideFlags = 0; + if (scalar(*CalleeNode, Callee, "callee name") || + unsignedValue(*FlagsNode, WideFlags, "32-bit called-global flags") || + WideFlags > std::numeric_limits<std::uint32_t>::max()) { + return true; + } + if (ActiveDocument->EmbeddedIR && !ActiveDocument->EmbeddedIR->hasGlobalValue(Callee)) { + return fail(CalleeNode->range(), "called global references an undefined embedded global"); + } + auto AddResult = + Function.records().addCalledGlobal({.CallInstruction = Call->stableID(), + .Callee = std::string(Callee), + .Flags = static_cast<std::uint32_t>(WideFlags)}); + if (!AddResult) + return importError(CalledNode->range(), AddResult.error()); + } + } + + if (const yaml::Node *Substitutions = mappingValue(Root, "debugValueSubstitutions")) { + if (requireKind(*Substitutions, yaml::NodeKind::Sequence, "debug value substitutions")) { + return true; + } + constexpr std::array Keys{ + std::string_view{"srcinst"}, std::string_view{"srcop"}, std::string_view{"dstinst"}, + std::string_view{"dstop"}, std::string_view{"subreg"}, + }; + for (const yaml::NodeID SubstitutionID : Substitutions->sequence()) { + const yaml::Node *Substitution = node(SubstitutionID); + if (Substitution == nullptr || + requireKind(*Substitution, yaml::NodeKind::Mapping, "debug value substitution") || + validateKeys(*Substitution, Keys, "debug value substitution")) { + return true; + } + std::array<std::uint32_t, 5> Values{}; + for (std::size_t Index = 0; Index < Keys.size(); ++Index) { + const yaml::Node *ValueNode = mappingValue(*Substitution, Keys[Index]); + if (ValueNode == nullptr) + return fail(Substitution->range(), "debug value substitution requires every fixed field"); + std::uint64_t Wide = 0; + if (unsignedValue(*ValueNode, Wide, "a 32-bit debug substitution value") || + Wide > std::numeric_limits<std::uint32_t>::max()) { + return true; + } + Values[Index] = static_cast<std::uint32_t>(Wide); + } + auto AddResult = Function.records().addDebugValueSubstitution( + {.Source = {Values[0], Values[1]}, + .Destination = {Values[2], Values[3]}, + .Subregister = SubRegIndex::fromValue(Values[4])}); + if (!AddResult) + return importError(Substitution->range(), AddResult.error()); + } + } + + if (const yaml::Node *PrefetchTargets = mappingValue(Root, "prefetch-targets")) { + if (requireKind(*PrefetchTargets, yaml::NodeKind::Sequence, "prefetch targets")) + return true; + for (const yaml::NodeID TargetID : PrefetchTargets->sequence()) { + const yaml::Node *TargetNode = node(TargetID); + std::string_view Text; + if (TargetNode == nullptr || scalar(*TargetNode, Text, "prefetch target")) + return true; + + std::size_t ContentOffset = TargetNode->range().Begin.Offset; + if (TargetNode->scalarStyle() == yaml::ScalarStyle::SingleQuoted || + TargetNode->scalarStyle() == yaml::ScalarStyle::DoubleQuoted) { + ++ContentOffset; + } + const auto ErrorRange = [this, ContentOffset](std::size_t Begin, std::size_t End = 1U) { + return Result.Source.range(ContentOffset + Begin, ContentOffset + Begin + End); + }; + const auto SkipSpace = [&Text](std::size_t &Cursor) { + while (Cursor < Text.size() && (Text[Cursor] == ' ' || Text[Cursor] == '\t')) + ++Cursor; + }; + const auto ParseNumber = [&Text, &SkipSpace](std::size_t &Cursor, std::uint32_t &Value) { + SkipSpace(Cursor); + const std::size_t Begin = Cursor; + while (Cursor < Text.size() && Text[Cursor] >= '0' && Text[Cursor] <= '9') + ++Cursor; + return Begin != Cursor && parseUnsigned32(Text.substr(Begin, Cursor - Begin), Value); + }; + + std::size_t Cursor = 0; + SkipSpace(Cursor); + if (!Text.substr(Cursor).starts_with("bb_id") || + (Cursor + std::string_view("bb_id").size() < Text.size() && + Text[Cursor + std::string_view("bb_id").size()] != ' ' && + Text[Cursor + std::string_view("bb_id").size()] != '\t')) { + return fail(ErrorRange(Cursor), "expected 'bb_id'"); + } + Cursor += std::string_view("bb_id").size(); + + std::array<std::uint32_t, 3> Values{}; + if (!ParseNumber(Cursor, Values[0])) + return fail(ErrorRange(0), "expected unsigned integer"); + + SkipSpace(Cursor); + if (Cursor < Text.size() && (Text[Cursor] == ',' || Text[Cursor] == '.')) { + ++Cursor; + if (!ParseNumber(Cursor, Values[1])) + return fail(ErrorRange(0), "expected unsigned integer"); + } else if (Cursor < Text.size() && Text[Cursor] >= '0' && Text[Cursor] <= '9') { + if (!ParseNumber(Cursor, Values[1])) + return fail(ErrorRange(0), "expected unsigned integer"); + } + + SkipSpace(Cursor); + if (Cursor >= Text.size() || Text[Cursor] != ',') + return fail(ErrorRange(Cursor), "expected ','"); + ++Cursor; + if (!ParseNumber(Cursor, Values[2])) + return fail(ErrorRange(0), "expected unsigned integer"); + SkipSpace(Cursor); + if (Cursor != Text.size()) + return fail(ErrorRange(0), "unexpected content after prefetch target"); + + auto AddResult = Function.records().addPrefetchTarget( + {.Block = {Values[0], Values[1]}, .CallsiteIndex = Values[2]}); + if (!AddResult) + return importError(TargetNode->range(), AddResult.error()); + } + } + + if (const yaml::Node *EntryValues = mappingValue(Root, "entry_values")) { + if (requireKind(*EntryValues, yaml::NodeKind::Sequence, "entry values")) + return true; + constexpr std::array Keys{ + std::string_view{"entry-value-register"}, + std::string_view{"debug-info-variable"}, + std::string_view{"debug-info-expression"}, + std::string_view{"debug-info-location"}, + }; + for (const yaml::NodeID EntryID : EntryValues->sequence()) { + const yaml::Node *Entry = node(EntryID); + if (Entry == nullptr || requireKind(*Entry, yaml::NodeKind::Mapping, "entry value") || + validateKeys(*Entry, Keys, "entry value")) { + return true; + } + const yaml::Node *RegNode = mappingValue(*Entry, "entry-value-register"); + const yaml::Node *VariableNode = mappingValue(*Entry, "debug-info-variable"); + const yaml::Node *ExpressionNode = mappingValue(*Entry, "debug-info-expression"); + const yaml::Node *LocationNode = mappingValue(*Entry, "debug-info-location"); + if (RegNode == nullptr || VariableNode == nullptr || ExpressionNode == nullptr || + LocationNode == nullptr) { + return fail(Entry->range(), "entry value requires all four fixed fields"); + } + PhysicalRegister Reg; + if (physicalRegister(*RegNode, Reg)) + return true; + MachineVariableDebugInfo DebugInfo; + if (parseVariableDebugInfo(*Entry, Function, DebugInfo)) + return true; + if (!DebugInfo.Variable || !DebugInfo.Expression || !DebugInfo.Location) { + return fail(Entry->range(), "entry value debug metadata cannot be empty", + "mir-debug-metadata"); + } + auto AddResult = + Function.records().addEntryValue({.Reg = Reg, .DebugInfo = std::move(DebugInfo)}); + if (!AddResult) + return importError(Entry->range(), AddResult.error()); + } + } + return false; +} + +} // namespace rcc::mir + +namespace rcc::mir { + +using parser_detail::parseBool; +using parser_detail::parsePrefixedID; +using parser_detail::parseSigned; +using parser_detail::parseUnsigned; +using parser_detail::parseUnsigned32; +using parser_detail::trim; + +Expected<MachineMetadataReference> +MIRDocumentParser::resolveMetadataReference(const MachineFunction &Function, + std::uint32_t Slot) const { + if (ActiveDocument != nullptr && ActiveDocument->EmbeddedIR && + ActiveDocument->EmbeddedIR->metadata(std::to_string(Slot)) != nullptr) { + return MachineMetadataReference::embeddedIR(Slot); + } + if (Function.metadata().node(Slot) != nullptr) + return MachineMetadataReference::machineFunction(Slot); + return makeUnexpected(ErrorCode::ParseError, + "metadata reference uses undefined slot '!" + std::to_string(Slot) + "'"); +} + +bool MIRDocumentParser::parseVariableDebugInfo(const yaml::Node &Entry, MachineFunction &Function, + MachineVariableDebugInfo &DebugInfo) { + const auto Resolve = [this, &Function](std::uint32_t Slot) { + return resolveMetadataReference(Function, Slot); + }; + if (const yaml::Node *VariableNode = mappingValue(Entry, "debug-info-variable")) { + std::string_view Text; + if (scalar(*VariableNode, Text, "debug variable")) + return true; + if (!Text.empty()) { + MetadataTextParser Parser(std::string(Text), Resolve); + auto Reference = Parser.parseReferenceOnly(); + if (!Reference) + return importError(VariableNode->range(), Reference.error(), "mir-debug-metadata"); + if (Reference->scope() != MachineMetadataScope::EmbeddedIR) { + return fail(VariableNode->range(), + "debug variable must reference embedded LLVM IR metadata", + "mir-debug-metadata"); + } + DebugInfo.Variable = *Reference; + } + } + if (const yaml::Node *ExpressionNode = mappingValue(Entry, "debug-info-expression")) { + std::string_view Text; + if (scalar(*ExpressionNode, Text, "debug expression")) + return true; + if (!Text.empty()) { + MetadataTextParser Parser(std::string(Text), Resolve); + auto Value = Parser.parseValue(); + if (!Value) + return importError(scalarDiagnosticRange(*ExpressionNode, Parser.diagnosticRange()), + Value.error(), "mir-debug-metadata"); + if (Value->reference() != nullptr && + Value->reference()->scope() != MachineMetadataScope::EmbeddedIR) { + return fail(ExpressionNode->range(), + "debug expression reference must use embedded LLVM IR metadata", + "mir-debug-metadata"); + } + DebugInfo.Expression = std::move(*Value); + } + } + if (const yaml::Node *LocationNode = mappingValue(Entry, "debug-info-location")) { + std::string_view Text; + if (scalar(*LocationNode, Text, "debug location")) + return true; + if (!Text.empty()) { + MetadataTextParser Parser(std::string(Text), Resolve); + auto Location = Parser.parseLocation(); + if (!Location) + return importError(LocationNode->range(), Location.error(), "mir-debug-metadata"); + if (Location->reference() != nullptr && + Location->reference()->scope() != MachineMetadataScope::EmbeddedIR) { + return fail(LocationNode->range(), + "variable debug location must reference embedded LLVM IR metadata", + "mir-debug-metadata"); + } + DebugInfo.Location = std::move(*Location); + } + } + if (!DebugInfo.Variable && !DebugInfo.Expression && !DebugInfo.Location) + return false; + auto Verification = DebugInfo.verifyStructure(); + if (!Verification) + return importError(Entry.range(), Verification.error(), "mir-debug-metadata"); + return false; +} + +bool MIRDocumentParser::parseMetadata(const yaml::Node &Root, MachineFunction &Function, + MIParserContext &Context) { + if (ActiveDocument->EmbeddedIR) { + for (const EmbeddedIRMetadataDefinition &Reference : + ActiveDocument->EmbeddedIR->metadataDefinitions()) { + std::uint32_t ID = 0; + if (!parseUnsigned32(Reference.Name, ID)) + continue; + if (std::ranges::any_of(Context.Metadata, + [ID](const MIMetadataSlot &Slot) { return Slot.TextID == ID; })) { + return fail(Root.range(), "embedded LLVM IR metadata slots are not unique"); + } + Context.Metadata.push_back( + {.TextID = ID, .Reference = MachineMetadataReference::embeddedIR(ID)}); + } + } + + const yaml::Node *Metadata = mappingValue(Root, "machineMetadataNodes"); + if (Metadata == nullptr) + return false; + if (requireKind(*Metadata, yaml::NodeKind::Sequence, "machine metadata nodes")) + return true; + + enum class ParsedElementKind : std::uint8_t { + Null, + Reference, + String, + }; + struct ParsedElement final { + ParsedElementKind Kind = ParsedElementKind::Reference; + std::uint32_t Reference = 0; + std::string String; + TextRange Range; + }; + struct Definition final { + const yaml::Node *Node = nullptr; + std::uint32_t Slot = 0; + bool IsDistinct = false; + std::vector<ParsedElement> Elements; + }; + std::vector<Definition> Definitions; + std::unordered_set<std::uint32_t> Slots; + + const auto TranslateRange = [this](const yaml::Node &Node, const TextRange &Range) { + std::size_t ContentBegin = Node.range().Begin.Offset; + if (Node.scalarStyle() == yaml::ScalarStyle::SingleQuoted || + Node.scalarStyle() == yaml::ScalarStyle::DoubleQuoted) { + ++ContentBegin; + } + return Result.Source.range(ContentBegin + Range.Begin.Offset, ContentBegin + Range.End.Offset); + }; + + for (const yaml::NodeID DefinitionID : Metadata->sequence()) { + const yaml::Node *DefinitionNode = node(DefinitionID); + std::string_view Text; + if (DefinitionNode == nullptr || scalar(*DefinitionNode, Text, "machine metadata definition")) + return true; + + MILexer Lexer("machine-metadata", std::string(Text)); + MIToken Token = Lexer.next(); + const auto LexerFailed = [this, &Lexer, DefinitionNode, &TranslateRange, &Token]() -> bool { + if (!Token.is(MITokenKind::Error)) + return false; + const auto Diagnostics = Lexer.diagnostics(); + const TextRange Range = Diagnostics.empty() + ? TranslateRange(*DefinitionNode, Token.range()) + : TranslateRange(*DefinitionNode, Diagnostics.front().Range); + return fail(Range, Diagnostics.empty() ? "invalid machine metadata node" + : Diagnostics.front().Message); + }; + const auto Advance = [&Lexer, &Token, &LexerFailed]() -> bool { + Token = Lexer.next(); + return LexerFailed(); + }; + const auto FailAtToken = [this, DefinitionNode, &TranslateRange](const MIToken &At, + std::string Message) { + return fail(TranslateRange(*DefinitionNode, At.range()), std::move(Message)); + }; + + if (LexerFailed()) + return true; + if (!Token.is(MITokenKind::Exclaim)) + return FailAtToken(Token, "expected a metadata node"); + if (Advance()) + return true; + std::uint32_t Slot = 0; + if (!Token.is(MITokenKind::IntegerLiteral) || !parseUnsigned32(Token.integerValue(), Slot)) { + return FailAtToken(Token, "expected metadata id after '!'"); + } + if (Advance()) + return true; + if (!Token.is(MITokenKind::Equal)) + return FailAtToken(Token, "expected '='"); + if (Advance()) + return true; + + const bool IsDistinct = Token.isKeyword("distinct"); + if (IsDistinct && Advance()) + return true; + if (!Token.is(MITokenKind::Exclaim)) + return FailAtToken(Token, "expected a metadata node"); + if (Advance()) + return true; + if (!Token.is(MITokenKind::LeftBrace)) + return FailAtToken(Token, "expected '{' here"); + if (Advance()) + return true; + + std::vector<ParsedElement> Elements; + if (!Token.is(MITokenKind::RightBrace)) { + while (true) { + if (Token.hasStringValue() && Token.stringValue() == "null") { + Elements.push_back({.Kind = ParsedElementKind::Null, + .Reference = 0, + .String = {}, + .Range = TranslateRange(*DefinitionNode, Token.range())}); + } else { + if (!Token.is(MITokenKind::Exclaim)) + return FailAtToken(Token, "expected '!' here"); + const TextRange ReferenceRange = TranslateRange(*DefinitionNode, Token.range()); + if (Advance()) + return true; + + if (Token.is(MITokenKind::StringConstant)) { + Elements.push_back({.Kind = ParsedElementKind::String, + .Reference = 0, + .String = std::string(Token.stringValue()), + .Range = TranslateRange(*DefinitionNode, Token.range())}); + } else { + std::uint32_t Reference = 0; + if (!Token.is(MITokenKind::IntegerLiteral) || + !parseUnsigned32(Token.integerValue(), Reference)) { + return FailAtToken(Token, "expected metadata id after '!'"); + } + Elements.push_back({.Kind = ParsedElementKind::Reference, + .Reference = Reference, + .String = {}, + .Range = ReferenceRange}); + } + } + if (Advance()) + return true; + if (!Token.is(MITokenKind::Comma)) + break; + if (Advance()) + return true; + } + } + if (!Token.is(MITokenKind::RightBrace)) + return FailAtToken(Token, "expected end of metadata node"); + if (Advance()) + return true; + if (!Token.is(MITokenKind::EndOfFile)) + return FailAtToken(Token, "expected end of metadata node"); + if (!Slots.insert(Slot).second) + return FailAtToken(Token, "Metadata id is already used"); + + Definitions.push_back({.Node = DefinitionNode, + .Slot = Slot, + .IsDistinct = IsDistinct, + .Elements = std::move(Elements)}); + Context.Metadata.push_back( + {.TextID = Slot, .Reference = MachineMetadataReference::machineFunction(Slot)}); + } + + for (const Definition &Definition : Definitions) { + MachineMetadataNode NewNode{ + .Slot = Definition.Slot, + .IsDistinct = Definition.IsDistinct, + .Elements = {}, + }; + for (const ParsedElement &Element : Definition.Elements) { + if (Element.Kind == ParsedElementKind::Null) { + NewNode.Elements.push_back(MachineMetadataElement::null()); + continue; + } + if (Element.Kind == ParsedElementKind::String) { + NewNode.Elements.push_back(MachineMetadataElement::string(Element.String)); + continue; + } + + MachineMetadataReference Reference; + if (ActiveDocument->EmbeddedIR != std::nullopt && + ActiveDocument->EmbeddedIR->metadata(std::to_string(Element.Reference)) != nullptr) { + Reference = MachineMetadataReference::embeddedIR(Element.Reference); + } else if (Slots.contains(Element.Reference)) { + Reference = MachineMetadataReference::machineFunction(Element.Reference); + } else { + return fail(Element.Range, + "use of undefined metadata '!" + std::to_string(Element.Reference) + "'"); + } + auto Resolved = MachineMetadataElement::reference(Reference); + if (!Resolved) + return importError(Element.Range, Resolved.error()); + NewNode.Elements.push_back(std::move(*Resolved)); + } + auto AddResult = Function.metadata().addNode(std::move(NewNode)); + if (!AddResult) + return importError(Definition.Node->range(), AddResult.error()); + } + return false; +} + +} // namespace rcc::mir diff --git a/backend/lib/CodeGen/MachineAnalyses.cpp b/backend/lib/CodeGen/MachineAnalyses.cpp new file mode 100644 index 0000000..3f09636 --- /dev/null +++ b/backend/lib/CodeGen/MachineAnalyses.cpp @@ -0,0 +1,851 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineAnalyses.h" + +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <array> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <map> +#include <optional> +#include <ranges> +#include <span> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] std::vector<MachineBasicBlock *> linkedBlocks(MachineFunction &Function) { + std::vector<MachineBasicBlock *> Result; + Result.reserve(Function.blockCount()); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + Result.push_back(Block); + return Result; +} + +[[nodiscard]] std::unordered_map<const MachineBasicBlock *, std::size_t> +blockOrder(std::span<MachineBasicBlock *const> Blocks) { + std::unordered_map<const MachineBasicBlock *, std::size_t> Result; + for (std::size_t Index = 0; Index < Blocks.size(); ++Index) + Result.emplace(Blocks[Index], Index); + return Result; +} + +[[nodiscard]] bool containsBlock(std::span<MachineBasicBlock *const> Blocks, + const MachineBasicBlock *Block) { + return std::ranges::find(Blocks, Block) != Blocks.end(); +} + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + if (std::numeric_limits<std::uint64_t>::max() - Left < Right) + return std::numeric_limits<std::uint64_t>::max(); + return Left + Right; +} + +[[nodiscard]] std::uint64_t saturatingMultiply(std::uint64_t Value, + std::uint64_t Multiplier) noexcept { + if (Value != 0 && Multiplier > std::numeric_limits<std::uint64_t>::max() / Value) + return std::numeric_limits<std::uint64_t>::max(); + return Value * Multiplier; +} + +[[nodiscard]] std::uint64_t scaleByProbability(std::uint64_t Value, + BranchProbability Probability) noexcept { + constexpr std::uint64_t Denominator = std::numeric_limits<std::uint32_t>::max(); + const std::uint64_t Numerator = Probability.rawValue(); + const std::uint64_t Quotient = Value / Denominator; + const std::uint64_t Remainder = Value % Denominator; + return saturatingAdd(saturatingMultiply(Quotient, Numerator), + (Remainder * Numerator) / Denominator); +} + +[[nodiscard]] bool isKnownDistinctObject(const MachineMemOperandDesc &Left, + const MachineMemOperandDesc &Right) noexcept { + const auto IsUniqueObject = [](MemoryBase::Kind Kind) { + return Kind == MemoryBase::Kind::FrameIndex || Kind == MemoryBase::Kind::ConstantPool || + Kind == MemoryBase::Kind::JumpTable; + }; + if (IsUniqueObject(Left.Base.kind()) && IsUniqueObject(Right.Base.kind()) && + Left.Base.kind() != Right.Base.kind()) { + return true; + } + if (Left.Base.kind() != Right.Base.kind()) + return false; + switch (Left.Base.kind()) { + case MemoryBase::Kind::FrameIndex: + return Left.Base.frameIndex() != Right.Base.frameIndex(); + case MemoryBase::Kind::ConstantPool: + return Left.Base.constantPoolIndex() != Right.Base.constantPoolIndex(); + case MemoryBase::Kind::JumpTable: + return Left.Base.jumpTableIndex() != Right.Base.jumpTableIndex(); + default: + return false; + } +} + +[[nodiscard]] bool hasOrderedMemory(const MachineInstr &Instruction) noexcept { + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory != nullptr && (Memory->isAtomic() || Memory->isVolatile())) + return true; + } + return false; +} + +} // namespace + +Expected<void> MachineDominatorTreeAnalysis::Result::build(MachineFunction &Function) { + Blocks = linkedBlocks(Function); + BlockToIndex = blockOrder(Blocks); + Dominators.assign(Blocks.size(), std::vector<bool>(Blocks.size(), false)); + ImmediateDominators.assign(Blocks.size(), nullptr); + Reachable.assign(Blocks.size(), false); + UnreachableBlocks.clear(); + if (Blocks.empty()) + return {}; + + std::vector<MachineBasicBlock *> Worklist{Blocks.front()}; + Reachable[0] = true; + for (std::size_t Position = 0; Position < Worklist.size(); ++Position) { + MachineBasicBlock *Block = Worklist[Position]; + for (const MachineSuccessor &Successor : Block->successors()) { + const auto Found = BlockToIndex.find(Successor.Block); + if (Successor.Block == nullptr || Found == BlockToIndex.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "dominator analysis encountered an out-of-function CFG edge"); + } + if (!Reachable[Found->second]) { + Reachable[Found->second] = true; + Worklist.push_back(Successor.Block); + } + } + } + + for (std::size_t BlockIndex = 0; BlockIndex < Blocks.size(); ++BlockIndex) { + if (!Reachable[BlockIndex]) { + UnreachableBlocks.push_back(Blocks[BlockIndex]); + Dominators[BlockIndex][BlockIndex] = true; + continue; + } + if (BlockIndex == 0) { + Dominators[BlockIndex][BlockIndex] = true; + continue; + } + for (std::size_t Candidate = 0; Candidate < Blocks.size(); ++Candidate) + Dominators[BlockIndex][Candidate] = Reachable[Candidate]; + } + + bool Changed = true; + while (Changed) { + Changed = false; + for (std::size_t BlockIndex = 1; BlockIndex < Blocks.size(); ++BlockIndex) { + if (!Reachable[BlockIndex]) + continue; + std::vector<bool> NewDominators(Blocks.size(), false); + bool HasReachablePredecessor = false; + for (MachineBasicBlock *Predecessor : Blocks[BlockIndex]->predecessors()) { + const auto Found = BlockToIndex.find(Predecessor); + if (Found == BlockToIndex.end() || !Reachable[Found->second]) + continue; + if (!HasReachablePredecessor) { + NewDominators = Dominators[Found->second]; + HasReachablePredecessor = true; + } else { + for (std::size_t Candidate = 0; Candidate < Blocks.size(); ++Candidate) + NewDominators[Candidate] = + NewDominators[Candidate] && Dominators[Found->second][Candidate]; + } + } + if (!HasReachablePredecessor) { + return makeUnexpected(ErrorCode::VerificationError, + "reachable non-entry block has no reachable predecessor"); + } + NewDominators[BlockIndex] = true; + if (NewDominators != Dominators[BlockIndex]) { + Dominators[BlockIndex] = std::move(NewDominators); + Changed = true; + } + } + } + + for (std::size_t BlockIndex = 1; BlockIndex < Blocks.size(); ++BlockIndex) { + if (!Reachable[BlockIndex]) + continue; + std::size_t Best = Blocks.size(); + std::size_t BestDepth = 0; + for (std::size_t Candidate = 0; Candidate < Blocks.size(); ++Candidate) { + if (Candidate == BlockIndex || !Dominators[BlockIndex][Candidate]) + continue; + const std::size_t Depth = + static_cast<std::size_t>(std::ranges::count(Dominators[Candidate], true)); + if (Best == Blocks.size() || Depth > BestDepth) { + Best = Candidate; + BestDepth = Depth; + } + } + if (Best == Blocks.size()) + return makeUnexpected(ErrorCode::InternalError, "reachable block has no immediate dominator"); + ImmediateDominators[BlockIndex] = Blocks[Best]; + } + return {}; +} + +Expected<void> +MachineDominatorTreeAnalysis::Result::updateAfterCFGChange(MachineFunction &Function) { + return build(Function); +} + +bool MachineDominatorTreeAnalysis::Result::isReachable( + const MachineBasicBlock &Block) const noexcept { + const auto Found = BlockToIndex.find(&Block); + return Found != BlockToIndex.end() && Reachable[Found->second]; +} + +bool MachineDominatorTreeAnalysis::Result::dominates( + const MachineBasicBlock &Dominator, const MachineBasicBlock &Block) const noexcept { + const auto DominatorPosition = BlockToIndex.find(&Dominator); + const auto BlockPosition = BlockToIndex.find(&Block); + if (DominatorPosition == BlockToIndex.end() || BlockPosition == BlockToIndex.end() || + !Reachable[BlockPosition->second]) { + return false; + } + return Dominators[BlockPosition->second][DominatorPosition->second]; +} + +bool MachineDominatorTreeAnalysis::Result::dominates( + const MachineInstr &Dominator, const MachineInstr &Instruction) const noexcept { + const MachineBasicBlock *DominatorBlock = Dominator.parent(); + const MachineBasicBlock *InstructionBlock = Instruction.parent(); + if (DominatorBlock == nullptr || InstructionBlock == nullptr) + return false; + if (DominatorBlock != InstructionBlock) + return dominates(*DominatorBlock, *InstructionBlock); + for (const MachineInstr *Current = DominatorBlock->firstInstruction(); Current != nullptr; + Current = Current->next()) { + if (Current == &Dominator) + return true; + if (Current == &Instruction) + return false; + } + return false; +} + +const MachineBasicBlock *MachineDominatorTreeAnalysis::Result::immediateDominator( + const MachineBasicBlock &Block) const noexcept { + const auto Found = BlockToIndex.find(&Block); + return Found == BlockToIndex.end() ? nullptr : ImmediateDominators[Found->second]; +} + +const MachineBasicBlock *MachineDominatorTreeAnalysis::Result::nearestCommonDominator( + const MachineBasicBlock &Left, const MachineBasicBlock &Right) const noexcept { + if (!isReachable(Left) || !isReachable(Right)) + return nullptr; + std::unordered_set<const MachineBasicBlock *> LeftAncestors; + for (const MachineBasicBlock *Current = &Left; Current != nullptr; + Current = immediateDominator(*Current)) { + LeftAncestors.insert(Current); + } + for (const MachineBasicBlock *Current = &Right; Current != nullptr; + Current = immediateDominator(*Current)) { + if (LeftAncestors.contains(Current)) + return Current; + } + return nullptr; +} + +Expected<MachineDominatorTreeAnalysis::Result> +MachineDominatorTreeAnalysis::run(MachineFunction &Function, const TargetDescription &, + MachineFunctionAnalysisManager &) { + Result ResultValue; + auto Built = ResultValue.build(Function); + if (!Built) + return std::unexpected(Built.error()); + return ResultValue; +} + +const MachineLoop * +MachineLoopAnalysis::Result::loopFor(const MachineBasicBlock &Block) const noexcept { + const auto Found = InnermostLoop.find(&Block); + return Found == InnermostLoop.end() ? nullptr : &Loops[Found->second]; +} + +std::uint32_t +MachineLoopAnalysis::Result::loopDepth(const MachineBasicBlock &Block) const noexcept { + const MachineLoop *Loop = loopFor(Block); + return Loop == nullptr ? 0 : Loop->Depth; +} + +bool MachineLoopAnalysis::Result::isIrreducible(const MachineBasicBlock &Block) const noexcept { + return IrreducibleBlocks.contains(&Block); +} + +Expected<MachineLoopAnalysis::Result> +MachineLoopAnalysis::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + + Result ResultValue; + std::vector<MachineBasicBlock *> Blocks = linkedBlocks(Function); + const auto Order = blockOrder(Blocks); + std::map<MachineBasicBlock *, std::unordered_set<MachineBasicBlock *>> LoopBlocks; + std::unordered_map<MachineBasicBlock *, std::vector<MachineBasicBlock *>> LoopLatches; + + for (MachineBasicBlock *Latch : Blocks) { + if (!(*Dominators)->isReachable(*Latch)) + continue; + for (const MachineSuccessor &Successor : Latch->successors()) { + MachineBasicBlock *Header = Successor.Block; + if (Header == nullptr || !(*Dominators)->dominates(*Header, *Latch)) + continue; + auto &Members = LoopBlocks[Header]; + Members.insert(Header); + Members.insert(Latch); + LoopLatches[Header].push_back(Latch); + std::vector<MachineBasicBlock *> Worklist; + if (Latch != Header) + Worklist.push_back(Latch); + for (std::size_t Position = 0; Position < Worklist.size(); ++Position) { + MachineBasicBlock *Current = Worklist[Position]; + for (MachineBasicBlock *Predecessor : Current->predecessors()) { + if (!(*Dominators)->isReachable(*Predecessor) || + !(*Dominators)->dominates(*Header, *Predecessor)) { + continue; + } + if (Members.insert(Predecessor).second && Predecessor != Header) + Worklist.push_back(Predecessor); + } + } + } + } + + std::vector<MachineBasicBlock *> Headers; + Headers.reserve(LoopBlocks.size()); + for (const auto &[Header, Members] : LoopBlocks) { + (void)Members; + Headers.push_back(Header); + } + std::ranges::sort(Headers, + [&Order](const MachineBasicBlock *Left, const MachineBasicBlock *Right) { + return Order.at(Left) < Order.at(Right); + }); + for (MachineBasicBlock *Header : Headers) { + MachineLoop Loop; + Loop.Header = Header; + Loop.Blocks.assign(LoopBlocks[Header].begin(), LoopBlocks[Header].end()); + Loop.Latches = LoopLatches[Header]; + std::ranges::sort(Loop.Blocks, + [&Order](const MachineBasicBlock *Left, const MachineBasicBlock *Right) { + return Order.at(Left) < Order.at(Right); + }); + std::ranges::sort(Loop.Latches, + [&Order](const MachineBasicBlock *Left, const MachineBasicBlock *Right) { + return Order.at(Left) < Order.at(Right); + }); + Loop.Latches.erase(std::ranges::unique(Loop.Latches).begin(), Loop.Latches.end()); + ResultValue.Loops.push_back(std::move(Loop)); + } + + for (std::size_t Index = 0; Index < ResultValue.Loops.size(); ++Index) { + const MachineLoop &Loop = ResultValue.Loops[Index]; + std::size_t Parent = static_cast<std::size_t>(-1); + for (std::size_t Candidate = 0; Candidate < ResultValue.Loops.size(); ++Candidate) { + if (Candidate == Index || ResultValue.Loops[Candidate].Blocks.size() <= Loop.Blocks.size()) { + continue; + } + const bool Contains = std::ranges::all_of(Loop.Blocks, [&](MachineBasicBlock *Block) { + return containsBlock(ResultValue.Loops[Candidate].Blocks, Block); + }); + if (Contains && + (Parent == static_cast<std::size_t>(-1) || + ResultValue.Loops[Candidate].Blocks.size() < ResultValue.Loops[Parent].Blocks.size())) { + Parent = Candidate; + } + } + ResultValue.Loops[Index].Parent = Parent; + } + for (std::size_t Index = 0; Index < ResultValue.Loops.size(); ++Index) { + const std::size_t Parent = ResultValue.Loops[Index].Parent; + if (Parent != static_cast<std::size_t>(-1)) + ResultValue.Loops[Parent].Children.push_back(Index); + std::uint32_t Depth = 1; + for (std::size_t Ancestor = Parent; Ancestor != static_cast<std::size_t>(-1); + Ancestor = ResultValue.Loops[Ancestor].Parent) { + ++Depth; + } + ResultValue.Loops[Index].Depth = Depth; + } + for (std::size_t Index = 0; Index < ResultValue.Loops.size(); ++Index) { + for (MachineBasicBlock *Block : ResultValue.Loops[Index].Blocks) { + const auto Existing = ResultValue.InnermostLoop.find(Block); + if (Existing == ResultValue.InnermostLoop.end() || + ResultValue.Loops[Existing->second].Depth < ResultValue.Loops[Index].Depth) { + ResultValue.InnermostLoop[Block] = Index; + } + } + } + + std::unordered_set<MachineBasicBlock *> Visited; + std::vector<MachineBasicBlock *> FinishOrder; + for (MachineBasicBlock *Root : Blocks) { + if (!(*Dominators)->isReachable(*Root) || Visited.contains(Root)) + continue; + std::vector<std::pair<MachineBasicBlock *, std::size_t>> Stack{{Root, 0}}; + Visited.insert(Root); + while (!Stack.empty()) { + auto &[Current, NextSuccessor] = Stack.back(); + if (NextSuccessor < Current->successors().size()) { + MachineBasicBlock *Next = Current->successors()[NextSuccessor++].Block; + if (Next != nullptr && (*Dominators)->isReachable(*Next) && Visited.insert(Next).second) + Stack.emplace_back(Next, 0); + continue; + } + FinishOrder.push_back(Current); + Stack.pop_back(); + } + } + + Visited.clear(); + for (auto RootPosition = FinishOrder.rbegin(); RootPosition != FinishOrder.rend(); + ++RootPosition) { + MachineBasicBlock *Root = *RootPosition; + if (!Visited.insert(Root).second) + continue; + std::vector<MachineBasicBlock *> Component; + std::vector<MachineBasicBlock *> Worklist{Root}; + for (std::size_t Position = 0; Position < Worklist.size(); ++Position) { + MachineBasicBlock *Current = Worklist[Position]; + Component.push_back(Current); + for (MachineBasicBlock *Predecessor : Current->predecessors()) { + if ((*Dominators)->isReachable(*Predecessor) && Visited.insert(Predecessor).second) + Worklist.push_back(Predecessor); + } + } + bool Cyclic = Component.size() > 1; + if (!Cyclic) { + Cyclic = std::ranges::any_of( + Component.front()->successors(), + [Root](const MachineSuccessor &Successor) { return Successor.Block == Root; }); + } + if (!Cyclic) + continue; + + std::unordered_set<MachineBasicBlock *> ComponentSet(Component.begin(), Component.end()); + std::unordered_set<MachineBasicBlock *> EntrySet; + for (MachineBasicBlock *Block : Component) { + if (std::ranges::any_of(Block->predecessors(), [&](MachineBasicBlock *Predecessor) { + return !ComponentSet.contains(Predecessor); + })) { + EntrySet.insert(Block); + } + } + if (EntrySet.size() <= 1) + continue; + + std::ranges::sort(Component, + [&Order](const MachineBasicBlock *Left, const MachineBasicBlock *Right) { + return Order.at(Left) < Order.at(Right); + }); + IrreducibleMachineRegion Region; + Region.Blocks = std::move(Component); + Region.Entries.assign(EntrySet.begin(), EntrySet.end()); + std::ranges::sort(Region.Entries, + [&Order](const MachineBasicBlock *Left, const MachineBasicBlock *Right) { + return Order.at(Left) < Order.at(Right); + }); + for (MachineBasicBlock *Block : Region.Blocks) + ResultValue.IrreducibleBlocks.insert(Block); + ResultValue.IrreducibleRegions.push_back(std::move(Region)); + } + std::ranges::sort( + ResultValue.IrreducibleRegions, + [&Order](const IrreducibleMachineRegion &Left, const IrreducibleMachineRegion &Right) { + return Order.at(Left.Blocks.front()) < Order.at(Right.Blocks.front()); + }); + return ResultValue; +} + +BranchProbability +MachineBranchProbabilityAnalysis::Result::probability(const MachineBasicBlock &From, + const MachineBasicBlock &To) const noexcept { + const auto Found = Probabilities.find(&From); + if (Found == Probabilities.end()) + return BranchProbability::zero(); + for (const MachineSuccessor &Successor : Found->second) { + if (Successor.Block == &To) + return Successor.Probability; + } + return BranchProbability::zero(); +} + +std::span<const MachineSuccessor> MachineBranchProbabilityAnalysis::Result::successors( + const MachineBasicBlock &Block) const noexcept { + const auto Found = Probabilities.find(&Block); + return Found == Probabilities.end() ? std::span<const MachineSuccessor>{} + : std::span<const MachineSuccessor>{Found->second}; +} + +Expected<MachineBranchProbabilityAnalysis::Result> +MachineBranchProbabilityAnalysis::run(MachineFunction &Function, const TargetDescription &, + MachineFunctionAnalysisManager &) { + Result ResultValue; + const std::vector<MachineBasicBlock *> Blocks = linkedBlocks(Function); + const auto Order = blockOrder(Blocks); + for (MachineBasicBlock *Block : Blocks) { + std::vector<MachineSuccessor> Successors(Block->successors().begin(), + Block->successors().end()); + for (const MachineSuccessor &Successor : Successors) { + if (Successor.Block == nullptr || !Order.contains(Successor.Block)) { + return makeUnexpected(ErrorCode::VerificationError, + "branch-probability analysis encountered an invalid CFG edge"); + } + } + ResultValue.Probabilities.emplace(Block, std::move(Successors)); + } + return ResultValue; +} + +std::uint64_t +MachineBlockFrequencyAnalysis::Result::frequency(const MachineBasicBlock &Block) const noexcept { + const auto Found = Frequencies.find(&Block); + return Found == Frequencies.end() ? 0 : Found->second; +} + +Expected<MachineBlockFrequencyAnalysis::Result> +MachineBlockFrequencyAnalysis::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + auto Loops = Analyses.getResult<MachineLoopAnalysis>(Function, Target); + auto Probabilities = Analyses.getResult<MachineBranchProbabilityAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + if (!Loops) + return std::unexpected(Loops.error()); + if (!Probabilities) + return std::unexpected(Probabilities.error()); + + Result ResultValue; + MachineBasicBlock *Entry = Function.firstBlock(); + if (Entry == nullptr) + return ResultValue; + + std::unordered_set<MachineBasicBlock *> Visited; + std::vector<MachineBasicBlock *> PostOrder; + std::vector<std::pair<MachineBasicBlock *, std::size_t>> Stack{{Entry, 0}}; + Visited.insert(Entry); + while (!Stack.empty()) { + auto &[Current, NextSuccessor] = Stack.back(); + const auto Successors = (*Probabilities)->successors(*Current); + if (NextSuccessor < Successors.size()) { + MachineBasicBlock *Next = Successors[NextSuccessor++].Block; + if (Next != nullptr && (*Dominators)->isReachable(*Next) && Visited.insert(Next).second) + Stack.emplace_back(Next, 0); + continue; + } + PostOrder.push_back(Current); + Stack.pop_back(); + } + std::ranges::reverse(PostOrder); + std::unordered_map<MachineBasicBlock *, std::uint64_t> BaseFrequencies; + BaseFrequencies[Entry] = EntryFrequency; + for (MachineBasicBlock *Block : PostOrder) { + const std::uint64_t Frequency = BaseFrequencies[Block]; + + const auto Successors = (*Probabilities)->successors(*Block); + const bool UseUniformProbability = + !Successors.empty() && std::ranges::all_of(Successors, [](const MachineSuccessor &Edge) { + return Edge.Probability == BranchProbability::zero(); + }); + for (std::size_t SuccessorIndex = 0; SuccessorIndex < Successors.size(); ++SuccessorIndex) { + const MachineSuccessor &Successor = Successors[SuccessorIndex]; + if (Successor.Block == nullptr || (*Dominators)->dominates(*Successor.Block, *Block)) + continue; + std::uint64_t Contribution = 0; + if (UseUniformProbability) { + const std::uint64_t Count = Successors.size(); + Contribution = Frequency / Count; + if (SuccessorIndex < Frequency % Count) + ++Contribution; + } else { + Contribution = scaleByProbability(Frequency, Successor.Probability); + } + BaseFrequencies[Successor.Block] = + saturatingAdd(BaseFrequencies[Successor.Block], Contribution); + } + } + for (MachineBasicBlock *Block : linkedBlocks(Function)) { + if (!(*Dominators)->isReachable(*Block)) { + ResultValue.Frequencies[Block] = 0; + continue; + } + std::uint64_t Frequency = std::max<std::uint64_t>(BaseFrequencies[Block], 1); + for (std::uint32_t Level = 0; Level < (*Loops)->loopDepth(*Block); ++Level) + Frequency = saturatingMultiply(Frequency, 8); + ResultValue.Frequencies[Block] = Frequency; + } + return ResultValue; +} + +MachineAliasResult +MachineAliasAnalysis::Result::alias(const MachineMemOperand &Left, + const MachineMemOperand &Right) const noexcept { + const MachineMemOperandDesc &LeftDesc = Left.description(); + const MachineMemOperandDesc &RightDesc = Right.description(); + if (isKnownDistinctObject(LeftDesc, RightDesc)) + return MachineAliasResult::NoAlias; + + const bool SameBase = + LeftDesc.Base == RightDesc.Base && LeftDesc.Base.kind() != MemoryBase::Kind::Unknown; + const bool SameAliasSource = LeftDesc.AliasSource && RightDesc.AliasSource && + LeftDesc.AliasSource == RightDesc.AliasSource; + if (!SameBase && !SameAliasSource) + return MachineAliasResult::MayAlias; + if (!LeftDesc.Size.isKnown() || !RightDesc.Size.isKnown() || LeftDesc.Size.isScalable() || + RightDesc.Size.isScalable()) { + return MachineAliasResult::MayAlias; + } + + const std::uint64_t LeftSize = LeftDesc.Size.minimumBytes(); + const std::uint64_t RightSize = RightDesc.Size.minimumBytes(); + if (LeftDesc.Offset == RightDesc.Offset) + return LeftSize == RightSize ? MachineAliasResult::MustAlias : MachineAliasResult::PartialAlias; + + const bool LeftFirst = LeftDesc.Offset < RightDesc.Offset; + const std::int64_t Lower = LeftFirst ? LeftDesc.Offset : RightDesc.Offset; + const std::int64_t Higher = LeftFirst ? RightDesc.Offset : LeftDesc.Offset; + const std::uint64_t LowerSize = LeftFirst ? LeftSize : RightSize; + const std::uint64_t Distance = + static_cast<std::uint64_t>(Higher) - static_cast<std::uint64_t>(Lower); + return Distance >= LowerSize ? MachineAliasResult::NoAlias : MachineAliasResult::PartialAlias; +} + +Expected<MachineAliasAnalysis::Result> MachineAliasAnalysis::run(MachineFunction &, + const TargetDescription &, + MachineFunctionAnalysisManager &) { + return Result{}; +} + +std::span<const MachineMemoryDependency> MachineMemoryDependenceAnalysis::Result::dependencies( + const MachineInstr &Instruction) const noexcept { + const auto Found = DestinationRanges.find(Instruction.stableID()); + if (Found == DestinationRanges.end()) + return {}; + return std::span<const MachineMemoryDependency>(Dependencies) + .subspan(Found->second.Begin, Found->second.Size); +} + +const MachineMemoryDependenceRegion * +MachineMemoryDependenceAnalysis::Result::regionFor(const MachineBasicBlock &Block) const noexcept { + const auto Found = BlockToRegion.find(&Block); + return Found == BlockToRegion.end() ? nullptr : &Regions[Found->second]; +} + +std::span<const MachineMemoryDependency> MachineMemoryDependenceAnalysis::Result::dependencies( + const MachineBasicBlock &Block) const noexcept { + const auto Found = BlockRanges.find(&Block); + if (Found == BlockRanges.end()) + return {}; + return std::span<const MachineMemoryDependency>(Dependencies) + .subspan(Found->second.Begin, Found->second.Size); +} + +Expected<MachineMemoryDependenceAnalysis::Result> +MachineMemoryDependenceAnalysis::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Alias = Analyses.getResult<MachineAliasAnalysis>(Function, Target); + if (!Alias) + return std::unexpected(Alias.error()); + + struct MemoryNode final { + MachineInstr *Instruction = nullptr; + bool Reads = false; + bool Writes = false; + bool Ordered = false; + bool Barrier = false; + }; + + Result ResultValue; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + std::vector<MemoryNode> Nodes; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory-dependence analysis encountered an unknown opcode"); + } + MemoryNode Node; + Node.Instruction = Instruction; + Node.Reads = Descriptor->Flags.has(InstructionFlag::MayLoad); + Node.Writes = Descriptor->Flags.has(InstructionFlag::MayStore); + Node.Ordered = hasOrderedMemory(*Instruction); + Node.Barrier = Descriptor->Flags.has(InstructionFlag::Call) || + Descriptor->Flags.has(InstructionFlag::UnmodeledSideEffects) || + (Descriptor->Flags.has(InstructionFlag::HasSideEffects) && + Instruction->memOperands().empty()); + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + if (Memory == nullptr) + continue; + Node.Reads = Node.Reads || Memory->isLoad(); + Node.Writes = Node.Writes || Memory->isStore(); + } + if (Node.Reads || Node.Writes || Node.Ordered || Node.Barrier) + Nodes.push_back(Node); + } + + MachineMemoryDependenceRegion Region; + Region.Block = Block; + Region.Instructions.reserve(Nodes.size()); + for (const MemoryNode &Node : Nodes) + Region.Instructions.push_back(Node.Instruction->stableID()); + ResultValue.BlockToRegion.emplace(Block, ResultValue.Regions.size()); + ResultValue.Regions.push_back(std::move(Region)); + + const std::size_t RegionBegin = ResultValue.Dependencies.size(); + for (std::size_t Destination = 0; Destination < Nodes.size(); ++Destination) { + for (std::size_t Source = 0; Source < Destination; ++Source) { + const MemoryNode &From = Nodes[Source]; + const MemoryNode &To = Nodes[Destination]; + if (!From.Writes && !To.Writes && !From.Ordered && !To.Ordered && !From.Barrier && + !To.Barrier) { + continue; + } + + bool MayAlias = From.Barrier || To.Barrier || From.Ordered || To.Ordered || + From.Instruction->memOperands().empty() || + To.Instruction->memOperands().empty(); + if (!MayAlias) { + for (const MachineMemOperand *Left : From.Instruction->memOperands()) { + for (const MachineMemOperand *Right : To.Instruction->memOperands()) { + if (Left != nullptr && Right != nullptr && + (*Alias)->alias(*Left, *Right) != MachineAliasResult::NoAlias) { + MayAlias = true; + break; + } + } + if (MayAlias) + break; + } + } + if (!MayAlias) + continue; + + MachineMemoryDependencyKind Kind = MachineMemoryDependencyKind::Ordered; + if (From.Writes && To.Reads) + Kind = MachineMemoryDependencyKind::Flow; + else if (From.Reads && To.Writes) + Kind = MachineMemoryDependencyKind::Anti; + else if (From.Writes && To.Writes) + Kind = MachineMemoryDependencyKind::Output; + ResultValue.Dependencies.push_back( + {From.Instruction->stableID(), To.Instruction->stableID(), Kind}); + } + } + + const std::size_t RegionEnd = ResultValue.Dependencies.size(); + ResultValue.BlockRanges.emplace(Block, Result::Range{RegionBegin, RegionEnd - RegionBegin}); + for (std::size_t Begin = RegionBegin; Begin < RegionEnd;) { + std::size_t End = Begin + 1; + while (End < RegionEnd && ResultValue.Dependencies[End].Destination == + ResultValue.Dependencies[Begin].Destination) { + ++End; + } + ResultValue.DestinationRanges.emplace(ResultValue.Dependencies[Begin].Destination, + Result::Range{Begin, End - Begin}); + Begin = End; + } + } + return ResultValue; +} + +Expected<void> registerStage8MachineAnalyses(MachineFunctionAnalysisManager &Analyses) { + const std::array IDs{ + MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineFunctionAnalysisManager::id<MachineLoopAnalysis>(), + MachineFunctionAnalysisManager::id<MachineBranchProbabilityAnalysis>(), + MachineFunctionAnalysisManager::id<MachineBlockFrequencyAnalysis>(), + MachineFunctionAnalysisManager::id<MachineAliasAnalysis>(), + MachineFunctionAnalysisManager::id<MachineMemoryDependenceAnalysis>(), + }; + const std::size_t Registered = static_cast<std::size_t>( + std::ranges::count_if(IDs, [&Analyses](AnalysisID ID) { return Analyses.isRegistered(ID); })); + if (Registered == IDs.size()) + return {}; + if (Registered != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "standard machine analyses are only partially registered"); + } + + auto Candidate = Analyses.forkEmpty(); + if (!Candidate) + return std::unexpected(Candidate.error()); + MachineFunctionAnalysisManager &Staging = **Candidate; + + auto Dominators = Staging.registerAnalysis<MachineDominatorTreeAnalysis>({ + .Name = "MachineDominatorTree", + .Dependencies = {}, + .InvalidatedBy = {MachinePassChange::CFG, MachinePassChange::BlockLayout}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!Dominators) + return std::unexpected(Dominators.error()); + auto Loops = Staging.registerAnalysis<MachineLoopAnalysis>({ + .Name = "MachineLoopInfo", + .Dependencies = {MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>()}, + .InvalidatedBy = {MachinePassChange::CFG, MachinePassChange::BlockLayout}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!Loops) + return std::unexpected(Loops.error()); + auto Probabilities = Staging.registerAnalysis<MachineBranchProbabilityAnalysis>({ + .Name = "MachineBranchProbability", + .Dependencies = {}, + .InvalidatedBy = {MachinePassChange::CFG}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!Probabilities) + return std::unexpected(Probabilities.error()); + auto Frequencies = Staging.registerAnalysis<MachineBlockFrequencyAnalysis>({ + .Name = "MachineBlockFrequency", + .Dependencies = {MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineFunctionAnalysisManager::id<MachineLoopAnalysis>(), + MachineFunctionAnalysisManager::id<MachineBranchProbabilityAnalysis>()}, + .InvalidatedBy = {MachinePassChange::CFG, MachinePassChange::BlockLayout}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Alias = Staging.registerAnalysis<MachineAliasAnalysis>({ + .Name = "MachineAliasAnalysis", + .Dependencies = {}, + .InvalidatedBy = {MachinePassChange::Operands, MachinePassChange::Frame, + MachinePassChange::ConstantPool, MachinePassChange::JumpTable, + MachinePassChange::TargetState}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!Alias) + return std::unexpected(Alias.error()); + auto MemoryDependence = Staging.registerAnalysis<MachineMemoryDependenceAnalysis>({ + .Name = "MachineMemoryDependence", + .Dependencies = {MachineFunctionAnalysisManager::id<MachineAliasAnalysis>()}, + .InvalidatedBy = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout, + MachinePassChange::Frame, MachinePassChange::ConstantPool, + MachinePassChange::JumpTable, MachinePassChange::TargetState}, + .ThreadSafety = PassThreadSafety::ThreadSafe, + }); + if (!MemoryDependence) + return std::unexpected(MemoryDependence.error()); + return Analyses.replaceRegistrations(std::move(Staging)); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBasicBlock.cpp b/backend/lib/CodeGen/MachineBasicBlock.cpp new file mode 100644 index 0000000..2de8f89 --- /dev/null +++ b/backend/lib/CodeGen/MachineBasicBlock.cpp @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBasicBlock.h" + +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" + +#include <algorithm> +#include <cstddef> + +namespace rcc { + +std::size_t MachineBasicBlock::instructionCount() const noexcept { + std::size_t Count = 0; + for (const MachineInstr *Instruction = FirstInstruction; Instruction != nullptr; + Instruction = Instruction->next()) { + ++Count; + } + return Count; +} + +Expected<void> MachineBasicBlock::appendInstruction(MachineInstr &Instruction) { + if (&Instruction.function() != OwnerFunction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction belongs to another machine function"); + } + if (Instruction.isLinked()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction is already linked to a basic block"); + } + + auto AttachResult = Instruction.attachToBlock(*this); + if (!AttachResult) + return std::unexpected(AttachResult.error()); + + Instruction.PreviousInstruction = LastInstruction; + Instruction.NextInstruction = nullptr; + if (LastInstruction != nullptr) + LastInstruction->NextInstruction = &Instruction; + else + FirstInstruction = &Instruction; + LastInstruction = &Instruction; + if (OwnerFunction->TheDelegate != nullptr) + OwnerFunction->TheDelegate->handleInsertion(Instruction); + return {}; +} + +Expected<void> MachineBasicBlock::insertInstructionBefore(MachineInstr &Position, + MachineInstr &Instruction) { + if (Position.parent() != this) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction insertion position is not in this block"); + } + if (&Instruction.function() != OwnerFunction || Instruction.isLinked()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "inserted instruction is linked or belongs to another function"); + } + + auto AttachResult = Instruction.attachToBlock(*this); + if (!AttachResult) + return std::unexpected(AttachResult.error()); + + Instruction.PreviousInstruction = Position.PreviousInstruction; + Instruction.NextInstruction = &Position; + if (Position.PreviousInstruction != nullptr) + Position.PreviousInstruction->NextInstruction = &Instruction; + else + FirstInstruction = &Instruction; + Position.PreviousInstruction = &Instruction; + if (OwnerFunction->TheDelegate != nullptr) + OwnerFunction->TheDelegate->handleInsertion(Instruction); + return {}; +} + +Expected<void> MachineBasicBlock::insertInstructionAfter(MachineInstr &Position, + MachineInstr &Instruction) { + if (Position.parent() != this) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction insertion position is not in this block"); + } + if (&Instruction.function() != OwnerFunction || Instruction.isLinked()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "inserted instruction is linked or belongs to another function"); + } + + auto AttachResult = Instruction.attachToBlock(*this); + if (!AttachResult) + return std::unexpected(AttachResult.error()); + + Instruction.PreviousInstruction = &Position; + Instruction.NextInstruction = Position.NextInstruction; + if (Position.NextInstruction != nullptr) + Position.NextInstruction->PreviousInstruction = &Instruction; + else + LastInstruction = &Instruction; + Position.NextInstruction = &Instruction; + if (OwnerFunction->TheDelegate != nullptr) + OwnerFunction->TheDelegate->handleInsertion(Instruction); + return {}; +} + +Expected<void> MachineBasicBlock::spliceInstructionBefore(MachineInstr &Position, + MachineInstr &Instruction) { + if (Position.parent() != this || Instruction.parent() != this) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction splice requires two instructions in this block"); + } + if (&Position == &Instruction || Instruction.NextInstruction == &Position) + return {}; + + if (Instruction.PreviousInstruction != nullptr) + Instruction.PreviousInstruction->NextInstruction = Instruction.NextInstruction; + else + FirstInstruction = Instruction.NextInstruction; + if (Instruction.NextInstruction != nullptr) + Instruction.NextInstruction->PreviousInstruction = Instruction.PreviousInstruction; + else + LastInstruction = Instruction.PreviousInstruction; + + Instruction.PreviousInstruction = Position.PreviousInstruction; + Instruction.NextInstruction = &Position; + if (Position.PreviousInstruction != nullptr) + Position.PreviousInstruction->NextInstruction = &Instruction; + else + FirstInstruction = &Instruction; + Position.PreviousInstruction = &Instruction; + return {}; +} + +Expected<void> MachineBasicBlock::removeInstruction(MachineInstr &Instruction) { + if (Instruction.parent() != this) { + return makeUnexpected(ErrorCode::InvalidArgument, "instruction does not belong to this block"); + } + + if (OwnerFunction->TheDelegate != nullptr) + OwnerFunction->TheDelegate->handleRemoval(Instruction); + + if (Instruction.PreviousInstruction != nullptr) + Instruction.PreviousInstruction->NextInstruction = Instruction.NextInstruction; + else + FirstInstruction = Instruction.NextInstruction; + + if (Instruction.NextInstruction != nullptr) + Instruction.NextInstruction->PreviousInstruction = Instruction.PreviousInstruction; + else + LastInstruction = Instruction.PreviousInstruction; + + Instruction.detachFromBlock(); + return {}; +} + +void MachineBasicBlock::clearInstructions() noexcept { + MachineInstr *Instruction = FirstInstruction; + while (Instruction != nullptr) { + MachineInstr *NextInstruction = Instruction->NextInstruction; + Instruction->detachFromBlock(); + Instruction = NextInstruction; + } + FirstInstruction = nullptr; + LastInstruction = nullptr; +} + +Expected<void> MachineBasicBlock::addLiveIn(PhysicalRegister PhysReg, LaneMask Lanes) { + if (!PhysReg.isValid() || Lanes.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "block live-in requires a physical register and nonempty lanes"); + } + + const auto Position = + std::ranges::find_if(LiveIns, [PhysReg](const MachineBasicBlockLiveIn &LiveIn) { + return LiveIn.PhysReg == PhysReg; + }); + if (Position != LiveIns.end()) { + Position->Lanes = Position->Lanes | Lanes; + return {}; + } + + LiveIns.push_back(MachineBasicBlockLiveIn{PhysReg, Lanes}); + return {}; +} + +Expected<void> MachineBasicBlock::removeLiveIn(PhysicalRegister PhysReg) { + const auto Position = + std::ranges::find_if(LiveIns, [PhysReg](const MachineBasicBlockLiveIn &LiveIn) { + return LiveIn.PhysReg == PhysReg; + }); + if (Position == LiveIns.end()) { + return makeUnexpected(ErrorCode::InvalidArgument, "block physical live-in is not present"); + } + LiveIns.erase(Position); + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBasicBlockAttributes.cpp b/backend/lib/CodeGen/MachineBasicBlockAttributes.cpp new file mode 100644 index 0000000..8cb6d48 --- /dev/null +++ b/backend/lib/CodeGen/MachineBasicBlockAttributes.cpp @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" + +#include <cstdint> +#include <string> +#include <string_view> +#include <utility> + +namespace rcc { + +std::string_view getMachineBasicBlockAttributeName(MachineBasicBlockAttribute Attribute) noexcept { + switch (Attribute) { + case MachineBasicBlockAttribute::MachineBlockAddressTaken: + return "MachineBlockAddressTaken"; + case MachineBasicBlockAttribute::IRBlockAddressTaken: + return "IRBlockAddressTaken"; + case MachineBasicBlockAttribute::LandingPad: + return "LandingPad"; + case MachineBasicBlockAttribute::InlineAsmBrIndirectTarget: + return "InlineAsmBrIndirectTarget"; + case MachineBasicBlockAttribute::EHFuncletEntry: + return "EHFuncletEntry"; + case MachineBasicBlockAttribute::EHScopeEntry: + return "EHScopeEntry"; + case MachineBasicBlockAttribute::Count: + return "Invalid"; + } + return "Invalid"; +} + +Expected<MachineBasicBlockAttributes> +MachineBasicBlockAttributes::fromRawFlags(std::uint8_t RawFlags) { + if ((RawFlags & static_cast<std::uint8_t>(~ValidFlags)) != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine basic block attributes contain unknown flag bits"); + } + MachineBasicBlockAttributes Result; + Result.Flags = RawFlags; + return Result; +} + +Expected<void> MachineBasicBlockAttributes::setName(std::string NewName) { + if (NewName.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, "machine basic block name cannot be empty"); + } + Name = std::move(NewName); + return {}; +} + +Expected<void> MachineBasicBlockAttributes::setIRBlockName(std::string NewName) { + if (NewName.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine basic block IR name cannot be empty"); + } + IRBlockName = std::move(NewName); + return {}; +} + +Expected<void> MachineBasicBlockAttributes::verify() const { + if ((Flags & static_cast<std::uint8_t>(~ValidFlags)) != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "machine basic block attributes contain unknown flag bits"); + } + if (Name && Name->empty()) { + return makeUnexpected(ErrorCode::VerificationError, "machine basic block name cannot be empty"); + } + if (IRBlockName && IRBlockName->empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine basic block IR name cannot be empty"); + } + if (has(MachineBasicBlockAttribute::IRBlockAddressTaken) && !IRBlockName) { + return makeUnexpected(ErrorCode::VerificationError, + "IR-block-address-taken block requires an IR block reference"); + } + if (BlockAlignment.bytes() == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "machine basic block alignment must be nonzero"); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBlockPlacement.cpp b/backend/lib/CodeGen/MachineBlockPlacement.cpp new file mode 100644 index 0000000..1b09f7d --- /dev/null +++ b/backend/lib/CodeGen/MachineBlockPlacement.cpp @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBlockPlacement.h" + +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineFunction.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +enum class PlacementClass : std::uint8_t { + Hot, + Normal, + Cold, + Exception, +}; + +[[nodiscard]] const MachinePassContract &placementContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-block-placement", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = + { + MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineFunctionAnalysisManager::id<MachineBranchProbabilityAnalysis>(), + MachineFunctionAnalysisManager::id<MachineBlockFrequencyAnalysis>(), + MachineFunctionAnalysisManager::id<MachineLoopAnalysis>(), + }, + .VerificationStage = std::nullopt, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] PlacementClass classify(const MachineBasicBlock &Block, + std::uint64_t Frequency) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + if (Attributes.sectionID().kind() == MachineBasicBlockSectionKind::Exception || + Attributes.has(MachineBasicBlockAttribute::LandingPad) || + Attributes.has(MachineBasicBlockAttribute::EHFuncletEntry) || + Attributes.has(MachineBasicBlockAttribute::EHScopeEntry)) { + return PlacementClass::Exception; + } + if (Attributes.sectionID().kind() == MachineBasicBlockSectionKind::Cold) + return PlacementClass::Cold; + if (Frequency >= MachineBlockFrequencyAnalysis::EntryFrequency) + return PlacementClass::Hot; + if (Frequency < MachineBlockFrequencyAnalysis::EntryFrequency / 1024U) + return PlacementClass::Cold; + return PlacementClass::Normal; +} + +[[nodiscard]] std::uint32_t classRank(PlacementClass Class) noexcept { + return static_cast<std::uint32_t>(Class); +} + +struct PlacementFacts final { + MachineBasicBlock *Block = nullptr; + PlacementClass Class = PlacementClass::Normal; + std::uint64_t Frequency = 0; + std::uint32_t LoopDepth = 0; +}; + +[[nodiscard]] bool betterRoot(const PlacementFacts &Left, const PlacementFacts &Right) noexcept { + if (Left.Class != Right.Class) + return classRank(Left.Class) < classRank(Right.Class); + if (Left.Frequency != Right.Frequency) + return Left.Frequency > Right.Frequency; + if (Left.LoopDepth != Right.LoopDepth) + return Left.LoopDepth > Right.LoopDepth; + return Left.Block->stableID().value() < Right.Block->stableID().value(); +} + +} // namespace + +const MachinePassContract &MachineBlockPlacementPass::contract() const noexcept { + return placementContract(); +} + +Expected<MachinePassResult> +MachineBlockPlacementPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +MachineBlockPlacementPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) const { + auto Probabilities = Analyses.getResult<MachineBranchProbabilityAnalysis>(Function, Target); + if (!Probabilities) + return std::unexpected(Probabilities.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Loops = Analyses.getResult<MachineLoopAnalysis>(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + + std::vector<PlacementFacts> Facts; + Facts.reserve(Function.blockCount()); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + const std::uint64_t Frequency = (*Frequencies)->frequency(*Block); + Facts.push_back({Block, classify(*Block, Frequency), Frequency, (*Loops)->loopDepth(*Block)}); + } + if (Facts.size() < 2) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + auto FactsFor = [&Facts](const MachineBasicBlock &Block) -> const PlacementFacts & { + return *std::ranges::find_if( + Facts, [&Block](const PlacementFacts &Value) { return Value.Block == &Block; }); + }; + std::unordered_set<MachineBasicBlock *> Placed; + std::vector<MachineBasicBlock *> Order; + Order.reserve(Facts.size()); + MachineBasicBlock *Root = Function.firstBlock(); + while (Order.size() != Facts.size()) { + if (Root == nullptr || Placed.contains(Root)) { + const auto Best = std::ranges::min_element( + Facts, [&Placed](const PlacementFacts &Left, const PlacementFacts &Right) { + if (Placed.contains(Left.Block)) + return false; + if (Placed.contains(Right.Block)) + return true; + return betterRoot(Left, Right); + }); + if (Best == Facts.end() || Placed.contains(Best->Block)) + return makeUnexpected(ErrorCode::InternalError, "block placement lost an unplaced block"); + Root = Best->Block; + } + + while (Root != nullptr && Placed.insert(Root).second) { + Order.push_back(Root); + MachineBasicBlock *BestSuccessor = nullptr; + BranchProbability BestProbability = BranchProbability::zero(); + for (const MachineSuccessor &Successor : (*Probabilities)->successors(*Root)) { + if (Successor.Block == nullptr || Placed.contains(Successor.Block)) + continue; + if (BestSuccessor == nullptr || BestProbability < Successor.Probability || + (BestProbability == Successor.Probability && + betterRoot(FactsFor(*Successor.Block), FactsFor(*BestSuccessor)))) { + BestSuccessor = Successor.Block; + BestProbability = Successor.Probability; + } + } + Root = BestSuccessor; + } + Root = nullptr; + } + + const auto &SavePoints = Function.frameInfo().state().SavePoints; + if (!SavePoints.empty() && SavePoints.front().Block != Function.firstBlock()->stableID()) { + const auto SavePosition = std::ranges::find_if(Order, [&](const MachineBasicBlock *Block) { + return Block->stableID() == SavePoints.front().Block; + }); + if (SavePosition == Order.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "block placement cannot find the frame save point"); + } + MachineBasicBlock *SaveBlock = *SavePosition; + std::vector<MachineBasicBlock *> OutsideFrame; + std::vector<MachineBasicBlock *> InsideFrame; + for (MachineBasicBlock *Block : Order) { + ((*Dominators)->dominates(*SaveBlock, *Block) ? InsideFrame : OutsideFrame).push_back(Block); + } + const auto SaveInRegion = std::ranges::find(InsideFrame, SaveBlock); + if (SaveInRegion == InsideFrame.end()) { + return makeUnexpected(ErrorCode::InternalError, "block placement lost the frame save region"); + } + std::rotate(InsideFrame.begin(), SaveInRegion, std::next(SaveInRegion)); + Order.clear(); + Order.insert(Order.end(), OutsideFrame.begin(), OutsideFrame.end()); + Order.insert(Order.end(), InsideFrame.begin(), InsideFrame.end()); + } + + bool Changed = false; + for (std::size_t Index = 1; Index < Order.size(); ++Index) { + if (Order[Index - 1U]->next() != Order[Index]) { + Changed = true; + break; + } + } + if (Changed) { + auto Accepted = Analyses.requestAction("apply-block-layout", Function); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + } + for (std::size_t Index = 1; Index < Order.size(); ++Index) { + auto Moved = Function.moveBlockAfter(*Order[Index - 1U], *Order[Index]); + if (!Moved) + return std::unexpected(Moved.error()); + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBranchOptimization.cpp b/backend/lib/CodeGen/MachineBranchOptimization.cpp new file mode 100644 index 0000000..6e3bc17 --- /dev/null +++ b/backend/lib/CodeGen/MachineBranchOptimization.cpp @@ -0,0 +1,622 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBranchOptimization.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetInstrInfo.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <ranges> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &branchOptimizationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-branch-optimization", + .Requires = {MachineFunctionProperty::NoPHIs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::FrameFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasInstructionRecord(const MachineInstr &Instruction, + const MachineFunction &Function) noexcept { + const MachineInstrStableID ID = Instruction.stableID(); + return std::ranges::any_of(Function.records().callSites(), + [ID](const auto &Record) { return Record.CallInstruction == ID; }) || + std::ranges::any_of(Function.records().calledGlobals(), + [ID](const auto &Record) { return Record.CallInstruction == ID; }); +} + +[[nodiscard]] bool hasObservableBlockIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +template <typename ValueT> +[[nodiscard]] bool payloadEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const ValueT *LeftValue = Left.getIf<ValueT>(); + const ValueT *RightValue = Right.getIf<ValueT>(); + return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue; +} + +template <typename MaskOperandT> +[[nodiscard]] bool maskEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const MaskOperandT *LeftValue = Left.getIf<MaskOperandT>(); + const MaskOperandT *RightValue = Right.getIf<MaskOperandT>(); + if (LeftValue == nullptr || RightValue == nullptr) + return false; + if (!LeftValue->Mask || !RightValue->Mask) + return LeftValue->Mask == RightValue->Mask; + return *LeftValue->Mask == *RightValue->Mask; +} + +[[nodiscard]] bool operandEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags() || + Left.tiedOperandIndex() != Right.tiedOperandIndex()) { + return false; + } + if (Left.kind() == MachineOperand::Kind::Register) { + const Register *LeftReg = Left.getIf<Register>(); + const Register *RightReg = Right.getIf<Register>(); + if (LeftReg == nullptr || RightReg == nullptr || *LeftReg != *RightReg || + Left.subReg() != Right.subReg()) { + return false; + } + RegisterOperandFlags LeftFlags = Left.registerFlags(); + RegisterOperandFlags RightFlags = Right.registerFlags(); + for (const RegisterOperandFlag Flag : + {RegisterOperandFlag::Kill, RegisterOperandFlag::Dead, RegisterOperandFlag::Renamable}) { + LeftFlags.clear(Flag); + RightFlags.clear(Flag); + } + return LeftFlags == RightFlags; + } + switch (Left.kind()) { + case MachineOperand::Kind::Register: + return false; + case MachineOperand::Kind::Immediate: + return payloadEquals<std::int64_t>(Left, Right); + case MachineOperand::Kind::IntegerImmediate: + return payloadEquals<IntegerImmediateOperand>(Left, Right); + case MachineOperand::Kind::FloatingImmediate: + return payloadEquals<FloatingImmediateOperand>(Left, Right); + case MachineOperand::Kind::BasicBlock: + return payloadEquals<BasicBlockOperand>(Left, Right); + case MachineOperand::Kind::FrameIndex: + return payloadEquals<FrameIndexOperand>(Left, Right); + case MachineOperand::Kind::ConstantPoolIndex: + return payloadEquals<ConstantPoolOperand>(Left, Right); + case MachineOperand::Kind::JumpTableIndex: + return payloadEquals<JumpTableOperand>(Left, Right); + case MachineOperand::Kind::GlobalSymbol: + return payloadEquals<GlobalSymbolOperand>(Left, Right); + case MachineOperand::Kind::ExternalSymbol: + return payloadEquals<ExternalSymbolOperand>(Left, Right); + case MachineOperand::Kind::BlockAddress: + return payloadEquals<BlockAddressOperand>(Left, Right); + case MachineOperand::Kind::RegisterMask: + return maskEquals<RegisterMaskOperand>(Left, Right); + case MachineOperand::Kind::LiveOutRegisterMask: + return maskEquals<LiveOutRegisterMaskOperand>(Left, Right); + case MachineOperand::Kind::Metadata: + return payloadEquals<MetadataOperand>(Left, Right); + case MachineOperand::Kind::TargetIndex: + return payloadEquals<TargetIndexOperand>(Left, Right); + case MachineOperand::Kind::MCSymbol: + return payloadEquals<MCSymbolOperand>(Left, Right); + case MachineOperand::Kind::CFIIndex: + return payloadEquals<CFIIndexOperand>(Left, Right); + case MachineOperand::Kind::Intrinsic: + return payloadEquals<IntrinsicOperand>(Left, Right); + case MachineOperand::Kind::Predicate: + return payloadEquals<PredicateOperand>(Left, Right); + case MachineOperand::Kind::ShuffleMask: + return payloadEquals<ShuffleMaskOperand>(Left, Right); + case MachineOperand::Kind::DebugInstrRef: + return payloadEquals<DebugInstrRefOperand>(Left, Right); + case MachineOperand::Kind::LaneMask: + return payloadEquals<LaneMaskOperand>(Left, Right); + case MachineOperand::Kind::Count: + return false; + } + return false; +} + +[[nodiscard]] Expected<bool> instructionEquals(const MachineInstr &Left, const MachineInstr &Right, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Left.opcode()); + if (Descriptor == nullptr || Target.instructionDescriptor(Right.opcode()) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "branch folding encountered an unknown instruction"); + } + if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() || + Left.operandCount() != Right.operandCount() || + Left.memOperands().size() != Right.memOperands().size() || + Left.sourceLocation() != Right.sourceLocation() || + Left.debugLocation() != Right.debugLocation() || Left.origin() != Right.origin() || + hasProtectedAttachment(Left) || hasProtectedAttachment(Right) || + Descriptor->Flags.has(InstructionFlag::Call) || + Descriptor->Flags.has(InstructionFlag::Convergent) || + Descriptor->Flags.has(InstructionFlag::NotDuplicable) || + Left.flags().has(MachineInstrFlag::NoMerge) || + Left.flags().has(MachineInstrFlag::BundledWithPred) || + Left.flags().has(MachineInstrFlag::BundledWithSucc)) { + return false; + } + for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) { + const MachineOperand *LeftOperand = Left.operandAt(Index); + const MachineOperand *RightOperand = Right.operandAt(Index); + if (LeftOperand == nullptr || RightOperand == nullptr || + !operandEquals(*LeftOperand, *RightOperand)) { + return false; + } + } + for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) { + const MachineMemOperand *LeftMemory = Left.memOperands()[Index]; + const MachineMemOperand *RightMemory = Right.memOperands()[Index]; + if (LeftMemory == nullptr || RightMemory == nullptr || + LeftMemory->description() != RightMemory->description()) { + return false; + } + } + return true; +} + +[[nodiscard]] Expected<bool> blockEquals(const MachineBasicBlock &Left, + const MachineBasicBlock &Right, + const MachineFunction &Function, + const TargetDescription &Target) { + if (&Left == &Right || &Left == Function.firstBlock() || &Right == Function.firstBlock() || + hasObservableBlockIdentity(Left, Function) || hasObservableBlockIdentity(Right, Function) || + Left.attributes().alignment() != Right.attributes().alignment() || + Left.attributes().sectionID() != Right.attributes().sectionID() || + Left.attributes().callFrameSize() != Right.attributes().callFrameSize() || + !std::ranges::equal(Left.liveIns(), Right.liveIns()) || + !std::ranges::equal(Left.successors(), Right.successors()) || + Left.instructionCount() != Right.instructionCount()) { + return false; + } + const MachineInstr *LeftInstruction = Left.firstInstruction(); + const MachineInstr *RightInstruction = Right.firstInstruction(); + while (LeftInstruction != nullptr && RightInstruction != nullptr) { + auto Equal = instructionEquals(*LeftInstruction, *RightInstruction, Target); + if (!Equal) + return std::unexpected(Equal.error()); + if (!*Equal) + return false; + LeftInstruction = LeftInstruction->next(); + RightInstruction = RightInstruction->next(); + } + return LeftInstruction == nullptr && RightInstruction == nullptr; +} + +[[nodiscard]] Expected<bool> canRedirectPredecessor(MachineBasicBlock &Predecessor, + MachineBasicBlock &Duplicate, + MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &Info) { + auto Analysis = Info.analyzeBranch(Predecessor, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator) + return false; + if ((Analysis->Conditional != nullptr && Analysis->TrueTarget == &Duplicate && + hasProtectedAttachment(*Analysis->Conditional)) || + (Analysis->Unconditional != nullptr && Analysis->TrueTarget == &Duplicate && + hasProtectedAttachment(*Analysis->Unconditional))) { + return false; + } + if (Analysis->TrueTarget == &Duplicate || Analysis->FalseTarget == &Duplicate) + return true; + return Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Predecessor.successors().size() == 1 && Predecessor.next() == &Duplicate; +} + +[[nodiscard]] Expected<bool> +redirectPredecessor(MachineBasicBlock &Predecessor, MachineBasicBlock &Duplicate, + MachineBasicBlock &Canonical, MachineFunction &Function, + const TargetDescription &Target, const TargetInstrInfo &Info) { + auto Analysis = Info.analyzeBranch(Predecessor, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + auto Rewritten = Info.rewriteBranchTarget(Predecessor, Duplicate, Canonical, Function, Target); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + if (!*Rewritten) { + const bool IsImplicitEdge = + (Analysis->Conditional == nullptr && Analysis->Unconditional == nullptr && + Predecessor.successors().size() == 1 && Predecessor.next() == &Duplicate) || + (Analysis->Conditional != nullptr && Analysis->Unconditional == nullptr && + Analysis->FalseTarget == &Duplicate); + if (!IsImplicitEdge) + return false; + auto Inserted = Info.insertBranch(Predecessor, Canonical, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Redirected = CFG.replaceSuccessor(Predecessor, Duplicate, Canonical); + if (!Redirected) + return std::unexpected(Redirected.error()); + return true; +} + +[[nodiscard]] bool canDeleteBlock(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + if (hasObservableBlockIdentity(Block, Function)) + return false; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (hasProtectedAttachment(*Instruction) || hasInstructionRecord(*Instruction, Function)) { + return false; + } + } + return true; +} + +[[nodiscard]] bool hasSuccessor(const MachineBasicBlock &Block, + const MachineBasicBlock &Target) noexcept { + return std::ranges::any_of(Block.successors(), [&Target](const MachineSuccessor &Successor) { + return Successor.Block == &Target; + }); +} + +[[nodiscard]] Expected<bool> foldConstantBranches(MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &Info) { + bool Changed = false; + MachineCFGUpdater CFG = Function.cfgUpdater(); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Analysis = Info.analyzeBranch(*Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator || Analysis->Conditional == nullptr || + hasProtectedAttachment(*Analysis->Conditional)) { + continue; + } + auto Condition = Info.evaluateBranchCondition(*Analysis->Conditional, Function, Target); + if (!Condition) + return std::unexpected(Condition.error()); + if (!Condition->has_value()) + continue; + + MachineBasicBlock *Selected = **Condition ? Analysis->TrueTarget : Analysis->FalseTarget; + MachineBasicBlock *Rejected = **Condition ? Analysis->FalseTarget : Analysis->TrueTarget; + if (Selected == nullptr || !hasSuccessor(*Block, *Selected)) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "constant branch has no selected CFG successor"); + } + if (Analysis->Unconditional != nullptr && hasProtectedAttachment(*Analysis->Unconditional)) { + continue; + } + + auto RemovedConditional = Info.removeBranch(*Analysis->Conditional, Function, Target); + if (!RemovedConditional) + return std::unexpected(RemovedConditional.error()); + if (Analysis->Unconditional != nullptr) { + auto RemovedUnconditional = Info.removeBranch(*Analysis->Unconditional, Function, Target); + if (!RemovedUnconditional) + return std::unexpected(RemovedUnconditional.error()); + } + auto Inserted = Info.insertBranch(*Block, *Selected, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + + if (Rejected != nullptr && Rejected != Selected && hasSuccessor(*Block, *Rejected)) { + auto RemovedEdge = CFG.removeSuccessor(*Block, *Rejected); + if (!RemovedEdge) + return std::unexpected(RemovedEdge.error()); + } + auto Probability = CFG.updateBranchProbability(*Block, *Selected, BranchProbability::one()); + if (!Probability) + return std::unexpected(Probability.error()); + Changed = true; + } + return Changed; +} + +[[nodiscard]] std::unordered_set<MachineBasicBlock *> reachableBlocks(MachineFunction &Function) { + std::unordered_set<MachineBasicBlock *> Reachable; + std::vector<MachineBasicBlock *> Worklist; + if (Function.firstBlock() != nullptr) + Worklist.push_back(Function.firstBlock()); + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!canDeleteBlock(*Block, Function)) + Worklist.push_back(Block); + } + while (!Worklist.empty()) { + MachineBasicBlock *Block = Worklist.back(); + Worklist.pop_back(); + if (Block == nullptr || !Reachable.insert(Block).second) + continue; + for (const MachineSuccessor &Successor : Block->successors()) + Worklist.push_back(Successor.Block); + } + return Reachable; +} + +[[nodiscard]] Expected<bool> removeUnreachableBlocks(MachineFunction &Function) { + const std::unordered_set<MachineBasicBlock *> Reachable = reachableBlocks(Function); + MachineCFGUpdater CFG = Function.cfgUpdater(); + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;) { + MachineBasicBlock *Next = Block->next(); + if (!Reachable.contains(Block) && canDeleteBlock(*Block, Function)) { + auto Deleted = CFG.deleteBlock(*Block); + if (!Deleted) + return std::unexpected(Deleted.error()); + Changed = true; + } + Block = Next; + } + return Changed; +} + +[[nodiscard]] Expected<bool> isForwardingBlock(MachineBasicBlock &Block, MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &Info) { + if (&Block == Function.firstBlock() || Block.successors().size() != 1 || + Block.successors().front().Block == nullptr || Block.successors().front().Block == &Block || + hasObservableBlockIdentity(Block, Function)) { + return false; + } + auto Analysis = Info.analyzeBranch(Block, Function, Target); + if (!Analysis) + return std::unexpected(Analysis.error()); + if (Analysis->HasUnanalyzableTerminator || Analysis->Conditional != nullptr) + return false; + MachineBasicBlock *Successor = Block.successors().front().Block; + if (Analysis->Unconditional == nullptr) { + return Block.firstInstruction() == nullptr && Block.next() == Successor; + } + if (Analysis->TrueTarget != Successor || hasProtectedAttachment(*Analysis->Unconditional)) { + return false; + } + return Block.firstInstruction() == Analysis->Unconditional && + Analysis->Unconditional->next() == nullptr; +} + +[[nodiscard]] Expected<bool> threadForwardingBlocks(MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &Info, + std::size_t MaximumEdges) { + MachineCFGUpdater CFG = Function.cfgUpdater(); + std::size_t ThreadedEdges = 0; + bool AnyChange = false; + bool LocalChange = true; + while (LocalChange && ThreadedEdges < MaximumEdges) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Forwarding = isForwardingBlock(*Block, Function, Target, Info); + if (!Forwarding) + return std::unexpected(Forwarding.error()); + if (!*Forwarding) + continue; + MachineBasicBlock *Successor = Block->successors().front().Block; + const std::vector<MachineBasicBlock *> Predecessors(Block->predecessors().begin(), + Block->predecessors().end()); + for (MachineBasicBlock *Predecessor : Predecessors) { + if (ThreadedEdges == MaximumEdges) + break; + if (Predecessor == nullptr || Predecessor->successors().size() != 1 || + Predecessor->successors().front().Block != Block) { + continue; + } + auto PredAnalysis = Info.analyzeBranch(*Predecessor, Function, Target); + if (!PredAnalysis) + return std::unexpected(PredAnalysis.error()); + if (PredAnalysis->HasUnanalyzableTerminator || PredAnalysis->Conditional != nullptr) + continue; + + if (PredAnalysis->Unconditional != nullptr) { + if (PredAnalysis->TrueTarget != Block || + hasProtectedAttachment(*PredAnalysis->Unconditional)) { + continue; + } + auto Rewritten = + Info.rewriteBranchTarget(*Predecessor, *Block, *Successor, Function, Target); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + if (!*Rewritten) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "branch threading did not rewrite an explicit branch"); + } + } else { + if (Predecessor->next() != Block) + continue; + auto Inserted = Info.insertBranch(*Predecessor, *Successor, Function, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + auto Redirected = CFG.replaceSuccessor(*Predecessor, *Block, *Successor); + if (!Redirected) + return std::unexpected(Redirected.error()); + ++ThreadedEdges; + AnyChange = true; + LocalChange = true; + } + if (Block->predecessors().empty() && canDeleteBlock(*Block, Function)) { + auto Deleted = CFG.deleteBlock(*Block); + if (!Deleted) + return std::unexpected(Deleted.error()); + AnyChange = true; + LocalChange = true; + break; + } + } + } + return AnyChange; +} + +[[nodiscard]] Expected<bool> mergeIdenticalTails(MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &Info, + std::size_t MaximumMergedBlocks) { + std::size_t MergedBlocks = 0; + bool Changed = false; + bool LocalChange = true; + while (LocalChange && MergedBlocks < MaximumMergedBlocks) { + LocalChange = false; + std::vector<MachineBasicBlock *> Blocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + Blocks.push_back(Block); + for (std::size_t DuplicateIndex = 1; DuplicateIndex < Blocks.size(); ++DuplicateIndex) { + MachineBasicBlock *Duplicate = Blocks[DuplicateIndex]; + if (Duplicate == nullptr || !Duplicate->isLinked()) + continue; + for (std::size_t CanonicalIndex = 1; CanonicalIndex < DuplicateIndex; ++CanonicalIndex) { + MachineBasicBlock *Canonical = Blocks[CanonicalIndex]; + if (Canonical == nullptr || !Canonical->isLinked()) + continue; + auto Equal = blockEquals(*Canonical, *Duplicate, Function, Target); + if (!Equal) + return std::unexpected(Equal.error()); + if (!*Equal) + continue; + + const std::vector<MachineBasicBlock *> Predecessors(Duplicate->predecessors().begin(), + Duplicate->predecessors().end()); + bool RedirectedAll = true; + for (MachineBasicBlock *Predecessor : Predecessors) { + if (Predecessor == nullptr || Predecessor == Canonical || Predecessor == Duplicate) { + RedirectedAll = false; + break; + } + auto CanRedirect = + canRedirectPredecessor(*Predecessor, *Duplicate, Function, Target, Info); + if (!CanRedirect) + return std::unexpected(CanRedirect.error()); + if (!*CanRedirect) { + RedirectedAll = false; + break; + } + } + if (!RedirectedAll) + continue; + for (MachineBasicBlock *Predecessor : Predecessors) { + auto Redirected = + redirectPredecessor(*Predecessor, *Duplicate, *Canonical, Function, Target, Info); + if (!Redirected) + return std::unexpected(Redirected.error()); + if (!*Redirected) { + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "tail merge predecessor changed after redirect preflight"); + } + } + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Deleted = CFG.deleteBlock(*Duplicate); + if (!Deleted) + return std::unexpected(Deleted.error()); + ++MergedBlocks; + Changed = true; + LocalChange = true; + break; + } + if (LocalChange) + break; + } + } + return Changed; +} + +} // namespace + +const MachinePassContract &MachineBranchOptimizationPass::contract() const noexcept { + return branchOptimizationContract(); +} + +Expected<MachinePassResult> MachineBranchOptimizationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + return runImplementation(Function, Target); +} + +Expected<MachinePassResult> +MachineBranchOptimizationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + auto Folded = foldConstantBranches(Function, Target, *Info); + if (!Folded) + return std::unexpected(Folded.error()); + auto RemovedBeforeThreading = removeUnreachableBlocks(Function); + if (!RemovedBeforeThreading) + return std::unexpected(RemovedBeforeThreading.error()); + auto Threaded = threadForwardingBlocks(Function, Target, *Info, Options.MaximumThreadedEdges); + if (!Threaded) + return std::unexpected(Threaded.error()); + auto Merged = mergeIdenticalTails(Function, Target, *Info, Options.MaximumMergedBlocks); + if (!Merged) + return std::unexpected(Merged.error()); + auto RemovedAfterThreading = removeUnreachableBlocks(Function); + if (!RemovedAfterThreading) + return std::unexpected(RemovedAfterThreading.error()); + + MachinePassResult Result; + Result.Changed = + *Folded || *RemovedBeforeThreading || *Threaded || *Merged || *RemovedAfterThreading; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineBranchRelaxation.cpp b/backend/lib/CodeGen/MachineBranchRelaxation.cpp new file mode 100644 index 0000000..f6c7df1 --- /dev/null +++ b/backend/lib/CodeGen/MachineBranchRelaxation.cpp @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineBranchRelaxation.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetInstrInfo.h" + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <string> +#include <unordered_map> +#include <utility> + +namespace rcc { +namespace { + +struct MachineLayout final { + std::unordered_map<const MachineBasicBlock *, std::uint64_t> BlockOffsets; + std::unordered_map<const MachineInstr *, std::uint64_t> InstructionOffsets; +}; + +[[nodiscard]] const MachinePassContract &relaxationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "branch-relaxation", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = std::nullopt, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Expected<std::uint64_t> alignTo(std::uint64_t Value, std::uint64_t Alignment) { + if (Alignment == 0 || (Alignment & (Alignment - 1U)) != 0) + return makeUnexpected(ErrorCode::VerificationError, "block alignment is not a power of two"); + const std::uint64_t Mask = Alignment - 1U; + if (Value > std::numeric_limits<std::uint64_t>::max() - Mask) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, "machine layout offset overflowed"); + return (Value + Mask) & ~Mask; +} + +[[nodiscard]] Expected<MachineLayout> computeLayout(MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &Info) { + MachineLayout Result; + std::uint64_t Offset = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Aligned = alignTo(Offset, Block->attributes().alignment().bytes()); + if (!Aligned) + return std::unexpected(Aligned.error()); + Offset = *Aligned; + Result.BlockOffsets.emplace(Block, Offset); + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Result.InstructionOffsets.emplace(Instruction, Offset); + auto Size = Info.instructionSize(*Instruction, Function, Target); + if (!Size) + return std::unexpected(Size.error()); + if (Offset > std::numeric_limits<std::uint64_t>::max() - *Size) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, "machine layout size overflowed"); + Offset += *Size; + } + } + return Result; +} + +[[nodiscard]] Expected<std::int64_t> displacement(std::uint64_t Source, std::uint64_t Target) { + if (Target >= Source) { + const std::uint64_t Difference = Target - Source; + if (Difference > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "forward branch displacement overflowed"); + return static_cast<std::int64_t>(Difference); + } + const std::uint64_t Difference = Source - Target; + if (Difference > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "backward branch displacement overflowed"); + return -static_cast<std::int64_t>(Difference); +} + +} // namespace + +const MachinePassContract &MachineBranchRelaxationPass::contract() const noexcept { + return relaxationContract(); +} + +Expected<MachinePassResult> +MachineBranchRelaxationPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +MachineBranchRelaxationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) const { + if (Options.ConvergenceWatchdog == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "branch relaxation watchdog must be nonzero"); + MachinePassResult Result; + for (std::size_t Iteration = 0; Iteration < Options.ConvergenceWatchdog; ++Iteration) { + auto Layout = computeLayout(Function, Target, *Info); + if (!Layout) + return std::unexpected(Layout.error()); + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !Changed; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto BranchTarget = Info->branchTarget(*Instruction, Function, Target); + if (!BranchTarget) + return std::unexpected(BranchTarget.error()); + if (*BranchTarget == nullptr) + continue; + const auto SourcePosition = Layout->InstructionOffsets.find(Instruction); + const auto TargetPosition = Layout->BlockOffsets.find(*BranchTarget); + if (SourcePosition == Layout->InstructionOffsets.end() || + TargetPosition == Layout->BlockOffsets.end()) { + return makeUnexpected(ErrorCode::InternalError, + "branch relaxation lost an instruction or target offset"); + } + auto Offset = displacement(SourcePosition->second, TargetPosition->second); + if (!Offset) + return std::unexpected(Offset.error()); + if (Info->isBranchOffsetInRange(*Instruction, *Offset)) + continue; + auto Accepted = Analyses.requestAction("relax-branch", Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "required branch relaxation action was rejected"); + } + auto Relaxed = Info->relaxBranch(*Instruction, *Offset, Function, Target); + if (!Relaxed) + return std::unexpected(Relaxed.error()); + if (!*Relaxed) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "target did not relax an out-of-range branch"); + } + Changed = true; + Result.Changed = true; + break; + } + } + if (Changed) + continue; + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; + } + return makeUnexpected(ErrorCode::InternalError, + "branch relaxation failed to reach a monotonic fixed point"); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCFGUpdater.cpp b/backend/lib/CodeGen/MachineCFGUpdater.cpp new file mode 100644 index 0000000..050071e --- /dev/null +++ b/backend/lib/CodeGen/MachineCFGUpdater.cpp @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCFGUpdater.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <span> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct PhiIncomingEdit final { + MachineInstr *Phi = nullptr; + std::size_t ValueIndex = 0; +}; + +[[nodiscard]] bool isKnownPhi(const MachineInstr &Instruction) noexcept { + return Instruction.opcode() == opcode(GenericOpcode::PHI) || + Instruction.opcode() == opcode(GenericOpcode::G_PHI); +} + +[[nodiscard]] bool containsKnownPhi(const MachineBasicBlock &Block) noexcept { + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (isKnownPhi(*Instruction)) + return true; + } + return false; +} + +[[nodiscard]] Expected<std::vector<PhiIncomingEdit>> +collectPhiIncomingEdits(MachineBasicBlock &Block, const MachineBasicBlock &Predecessor) { + std::vector<PhiIncomingEdit> Result; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (!isKnownPhi(*Instruction)) + continue; + if (Instruction->operandCount() < 3 || (Instruction->operandCount() - 1U) % 2U != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "CFG update encountered a malformed PHI instruction"); + } + + std::optional<std::size_t> Match; + for (std::size_t Index = 1; Index < Instruction->operandCount(); Index += 2U) { + const MachineOperand *Value = Instruction->operandAt(Index); + const MachineOperand *BlockOperand = Instruction->operandAt(Index + 1U); + const Register *Reg = Value == nullptr ? nullptr : Value->getIf<Register>(); + const BasicBlockOperand *Incoming = + BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>(); + if (Reg == nullptr || Incoming == nullptr || Incoming->Block == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "CFG update encountered malformed PHI incoming operands"); + } + if (Incoming->Block != &Predecessor) + continue; + if (Match) { + return makeUnexpected(ErrorCode::VerificationError, + "CFG update encountered duplicate PHI incoming blocks"); + } + Match = Index; + } + if (!Match) { + return makeUnexpected(ErrorCode::VerificationError, + "CFG edge has no corresponding PHI incoming value"); + } + Result.push_back(PhiIncomingEdit{Instruction, *Match}); + } + return Result; +} + +[[nodiscard]] Expected<void> erasePhiIncoming(std::span<const PhiIncomingEdit> Edits) { + for (const PhiIncomingEdit &Edit : Edits) { + auto BlockErased = Edit.Phi->eraseOperand(Edit.ValueIndex + 1U); + if (!BlockErased) + return std::unexpected(BlockErased.error()); + auto ValueErased = Edit.Phi->eraseOperand(Edit.ValueIndex); + if (!ValueErased) + return std::unexpected(ValueErased.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> rewritePhiIncoming(std::span<const PhiIncomingEdit> Edits, + MachineBasicBlock &NewPredecessor) { + for (const PhiIncomingEdit &Edit : Edits) { + MachineOperand *BlockOperand = Edit.Phi->operandAt(Edit.ValueIndex + 1U); + if (BlockOperand == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "PHI incoming operand disappeared during CFG update"); + } + auto Rewritten = BlockOperand->setBasicBlock(NewPredecessor); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + } + return {}; +} + +[[nodiscard]] std::optional<std::size_t> findSuccessorIndex(const MachineBasicBlock &From, + const MachineBasicBlock &To) { + const auto Successors = From.successors(); + for (std::size_t Index = 0; Index < Successors.size(); ++Index) { + if (Successors[Index].Block == &To) + return Index; + } + return std::nullopt; +} + +[[nodiscard]] Expected<void> validateOutgoingEdges(MachineBasicBlock &From) { + for (const MachineSuccessor &Successor : From.successors()) { + if (Successor.Block == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "CFG successor is null"); + } + if (std::ranges::find(Successor.Block->predecessors(), &From) == + Successor.Block->predecessors().end()) { + return makeUnexpected(ErrorCode::VerificationError, + "CFG successor has no reciprocal predecessor"); + } + auto PhiEdits = collectPhiIncomingEdits(*Successor.Block, From); + if (!PhiEdits) + return std::unexpected(PhiEdits.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> validateIncomingEdges(MachineBasicBlock &To) { + for (MachineBasicBlock *Predecessor : To.predecessors()) { + if (Predecessor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "CFG predecessor is null"); + } + if (!findSuccessorIndex(*Predecessor, To)) { + return makeUnexpected(ErrorCode::VerificationError, + "CFG predecessor has no reciprocal successor"); + } + auto PhiEdits = collectPhiIncomingEdits(To, *Predecessor); + if (!PhiEdits) + return std::unexpected(PhiEdits.error()); + } + return {}; +} + +} // namespace + +Expected<void> MachineCFGUpdater::verifyOwnership(const MachineBasicBlock &Block) const { + if (&Block.function() != Function) { + return makeUnexpected(ErrorCode::InvalidArgument, + "CFG block belongs to another machine function"); + } + return {}; +} + +Expected<void> MachineCFGUpdater::addSuccessor(MachineBasicBlock &From, MachineBasicBlock &To, + BranchProbability Probability) { + auto FromOwnership = verifyOwnership(From); + if (!FromOwnership) + return std::unexpected(FromOwnership.error()); + auto ToOwnership = verifyOwnership(To); + if (!ToOwnership) + return std::unexpected(ToOwnership.error()); + + const auto Existing = findSuccessorIndex(From, To); + if (Existing) { + From.Successors[*Existing].Probability = + From.Successors[*Existing].Probability.saturatingAdd(Probability); + return {}; + } + + if (containsKnownPhi(To)) { + auto PhiEdits = collectPhiIncomingEdits(To, From); + if (!PhiEdits) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "adding a predecessor to a PHI block requires explicit incoming values"); + } + } + + From.Successors.push_back(MachineSuccessor{&To, Probability}); + if (std::ranges::find(To.Predecessors, &From) == To.Predecessors.end()) + To.Predecessors.push_back(&From); + return {}; +} + +Expected<void> MachineCFGUpdater::removeSuccessor(MachineBasicBlock &From, MachineBasicBlock &To) { + auto FromOwnership = verifyOwnership(From); + if (!FromOwnership) + return std::unexpected(FromOwnership.error()); + auto ToOwnership = verifyOwnership(To); + if (!ToOwnership) + return std::unexpected(ToOwnership.error()); + + const auto Successor = findSuccessorIndex(From, To); + if (!Successor) { + return makeUnexpected(ErrorCode::InvalidArgument, "CFG successor edge is not present"); + } + const auto Predecessor = std::ranges::find(To.Predecessors, &From); + if (Predecessor == To.Predecessors.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "CFG successor has no reciprocal predecessor"); + } + + auto PhiEdits = collectPhiIncomingEdits(To, From); + if (!PhiEdits) + return std::unexpected(PhiEdits.error()); + auto PhiResult = erasePhiIncoming(*PhiEdits); + if (!PhiResult) + return std::unexpected(PhiResult.error()); + + From.Successors.erase(From.Successors.begin() + static_cast<std::ptrdiff_t>(*Successor)); + To.Predecessors.erase(Predecessor); + return {}; +} + +Expected<void> MachineCFGUpdater::replaceSuccessor(MachineBasicBlock &From, + MachineBasicBlock &OldTarget, + MachineBasicBlock &NewTarget) { + auto FromOwnership = verifyOwnership(From); + if (!FromOwnership) + return std::unexpected(FromOwnership.error()); + auto OldOwnership = verifyOwnership(OldTarget); + if (!OldOwnership) + return std::unexpected(OldOwnership.error()); + auto NewOwnership = verifyOwnership(NewTarget); + if (!NewOwnership) + return std::unexpected(NewOwnership.error()); + + if (&OldTarget == &NewTarget) + return {}; + + const auto Existing = findSuccessorIndex(From, OldTarget); + if (!Existing) { + return makeUnexpected(ErrorCode::InvalidArgument, "replaced CFG successor edge is not present"); + } + const BranchProbability Probability = From.Successors[*Existing].Probability; + + const auto OldPredecessor = std::ranges::find(OldTarget.Predecessors, &From); + if (OldPredecessor == OldTarget.Predecessors.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "replaced CFG successor has no reciprocal predecessor"); + } + auto OldPhiEdits = collectPhiIncomingEdits(OldTarget, From); + if (!OldPhiEdits) + return std::unexpected(OldPhiEdits.error()); + + const auto NewEdge = findSuccessorIndex(From, NewTarget); + const auto NewPredecessor = std::ranges::find(NewTarget.Predecessors, &From); + if (NewEdge) { + if (NewPredecessor == NewTarget.Predecessors.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "replacement target has no reciprocal predecessor"); + } + auto NewPhiEdits = collectPhiIncomingEdits(NewTarget, From); + if (!NewPhiEdits) + return std::unexpected(NewPhiEdits.error()); + } else { + if (NewPredecessor != NewTarget.Predecessors.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "replacement target has a predecessor without a reciprocal edge"); + } + if (containsKnownPhi(NewTarget)) { + auto NewPhiEdits = collectPhiIncomingEdits(NewTarget, From); + if (!NewPhiEdits) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "redirecting an edge to a PHI block requires explicit incoming values"); + } + } + } + auto PhiResult = erasePhiIncoming(*OldPhiEdits); + if (!PhiResult) + return std::unexpected(PhiResult.error()); + OldTarget.Predecessors.erase(OldPredecessor); + if (NewEdge) { + From.Successors[*NewEdge].Probability = + From.Successors[*NewEdge].Probability.saturatingAdd(Probability); + From.Successors.erase(From.Successors.begin() + static_cast<std::ptrdiff_t>(*Existing)); + } else { + From.Successors[*Existing].Block = &NewTarget; + if (std::ranges::find(NewTarget.Predecessors, &From) == NewTarget.Predecessors.end()) + NewTarget.Predecessors.push_back(&From); + } + return {}; +} + +Expected<void> MachineCFGUpdater::splitEdge(MachineBasicBlock &From, MachineBasicBlock &To, + MachineBasicBlock &EdgeBlock) { + auto FromOwnership = verifyOwnership(From); + if (!FromOwnership) + return std::unexpected(FromOwnership.error()); + auto ToOwnership = verifyOwnership(To); + if (!ToOwnership) + return std::unexpected(ToOwnership.error()); + auto EdgeOwnership = verifyOwnership(EdgeBlock); + if (!EdgeOwnership) + return std::unexpected(EdgeOwnership.error()); + if (&From == &To || &From == &EdgeBlock || &To == &EdgeBlock) { + return makeUnexpected(ErrorCode::InvalidArgument, + "edge split requires three distinct basic blocks"); + } + if (!EdgeBlock.isLinked() || EdgeBlock.firstInstruction() != nullptr || + !EdgeBlock.predecessors().empty() || !EdgeBlock.successors().empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "edge split block must be linked, empty, and disconnected"); + } + + const auto Successor = findSuccessorIndex(From, To); + if (!Successor) + return makeUnexpected(ErrorCode::InvalidArgument, "split CFG edge is not present"); + auto Predecessor = std::ranges::find(To.Predecessors, &From); + if (Predecessor == To.Predecessors.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "split CFG edge has no reciprocal predecessor"); + } + auto PhiEdits = collectPhiIncomingEdits(To, From); + if (!PhiEdits) + return std::unexpected(PhiEdits.error()); + + auto PhiResult = rewritePhiIncoming(*PhiEdits, EdgeBlock); + if (!PhiResult) + return std::unexpected(PhiResult.error()); + From.Successors[*Successor].Block = &EdgeBlock; + *Predecessor = &EdgeBlock; + EdgeBlock.Predecessors.push_back(&From); + EdgeBlock.Successors.push_back(MachineSuccessor{&To, BranchProbability::one()}); + return {}; +} + +Expected<void> MachineCFGUpdater::updateBranchProbability(MachineBasicBlock &From, + MachineBasicBlock &To, + BranchProbability Probability) { + auto FromOwnership = verifyOwnership(From); + if (!FromOwnership) + return std::unexpected(FromOwnership.error()); + auto ToOwnership = verifyOwnership(To); + if (!ToOwnership) + return std::unexpected(ToOwnership.error()); + + const auto Successor = findSuccessorIndex(From, To); + if (!Successor) { + return makeUnexpected(ErrorCode::InvalidArgument, "updated CFG successor edge is not present"); + } + From.Successors[*Successor].Probability = Probability; + return {}; +} + +Expected<void> MachineCFGUpdater::transferSuccessors(MachineBasicBlock &From, + MachineBasicBlock &To) { + auto FromOwnership = verifyOwnership(From); + if (!FromOwnership) + return std::unexpected(FromOwnership.error()); + auto ToOwnership = verifyOwnership(To); + if (!ToOwnership) + return std::unexpected(ToOwnership.error()); + if (&From == &To) + return {}; + + struct Transfer final { + MachineSuccessor Successor; + std::vector<PhiIncomingEdit> PhiEdits; + bool MergeExisting = false; + }; + std::vector<Transfer> Transfers; + Transfers.reserve(From.Successors.size()); + for (const MachineSuccessor &Successor : From.Successors) { + if (Successor.Block == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "transferred CFG successor is null"); + } + if (std::ranges::find(Successor.Block->Predecessors, &From) == + Successor.Block->Predecessors.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "transferred CFG successor has no reciprocal predecessor"); + } + auto PhiEdits = collectPhiIncomingEdits(*Successor.Block, From); + if (!PhiEdits) + return std::unexpected(PhiEdits.error()); + const bool MergeExisting = findSuccessorIndex(To, *Successor.Block).has_value(); + const auto ToPredecessor = std::ranges::find(Successor.Block->Predecessors, &To); + if (MergeExisting) { + if (ToPredecessor == Successor.Block->Predecessors.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "transfer target edge has no reciprocal predecessor"); + } + auto ExistingPhiEdits = collectPhiIncomingEdits(*Successor.Block, To); + if (!ExistingPhiEdits) + return std::unexpected(ExistingPhiEdits.error()); + } else if (ToPredecessor != Successor.Block->Predecessors.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "transfer target is a predecessor without a reciprocal edge"); + } + Transfers.push_back(Transfer{Successor, std::move(*PhiEdits), MergeExisting}); + } + + for (Transfer &TransferValue : Transfers) { + MachineBasicBlock &Successor = *TransferValue.Successor.Block; + auto OldPredecessor = std::ranges::find(Successor.Predecessors, &From); + if (TransferValue.MergeExisting) { + auto PhiResult = erasePhiIncoming(TransferValue.PhiEdits); + if (!PhiResult) + return std::unexpected(PhiResult.error()); + Successor.Predecessors.erase(OldPredecessor); + const std::optional<std::size_t> Existing = findSuccessorIndex(To, Successor); + if (!Existing) { + return makeUnexpected(ErrorCode::InternalError, + "preflighted transfer target edge disappeared"); + } + To.Successors[Existing.value()].Probability = + To.Successors[Existing.value()].Probability.saturatingAdd( + TransferValue.Successor.Probability); + } else { + auto PhiResult = rewritePhiIncoming(TransferValue.PhiEdits, To); + if (!PhiResult) + return std::unexpected(PhiResult.error()); + *OldPredecessor = &To; + To.Successors.push_back(TransferValue.Successor); + } + } + From.Successors.clear(); + return {}; +} + +Expected<void> MachineCFGUpdater::disconnectBlock(MachineBasicBlock &Block) { + auto Ownership = verifyOwnership(Block); + if (!Ownership) + return std::unexpected(Ownership.error()); + + auto Outgoing = validateOutgoingEdges(Block); + if (!Outgoing) + return std::unexpected(Outgoing.error()); + auto Incoming = validateIncomingEdges(Block); + if (!Incoming) + return std::unexpected(Incoming.error()); + + const std::vector<MachineSuccessor> Successors = Block.Successors; + for (const MachineSuccessor &Successor : Successors) { + auto Result = removeSuccessor(Block, *Successor.Block); + if (!Result) + return std::unexpected(Result.error()); + } + + const std::vector<MachineBasicBlock *> Predecessors = Block.Predecessors; + for (MachineBasicBlock *Predecessor : Predecessors) { + auto Result = removeSuccessor(*Predecessor, Block); + if (!Result) + return std::unexpected(Result.error()); + } + return {}; +} + +Expected<MachineBasicBlock *> MachineCFGUpdater::splitBlock(MachineBasicBlock &Block, + MachineInstr &SplitBefore) { + auto Ownership = verifyOwnership(Block); + if (!Ownership) + return std::unexpected(Ownership.error()); + if (!Block.isLinked() || SplitBefore.parent() != &Block) { + return makeUnexpected(ErrorCode::InvalidArgument, + "split point is not linked to the requested basic block"); + } + bool MovesPhi = false; + for (const MachineInstr *Instruction = &SplitBefore; Instruction != nullptr; + Instruction = Instruction->next()) { + MovesPhi |= isKnownPhi(*Instruction); + } + if (MovesPhi) { + return makeUnexpected(ErrorCode::InvalidArgument, + "a basic block cannot be split inside its PHI region"); + } + + auto Outgoing = validateOutgoingEdges(Block); + if (!Outgoing) + return std::unexpected(Outgoing.error()); + + auto NewBlockResult = Function->createBasicBlock(); + if (!NewBlockResult) + return std::unexpected(NewBlockResult.error()); + MachineBasicBlock *NewBlock = *NewBlockResult; + Expected<void> Inserted = Block.next() == nullptr + ? Function->appendBlock(*NewBlock) + : Function->insertBlockBefore(*Block.next(), *NewBlock); + if (!Inserted) + return std::unexpected(Inserted.error()); + + MachineInstr *Instruction = &SplitBefore; + while (Instruction != nullptr) { + MachineInstr *Next = Instruction->next(); + auto Removed = Instruction->eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + auto Appended = NewBlock->appendInstruction(*Instruction); + if (!Appended) + return std::unexpected(Appended.error()); + Instruction = Next; + } + + auto Transferred = transferSuccessors(Block, *NewBlock); + if (!Transferred) + return std::unexpected(Transferred.error()); + auto Connected = addSuccessor(Block, *NewBlock, BranchProbability::one()); + if (!Connected) + return std::unexpected(Connected.error()); + return NewBlock; +} + +Expected<void> MachineCFGUpdater::deleteBlock(MachineBasicBlock &Block) { + auto Ownership = verifyOwnership(Block); + if (!Ownership) + return std::unexpected(Ownership.error()); + if (!Block.isLinked()) + return makeUnexpected(ErrorCode::InvalidArgument, "deleted CFG block is not linked"); + auto Disconnected = disconnectBlock(Block); + if (!Disconnected) + return std::unexpected(Disconnected.error()); + return Function->removeBlock(Block); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCFIInfo.cpp b/backend/lib/CodeGen/MachineCFIInfo.cpp new file mode 100644 index 0000000..db9f319 --- /dev/null +++ b/backend/lib/CodeGen/MachineCFIInfo.cpp @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCFIInfo.h" + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <string> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] Expected<void> verifyVectorRegisters(const MachineCFIVectorRegistersRule &Rule, + ErrorCode Code) { + if (Rule.VectorRegisters.empty()) { + return makeUnexpected(Code, "CFI vector-register rule must contain at least one lane"); + } + return {}; +} + +} // namespace + +MachineCFIInstruction MachineCFIInstruction::sameValue(DwarfRegisterNumber Register) { + return MachineCFIInstruction(MachineCFIOperation::SameValue, MachineCFIRegisterRule{Register}); +} + +MachineCFIInstruction MachineCFIInstruction::rememberState() { + return MachineCFIInstruction(MachineCFIOperation::RememberState, MachineCFINoOperands{}); +} + +MachineCFIInstruction MachineCFIInstruction::restoreState() { + return MachineCFIInstruction(MachineCFIOperation::RestoreState, MachineCFINoOperands{}); +} + +MachineCFIInstruction MachineCFIInstruction::offset(DwarfRegisterNumber Register, + std::int64_t Offset) { + return MachineCFIInstruction(MachineCFIOperation::Offset, + MachineCFIRegisterOffsetRule{Register, Offset}); +} + +MachineCFIInstruction MachineCFIInstruction::llvmDefAddressSpaceCFA(DwarfRegisterNumber Register, + std::int64_t Offset, + std::uint32_t AddressSpace) { + return MachineCFIInstruction(MachineCFIOperation::LLVMDefAddressSpaceCFA, + MachineCFIAddressSpaceCFARule{Register, Offset, AddressSpace}); +} + +MachineCFIInstruction MachineCFIInstruction::defCFARegister(DwarfRegisterNumber Register) { + return MachineCFIInstruction(MachineCFIOperation::DefCFARegister, + MachineCFIRegisterRule{Register}); +} + +MachineCFIInstruction MachineCFIInstruction::defCFAOffset(std::int64_t Offset) { + return MachineCFIInstruction(MachineCFIOperation::DefCFAOffset, MachineCFIOffsetRule{Offset}); +} + +MachineCFIInstruction MachineCFIInstruction::defCFA(DwarfRegisterNumber Register, + std::int64_t Offset) { + return MachineCFIInstruction(MachineCFIOperation::DefCFA, + MachineCFIRegisterOffsetRule{Register, Offset}); +} + +MachineCFIInstruction MachineCFIInstruction::relativeOffset(DwarfRegisterNumber Register, + std::int64_t Offset) { + return MachineCFIInstruction(MachineCFIOperation::RelativeOffset, + MachineCFIRegisterOffsetRule{Register, Offset}); +} + +MachineCFIInstruction MachineCFIInstruction::adjustCFAOffset(std::int64_t Offset) { + return MachineCFIInstruction(MachineCFIOperation::AdjustCFAOffset, MachineCFIOffsetRule{Offset}); +} + +Expected<MachineCFIInstruction> MachineCFIInstruction::escape(std::vector<std::uint8_t> Bytes) { + if (Bytes.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "CFI escape byte sequence cannot be empty"); + return MachineCFIInstruction(MachineCFIOperation::Escape, MachineCFIEscapeRule{std::move(Bytes)}); +} + +MachineCFIInstruction MachineCFIInstruction::restore(DwarfRegisterNumber Register) { + return MachineCFIInstruction(MachineCFIOperation::Restore, MachineCFIRegisterRule{Register}); +} + +MachineCFIInstruction MachineCFIInstruction::undefined(DwarfRegisterNumber Register) { + return MachineCFIInstruction(MachineCFIOperation::Undefined, MachineCFIRegisterRule{Register}); +} + +MachineCFIInstruction MachineCFIInstruction::registerRule(DwarfRegisterNumber Register, + DwarfRegisterNumber SourceRegister) { + return MachineCFIInstruction(MachineCFIOperation::Register, + MachineCFIRegisterRegisterRule{Register, SourceRegister}); +} + +MachineCFIInstruction MachineCFIInstruction::windowSave() { + return MachineCFIInstruction(MachineCFIOperation::WindowSave, MachineCFINoOperands{}); +} + +MachineCFIInstruction MachineCFIInstruction::negateReturnAddressState() { + return MachineCFIInstruction(MachineCFIOperation::NegateReturnAddressState, + MachineCFINoOperands{}); +} + +MachineCFIInstruction MachineCFIInstruction::negateReturnAddressStateWithPC() { + return MachineCFIInstruction(MachineCFIOperation::NegateReturnAddressStateWithPC, + MachineCFINoOperands{}); +} + +MachineCFIInstruction MachineCFIInstruction::gnuArgsSize(std::int64_t Size) { + return MachineCFIInstruction(MachineCFIOperation::GNUArgsSize, MachineCFIOffsetRule{Size}); +} + +Expected<MachineCFIInstruction> MachineCFIInstruction::label(std::string Symbol) { + if (Symbol.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "CFI label symbol cannot be empty"); + return MachineCFIInstruction(MachineCFIOperation::Label, MachineCFILabelRule{std::move(Symbol)}); +} + +MachineCFIInstruction MachineCFIInstruction::valueOffset(DwarfRegisterNumber Register, + std::int64_t Offset) { + return MachineCFIInstruction(MachineCFIOperation::ValueOffset, + MachineCFIRegisterOffsetRule{Register, Offset}); +} + +MachineCFIInstruction MachineCFIInstruction::llvmRegisterPair(MachineCFIRegisterPairRule Rule) { + return MachineCFIInstruction(MachineCFIOperation::LLVMRegisterPair, Rule); +} + +Expected<MachineCFIInstruction> +MachineCFIInstruction::llvmVectorRegisters(MachineCFIVectorRegistersRule Rule) { + auto Verification = verifyVectorRegisters(Rule, ErrorCode::InvalidArgument); + if (!Verification) + return std::unexpected(Verification.error()); + return MachineCFIInstruction(MachineCFIOperation::LLVMVectorRegisters, std::move(Rule)); +} + +MachineCFIInstruction MachineCFIInstruction::llvmVectorOffset(MachineCFIVectorOffsetRule Rule) { + return MachineCFIInstruction(MachineCFIOperation::LLVMVectorOffset, Rule); +} + +MachineCFIInstruction +MachineCFIInstruction::llvmVectorRegisterMask(MachineCFIVectorRegisterMaskRule Rule) { + return MachineCFIInstruction(MachineCFIOperation::LLVMVectorRegisterMask, Rule); +} + +Expected<void> MachineCFIInstruction::verify() const { + switch (Operation) { + case MachineCFIOperation::SameValue: + case MachineCFIOperation::DefCFARegister: + case MachineCFIOperation::Restore: + case MachineCFIOperation::Undefined: + if (getIf<MachineCFIRegisterRule>() == nullptr) + break; + return {}; + case MachineCFIOperation::RememberState: + case MachineCFIOperation::RestoreState: + case MachineCFIOperation::WindowSave: + case MachineCFIOperation::NegateReturnAddressState: + case MachineCFIOperation::NegateReturnAddressStateWithPC: + if (getIf<MachineCFINoOperands>() == nullptr) + break; + return {}; + case MachineCFIOperation::Offset: + case MachineCFIOperation::DefCFA: + case MachineCFIOperation::RelativeOffset: + case MachineCFIOperation::ValueOffset: + if (getIf<MachineCFIRegisterOffsetRule>() == nullptr) + break; + return {}; + case MachineCFIOperation::LLVMDefAddressSpaceCFA: + if (getIf<MachineCFIAddressSpaceCFARule>() == nullptr) + break; + return {}; + case MachineCFIOperation::DefCFAOffset: + case MachineCFIOperation::AdjustCFAOffset: + case MachineCFIOperation::GNUArgsSize: + if (getIf<MachineCFIOffsetRule>() == nullptr) + break; + return {}; + case MachineCFIOperation::Escape: { + const auto *Rule = getIf<MachineCFIEscapeRule>(); + if (Rule == nullptr) + break; + if (Rule->Bytes.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "CFI escape byte sequence cannot be empty"); + } + return {}; + } + case MachineCFIOperation::Register: + if (getIf<MachineCFIRegisterRegisterRule>() == nullptr) + break; + return {}; + case MachineCFIOperation::Label: { + const auto *Rule = getIf<MachineCFILabelRule>(); + if (Rule == nullptr) + break; + if (Rule->Symbol.empty()) + return makeUnexpected(ErrorCode::VerificationError, "CFI label symbol cannot be empty"); + return {}; + } + case MachineCFIOperation::LLVMRegisterPair: { + const auto *Rule = getIf<MachineCFIRegisterPairRule>(); + if (Rule == nullptr) + break; + return {}; + } + case MachineCFIOperation::LLVMVectorRegisters: { + const auto *Rule = getIf<MachineCFIVectorRegistersRule>(); + if (Rule == nullptr) + break; + return verifyVectorRegisters(*Rule, ErrorCode::VerificationError); + } + case MachineCFIOperation::LLVMVectorOffset: { + const auto *Rule = getIf<MachineCFIVectorOffsetRule>(); + if (Rule == nullptr) + break; + return {}; + } + case MachineCFIOperation::LLVMVectorRegisterMask: { + const auto *Rule = getIf<MachineCFIVectorRegisterMaskRule>(); + if (Rule == nullptr) + break; + return {}; + } + case MachineCFIOperation::Count: + break; + } + return makeUnexpected(ErrorCode::VerificationError, + "CFI instruction operation and payload do not match"); +} + +Expected<CFIInstructionIndex> MachineCFIInfo::addInstruction(MachineCFIInstruction Instruction) { + auto Verification = Instruction.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + if (Instructions.size() > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::Unsupported, "CFI instruction index space is exhausted"); + } + + const auto Index = + CFIInstructionIndex::fromValue(static_cast<std::uint32_t>(Instructions.size())); + Instructions.push_back(std::move(Instruction)); + return Index; +} + +const MachineCFIInstruction *MachineCFIInfo::instruction(CFIInstructionIndex Index) const noexcept { + if (Index.value() >= Instructions.size()) + return nullptr; + return &Instructions[Index.value()]; +} + +Expected<void> MachineCFIInfo::verify() const { + for (const MachineCFIInstruction &Instruction : Instructions) { + auto Verification = Instruction.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + } + return {}; +} + +Expected<MachineCFIInfo> MachineCFIInfo::clone() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + + MachineCFIInfo Result; + for (const MachineCFIInstruction &Instruction : Instructions) { + auto Index = Result.addInstruction(Instruction); + if (!Index) + return std::unexpected(Index.error()); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCSE.cpp b/backend/lib/CodeGen/MachineCSE.cpp new file mode 100644 index 0000000..353cbaa --- /dev/null +++ b/backend/lib/CodeGen/MachineCSE.cpp @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCSE.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cstddef> +#include <cstdint> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &cseContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-cse", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +template <typename ValueT> +[[nodiscard]] bool payloadEquals(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const ValueT *LeftValue = Left.getIf<ValueT>(); + const ValueT *RightValue = Right.getIf<ValueT>(); + return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue; +} + +[[nodiscard]] bool registerOperandEquals(const MachineOperand &Left, const MachineOperand &Right, + bool IgnoreDefinitionValue) noexcept { + const Register *LeftReg = Left.getIf<Register>(); + const Register *RightReg = Right.getIf<Register>(); + if (LeftReg == nullptr || RightReg == nullptr || Left.subReg() != Right.subReg() || + Left.targetFlags() != Right.targetFlags()) { + return false; + } + RegisterOperandFlags LeftFlags = Left.registerFlags(); + RegisterOperandFlags RightFlags = Right.registerFlags(); + for (const RegisterOperandFlag Flag : + {RegisterOperandFlag::Kill, RegisterOperandFlag::Dead, RegisterOperandFlag::Renamable}) { + LeftFlags.clear(Flag); + RightFlags.clear(Flag); + } + return LeftFlags == RightFlags && (IgnoreDefinitionValue || *LeftReg == *RightReg); +} + +[[nodiscard]] bool operandEquals(const MachineOperand &Left, const MachineOperand &Right, + bool IgnoreDefinitionValue) noexcept { + if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags()) + return false; + switch (Left.kind()) { + case MachineOperand::Kind::Register: + return registerOperandEquals(Left, Right, IgnoreDefinitionValue); + case MachineOperand::Kind::Immediate: + return payloadEquals<std::int64_t>(Left, Right); + case MachineOperand::Kind::IntegerImmediate: + return payloadEquals<IntegerImmediateOperand>(Left, Right); + case MachineOperand::Kind::FloatingImmediate: + return payloadEquals<FloatingImmediateOperand>(Left, Right); + case MachineOperand::Kind::BasicBlock: + return payloadEquals<BasicBlockOperand>(Left, Right); + case MachineOperand::Kind::FrameIndex: + return payloadEquals<FrameIndexOperand>(Left, Right); + case MachineOperand::Kind::ConstantPoolIndex: + return payloadEquals<ConstantPoolOperand>(Left, Right); + case MachineOperand::Kind::JumpTableIndex: + return payloadEquals<JumpTableOperand>(Left, Right); + case MachineOperand::Kind::GlobalSymbol: + return payloadEquals<GlobalSymbolOperand>(Left, Right); + case MachineOperand::Kind::ExternalSymbol: + return payloadEquals<ExternalSymbolOperand>(Left, Right); + case MachineOperand::Kind::BlockAddress: + return payloadEquals<BlockAddressOperand>(Left, Right); + case MachineOperand::Kind::RegisterMask: + case MachineOperand::Kind::LiveOutRegisterMask: + return false; + case MachineOperand::Kind::Metadata: + return payloadEquals<MetadataOperand>(Left, Right); + case MachineOperand::Kind::TargetIndex: + return payloadEquals<TargetIndexOperand>(Left, Right); + case MachineOperand::Kind::MCSymbol: + return payloadEquals<MCSymbolOperand>(Left, Right); + case MachineOperand::Kind::CFIIndex: + return payloadEquals<CFIIndexOperand>(Left, Right); + case MachineOperand::Kind::Intrinsic: + return payloadEquals<IntrinsicOperand>(Left, Right); + case MachineOperand::Kind::Predicate: + return payloadEquals<PredicateOperand>(Left, Right); + case MachineOperand::Kind::ShuffleMask: + return payloadEquals<ShuffleMaskOperand>(Left, Right); + case MachineOperand::Kind::DebugInstrRef: + return payloadEquals<DebugInstrRefOperand>(Left, Right); + case MachineOperand::Kind::LaneMask: + return payloadEquals<LaneMaskOperand>(Left, Right); + case MachineOperand::Kind::Count: + return false; + } + return false; +} + +[[nodiscard]] bool isConstantMemoryBase(MemoryBase::Kind Kind) noexcept { + return Kind == MemoryBase::Kind::ConstantPool || Kind == MemoryBase::Kind::ConstantPoolPseudo; +} + +[[nodiscard]] bool hasSafeInvariantMemory(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().empty()) + return false; + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory == nullptr || Memory->isStore() || Memory->isAtomic() || Memory->isVolatile()) + return false; + const MachineMemOperandDesc &Description = Memory->description(); + const bool ExplicitInvariant = Description.Accesses.has(MemoryAccessFlag::Invariant) && + Description.Accesses.has(MemoryAccessFlag::Dereferenceable); + if (!ExplicitInvariant && !isConstantMemoryBase(Description.Base.kind())) + return false; + } + return true; +} + +[[nodiscard]] Expected<bool> isCandidate(const MachineInstr &Instruction, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine CSE encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Pseudo) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction)) { + return false; + } + if (Flags.has(InstructionFlag::MayLoad) && !hasSafeInvariantMemory(Instruction)) + return false; + if (!Flags.has(InstructionFlag::MayLoad) && !Instruction.memOperands().empty()) + return false; + if (Flags.has(InstructionFlag::MayRaiseFPException) && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) { + return false; + } + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr) + continue; + if (Reg->isPhysical() || Reg->isNoRegister() || Operand.isDebugUse() || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Operand.isDef()) + HasDefinition = true; + } + return HasDefinition; +} + +[[nodiscard]] bool expressionEquals(const MachineInstr &Left, const MachineInstr &Right) noexcept { + if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() || + Left.operandCount() != Right.operandCount() || + Left.memOperands().size() != Right.memOperands().size()) { + return false; + } + for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) { + const MachineOperand *LeftOperand = Left.operandAt(Index); + const MachineOperand *RightOperand = Right.operandAt(Index); + if (LeftOperand == nullptr || RightOperand == nullptr || + LeftOperand->isDef() != RightOperand->isDef() || + !operandEquals(*LeftOperand, *RightOperand, LeftOperand->isDef())) { + return false; + } + } + for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) { + const MachineMemOperand *LeftMemory = Left.memOperands()[Index]; + const MachineMemOperand *RightMemory = Right.memOperands()[Index]; + if (LeftMemory == nullptr || RightMemory == nullptr || *LeftMemory != *RightMemory) + return false; + } + return true; +} + +[[nodiscard]] Expected<std::vector<std::pair<Register, Register>>> +replacementPairs(const MachineInstr &Available, const MachineInstr &Redundant, + const MachineFunction &Function, const TargetDescription &Target) { + std::vector<std::pair<Register, Register>> Result; + std::unordered_set<Register> RedundantDefinitions; + const MachineRegisterInfo &Registers = Function.registerInfo(); + for (std::size_t Index = 0; Index < Available.operandCount(); ++Index) { + const MachineOperand *AvailableOperand = Available.operandAt(Index); + const MachineOperand *RedundantOperand = Redundant.operandAt(Index); + if (AvailableOperand == nullptr || RedundantOperand == nullptr || !AvailableOperand->isDef()) { + continue; + } + const Register *AvailableReg = AvailableOperand->getIf<Register>(); + const Register *RedundantReg = RedundantOperand->getIf<Register>(); + if (AvailableReg == nullptr || RedundantReg == nullptr || !AvailableReg->isVirtual() || + !RedundantReg->isVirtual() || !RedundantDefinitions.insert(*RedundantReg).second) { + return std::vector<std::pair<Register, Register>>{}; + } + auto AvailableInfo = Registers.virtualRegisterInfo(*AvailableReg); + if (!AvailableInfo) + return std::unexpected(AvailableInfo.error()); + auto RedundantInfo = Registers.virtualRegisterInfo(*RedundantReg); + if (!RedundantInfo) + return std::unexpected(RedundantInfo.error()); + const VirtualRegisterInfo &Left = AvailableInfo->get(); + const VirtualRegisterInfo &Right = RedundantInfo->get(); + if (!Left.Type || !Right.Type || Left.Type != Right.Type || + Left.TargetFlags != Right.TargetFlags) + return std::vector<std::pair<Register, Register>>{}; + if (Left.Bank && Right.Bank && + !Target.registers().registerBanksOverlap(*Left.Bank, *Right.Bank)) { + return std::vector<std::pair<Register, Register>>{}; + } + if (Left.RegClass && Right.RegClass && + !Target.registers().registerClassesOverlap(*Left.RegClass, *Right.RegClass)) { + return std::vector<std::pair<Register, Register>>{}; + } + Result.emplace_back(*RedundantReg, *AvailableReg); + } + return Result; +} + +[[nodiscard]] Expected<void> +replaceAndErase(MachineInstr &Redundant, + const std::vector<std::pair<Register, Register>> &Replacements, + MachineRegisterInfo &Registers) { + for (const auto &[OldReg, NewReg] : Replacements) { + auto Replaced = Registers.replaceRegWith(OldReg, NewReg); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + auto Removed = Redundant.eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + for (const auto &[OldReg, NewReg] : Replacements) { + (void)NewReg; + if (!Registers.useEmpty(OldReg, true) || Registers.getUniqueVRegDef(OldReg) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine CSE left references to a redundant result"); + } + auto Erased = Registers.eraseVirtualRegister(OldReg); + if (!Erased) + return std::unexpected(Erased.error()); + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineCSEPass::contract() const noexcept { + return cseContract(); +} + +Expected<MachinePassResult> MachineCSEPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +MachineCSEPass::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + + MachinePassResult Result; + MachineRegisterInfo &Registers = Function.registerInfo(); + std::unordered_map<Opcode, std::vector<MachineInstr *>> Available; + std::size_t Comparisons = 0; + bool BudgetExhausted = false; + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !BudgetExhausted; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Candidate = isCandidate(*Instruction, Target); + if (!Candidate) + return std::unexpected(Candidate.error()); + if (!*Candidate) { + Instruction = Next; + continue; + } + + MachineInstr *Match = nullptr; + auto &OpcodeCandidates = Available[Instruction->opcode()]; + for (MachineInstr *Prior : OpcodeCandidates) { + if (Comparisons >= Options.MaximumComparisons) { + BudgetExhausted = true; + break; + } + ++Comparisons; + if (!(*Dominators)->dominates(*Prior, *Instruction) || + !expressionEquals(*Prior, *Instruction)) { + continue; + } + Match = Prior; + break; + } + if (BudgetExhausted) + break; + if (Match == nullptr) { + OpcodeCandidates.push_back(Instruction); + Instruction = Next; + continue; + } + + auto Replacements = replacementPairs(*Match, *Instruction, Function, Target); + if (!Replacements) + return std::unexpected(Replacements.error()); + if (Replacements->empty()) { + OpcodeCandidates.push_back(Instruction); + Instruction = Next; + continue; + } + auto Replaced = replaceAndErase(*Instruction, *Replacements, Registers); + if (!Replaced) + return std::unexpected(Replaced.error()); + Result.Changed = true; + Instruction = Next; + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved.preserve<MachineDominatorTreeAnalysis>(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCombiner.cpp b/backend/lib/CodeGen/MachineCombiner.cpp new file mode 100644 index 0000000..4341700 --- /dev/null +++ b/backend/lib/CodeGen/MachineCombiner.cpp @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCombiner.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <cstddef> +#include <cstdint> +#include <string> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &combinerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-combiner", + .Requires = {MachineFunctionProperty::IsSSA}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.flags().has(MachineInstrFlag::FrameSetup) || + Instruction.flags().has(MachineInstrFlag::FrameDestroy) || + Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] MachineInstrFlags intersectFlags(MachineInstrFlags Left, + MachineInstrFlags Right) noexcept { + MachineInstrFlags Result; + for (std::uint32_t Index = 0; Index < static_cast<std::uint32_t>(MachineInstrFlag::Count); + ++Index) { + const auto Flag = static_cast<MachineInstrFlag>(Index); + if (Left.has(Flag) && Right.has(Flag)) + Result.set(Flag); + } + return Result; +} + +[[nodiscard]] bool hasEqualRoundingMode(const MachineInstr &Root, + const MachineInstr &Producer) noexcept { + const MachineOperand *RootMode = Root.operandAt(3); + const MachineOperand *ProducerMode = Producer.operandAt(3); + const auto *RootImmediate = RootMode == nullptr ? nullptr : RootMode->getIf<std::int64_t>(); + const auto *ProducerImmediate = + ProducerMode == nullptr ? nullptr : ProducerMode->getIf<std::int64_t>(); + return RootImmediate != nullptr && ProducerImmediate != nullptr && + *RootImmediate == *ProducerImmediate; +} + +[[nodiscard]] MachineInstr *findProducer(MachineInstr &Root, + const MachineCombinerPattern &Pattern) noexcept { + const MachineOperand *ProducerUse = Root.operandAt(Pattern.ProducerOperand); + const Register *ProducerRegister = + ProducerUse == nullptr ? nullptr : ProducerUse->getIf<Register>(); + if (ProducerRegister == nullptr || !ProducerUse->isUse() || !ProducerRegister->isVirtual()) { + return nullptr; + } + MachineInstr *Producer = Root.function().registerInfo().getVRegDef(*ProducerRegister); + if (Producer == nullptr || Producer->opcode() != Pattern.ProducerOpcode || + Producer->parent() != Root.parent() || Producer->operandCount() < 4 || + hasProtectedAttachment(*Producer)) { + return nullptr; + } + const MachineOperand *Definition = Producer->operandAt(0); + const Register *DefinedRegister = Definition == nullptr ? nullptr : Definition->getIf<Register>(); + if (DefinedRegister == nullptr || !Definition->isDef() || Definition->isUse() || + *DefinedRegister != *ProducerRegister) { + return nullptr; + } + if (!Root.flags().has(MachineInstrFlag::FastMathAllowContract) || + !Producer->flags().has(MachineInstrFlag::FastMathAllowContract) || + !hasEqualRoundingMode(Root, *Producer)) { + return nullptr; + } + return Producer; +} + +[[nodiscard]] Expected<void> clearKillFlags(MachineRegisterInfo &Registers, Register Reg) { + for (MachineOperand &Operand : Registers.regOperands(Reg)) { + RegisterOperandFlags Flags = Operand.registerFlags(); + if (!Flags.has(RegisterOperandFlag::Kill)) + continue; + Flags.clear(RegisterOperandFlag::Kill); + auto Changed = Operand.setRegisterFlags(Flags); + if (!Changed) + return std::unexpected(Changed.error()); + } + return {}; +} + +[[nodiscard]] Expected<bool> combineFusedMultiplyAdd(MachineInstr &Root, MachineInstr &Producer, + const MachineCombinerPattern &Pattern, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *RootDefinition = Root.operandAt(0); + const MachineOperand *MultiplyLeft = Producer.operandAt(1); + const MachineOperand *MultiplyRight = Producer.operandAt(2); + const MachineOperand *Addend = Root.operandAt(Pattern.AddendOperand); + const Register *ProducerResult = Producer.operandAt(0)->getIf<Register>(); + const Register *MultiplyLeftRegister = + MultiplyLeft == nullptr ? nullptr : MultiplyLeft->getIf<Register>(); + const Register *MultiplyRightRegister = + MultiplyRight == nullptr ? nullptr : MultiplyRight->getIf<Register>(); + if (RootDefinition == nullptr || MultiplyLeftRegister == nullptr || + MultiplyRightRegister == nullptr || Addend == nullptr || ProducerResult == nullptr) { + return false; + } + + MachineRegisterInfo &Registers = Function.registerInfo(); + const bool EraseProducer = Registers.hasOneUse(*ProducerResult, false); + auto Cleared = clearKillFlags(Registers, *MultiplyLeftRegister); + if (!Cleared) + return std::unexpected(Cleared.error()); + Cleared = clearKillFlags(Registers, *MultiplyRightRegister); + if (!Cleared) + return std::unexpected(Cleared.error()); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Root); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(Pattern.ReplacementOpcode); + if (!Pending) + return std::unexpected(Pending.error()); + + auto Added = Pending->addOperand(RootDefinition->cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + Added = Pending->addOperand(MultiplyLeft->cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + Added = Pending->addOperand(MultiplyRight->cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + Added = Pending->addOperand(Addend->cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + Added = Pending->addOperand(Root.operandAt(3)->cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + for (std::size_t Index = 4; Index < Root.operandCount(); ++Index) { + Added = Pending->addImplicitOperand(Root.operandAt(Index)->cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + } + Added = Pending->setFlags(intersectFlags(Root.flags(), Producer.flags())); + if (!Added) + return std::unexpected(Added.error()); + if (Root.debugLocation()) { + Added = Pending->setDebugLocation(*Root.debugLocation()); + if (!Added) + return std::unexpected(Added.error()); + } + auto Fused = Pending->build(); + if (!Fused) + return std::unexpected(Fused.error()); + + auto Removed = Root.eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + if (EraseProducer) { + Removed = Producer.eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + } + return true; +} + +} // namespace + +const MachinePassContract &MachineCombinerPass::contract() const noexcept { + return combinerContract(); +} + +Expected<MachinePassResult> MachineCombinerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +MachineCombinerPass::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) const { + if (Options.MaximumIterations == 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "machine combiner iteration budget must be nonzero"); + MachinePassResult Result; + std::size_t Rewrites = 0; + for (std::size_t Iteration = 0; Iteration < Options.MaximumIterations; ++Iteration) { + bool IterationChanged = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + for (const MachineCombinerPattern &Pattern : Info->patterns(Instruction->opcode())) { + if (Rewrites == Options.MaximumRewrites) + break; + if (hasProtectedAttachment(*Instruction) || Instruction->operandCount() < 4) + continue; + MachineInstr *Producer = findProducer(*Instruction, Pattern); + if (Producer == nullptr) + continue; + std::string Rule = "combine."; + Rule.append(Pattern.Name); + auto Accepted = Analyses.requestAction(Rule, Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + continue; + auto Executed = + combineFusedMultiplyAdd(*Instruction, *Producer, Pattern, Function, Target); + if (!Executed) + return std::unexpected(Executed.error()); + if (!*Executed) + continue; + ++Rewrites; + IterationChanged = true; + Result.Changed = true; + break; + } + Instruction = Next; + } + } + if (!IterationChanged || Rewrites == Options.MaximumRewrites) + break; + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCombinerPattern.cpp b/backend/lib/CodeGen/MachineCombinerPattern.cpp new file mode 100644 index 0000000..f13ba73 --- /dev/null +++ b/backend/lib/CodeGen/MachineCombinerPattern.cpp @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCombinerPattern.h" + +#include <string> +#include <unordered_set> +#include <utility> + +namespace rcc { + +Expected<TargetMachineCombinerInfo> +TargetMachineCombinerInfo::create(std::vector<MachineCombinerPattern> Patterns) { + TargetMachineCombinerInfo Result(std::move(Patterns)); + std::unordered_set<std::string> Names; + for (const MachineCombinerPattern &Pattern : Result.Patterns) { + if (Pattern.Name.empty() || !Pattern.RootOpcode.isValid() || + !Pattern.ProducerOpcode.isValid() || !Pattern.ReplacementOpcode.isValid() || + Pattern.ProducerOperand == 0 || Pattern.AddendOperand == 0 || + Pattern.ProducerOperand == Pattern.AddendOperand || !Names.insert(Pattern.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine combiner pattern has an invalid target pattern"); + } + Result.MatchTable[Pattern.RootOpcode].push_back(Pattern); + } + return Result; +} + +std::span<const MachineCombinerPattern> +TargetMachineCombinerInfo::patterns(Opcode RootOpcode) const noexcept { + const auto Position = MatchTable.find(RootOpcode); + return Position == MatchTable.end() ? std::span<const MachineCombinerPattern>{} + : std::span<const MachineCombinerPattern>(Position->second); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCompilationStatus.cpp b/backend/lib/CodeGen/MachineCompilationStatus.cpp new file mode 100644 index 0000000..df009a4 --- /dev/null +++ b/backend/lib/CodeGen/MachineCompilationStatus.cpp @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCompilationStatus.h" + +#include "rcc/CodeGen/MachineFunctionCloner.h" + +#include <array> +#include <cstddef> +#include <optional> +#include <string> +#include <string_view> +#include <utility> + +namespace rcc { +namespace { + +constexpr std::array StateNames{ + std::string_view{"Success"}, + std::string_view{"Unsupported"}, + std::string_view{"MIRParseFailed"}, + std::string_view{"LegalizationFailed"}, + std::string_view{"SelectionFailed"}, + std::string_view{"RegisterAllocationFailed"}, + std::string_view{"CallLoweringFailed"}, + std::string_view{"FrameLayoutFailed"}, + std::string_view{"FinalizationFailed"}, + std::string_view{"AssemblyLoweringFailed"}, + std::string_view{"InternalError"}, + std::string_view{"SchedulingFailed"}, +}; + +static_assert(StateNames.size() == static_cast<std::size_t>(MachineCompilationState::Count)); + +[[nodiscard]] bool +isKnownDiagnosticSeverity(MachineCompilationDiagnosticSeverity Severity) noexcept { + return static_cast<std::size_t>(Severity) < + static_cast<std::size_t>(MachineCompilationDiagnosticSeverity::Count); +} + +[[nodiscard]] bool isKnownErrorCode(ErrorCode Code) noexcept { + switch (Code) { + case ErrorCode::InvalidArgument: + case ErrorCode::ParseError: + case ErrorCode::VerificationError: + case ErrorCode::Unsupported: + case ErrorCode::RegisterAllocationFailed: + case ErrorCode::FrameLayoutFailed: + case ErrorCode::SchedulingFailed: + case ErrorCode::BranchRelaxationFailed: + case ErrorCode::PseudoExpansionFailed: + case ErrorCode::FinalEncodabilityFailed: + case ErrorCode::CFGOptimizationFailed: + case ErrorCode::AssemblyLoweringFailed: + case ErrorCode::InstructionEncodingFailed: + case ErrorCode::FixupApplicationFailed: + case ErrorCode::RelocationFailed: + case ErrorCode::ObjectWritingFailed: + case ErrorCode::InvalidInvocation: + case ErrorCode::ManifestError: + case ErrorCode::CacheMiss: + case ErrorCode::ReproducerError: + case ErrorCode::ReductionFailed: + case ErrorCode::FuzzingFailure: + case ErrorCode::IoError: + case ErrorCode::InternalError: + return true; + } + return false; +} + +[[nodiscard]] bool isKnownPipelineStage(PipelineStage Stage) noexcept { + switch (Stage) { + case PipelineStage::MachineIRInput: + case PipelineStage::Legalized: + case PipelineStage::RegBankSelected: + case PipelineStage::Selected: + case PipelineStage::SSAEliminated: + case PipelineStage::TiedOpsRewritten: + case PipelineStage::RegisterAllocated: + case PipelineStage::FrameFinalized: + case PipelineStage::PostRAFinalized: + case PipelineStage::Emission: + return true; + } + return false; +} + +[[nodiscard]] Expected<void> verifyOptionalText(const std::optional<std::string> &Text, + std::string_view FieldName) { + if (Text && Text->empty()) { + return makeUnexpected(ErrorCode::VerificationError, "machine compilation failure " + + std::string(FieldName) + + " cannot be empty"); + } + return {}; +} + +} // namespace + +bool isKnownMachineCompilationState(MachineCompilationState State) noexcept { + return static_cast<std::size_t>(State) < StateNames.size(); +} + +std::string_view getMachineCompilationStateName(MachineCompilationState State) noexcept { + const auto Index = static_cast<std::size_t>(State); + if (Index >= StateNames.size()) + return "Unknown"; + return StateNames[Index]; +} + +Expected<void> MachineCompilationStatus::fail(MachineCompilationState NewState, + MachineCompilationFailure NewFailure) { + if (!succeeded()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine compilation status already contains a failure"); + } + if (!isKnownMachineCompilationState(NewState) || NewState == MachineCompilationState::Success) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine compilation failure requires a known failure state"); + } + + MachineCompilationStatus Candidate; + Candidate.State = NewState; + Candidate.Failure = std::move(NewFailure); + auto Verification = Candidate.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + + *this = std::move(Candidate); + return {}; +} + +Expected<void> MachineCompilationStatus::verify() const { + if (!isKnownMachineCompilationState(State)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine compilation status has an unknown state"); + } + if (State == MachineCompilationState::Success) { + if (Failure) { + return makeUnexpected(ErrorCode::VerificationError, + "successful machine compilation status contains failure details"); + } + return {}; + } + if (!Failure) { + return makeUnexpected(ErrorCode::VerificationError, + "failed machine compilation status lacks failure details"); + } + + if (!isKnownDiagnosticSeverity(Failure->Diagnostic.Severity)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine compilation diagnostic has an unknown severity"); + } + if (!isKnownErrorCode(Failure->Diagnostic.Code)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine compilation diagnostic has an unknown error code"); + } + if (Failure->Diagnostic.Message.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine compilation diagnostic message cannot be empty"); + } + if (Failure->Diagnostic.Location && !Failure->Diagnostic.Location->isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine compilation diagnostic has an invalid source location"); + } + if (Failure->FailureInstruction && !Failure->FailureInstruction->isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine compilation failure has an invalid instruction StableID"); + } + if (Failure->FailureStage && !isKnownPipelineStage(*Failure->FailureStage)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine compilation failure has an unknown pipeline stage"); + } + + for (const auto &[Text, Name] : + {std::pair{&Failure->FailurePass, std::string_view{"pass"}}, + std::pair{&Failure->FailureRule, std::string_view{"rule or pattern"}}, + std::pair{&Failure->Target, std::string_view{"target"}}, + std::pair{&Failure->Subtarget, std::string_view{"subtarget"}}}) { + auto TextVerification = verifyOptionalText(*Text, Name); + if (!TextVerification) + return TextVerification; + } + return {}; +} + +Expected<MachineCompilationStatus> +MachineCompilationStatus::clone(const MachineFunctionCloneContext &CloneContext) const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + if (succeeded()) + return MachineCompilationStatus{}; + if (!Failure) { + return makeUnexpected(ErrorCode::InternalError, + "verified failed compilation status has no failure details"); + } + + MachineCompilationFailure ClonedFailure = *Failure; + if (ClonedFailure.FailureInstruction) { + auto Mapped = CloneContext.mapInstructionStableID(*ClonedFailure.FailureInstruction); + if (!Mapped) + return std::unexpected(Mapped.error()); + ClonedFailure.FailureInstruction = *Mapped; + } + + MachineCompilationStatus Result; + auto FailureResult = Result.fail(State, std::move(ClonedFailure)); + if (!FailureResult) + return std::unexpected(FailureResult.error()); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineConstantPool.cpp b/backend/lib/CodeGen/MachineConstantPool.cpp new file mode 100644 index 0000000..bd46f2d --- /dev/null +++ b/backend/lib/CodeGen/MachineConstantPool.cpp @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineConstantPool.h" + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] const TargetConstant * +getTargetConstant(const MachineConstantPoolValue &Value) noexcept { + if (const auto *Target = std::get_if<TargetConstant>(&Value)) + return Target; + if (const auto *Expression = std::get_if<TargetExpressionConstant>(&Value)) + return &Expression->Expression; + return nullptr; +} + +[[nodiscard]] Expected<void> verifyTargetConstant(const MachineConstantPoolEntryDesc &Description, + const TargetConstant &Constant) { + auto Verification = Constant.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + + auto RequiredAlignment = Constant.requiredAlignment(); + if (!RequiredAlignment) + return std::unexpected(RequiredAlignment.error()); + if (Description.Align.bytes() < RequiredAlignment->bytes()) { + return makeUnexpected(ErrorCode::VerificationError, + "constant pool alignment is weaker than target constant requirement"); + } + + auto Emission = Constant.emit(); + if (!Emission) + return std::unexpected(Emission.error()); + if (Emission->Bytes.size() != Description.Size) { + return makeUnexpected(ErrorCode::VerificationError, + "target constant emitted size does not match its pool entry size"); + } + + auto RelocationKind = Constant.relocationKind(); + if (!RelocationKind) + return std::unexpected(RelocationKind.error()); + const bool HasRelocation = RelocationKind->has_value() || !Emission->Fixups.empty(); + if (HasRelocation == (Description.Relocation == ConstantPoolRelocationMode::None)) { + return makeUnexpected(ErrorCode::VerificationError, + "target constant relocation contract disagrees with its pool entry"); + } + return {}; +} + +[[nodiscard]] std::optional<std::uint64_t> +fixedGenericStoreSize(const GenericConstantType &Type) noexcept { + switch (Type.Kind) { + case GenericConstantTypeKind::Integer: + return (static_cast<std::uint64_t>(Type.BitWidth) + 7U) / 8U; + case GenericConstantTypeKind::Byte: + return 1; + case GenericConstantTypeKind::Floating: + return (static_cast<std::uint64_t>(getFloatSemanticsBitWidth(Type.Semantics).value_or(0)) + + 7U) / + 8U; + case GenericConstantTypeKind::Array: + case GenericConstantTypeKind::Vector: { + if (Type.Scalable || Type.Elements.size() != 1) + return std::nullopt; + const auto ElementSize = fixedGenericStoreSize(Type.Elements.front()); + if (!ElementSize || + (*ElementSize != 0 && + Type.ElementCount > std::numeric_limits<std::uint64_t>::max() / *ElementSize)) { + return std::nullopt; + } + return *ElementSize * Type.ElementCount; + } + default: + return std::nullopt; + } +} + +} // namespace + +ConstantPoolEntryKind MachineConstantPoolEntryDesc::kind() const noexcept { + switch (Value.index()) { + case 0: + return ConstantPoolEntryKind::Generic; + case 1: + return ConstantPoolEntryKind::Target; + case 2: + return ConstantPoolEntryKind::SymbolDifference; + case 3: + return ConstantPoolEntryKind::BlockAddress; + case 4: + return ConstantPoolEntryKind::TargetExpression; + default: + return ConstantPoolEntryKind::Generic; + } +} + +Expected<void> MachineConstantPoolEntryDesc::verify() const { + if (Size == 0) + return makeUnexpected(ErrorCode::VerificationError, "constant pool entry size must be nonzero"); + if (Placement.Section >= ConstantPoolSectionKind::Count) { + return makeUnexpected(ErrorCode::VerificationError, "constant pool section kind is invalid"); + } + if (Placement.MaximumDistance && *Placement.MaximumDistance == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "constant pool maximum distance must be nonzero"); + } + if ((Placement.Section == ConstantPoolSectionKind::Target) != (Placement.TargetFlags != 0)) { + return makeUnexpected(ErrorCode::VerificationError, + "target constant pool placement requires exactly one target identity"); + } + if (Placement.Section == ConstantPoolSectionKind::MergeableReadOnly && + Merge != ConstantPoolMergeKind::Exact) { + return makeUnexpected(ErrorCode::VerificationError, + "mergeable constant section requires exact entry merging"); + } + + switch (kind()) { + case ConstantPoolEntryKind::Generic: { + const auto *Generic = std::get_if<GenericConstantData>(&Value); + if (Generic == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "generic constant pool entry has an invalid payload"); + } + auto Verification = Generic->verify(); + if (!Verification) + return Verification; + if (const auto FixedSize = fixedGenericStoreSize(Generic->Type); + FixedSize && *FixedSize != Size) { + return makeUnexpected(ErrorCode::VerificationError, + "generic constant store size does not match its pool entry size"); + } + if (!Generic->referencesSymbol() && Relocation != ConstantPoolRelocationMode::None) { + return makeUnexpected(ErrorCode::VerificationError, + "symbol-free generic constant cannot require relocation"); + } + return {}; + } + case ConstantPoolEntryKind::Target: + case ConstantPoolEntryKind::TargetExpression: { + const TargetConstant *Target = getTargetConstant(Value); + if (Target == nullptr) + return makeUnexpected(ErrorCode::InternalError, "target constant variant is inconsistent"); + return verifyTargetConstant(*this, *Target); + } + case ConstantPoolEntryKind::SymbolDifference: { + const auto *Difference = std::get_if<SymbolDifferenceConstant>(&Value); + if (Difference == nullptr || Difference->Minuend.empty() || Difference->Subtrahend.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "symbol-difference constant requires two symbols"); + } + if (Relocation == ConstantPoolRelocationMode::None) { + return makeUnexpected(ErrorCode::VerificationError, + "symbol-difference constant requires a relocation mode"); + } + return {}; + } + case ConstantPoolEntryKind::BlockAddress: { + const auto *Address = std::get_if<BlockAddressConstant>(&Value); + if (Address == nullptr || Address->FunctionName.empty() || Address->BlockName.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "block-address constant requires a function and block name"); + } + if (Relocation == ConstantPoolRelocationMode::None) { + return makeUnexpected(ErrorCode::VerificationError, + "block-address constant requires a relocation mode"); + } + return {}; + } + } + return makeUnexpected(ErrorCode::InternalError, "constant pool entry kind is unhandled"); +} + +Expected<MachineConstantPoolEntryDesc> MachineConstantPoolEntryDesc::clone() const { + MachineConstantPoolEntryDesc Result; + Result.Size = Size; + Result.Align = Align; + Result.Merge = Merge; + Result.Relocation = Relocation; + Result.Placement = Placement; + + if (const auto *Generic = std::get_if<GenericConstantData>(&Value)) { + Result.Value = *Generic; + } else if (const auto *Target = std::get_if<TargetConstant>(&Value)) { + auto Cloned = Target->clone(); + if (!Cloned) + return std::unexpected(Cloned.error()); + Result.Value = std::move(*Cloned); + } else if (const auto *Difference = std::get_if<SymbolDifferenceConstant>(&Value)) { + Result.Value = *Difference; + } else if (const auto *Address = std::get_if<BlockAddressConstant>(&Value)) { + Result.Value = *Address; + } else if (const auto *Expression = std::get_if<TargetExpressionConstant>(&Value)) { + auto Cloned = Expression->Expression.clone(); + if (!Cloned) + return std::unexpected(Cloned.error()); + Result.Value = TargetExpressionConstant{std::move(*Cloned)}; + } else { + return makeUnexpected(ErrorCode::InternalError, "constant pool value is unhandled"); + } + return Result; +} + +bool MachineConstantPoolEntryDesc::semanticallyEquals( + const MachineConstantPoolEntryDesc &Other) const noexcept { + if (Size != Other.Size || Align != Other.Align || Merge != Other.Merge || + Relocation != Other.Relocation || Placement != Other.Placement || kind() != Other.kind()) { + return false; + } + + if (const auto *Generic = std::get_if<GenericConstantData>(&Value)) { + const auto *OtherGeneric = std::get_if<GenericConstantData>(&Other.Value); + return OtherGeneric != nullptr && *Generic == *OtherGeneric; + } + if (const auto *Target = std::get_if<TargetConstant>(&Value)) { + const auto *OtherTarget = std::get_if<TargetConstant>(&Other.Value); + return OtherTarget != nullptr && Target->equals(*OtherTarget); + } + if (const auto *Difference = std::get_if<SymbolDifferenceConstant>(&Value)) { + const auto *OtherDifference = std::get_if<SymbolDifferenceConstant>(&Other.Value); + return OtherDifference != nullptr && *Difference == *OtherDifference; + } + if (const auto *Address = std::get_if<BlockAddressConstant>(&Value)) { + const auto *OtherAddress = std::get_if<BlockAddressConstant>(&Other.Value); + return OtherAddress != nullptr && *Address == *OtherAddress; + } + if (const auto *Expression = std::get_if<TargetExpressionConstant>(&Value)) { + const auto *OtherExpression = std::get_if<TargetExpressionConstant>(&Other.Value); + return OtherExpression != nullptr && Expression->Expression.equals(OtherExpression->Expression); + } + return false; +} + +Expected<MachineConstantPoolEntryDesc> MachineConstantPoolEntry::cloneDescription() const { + const MachineConstantPoolEntryDesc *Desc = description(); + if (Desc == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "constant pool tombstone has no description to clone"); + } + return Desc->clone(); +} + +const MachineConstantPoolEntry * +MachineConstantPool::findEquivalent(const MachineConstantPoolEntryDesc &Description, + std::optional<ConstantPoolIndex> Excluded) const noexcept { + for (const MachineConstantPoolEntry &Entry : Entries) { + if (Entry.isTombstone() || (Excluded && Entry.index() == *Excluded)) + continue; + const MachineConstantPoolEntryDesc *Existing = Entry.description(); + if (Existing != nullptr && Existing->semanticallyEquals(Description)) + return &Entry; + } + return nullptr; +} + +Expected<ConstantPoolIndex> +MachineConstantPool::addEntry(MachineConstantPoolEntryDesc Description) { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, "frozen constant pool cannot add entries"); + auto Verification = Description.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + if (Description.Merge == ConstantPoolMergeKind::Exact) { + if (const MachineConstantPoolEntry *Existing = findEquivalent(Description)) + return Existing->index(); + } + if (Entries.size() > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::Unsupported, "constant pool index space is exhausted"); + } + + const ConstantPoolIndex Index = + ConstantPoolIndex::fromValue(static_cast<std::uint32_t>(Entries.size())); + Entries.push_back(MachineConstantPoolEntry(Index, std::move(Description))); + return Index; +} + +Expected<MachineConstantPoolEntry *> MachineConstantPool::mutableEntry(ConstantPoolIndex Index) { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, "frozen constant pool cannot modify entries"); + if (Index.value() >= Entries.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "constant pool index is out of range"); + MachineConstantPoolEntry &Entry = Entries[Index.value()]; + if (Entry.isTombstone()) + return makeUnexpected(ErrorCode::InvalidArgument, "constant pool index is a tombstone"); + return &Entry; +} + +Expected<void> MachineConstantPool::updateEntry(ConstantPoolIndex Index, + MachineConstantPoolEntryDesc Description) { + auto Verification = Description.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + auto Entry = mutableEntry(Index); + if (!Entry) + return std::unexpected(Entry.error()); + if (Description.Merge == ConstantPoolMergeKind::Exact && + findEquivalent(Description, Index) != nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "constant pool update would create duplicate mergeable entries"); + } + (*Entry)->Description.emplace<MachineConstantPoolEntryDesc>(std::move(Description)); + return {}; +} + +Expected<void> MachineConstantPool::eraseEntry(ConstantPoolIndex Index) { + auto Entry = mutableEntry(Index); + if (!Entry) + return std::unexpected(Entry.error()); + (*Entry)->Description.emplace<std::monostate>(); + return {}; +} + +const MachineConstantPoolEntry *MachineConstantPool::entry(ConstantPoolIndex Index) const noexcept { + if (Index.value() >= Entries.size() || Entries[Index.value()].isTombstone()) + return nullptr; + return &Entries[Index.value()]; +} + +std::size_t MachineConstantPool::liveEntryCount() const noexcept { + std::size_t Count = 0; + for (const MachineConstantPoolEntry &Entry : Entries) { + if (!Entry.isTombstone()) + ++Count; + } + return Count; +} + +Expected<void> MachineConstantPool::freeze() { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, "constant pool is already frozen"); + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + Frozen = true; + return {}; +} + +Expected<void> MachineConstantPool::verify() const { + for (std::size_t Position = 0; Position < Entries.size(); ++Position) { + const MachineConstantPoolEntry &Entry = Entries[Position]; + if (Entry.index() != ConstantPoolIndex::fromValue(static_cast<std::uint32_t>(Position))) { + return makeUnexpected(ErrorCode::VerificationError, + "constant pool index does not match its stable table position"); + } + const MachineConstantPoolEntryDesc *Description = Entry.description(); + if (Description == nullptr) + continue; + auto Verification = Description->verify(); + if (!Verification) + return std::unexpected(Verification.error()); + if (Description->Merge == ConstantPoolMergeKind::Exact && + findEquivalent(*Description, Entry.index()) != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "constant pool contains duplicate mergeable entries"); + } + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineCopyPropagation.cpp b/backend/lib/CodeGen/MachineCopyPropagation.cpp new file mode 100644 index 0000000..94d83e3 --- /dev/null +++ b/backend/lib/CodeGen/MachineCopyPropagation.cpp @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineCopyPropagation.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetInstrInfo.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract ©PropagationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-cp", + .Requires = {MachineFunctionProperty::NoVRegs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = std::nullopt, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +struct CopyOperands final { + MachineOperand *DestinationOperand = nullptr; + MachineOperand *SourceOperand = nullptr; + Register Destination; + Register Source; +}; + +[[nodiscard]] std::optional<CopyOperands> copyOperands(MachineInstr &Instruction, + const TargetInstrInfo &Info, + bool UseTargetCopyInstructions) { + if (hasProtectedAttachment(Instruction)) + return std::nullopt; + MachineOperand *DestinationOperand = nullptr; + MachineOperand *SourceOperand = nullptr; + if (UseTargetCopyInstructions) { + const std::optional<MachineCopyOperands> TargetCopy = Info.getCopyOperands(Instruction); + if (!TargetCopy) + return std::nullopt; + DestinationOperand = TargetCopy->Destination; + SourceOperand = TargetCopy->Source; + } else { + if (Instruction.opcode() != opcode(GenericOpcode::COPY) || Instruction.operandCount() < 2) + return std::nullopt; + DestinationOperand = Instruction.operandAt(0); + SourceOperand = Instruction.operandAt(1); + } + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Destination == nullptr || Source == nullptr || !DestinationOperand->isDef() || + !SourceOperand->isUse() || DestinationOperand->isImplicit() || SourceOperand->isImplicit()) { + return std::nullopt; + } + return CopyOperands{DestinationOperand, SourceOperand, *Destination, *Source}; +} + +[[nodiscard]] bool shareRegisterClass(PhysicalRegister Left, PhysicalRegister Right, + const TargetRegisterInfo &Registers) noexcept { + return std::ranges::any_of(Registers.registerClasses(), [&](const RegisterClassDesc &Class) { + return Registers.registerClassContains(Class.ID, Left) && + Registers.registerClassContains(Class.ID, Right); + }); +} + +[[nodiscard]] bool clobbersRegister(const MachineInstr &Instruction, PhysicalRegister Reg, + const TargetRegisterInfo &Registers) noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>()) { + if (Mask->Mask && Mask->Mask->clobbers(Reg)) + return true; + continue; + } + const Register *OperandReg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + OperandReg == nullptr ? std::nullopt : OperandReg->physicalRegister(); + if (Physical && Operand.isDef() && Registers.regsOverlap(*Physical, Reg)) + return true; + } + return false; +} + +[[nodiscard]] bool canReplacePhysicalUse(const MachineInstr &Instruction, std::size_t OperandIndex, + PhysicalRegister Source, const MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &Info) noexcept { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Operand == nullptr || Descriptor == nullptr || Operand->isImplicit() || + Operand->tiedOperandIndex() || Operand->subReg().value() != 0 || + OperandIndex >= Descriptor->Operands.size()) { + return false; + } + const OperandDescriptor &Constraint = Descriptor->Operands[OperandIndex]; + if (Constraint.RegClass && + !Target.registers().registerClassContains(*Constraint.RegClass, Source)) { + return false; + } + if (Constraint.RegBank && !Target.registers().bankContainsRegister(*Constraint.RegBank, Source)) { + return false; + } + if (Instruction.opcode() == opcode(GenericOpcode::COPY)) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr) + return false; + auto Valid = + Info.validateCopy(*Destination, DestinationOperand->subReg(), + Register::fromPhysical(Source), Operand->subReg(), Function, Target); + if (!Valid) + return false; + } + return true; +} + +[[nodiscard]] bool valueUsedAfter(const MachineInstr &Copy, PhysicalRegister Destination, + const TargetDescription &Target) noexcept { + for (const MachineInstr *Instruction = Copy.next(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>()) { + if (Mask->Mask && Mask->Mask->clobbers(Destination)) + return false; + continue; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical || !Target.registers().regsOverlap(*Physical, Destination)) + continue; + if (Operand.isUse()) + return true; + if (Operand.isDef()) + return false; + } + } + const MachineBasicBlock *Block = Copy.parent(); + if (Block == nullptr) + return true; + if (!Block->successors().empty() && + !Copy.function().properties().has(MachineFunctionProperty::TracksLiveness)) { + return true; + } + return std::ranges::any_of(Block->successors(), [&](const MachineSuccessor &Successor) { + return Successor.Block != nullptr && + std::ranges::any_of(Successor.Block->liveIns(), [&](const MachineBasicBlockLiveIn &In) { + return Target.registers().regsOverlap(In.PhysReg, Destination); + }); + }); +} + +[[nodiscard]] Expected<void> clearRegisterKills(MachineInstr &First, MachineInstr &Last, + PhysicalRegister Reg, + const TargetRegisterInfo &Registers) { + for (MachineInstr *Instruction = &First; Instruction != nullptr; + Instruction = Instruction->next()) { + if (Instruction == &Last) + return {}; + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const Register *OperandReg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + OperandReg == nullptr ? std::nullopt : OperandReg->physicalRegister(); + if (!Physical || !Operand->isUse() || + !Operand->registerFlags().has(RegisterOperandFlag::Kill) || + !Registers.regsOverlap(*Physical, Reg)) { + continue; + } + RegisterOperandFlags Flags = Operand->registerFlags(); + Flags.clear(RegisterOperandFlag::Kill); + auto Updated = Operand->setRegisterFlags(Flags); + if (!Updated) + return std::unexpected(Updated.error()); + } + } + return makeUnexpected(ErrorCode::InternalError, + "COPY propagation replacement is outside the COPY block"); +} + +[[nodiscard]] Expected<bool> propagatePhysicalCopy(MachineInstr &Copy, const CopyOperands &Operands, + MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &Info) { + const std::optional<PhysicalRegister> Destination = Operands.Destination.physicalRegister(); + const std::optional<PhysicalRegister> Source = Operands.Source.physicalRegister(); + if (!Destination || !Source || Operands.DestinationOperand->subReg().value() != 0 || + Operands.SourceOperand->subReg().value() != 0) { + return false; + } + auto ValidCopy = + Info.validateCopy(Operands.Destination, {}, Operands.Source, {}, Function, Target); + if (!ValidCopy) { + if (ValidCopy.error().code() == ErrorCode::Unsupported) + return false; + return std::unexpected(ValidCopy.error()); + } + MachineBasicBlock *Block = Copy.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "physical COPY propagation candidate is detached"); + const bool IsGenericCopy = Copy.opcode() == opcode(GenericOpcode::COPY); + if (*Destination == *Source) { + if (!IsGenericCopy) + return false; + auto Removed = Copy.eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + if (!shareRegisterClass(*Destination, *Source, Target.registers())) + return false; + + bool Changed = false; + for (MachineInstr *Instruction = Copy.next(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (clobbersRegister(*Instruction, *Destination, Target.registers()) || + clobbersRegister(*Instruction, *Source, Target.registers())) { + break; + } + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical || *Physical != *Destination || !Operand->isUse() || Operand->isDebugUse() || + !Operand->registerFlags().has(RegisterOperandFlag::Renamable) || + !canReplacePhysicalUse(*Instruction, Index, *Source, Function, Target, Info)) { + continue; + } + const bool DestinationWasKilled = Operand->registerFlags().has(RegisterOperandFlag::Kill); + auto Replaced = Operand->setRegister(Operands.Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + RegisterOperandFlags Flags = Operand->registerFlags(); + Flags.clear(RegisterOperandFlag::Kill); + if (DestinationWasKilled && (clobbersRegister(*Instruction, *Source, Target.registers()) || + !valueUsedAfter(*Instruction, *Source, Target))) { + Flags.set(RegisterOperandFlag::Kill); + } + if (!Operands.SourceOperand->registerFlags().has(RegisterOperandFlag::Renamable)) + Flags.clear(RegisterOperandFlag::Renamable); + if (Operands.SourceOperand->registerFlags().has(RegisterOperandFlag::Undef)) + Flags.set(RegisterOperandFlag::Undef); + else + Flags.clear(RegisterOperandFlag::Undef); + auto SetFlags = Operand->setRegisterFlags(Flags); + if (!SetFlags) + return std::unexpected(SetFlags.error()); + auto ClearedKills = clearRegisterKills(Copy, *Instruction, *Source, Target.registers()); + if (!ClearedKills) + return std::unexpected(ClearedKills.error()); + Changed = true; + } + } + if (IsGenericCopy && + Operands.DestinationOperand->registerFlags().has(RegisterOperandFlag::Renamable) && + !valueUsedAfter(Copy, *Destination, Target)) { + auto Removed = Copy.eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + } + return Changed; +} + +} // namespace + +const MachinePassContract &MachineCopyPropagationPass::contract() const noexcept { + return copyPropagationContract(); +} + +Expected<MachinePassResult> +MachineCopyPropagationPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +MachineCopyPropagationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) const { + MachinePassResult Result; + std::size_t PropagatedCopies = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); + Instruction != nullptr && PropagatedCopies < Options.MaximumPropagatedCopies;) { + MachineInstr *Next = Instruction->next(); + auto Simplified = Info->simplifyInstruction(*Instruction, Function, Target); + if (!Simplified) + return std::unexpected(Simplified.error()); + if (*Simplified) + Result.Changed = true; + const std::optional<CopyOperands> Operands = + copyOperands(*Instruction, *Info, Options.UseTargetCopyInstructions); + if (!Operands) { + Instruction = Next; + continue; + } + auto Accepted = Analyses.requestAction("propagate-copy", Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + Instruction = Next; + continue; + } + Expected<bool> Changed = + propagatePhysicalCopy(*Instruction, *Operands, Function, Target, *Info); + if (!Changed) + return std::unexpected(Changed.error()); + if (*Changed) { + ++PropagatedCopies; + Result.Changed = true; + } + Instruction = Next; + } + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineDCE.cpp b/backend/lib/CodeGen/MachineDCE.cpp new file mode 100644 index 0000000..734509f --- /dev/null +++ b/backend/lib/CodeGen/MachineDCE.cpp @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineDCE.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <optional> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &dceContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-dce", + .Requires = {MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.flags().has(MachineInstrFlag::FrameSetup) || + Instruction.flags().has(MachineInstrFlag::FrameDestroy) || + Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] Expected<std::optional<std::vector<Register>>> +deadDefinitions(MachineInstr &Instruction, MachineRegisterInfo &Registers, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine DCE encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || hasProtectedAttachment(Instruction)) { + return std::nullopt; + } + + std::vector<Register> VirtualDefinitions; + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Operand.isDef()) + continue; + HasDefinition = true; + if (Operand.isUse() || Operand.tiedOperandIndex()) + return std::nullopt; + if (Reg->isVirtual()) { + if (!Registers.useEmpty(*Reg, false)) + return std::nullopt; + VirtualDefinitions.push_back(*Reg); + continue; + } + if (!Reg->isPhysical() || !Operand.registerFlags().has(RegisterOperandFlag::Dead)) { + return std::nullopt; + } + } + if (!HasDefinition) + return std::nullopt; + return VirtualDefinitions; +} + +[[nodiscard]] Expected<void> clearDebugUses(MachineRegisterInfo &Registers, Register Reg) { + std::vector<MachineOperand *> Uses; + for (MachineOperand &Operand : Registers.useOperands(Reg, true)) { + if (Operand.isDebugUse()) + Uses.push_back(&Operand); + } + for (MachineOperand *Operand : Uses) { + auto Cleared = Operand->setRegister(Register{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineDCEPass::contract() const noexcept { + return dceContract(); +} + +Expected<MachinePassResult> MachineDCEPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + return runImplementation(Function, Target); +} + +Expected<MachinePassResult> MachineDCEPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) { + MachinePassResult Result; + MachineRegisterInfo &Registers = Function.registerInfo(); + + bool LocalChange = true; + while (LocalChange) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Definitions = deadDefinitions(*Instruction, Registers, Target); + if (!Definitions) + return std::unexpected(Definitions.error()); + if (!*Definitions) { + Instruction = Next; + continue; + } + std::vector<Register> DefinitionList = + std::move(*Definitions).value_or(std::vector<Register>{}); + + for (const Register Reg : DefinitionList) { + auto Cleared = clearDebugUses(Registers, Reg); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + auto Removed = Instruction->eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + for (const Register Reg : DefinitionList) { + if (!Registers.useEmpty(Reg, true) || Registers.getUniqueVRegDef(Reg) != nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine DCE left references to a removed definition"); + } + auto Erased = Registers.eraseVirtualRegister(Reg); + if (!Erased) + return std::unexpected(Erased.error()); + } + LocalChange = true; + Result.Changed = true; + Instruction = Next; + } + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineDebugInfo.cpp b/backend/lib/CodeGen/MachineDebugInfo.cpp new file mode 100644 index 0000000..35f68ca --- /dev/null +++ b/backend/lib/CodeGen/MachineDebugInfo.cpp @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineDebugInfo.h" + +#include <utility> + +namespace rcc { + +Expected<MachineMetadataValue> MachineMetadataValue::reference(MachineMetadataReference Reference) { + if (!Reference.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine metadata value reference must be valid"); + } + return MachineMetadataValue(Reference); +} + +MachineMetadataValue MachineMetadataValue::expression(MachineDIExpression Expression) { + return MachineMetadataValue(std::move(Expression)); +} + +Expected<void> MachineMetadataValue::verify(const MachineMetadataInfo &Metadata) const { + if (!isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine metadata value has no representation"); + } + const MachineMetadataReference *Reference = reference(); + if (Reference == nullptr) + return {}; + return verifyMachineMetadataReference(*Reference, Metadata); +} + +Expected<MachineDILocation> +MachineDILocation::create(std::vector<MachineDILocationFrame> InlineChain, + std::optional<MachineMetadataReference> TerminalInlinedAt) { + MachineDILocation Result(std::move(InlineChain), TerminalInlinedAt); + auto Verification = Result.verifyStructure(); + if (!Verification) + return std::unexpected(Verification.error()); + return Result; +} + +Expected<void> MachineDILocation::verifyStructure() const { + if (InlineChain.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "inline debug location requires at least one frame"); + } + for (const MachineDILocationFrame &Frame : InlineChain) { + if (!Frame.Scope.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "inline debug location frame has an invalid scope"); + } + if (Frame.Scope.scope() != MachineMetadataScope::EmbeddedIR) { + return makeUnexpected(ErrorCode::VerificationError, + "inline debug location scope must reference embedded IR metadata"); + } + if (Frame.AtomGroup > MaxAtomGroup) { + return makeUnexpected(ErrorCode::VerificationError, + "inline debug location atom group exceeds 61 bits"); + } + if (Frame.AtomRank > 7) { + return makeUnexpected(ErrorCode::VerificationError, + "inline debug location atom rank exceeds three bits"); + } + } + if (TerminalInlinedAt && !TerminalInlinedAt->isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "inline debug location has an invalid terminal inlined-at reference"); + } + if (TerminalInlinedAt && TerminalInlinedAt->scope() != MachineMetadataScope::EmbeddedIR) { + return makeUnexpected(ErrorCode::VerificationError, + "inline debug location inlined-at must reference embedded IR metadata"); + } + return {}; +} + +Expected<void> MachineDILocation::verify(const MachineMetadataInfo &Metadata) const { + auto Structure = verifyStructure(); + if (!Structure) + return Structure; + for (const MachineDILocationFrame &Frame : InlineChain) { + auto Scope = verifyMachineMetadataReference(Frame.Scope, Metadata); + if (!Scope) + return Scope; + } + if (TerminalInlinedAt) + return verifyMachineMetadataReference(*TerminalInlinedAt, Metadata); + return {}; +} + +Expected<MachineDebugLocation> MachineDebugLocation::reference(MachineMetadataReference Reference) { + if (!Reference.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine debug location reference must be valid"); + } + if (Reference.scope() != MachineMetadataScope::EmbeddedIR) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine debug location must reference embedded IR metadata"); + } + return MachineDebugLocation(Reference); +} + +MachineDebugLocation MachineDebugLocation::inlineLocation(MachineDILocation Location) { + return MachineDebugLocation(std::move(Location)); +} + +Expected<void> MachineDebugLocation::verify(const MachineMetadataInfo &Metadata) const { + if (!isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine debug location has no representation"); + } + if (const MachineMetadataReference *Reference = reference()) + return verifyMachineMetadataReference(*Reference, Metadata); + return inlineLocation()->verify(Metadata); +} + +Expected<void> MachineVariableDebugInfo::verifyStructure() const { + if (!Variable && !Expression && !Location) { + return makeUnexpected(ErrorCode::VerificationError, + "variable debug information has no metadata"); + } + if (Variable && !Variable->isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "variable debug information has an invalid variable reference"); + } + if (Variable && Variable->scope() != MachineMetadataScope::EmbeddedIR) { + return makeUnexpected(ErrorCode::VerificationError, + "variable debug information must reference embedded IR metadata"); + } + if (Expression && !Expression->isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "variable debug information has an invalid expression"); + } + if (Expression && Expression->reference() != nullptr && + Expression->reference()->scope() != MachineMetadataScope::EmbeddedIR) { + return makeUnexpected(ErrorCode::VerificationError, + "variable debug expression must reference embedded IR metadata"); + } + if (Location && !Location->isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "variable debug information has an invalid location"); + } + return {}; +} + +Expected<void> MachineVariableDebugInfo::verify(const MachineMetadataInfo &Metadata) const { + auto Structure = verifyStructure(); + if (!Structure) + return Structure; + if (Variable) { + auto VariableResult = verifyMachineMetadataReference(*Variable, Metadata); + if (!VariableResult) + return VariableResult; + } + if (Expression) { + auto ExpressionResult = Expression->verify(Metadata); + if (!ExpressionResult) + return ExpressionResult; + } + if (Location) + return Location->verify(Metadata); + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFinalization.cpp b/backend/lib/CodeGen/MachineFinalization.cpp new file mode 100644 index 0000000..6f16d97 --- /dev/null +++ b/backend/lib/CodeGen/MachineFinalization.cpp @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFinalization.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetInstrInfo.h" + +#include <cstdint> +#include <limits> +#include <string> +#include <unordered_map> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &finalizePseudosContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "finalize-emission-pseudos", + .Requires = {MachineFunctionProperty::NoPHIs, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {MachineFunctionProperty::NoExecutablePseudos}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Emission, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] const MachinePassContract &encodabilityContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "final-encodability-verifier", + .Requires = {MachineFunctionProperty::NoPHIs, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices, + MachineFunctionProperty::NoExecutablePseudos}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Emission, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] Expected<std::uint64_t> alignTo(std::uint64_t Value, std::uint64_t Alignment) { + if (Alignment == 0 || (Alignment & (Alignment - 1U)) != 0) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final block alignment is not a power of two"); + const std::uint64_t Mask = Alignment - 1U; + if (Value > std::numeric_limits<std::uint64_t>::max() - Mask) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final machine layout offset overflowed"); + return (Value + Mask) & ~Mask; +} + +[[nodiscard]] Expected<std::int64_t> displacement(std::uint64_t Source, std::uint64_t Target) { + if (Target >= Source) { + const std::uint64_t Difference = Target - Source; + if (Difference > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch displacement overflowed"); + return static_cast<std::int64_t>(Difference); + } + const std::uint64_t Difference = Source - Target; + if (Difference > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch displacement overflowed"); + return -static_cast<std::int64_t>(Difference); +} + +} // namespace + +const MachinePassContract &FinalizeEmissionPseudosPass::contract() const noexcept { + return finalizePseudosContract(); +} + +Expected<MachinePassResult> FinalizeEmissionPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + for (MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "emission-pseudo finalization encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::Pseudo)) + continue; + const bool NonEmitting = Descriptor->Flags.has(InstructionFlag::Meta) && + !Descriptor->Flags.has(InstructionFlag::ProducesCode); + const bool Terminal = Descriptor->Flags.has(InstructionFlag::TerminalEmission) && + Descriptor->Owner == PseudoOwner::AsmPrinter; + if (!NonEmitting && !Terminal) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "non-terminal executable pseudo survived all expansion owners"); + } + } + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; +} + +const MachinePassContract &FinalEncodabilityVerifierPass::contract() const noexcept { + return encodabilityContract(); +} + +Expected<MachinePassResult> FinalEncodabilityVerifierPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + std::unordered_map<const MachineBasicBlock *, std::uint64_t> BlockOffsets; + std::unordered_map<const MachineInstr *, std::uint64_t> InstructionOffsets; + std::uint64_t Offset = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + auto Aligned = alignTo(Offset, Block->attributes().alignment().bytes()); + if (!Aligned) + return std::unexpected(Aligned.error()); + Offset = *Aligned; + BlockOffsets.emplace(Block, Offset); + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction uses an unknown opcode"); + auto InstructionValidation = + Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!InstructionValidation) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction is not encodable: " + + std::string(InstructionValidation.error().message())); + } + const bool BundledWithPred = Instruction->flags().has(MachineInstrFlag::BundledWithPred); + const bool BundledWithSucc = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if ((BundledWithPred && + (Instruction->previous() == nullptr || + !Instruction->previous()->flags().has(MachineInstrFlag::BundledWithSucc))) || + (BundledWithSucc && + (Instruction->next() == nullptr || + !Instruction->next()->flags().has(MachineInstrFlag::BundledWithPred)))) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction has an inconsistent bundle boundary"); + } + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && !Reg->isNoRegister() && !Reg->isPhysical()) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction retains a virtual register"); + if (Operand.kind() == MachineOperand::Kind::FrameIndex) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction retains a frame index"); + } + auto Size = BranchInfo->instructionSize(*Instruction, Function, Target); + if (!Size) + return std::unexpected(Size.error()); + const bool ProducesCode = Descriptor->Flags.has(InstructionFlag::ProducesCode); + if ((ProducesCode && *Size == 0) || + (Descriptor->Flags.has(InstructionFlag::Meta) && *Size != 0)) { + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction violates its exact size contract"); + } + InstructionOffsets.emplace(Instruction, Offset); + if (Offset > std::numeric_limits<std::uint64_t>::max() - *Size) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final machine function size overflowed"); + Offset += *Size; + } + } + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto TargetBlock = BranchInfo->branchTarget(*Instruction, Function, Target); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + if (*TargetBlock == nullptr) + continue; + const auto Source = InstructionOffsets.find(Instruction); + const auto Destination = BlockOffsets.find(*TargetBlock); + if (Source == InstructionOffsets.end() || Destination == BlockOffsets.end()) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final branch target has no layout offset"); + auto Distance = displacement(Source->second, Destination->second); + if (!Distance) + return std::unexpected(Distance.error()); + if (!BranchInfo->isBranchOffsetInRange(*Instruction, *Distance)) + return makeUnexpected(ErrorCode::FinalEncodabilityFailed, + "final instruction contains an out-of-range branch"); + } + } + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFrameInfo.cpp b/backend/lib/CodeGen/MachineFrameInfo.cpp new file mode 100644 index 0000000..80e1ee0 --- /dev/null +++ b/backend/lib/CodeGen/MachineFrameInfo.cpp @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFrameInfo.h" + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <string> +#include <string_view> +#include <unordered_set> +#include <utility> + +namespace rcc { + +Expected<StackID> StackID::fromValue(std::uint32_t Value) { + StackID Result(Value); + if (!Result.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "stack ID is not serializable in LLVM MIR"); + return Result; +} + +std::optional<StackID> parseStackIDName(std::string_view Name) noexcept { + if (Name == "default") + return StackID::defaultStack(); + if (Name == "sgpr-spill") + return StackID::sgprSpillStack(); + if (Name == "scalable-vector") + return StackID::scalableVectorStack(); + if (Name == "scalable-predicate-vector") + return StackID::scalablePredicateVectorStack(); + if (Name == "wasm-local") + return StackID::wasmLocalStack(); + if (Name == "noalloc") + return StackID::noAllocStack(); + return std::nullopt; +} + +std::string_view getStackIDName(StackID ID) noexcept { + if (ID == StackID::defaultStack()) + return "default"; + if (ID == StackID::sgprSpillStack()) + return "sgpr-spill"; + if (ID == StackID::scalableVectorStack()) + return "scalable-vector"; + if (ID == StackID::scalablePredicateVectorStack()) + return "scalable-predicate-vector"; + if (ID == StackID::wasmLocalStack()) + return "wasm-local"; + if (ID == StackID::noAllocStack()) + return "noalloc"; + return "unknown"; +} + +std::optional<MachineFramePointerPolicy> +parseMachineFramePointerPolicy(std::string_view Name) noexcept { + if (Name == "none") + return MachineFramePointerPolicy::None; + if (Name == "non-leaf") + return MachineFramePointerPolicy::NonLeaf; + if (Name == "all") + return MachineFramePointerPolicy::All; + if (Name == "reserved") + return MachineFramePointerPolicy::Reserved; + if (Name == "non-leaf-no-reserve") + return MachineFramePointerPolicy::NonLeafNoReserve; + return std::nullopt; +} + +std::string_view getMachineFramePointerPolicyName(MachineFramePointerPolicy Policy) noexcept { + switch (Policy) { + case MachineFramePointerPolicy::None: + return "none"; + case MachineFramePointerPolicy::NonLeaf: + return "non-leaf"; + case MachineFramePointerPolicy::All: + return "all"; + case MachineFramePointerPolicy::Reserved: + return "reserved"; + case MachineFramePointerPolicy::NonLeafNoReserve: + return "non-leaf-no-reserve"; + case MachineFramePointerPolicy::Count: + return "unknown"; + } + return "unknown"; +} + +bool StackObject::isSpill() const noexcept { + const StackObjectDesc *Desc = description(); + if (Desc == nullptr) + return false; + return Desc->Kind == StackObjectKind::SpillSlot || Desc->Kind == StackObjectKind::CalleeSaved || + Desc->Kind == StackObjectKind::EmergencySpill || + Desc->Kind == StackObjectKind::FixedSpillSlot; +} + +bool StackObject::isFixed() const noexcept { + const StackObjectDesc *Desc = description(); + return Desc != nullptr && + (Desc->Kind == StackObjectKind::FixedABI || Desc->Kind == StackObjectKind::FixedSpillSlot); +} + +bool StackObject::isVariableSized() const noexcept { + const StackObjectDesc *Desc = description(); + return Desc != nullptr && (Desc->Kind == StackObjectKind::VariableSized || + Desc->Kind == StackObjectKind::DynamicAlloca); +} + +Expected<void> MachineFrameInfo::verifyDescription(const StackObjectDesc &Description) { + if (Description.Kind >= StackObjectKind::Count) + return makeUnexpected(ErrorCode::InvalidArgument, "stack object kind is invalid"); + if (!Description.Stack.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "stack object has an invalid stack ID"); + + const bool IsVariable = Description.Kind == StackObjectKind::VariableSized || + Description.Kind == StackObjectKind::DynamicAlloca; + if (IsVariable) { + if (Description.Size) { + return makeUnexpected(ErrorCode::InvalidArgument, + "variable-sized stack objects cannot have a fixed size"); + } + } else if (!Description.Size) { + return makeUnexpected(ErrorCode::InvalidArgument, + "fixed-size stack objects require an explicit size"); + } + + if ((Description.Kind == StackObjectKind::FixedABI || + Description.Kind == StackObjectKind::FixedSpillSlot) && + !Description.Offset) { + return makeUnexpected(ErrorCode::InvalidArgument, + "fixed ABI stack objects require a fixed offset"); + } + if (Description.AllocaName && Description.AllocaName->empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "named stack objects require a nonempty alloca name"); + } + if (Description.IsImmutable && Description.Kind != StackObjectKind::FixedABI) { + return makeUnexpected(ErrorCode::InvalidArgument, + "only fixed ABI stack objects may be immutable"); + } + if (Description.CalleeSavedRegister && Description.CalleeSavedRegister->number() == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "callee-saved stack objects require a nonzero physical register"); + } + if (Description.LocalOffset && (Description.Kind == StackObjectKind::FixedABI || + Description.Kind == StackObjectKind::FixedSpillSlot)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "fixed ABI stack objects cannot have a local frame offset"); + } + if (Description.Kind == StackObjectKind::Target && !Description.Stack.isSpecialStack() && + Description.TargetFlags == 0) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "target stack objects require a special stack ID or nonzero target flags"); + } + + std::unordered_set<MachineInstrStableID> LifetimeBoundaries; + for (const StackObjectLifetimeRange &Range : Description.Lifetime) { + if (!Range.Begin.isValid() || !Range.End.isValid() || Range.Begin == Range.End) { + return makeUnexpected(ErrorCode::InvalidArgument, + "stack object lifetime boundaries must be valid and distinct"); + } + if (!LifetimeBoundaries.insert(Range.Begin).second || + !LifetimeBoundaries.insert(Range.End).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "stack object lifetime boundaries must not be reused"); + } + } + if (Description.DebugInfo) { + auto DebugInfoResult = Description.DebugInfo->verifyStructure(); + if (!DebugInfoResult) + return DebugInfoResult; + } + return {}; +} + +Expected<void> MachineFrameInfo::verifyStateStructure(const MachineFrameState &State) { + if (State.FramePointerPolicy >= MachineFramePointerPolicy::Count) { + return makeUnexpected(ErrorCode::InvalidArgument, "frame-pointer policy is invalid"); + } + + auto VerifyPoints = [](std::span<const MachineFrameSaveRestorePoint> Points, + std::string_view Kind) -> Expected<void> { + std::unordered_set<MachineBlockStableID> Blocks; + for (const MachineFrameSaveRestorePoint &Point : Points) { + if (!Point.Block.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + std::string(Kind) + " point requires a valid basic-block identity"); + } + if (!Blocks.insert(Point.Block).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + std::string(Kind) + " points must use unique basic blocks"); + } + std::unordered_set<PhysicalRegister> Registers; + for (const PhysicalRegister Reg : Point.Registers) { + if (Reg.number() == 0 || !Registers.insert(Reg).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + std::string(Kind) + " point registers must be nonzero and unique"); + } + } + } + return {}; + }; + + auto SaveResult = VerifyPoints(State.SavePoints, "save"); + if (!SaveResult) + return SaveResult; + return VerifyPoints(State.RestorePoints, "restore"); +} + +Expected<FrameIndex> MachineFrameInfo::createObject(StackObjectDesc Description) { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, "frozen frame cannot create stack objects"); + auto DescriptionResult = verifyDescription(Description); + if (!DescriptionResult) + return std::unexpected(DescriptionResult.error()); + if (Objects.size() > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::Unsupported, "stack object index space is exhausted"); + } + + const FrameIndex Index = FrameIndex::fromValue(static_cast<std::uint32_t>(Objects.size())); + Objects.push_back(StackObject(Index, std::move(Description))); + return Index; +} + +Expected<StackObject *> MachineFrameInfo::mutableObject(FrameIndex Index) { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, "frozen frame cannot modify stack objects"); + if (Index.value() >= Objects.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "stack object index is out of range"); + StackObject &Object = Objects[Index.value()]; + if (Object.isTombstone()) + return makeUnexpected(ErrorCode::InvalidArgument, "stack object index is a tombstone"); + return &Object; +} + +Expected<void> MachineFrameInfo::updateObject(FrameIndex Index, StackObjectDesc Description) { + auto DescriptionResult = verifyDescription(Description); + if (!DescriptionResult) + return std::unexpected(DescriptionResult.error()); + auto Object = mutableObject(Index); + if (!Object) + return std::unexpected(Object.error()); + (*Object)->Description.emplace<StackObjectDesc>(std::move(Description)); + return {}; +} + +Expected<void> MachineFrameInfo::setObjectOffset(FrameIndex Index, std::int64_t Offset) { + auto Object = mutableObject(Index); + if (!Object) + return std::unexpected(Object.error()); + StackObjectDesc *CurrentDescription = std::get_if<StackObjectDesc>(&(*Object)->Description); + if (CurrentDescription == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "live stack object unexpectedly lacks a description"); + } + StackObjectDesc Description = *CurrentDescription; + Description.Offset = Offset; + (*Object)->Description.emplace<StackObjectDesc>(std::move(Description)); + return {}; +} + +Expected<void> MachineFrameInfo::markObjectDead(FrameIndex Index) { + auto Object = mutableObject(Index); + if (!Object) + return std::unexpected(Object.error()); + StackObjectDesc *Description = std::get_if<StackObjectDesc>(&(*Object)->Description); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "live stack object unexpectedly lacks a description"); + } + Description->IsDead = true; + return {}; +} + +Expected<void> MachineFrameInfo::eraseObject(FrameIndex Index) { + auto Object = mutableObject(Index); + if (!Object) + return std::unexpected(Object.error()); + (*Object)->Description.emplace<std::monostate>(); + return {}; +} + +const StackObject *MachineFrameInfo::object(FrameIndex Index) const noexcept { + if (Index.value() >= Objects.size() || Objects[Index.value()].isTombstone()) + return nullptr; + return &Objects[Index.value()]; +} + +std::size_t MachineFrameInfo::liveObjectCount() const noexcept { + std::size_t Count = 0; + for (const StackObject &Object : Objects) { + if (!Object.isTombstone()) + ++Count; + } + return Count; +} + +Expected<void> MachineFrameInfo::setMaxCallFrameSize(std::uint64_t Size) { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, + "frozen frame cannot change its maximum call-frame size"); + State.MaxCallFrameSize = Size; + return {}; +} + +Expected<void> MachineFrameInfo::clearMaxCallFrameSize() { + if (Frozen) { + return makeUnexpected(ErrorCode::InvalidArgument, + "frozen frame cannot change its maximum call-frame size"); + } + State.MaxCallFrameSize.reset(); + return {}; +} + +Expected<void> MachineFrameInfo::setState(MachineFrameState NewState) { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, "frozen frame cannot replace its state"); + auto Verification = verifyStateStructure(NewState); + if (!Verification) + return std::unexpected(Verification.error()); + State = std::move(NewState); + return {}; +} + +Expected<void> MachineFrameInfo::freeze(std::uint64_t Size, Alignment Align) { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, "frame layout is already frozen"); + for (const StackObject &Object : Objects) { + if (Object.isTombstone() || Object.description()->IsDead || Object.isVariableSized()) + continue; + if (!Object.description()->Offset) { + return makeUnexpected(ErrorCode::VerificationError, + "fixed-size stack object is missing its final offset"); + } + } + if (Size % Align.bytes() != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "static stack size must be aligned to the stack alignment"); + } + + State.StackSize = Size; + State.MaxAlignment = Align; + Frozen = true; + return {}; +} + +Expected<void> MachineFrameInfo::verify() const { + auto StateResult = verifyStateStructure(State); + if (!StateResult) + return StateResult; + + for (std::size_t Position = 0; Position < Objects.size(); ++Position) { + const StackObject &Object = Objects[Position]; + if (Object.index() != FrameIndex::fromValue(static_cast<std::uint32_t>(Position))) { + return makeUnexpected(ErrorCode::VerificationError, + "stack object index does not match its stable table position"); + } + if (Object.isTombstone()) + continue; + auto DescriptionResult = verifyDescription(*Object.description()); + if (!DescriptionResult) + return std::unexpected(DescriptionResult.error()); + if (Frozen && !Object.description()->IsDead && !Object.isVariableSized() && + !Object.description()->Offset) { + return makeUnexpected(ErrorCode::VerificationError, + "frozen fixed-size stack object has no final offset"); + } + } + + for (const std::optional<FrameIndex> Index : {State.StackProtector, State.FunctionContext}) { + if (Index && object(*Index) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "frame state references an unknown stack object"); + } + } + if (Frozen && !State.MaxAlignment) { + return makeUnexpected(ErrorCode::VerificationError, + "frozen frame is missing its maximum alignment"); + } + if (Frozen && State.StackSize % State.MaxAlignment->bytes() != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "frozen static stack size violates stack alignment"); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFunction.cpp b/backend/lib/CodeGen/MachineFunction.cpp new file mode 100644 index 0000000..b870858 --- /dev/null +++ b/backend/lib/CodeGen/MachineFunction.cpp @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFunction.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <memory> +#include <memory_resource> +#include <new> +#include <utility> +#include <vector> + +namespace rcc { + +class MachineArena final { +public: + MachineArena() = default; + MachineArena(const MachineArena &) = delete; + MachineArena &operator=(const MachineArena &) = delete; + MachineArena(MachineArena &&) = delete; + MachineArena &operator=(MachineArena &&) = delete; + + ~MachineArena() { + for (auto Position = Destructors.rbegin(); Position != Destructors.rend(); ++Position) + Position->Destroy(Position->Object); + } + + template <typename ObjectT, typename... ArgumentTs> + [[nodiscard]] ObjectT *create(ArgumentTs &&...Arguments) { + void *Storage = Resource.allocate(sizeof(ObjectT), alignof(ObjectT)); + ObjectT *Object = ::new (Storage) ObjectT(std::forward<ArgumentTs>(Arguments)...); + auto DestroyObject = [](ObjectT *TypedObject) noexcept { std::destroy_at(TypedObject); }; + std::unique_ptr<ObjectT, decltype(DestroyObject)> ConstructionGuard(Object, DestroyObject); + Destructors.push_back(DestructorRecord{ + Object, + [](void *RawObject) noexcept { std::destroy_at(static_cast<ObjectT *>(RawObject)); }, + }); + return ConstructionGuard.release(); + } + +private: + struct DestructorRecord final { + void *Object; + void (*Destroy)(void *) noexcept; + }; + + std::pmr::monotonic_buffer_resource Resource; + std::vector<DestructorRecord> Destructors; +}; + +Expected<std::unique_ptr<MachineFunction>> +MachineFunction::create(std::string Name, MachineFunctionStableID StableID) { + if (!StableID.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "machine function StableID must be valid"); + return std::unique_ptr<MachineFunction>(new MachineFunction(std::move(Name), StableID)); +} + +MachineFunction::MachineFunction(std::string Name, MachineFunctionStableID StableID) + : Name(std::move(Name)), StableID(StableID), JumpTables(*this), + Arena(std::make_unique<MachineArena>()) {} + +MachineFunction::~MachineFunction() { + for (MachineBasicBlock *Block = FirstBlock; Block != nullptr; Block = Block->NextBlock) { + Block->clearInstructions(); + Block->Predecessors.clear(); + Block->Successors.clear(); + } +} + +Expected<void> MachineFunction::setTargetState(TargetFunctionState State) { + if (TargetState.hasValue()) { + return makeUnexpected(ErrorCode::InvalidArgument, "machine function already has target state"); + } + auto Verification = State.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + TargetState = std::move(State); + return {}; +} + +Expected<std::uint32_t> MachineFunction::allocateDebugInstrNumber() { + if (DebugInstrNumberingCount == std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::InternalError, + "machine debug instruction number space is exhausted"); + } + ++DebugInstrNumberingCount; + return DebugInstrNumberingCount; +} + +void MachineFunction::observeDebugInstrNumber(std::uint32_t Number) noexcept { + DebugInstrNumberingCount = std::max(DebugInstrNumberingCount, Number); +} + +Expected<MachineBasicBlock *> MachineFunction::createBasicBlock() { + auto ID = BlockIDGenerator.allocate(); + if (!ID) + return std::unexpected(ID.error()); + MachineBasicBlock *Block = Arena->create<MachineBasicBlock>(*this, *ID); + AllBlocks.push_back(Block); + return Block; +} + +Expected<MachineInstr *> MachineFunction::createInstruction(Opcode InstructionOpcode) { + if (!InstructionOpcode.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "machine opcode must be nonzero"); + + auto ID = InstrIDGenerator.allocate(); + if (!ID) + return std::unexpected(ID.error()); + MachineInstr *Instruction = Arena->create<MachineInstr>(*this, InstructionOpcode, *ID); + AllInstructions.push_back(Instruction); + return Instruction; +} + +Expected<const MachineMemOperand *> +MachineFunction::createMemOperand(MachineMemOperandDesc Description) { + auto MemOperand = MachineMemOperand::create(std::move(Description)); + if (!MemOperand) + return std::unexpected(MemOperand.error()); + + const MachineMemOperand *Stored = Arena->create<MachineMemOperand>(std::move(*MemOperand)); + AllMemOperands.push_back(Stored); + return Stored; +} + +Expected<void> MachineFunction::appendBlock(MachineBasicBlock &Block) { + if (Block.OwnerFunction != this) + return makeUnexpected(ErrorCode::InvalidArgument, + "basic block belongs to another machine function"); + if (Block.IsLinked) + return makeUnexpected(ErrorCode::InvalidArgument, "basic block is already linked"); + + Block.PreviousBlock = LastBlock; + Block.NextBlock = nullptr; + if (LastBlock != nullptr) + LastBlock->NextBlock = &Block; + else + FirstBlock = &Block; + LastBlock = &Block; + Block.IsLinked = true; + renumberBlocks(); + return {}; +} + +Expected<void> MachineFunction::insertBlockBefore(MachineBasicBlock &Position, + MachineBasicBlock &Block) { + if (Position.OwnerFunction != this || !Position.IsLinked) + return makeUnexpected(ErrorCode::InvalidArgument, + "block insertion position is not linked to this function"); + if (Block.OwnerFunction != this || Block.IsLinked) + return makeUnexpected(ErrorCode::InvalidArgument, + "inserted block is linked or belongs to another function"); + + Block.PreviousBlock = Position.PreviousBlock; + Block.NextBlock = &Position; + if (Position.PreviousBlock != nullptr) + Position.PreviousBlock->NextBlock = &Block; + else + FirstBlock = &Block; + Position.PreviousBlock = &Block; + Block.IsLinked = true; + renumberBlocks(); + return {}; +} + +Expected<void> MachineFunction::moveBlockBefore(MachineBasicBlock &Position, + MachineBasicBlock &Block) { + if (Position.OwnerFunction != this || Block.OwnerFunction != this || !Position.IsLinked || + !Block.IsLinked) { + return makeUnexpected(ErrorCode::InvalidArgument, + "layout move requires two linked blocks in the same function"); + } + if (&Position == &Block || Block.NextBlock == &Position) + return {}; + + if (Block.PreviousBlock != nullptr) + Block.PreviousBlock->NextBlock = Block.NextBlock; + else + FirstBlock = Block.NextBlock; + if (Block.NextBlock != nullptr) + Block.NextBlock->PreviousBlock = Block.PreviousBlock; + else + LastBlock = Block.PreviousBlock; + + Block.PreviousBlock = Position.PreviousBlock; + Block.NextBlock = &Position; + if (Position.PreviousBlock != nullptr) + Position.PreviousBlock->NextBlock = &Block; + else + FirstBlock = &Block; + Position.PreviousBlock = &Block; + renumberBlocks(); + return {}; +} + +Expected<void> MachineFunction::moveBlockAfter(MachineBasicBlock &Position, + MachineBasicBlock &Block) { + if (Position.OwnerFunction != this || Block.OwnerFunction != this || !Position.IsLinked || + !Block.IsLinked) { + return makeUnexpected(ErrorCode::InvalidArgument, + "layout move requires two linked blocks in the same function"); + } + if (&Position == &Block || Position.NextBlock == &Block) + return {}; + + if (Block.PreviousBlock != nullptr) + Block.PreviousBlock->NextBlock = Block.NextBlock; + else + FirstBlock = Block.NextBlock; + if (Block.NextBlock != nullptr) + Block.NextBlock->PreviousBlock = Block.PreviousBlock; + else + LastBlock = Block.PreviousBlock; + + Block.PreviousBlock = &Position; + Block.NextBlock = Position.NextBlock; + if (Position.NextBlock != nullptr) + Position.NextBlock->PreviousBlock = &Block; + else + LastBlock = &Block; + Position.NextBlock = &Block; + renumberBlocks(); + return {}; +} + +Expected<void> MachineFunction::removeBlock(MachineBasicBlock &Block) { + if (Block.OwnerFunction != this || !Block.IsLinked) + return makeUnexpected(ErrorCode::InvalidArgument, + "removed block is not linked to this function"); + if (!Block.Predecessors.empty() || !Block.Successors.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "disconnect CFG edges before removing a basic block"); + } + + Block.clearInstructions(); + if (Block.PreviousBlock != nullptr) + Block.PreviousBlock->NextBlock = Block.NextBlock; + else + FirstBlock = Block.NextBlock; + if (Block.NextBlock != nullptr) + Block.NextBlock->PreviousBlock = Block.PreviousBlock; + else + LastBlock = Block.PreviousBlock; + + Block.IsLinked = false; + Block.PreviousBlock = nullptr; + Block.NextBlock = nullptr; + Block.DenseNumber.reset(); + Block.LayoutIndex.reset(); + renumberBlocks(); + return {}; +} + +std::size_t MachineFunction::blockCount() const noexcept { + std::size_t Count = 0; + for (const MachineBasicBlock *Block = FirstBlock; Block != nullptr; Block = Block->next()) { + ++Count; + } + return Count; +} + +bool MachineFunction::ownsMemOperand(const MachineMemOperand &MemOperand) const noexcept { + return std::ranges::find(AllMemOperands, &MemOperand) != AllMemOperands.end(); +} + +MachineCFGUpdater MachineFunction::cfgUpdater() noexcept { + return MachineCFGUpdater(*this); +} + +void MachineFunction::renumberBlocks() noexcept { + std::uint32_t Number = 0; + for (MachineBasicBlock *Block = FirstBlock; Block != nullptr; Block = Block->NextBlock) { + Block->DenseNumber = Number; + Block->LayoutIndex = Number; + ++Number; + } +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFunctionAttributes.cpp b/backend/lib/CodeGen/MachineFunctionAttributes.cpp new file mode 100644 index 0000000..afe18a9 --- /dev/null +++ b/backend/lib/CodeGen/MachineFunctionAttributes.cpp @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFunctionAttributes.h" + +#include <cstdint> +#include <string_view> + +namespace rcc { + +std::string_view getMachineFunctionAttributeName(MachineFunctionAttribute Attribute) noexcept { + switch (Attribute) { + case MachineFunctionAttribute::ExposesReturnsTwice: + return "ExposesReturnsTwice"; + case MachineFunctionAttribute::HasWinCFI: + return "HasWinCFI"; + case MachineFunctionAttribute::HasFakeUses: + return "HasFakeUses"; + case MachineFunctionAttribute::CallsEHReturn: + return "CallsEHReturn"; + case MachineFunctionAttribute::CallsUnwindInit: + return "CallsUnwindInit"; + case MachineFunctionAttribute::HasEHContTarget: + return "HasEHContTarget"; + case MachineFunctionAttribute::HasEHScopes: + return "HasEHScopes"; + case MachineFunctionAttribute::HasEHFunclets: + return "HasEHFunclets"; + case MachineFunctionAttribute::IsOutlined: + return "IsOutlined"; + case MachineFunctionAttribute::DebugInstrRef: + return "DebugInstrRef"; + case MachineFunctionAttribute::TracksDebugUserValues: + return "TracksDebugUserValues"; + case MachineFunctionAttribute::MinSize: + return "MinSize"; + case MachineFunctionAttribute::Count: + return "Invalid"; + } + return "Invalid"; +} + +Expected<MachineFunctionAttributes> +MachineFunctionAttributes::fromRawFlags(std::uint16_t RawFlags) { + if ((RawFlags & static_cast<std::uint16_t>(~ValidFlags)) != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine function attributes contain unknown flag bits"); + } + MachineFunctionAttributes Result; + Result.Flags = RawFlags; + return Result; +} + +Expected<void> MachineFunctionAttributes::verify() const { + if ((Flags & static_cast<std::uint16_t>(~ValidFlags)) != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "machine function attributes contain unknown flag bits"); + } + if (FunctionAlignment && FunctionAlignment->bytes() == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "machine function alignment must be nonzero"); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFunctionCloner.cpp b/backend/lib/CodeGen/MachineFunctionCloner.cpp new file mode 100644 index 0000000..bbd70b7 --- /dev/null +++ b/backend/lib/CodeGen/MachineFunctionCloner.cpp @@ -0,0 +1,757 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFunctionCloner.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineCompilationStatus.h" +#include "rcc/CodeGen/MachineConstantPool.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetFunctionState.h" + +#include <cstddef> +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <unordered_map> +#include <utility> + +namespace rcc { +namespace { + +struct CloneState final { + const MachineFunction &Source; + MachineFunction &Destination; + MachineFunctionCloneResult &Result; + std::unordered_map<const MachineBasicBlock *, MachineBasicBlock *> Blocks; + std::unordered_map<const MachineInstr *, MachineInstr *> Instructions; + std::unordered_map<const MachineMemOperand *, const MachineMemOperand *> MemOperands; +}; + +[[nodiscard]] Expected<MachineOperand> cloneOperand(const MachineOperand &SourceOperand, + CloneState &State) { + switch (SourceOperand.kind()) { + case MachineOperand::Kind::Register: { + const Register *SourceReg = SourceOperand.getIf<Register>(); + if (SourceReg == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "source register operand has inconsistent storage"); + } + Register DestinationReg = *SourceReg; + if (SourceReg->isVirtual()) { + const auto Position = State.Result.VirtualRegisters.find(*SourceReg); + if (Position == State.Result.VirtualRegisters.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source operand references an unmapped virtual register"); + } + DestinationReg = Position->second; + } + + MachineOperand Result = SourceOperand.cloneDetached(); + auto SetResult = Result.setRegister(DestinationReg); + if (!SetResult) + return std::unexpected(SetResult.error()); + return Result; + } + case MachineOperand::Kind::BasicBlock: { + const auto *SourceBlock = SourceOperand.getIf<BasicBlockOperand>(); + if (SourceBlock == nullptr || SourceBlock->Block == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "source basic-block operand is invalid"); + } + const auto Position = State.Blocks.find(SourceBlock->Block); + if (Position == State.Blocks.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source basic-block operand is not in live layout"); + } + return MachineOperand::basicBlock(*Position->second, SourceOperand.targetFlags()); + } + case MachineOperand::Kind::BlockAddress: { + const auto *Address = SourceOperand.getIf<BlockAddressOperand>(); + if (Address == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "source block-address operand is invalid"); + } + std::string FunctionName = Address->FunctionName; + if (FunctionName == State.Source.name()) + FunctionName = std::string(State.Destination.name()); + return MachineOperand::blockAddress(std::move(FunctionName), Address->BlockName, + Address->Offset, SourceOperand.targetFlags()); + } + case MachineOperand::Kind::FrameIndex: { + const auto *Frame = SourceOperand.getIf<FrameIndexOperand>(); + if (Frame == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "source frame-index operand is invalid"); + } + const auto Position = State.Result.FrameIndices.find(Frame->Index); + if (Position == State.Result.FrameIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source operand references an unmapped frame index"); + } + return MachineOperand::frameIndex(Position->second, Frame->Offset, SourceOperand.targetFlags()); + } + case MachineOperand::Kind::ConstantPoolIndex: { + const auto *Constant = SourceOperand.getIf<ConstantPoolOperand>(); + if (Constant == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "source constant-pool operand is invalid"); + } + const auto Position = State.Result.ConstantPoolIndices.find(Constant->Index); + if (Position == State.Result.ConstantPoolIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source operand references an unmapped constant pool entry"); + } + return MachineOperand::constantPool(Position->second, Constant->Offset, + SourceOperand.targetFlags()); + } + case MachineOperand::Kind::JumpTableIndex: { + const auto *JumpTable = SourceOperand.getIf<JumpTableOperand>(); + if (JumpTable == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "source jump-table operand is invalid"); + } + const auto Position = State.Result.JumpTableIndices.find(JumpTable->Index); + if (Position == State.Result.JumpTableIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source operand references an unmapped jump table"); + } + return MachineOperand::jumpTable(Position->second, SourceOperand.targetFlags()); + } + case MachineOperand::Kind::Immediate: + case MachineOperand::Kind::IntegerImmediate: + case MachineOperand::Kind::FloatingImmediate: + case MachineOperand::Kind::GlobalSymbol: + case MachineOperand::Kind::ExternalSymbol: + case MachineOperand::Kind::RegisterMask: + case MachineOperand::Kind::LiveOutRegisterMask: + case MachineOperand::Kind::Metadata: + case MachineOperand::Kind::TargetIndex: + case MachineOperand::Kind::MCSymbol: + case MachineOperand::Kind::CFIIndex: + case MachineOperand::Kind::Intrinsic: + case MachineOperand::Kind::Predicate: + case MachineOperand::Kind::ShuffleMask: + case MachineOperand::Kind::DebugInstrRef: + case MachineOperand::Kind::LaneMask: + return SourceOperand.cloneDetached(); + case MachineOperand::Kind::Count: + return makeUnexpected(ErrorCode::VerificationError, "source operand has an invalid kind"); + } + return makeUnexpected(ErrorCode::InternalError, "source operand kind is not handled"); +} + +[[nodiscard]] Expected<const MachineMemOperand *> +cloneMemOperand(const MachineMemOperand &SourceOperand, CloneState &State) { + const auto Existing = State.MemOperands.find(&SourceOperand); + if (Existing != State.MemOperands.end()) + return Existing->second; + + MachineMemOperandDesc Description = SourceOperand.description(); + if (const auto Frame = Description.Base.frameIndex()) { + const auto Position = State.Result.FrameIndices.find(*Frame); + if (Position == State.Result.FrameIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source MMO references an unmapped frame index"); + } + Description.Base = MemoryBase::frameIndex(Position->second); + } + if (const auto Constant = Description.Base.constantPoolIndex()) { + const auto Position = State.Result.ConstantPoolIndices.find(*Constant); + if (Position == State.Result.ConstantPoolIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source MMO references an unmapped constant pool entry"); + } + Description.Base = MemoryBase::constantPool(Position->second); + } + if (const auto JumpTable = Description.Base.jumpTableIndex()) { + const auto Position = State.Result.JumpTableIndices.find(*JumpTable); + if (Position == State.Result.JumpTableIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source MMO references an unmapped jump table"); + } + Description.Base = MemoryBase::jumpTable(Position->second); + } + auto DestinationOperand = State.Destination.createMemOperand(std::move(Description)); + if (!DestinationOperand) + return std::unexpected(DestinationOperand.error()); + State.MemOperands.emplace(&SourceOperand, *DestinationOperand); + return *DestinationOperand; +} + +[[nodiscard]] Expected<void> cloneVirtualRegisters(CloneState &State) { + const auto SourceRegisters = State.Source.registerInfo().virtualRegisters(); + for (std::size_t Index = 0; Index < SourceRegisters.size(); ++Index) { + const VirtualRegisterInfo &SourceInfo = SourceRegisters[Index]; + if (SourceInfo.IsTombstone) + continue; + + auto SourceReg = Register::makeVirtual(Index); + if (!SourceReg) + return std::unexpected(SourceReg.error()); + Expected<Register> DestinationReg = + SourceInfo.Type ? State.Destination.registerInfo().createVirtualRegister(*SourceInfo.Type) + : State.Destination.registerInfo().createVirtualRegister(); + if (!DestinationReg) + return std::unexpected(DestinationReg.error()); + + if (SourceInfo.Name) { + auto DestinationInfo = State.Destination.registerInfo().virtualRegisterInfo(*DestinationReg); + if (!DestinationInfo) + return std::unexpected(DestinationInfo.error()); + DestinationInfo->get().Name = SourceInfo.Name; + } + + if (SourceInfo.Bank) { + auto SetResult = + State.Destination.registerInfo().setRegisterBank(*DestinationReg, *SourceInfo.Bank); + if (!SetResult) + return std::unexpected(SetResult.error()); + } + if (SourceInfo.RegClass) { + auto SetResult = + State.Destination.registerInfo().setRegisterClass(*DestinationReg, *SourceInfo.RegClass); + if (!SetResult) + return std::unexpected(SetResult.error()); + } + if (SourceInfo.TargetFlags != 0) { + auto SetResult = State.Destination.registerInfo().setTargetRegisterFlags( + *DestinationReg, SourceInfo.TargetFlags); + if (!SetResult) + return std::unexpected(SetResult.error()); + } + for (const PhysicalRegister Hint : SourceInfo.AllocationHints) { + auto AddResult = State.Destination.registerInfo().addAllocationHint(*DestinationReg, Hint); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + State.Result.VirtualRegisters.emplace(*SourceReg, *DestinationReg); + } + return {}; +} + +[[nodiscard]] Expected<void> cloneRegisterConfiguration(CloneState &State) { + for (const PhysicalRegisterLiveIn &LiveIn : State.Source.registerInfo().liveIns()) { + std::optional<Register> Initial; + if (LiveIn.InitialVirtualRegister) { + const auto Position = State.Result.VirtualRegisters.find(*LiveIn.InitialVirtualRegister); + if (Position == State.Result.VirtualRegisters.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source live-in references an unmapped virtual register"); + } + Initial = Position->second; + } + auto AddResult = State.Destination.registerInfo().addLiveIn(LiveIn.PhysReg, Initial); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + for (const PhysicalRegister Reg : State.Source.registerInfo().reservedRegisters()) { + auto ReserveResult = State.Destination.registerInfo().reservePhysicalRegister(Reg); + if (!ReserveResult) + return std::unexpected(ReserveResult.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> cloneFrameObjects(CloneState &State) { + for (const StackObject &SourceObject : State.Source.frameInfo().objects()) { + const StackObjectDesc *Description = SourceObject.description(); + if (Description == nullptr) + continue; + auto DestinationIndex = State.Destination.frameInfo().createObject(*Description); + if (!DestinationIndex) + return std::unexpected(DestinationIndex.error()); + State.Result.FrameIndices.emplace(SourceObject.index(), *DestinationIndex); + } + MachineFrameState FrameState = State.Source.frameInfo().state(); + FrameState.SavePoints.clear(); + FrameState.RestorePoints.clear(); + auto MapFrameReference = [&State](std::optional<FrameIndex> &Reference) -> Expected<void> { + if (!Reference) + return {}; + const auto Position = State.Result.FrameIndices.find(*Reference); + if (Position == State.Result.FrameIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source frame state references an unmapped stack object"); + } + Reference = Position->second; + return {}; + }; + auto StackProtectorResult = MapFrameReference(FrameState.StackProtector); + if (!StackProtectorResult) + return StackProtectorResult; + auto FunctionContextResult = MapFrameReference(FrameState.FunctionContext); + if (!FunctionContextResult) + return FunctionContextResult; + auto StateResult = State.Destination.frameInfo().setState(std::move(FrameState)); + if (!StateResult) + return std::unexpected(StateResult.error()); + return {}; +} + +[[nodiscard]] Expected<void> cloneConstantPool(CloneState &State) { + for (const MachineConstantPoolEntry &SourceEntry : State.Source.constantPool().entries()) { + if (SourceEntry.isTombstone()) + continue; + auto Description = SourceEntry.cloneDescription(); + if (!Description) + return std::unexpected(Description.error()); + if (auto *Address = std::get_if<BlockAddressConstant>(&Description->Value); + Address != nullptr && Address->FunctionName == State.Source.name()) { + Address->FunctionName = std::string(State.Destination.name()); + } + + auto DestinationIndex = State.Destination.constantPool().addEntry(std::move(*Description)); + if (!DestinationIndex) + return std::unexpected(DestinationIndex.error()); + State.Result.ConstantPoolIndices.emplace(SourceEntry.index(), *DestinationIndex); + } + if (State.Source.constantPool().isFrozen()) { + auto FreezeResult = State.Destination.constantPool().freeze(); + if (!FreezeResult) + return std::unexpected(FreezeResult.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> cloneBlocks(CloneState &State) { + for (const MachineBasicBlock *SourceBlock = State.Source.firstBlock(); SourceBlock != nullptr; + SourceBlock = SourceBlock->next()) { + auto DestinationBlock = State.Destination.createBasicBlock(); + if (!DestinationBlock) + return std::unexpected(DestinationBlock.error()); + auto AppendResult = State.Destination.appendBlock(**DestinationBlock); + if (!AppendResult) + return std::unexpected(AppendResult.error()); + (*DestinationBlock)->attributes() = SourceBlock->attributes(); + State.Blocks.emplace(SourceBlock, *DestinationBlock); + State.Result.BlockStableIDs.emplace(SourceBlock->stableID(), (*DestinationBlock)->stableID()); + + for (const MachineBasicBlockLiveIn &LiveIn : SourceBlock->liveIns()) { + auto AddResult = (*DestinationBlock)->addLiveIn(LiveIn.PhysReg, LiveIn.Lanes); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + } + return {}; +} + +[[nodiscard]] Expected<void> cloneJumpTables(CloneState &State) { + for (const MachineJumpTableEntry &SourceEntry : State.Source.jumpTableInfo().tables()) { + const MachineJumpTableDesc *SourceDescription = SourceEntry.description(); + if (SourceDescription == nullptr) + continue; + MachineJumpTableDesc DestinationDescription = *SourceDescription; + for (MachineBasicBlock *&Target : DestinationDescription.Targets) { + const auto Position = State.Blocks.find(Target); + if (Position == State.Blocks.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source jump table references an unmapped block"); + } + Target = Position->second; + } + auto DestinationIndex = + State.Destination.jumpTableInfo().createTable(std::move(DestinationDescription)); + if (!DestinationIndex) + return std::unexpected(DestinationIndex.error()); + State.Result.JumpTableIndices.emplace(SourceEntry.index(), *DestinationIndex); + } + if (State.Source.jumpTableInfo().isFrozen()) { + auto FreezeResult = State.Destination.jumpTableInfo().freeze(); + if (!FreezeResult) + return std::unexpected(FreezeResult.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> cloneInstructions(CloneState &State) { + for (const MachineBasicBlock *SourceBlock = State.Source.firstBlock(); SourceBlock != nullptr; + SourceBlock = SourceBlock->next()) { + const auto DestinationBlockPosition = State.Blocks.find(SourceBlock); + if (DestinationBlockPosition == State.Blocks.end()) { + return makeUnexpected(ErrorCode::InternalError, + "source block was not mapped before instruction cloning"); + } + MachineBasicBlock *DestinationBlock = DestinationBlockPosition->second; + for (const MachineInstr *SourceInstruction = SourceBlock->firstInstruction(); + SourceInstruction != nullptr; SourceInstruction = SourceInstruction->next()) { + auto DestinationInstruction = + State.Destination.createInstruction(SourceInstruction->opcode()); + if (!DestinationInstruction) + return std::unexpected(DestinationInstruction.error()); + MachineInstr *NewInstruction = *DestinationInstruction; + State.Instructions.emplace(SourceInstruction, NewInstruction); + State.Result.InstructionStableIDs.emplace(SourceInstruction->stableID(), + NewInstruction->stableID()); + NewInstruction->reserveOperands(SourceInstruction->operandCount()); + + for (const MachineOperand &SourceOperand : SourceInstruction->operands()) { + auto DestinationOperand = cloneOperand(SourceOperand, State); + if (!DestinationOperand) + return std::unexpected(DestinationOperand.error()); + auto AddResult = NewInstruction->addOperand(std::move(*DestinationOperand)); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + + const auto SourceOperands = SourceInstruction->operands(); + for (std::size_t Index = 0; Index < SourceOperands.size(); ++Index) { + const auto Tied = SourceOperands[Index].tiedOperandIndex(); + if (!Tied || Index > *Tied) + continue; + const std::size_t DefIndex = SourceOperands[Index].isDef() ? Index : *Tied; + const std::size_t UseIndex = SourceOperands[Index].isDef() ? *Tied : Index; + auto TieResult = NewInstruction->tieOperands(DefIndex, UseIndex); + if (!TieResult) + return std::unexpected(TieResult.error()); + } + + for (const MachineMemOperand *SourceMemOperand : SourceInstruction->memOperands()) { + auto DestinationMemOperand = cloneMemOperand(*SourceMemOperand, State); + if (!DestinationMemOperand) + return std::unexpected(DestinationMemOperand.error()); + auto AddResult = NewInstruction->addMemOperand(**DestinationMemOperand); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + + NewInstruction->setFlags(SourceInstruction->flags()); + if (SourceInstruction->sourceLocation()) { + NewInstruction->setSourceLocation(*SourceInstruction->sourceLocation()); + } + if (SourceInstruction->debugLocation()) { + auto DebugLocationResult = + NewInstruction->setDebugLocation(*SourceInstruction->debugLocation()); + if (!DebugLocationResult) + return std::unexpected(DebugLocationResult.error()); + } + if (SourceInstruction->origin()) + NewInstruction->setOrigin(*SourceInstruction->origin()); + if (SourceInstruction->debugInstrNumber() != 0) { + auto DebugNumberResult = + NewInstruction->setDebugInstrNumber(SourceInstruction->debugInstrNumber()); + if (!DebugNumberResult) + return std::unexpected(DebugNumberResult.error()); + } + if (SourceInstruction->preInstrSymbol()) { + auto SymbolResult = + NewInstruction->setPreInstrSymbol(std::string(*SourceInstruction->preInstrSymbol())); + if (!SymbolResult) + return std::unexpected(SymbolResult.error()); + } + if (SourceInstruction->postInstrSymbol()) { + auto SymbolResult = + NewInstruction->setPostInstrSymbol(std::string(*SourceInstruction->postInstrSymbol())); + if (!SymbolResult) + return std::unexpected(SymbolResult.error()); + } + if (SourceInstruction->heapAllocMarker()) { + auto MarkerResult = + NewInstruction->setHeapAllocMarker(*SourceInstruction->heapAllocMarker()); + if (!MarkerResult) + return std::unexpected(MarkerResult.error()); + } + if (SourceInstruction->pcSections()) { + auto SectionsResult = NewInstruction->setPCSections(*SourceInstruction->pcSections()); + if (!SectionsResult) + return std::unexpected(SectionsResult.error()); + } + if (SourceInstruction->mmra()) { + auto MMRAResult = NewInstruction->setMMRA(*SourceInstruction->mmra()); + if (!MMRAResult) + return std::unexpected(MMRAResult.error()); + } + if (SourceInstruction->cfiType()) { + auto TypeResult = NewInstruction->setCFIType(*SourceInstruction->cfiType()); + if (!TypeResult) + return std::unexpected(TypeResult.error()); + } + if (SourceInstruction->deactivationSymbol()) { + auto SymbolResult = NewInstruction->setDeactivationSymbol( + std::string(*SourceInstruction->deactivationSymbol())); + if (!SymbolResult) + return std::unexpected(SymbolResult.error()); + } + + auto AppendResult = DestinationBlock->appendInstruction(*NewInstruction); + if (!AppendResult) + return std::unexpected(AppendResult.error()); + } + } + return {}; +} + +[[nodiscard]] Expected<void> finalizeFrameClone(CloneState &State) { + for (const StackObject &SourceObject : State.Source.frameInfo().objects()) { + const StackObjectDesc *SourceDescription = SourceObject.description(); + if (SourceDescription == nullptr) + continue; + const auto FramePosition = State.Result.FrameIndices.find(SourceObject.index()); + if (FramePosition == State.Result.FrameIndices.end()) { + return makeUnexpected(ErrorCode::InternalError, + "source stack object was not mapped before frame finalization"); + } + + StackObjectDesc DestinationDescription = *SourceDescription; + for (StackObjectLifetimeRange &Range : DestinationDescription.Lifetime) { + const auto Begin = State.Result.InstructionStableIDs.find(Range.Begin); + const auto End = State.Result.InstructionStableIDs.find(Range.End); + if (Begin == State.Result.InstructionStableIDs.end() || + End == State.Result.InstructionStableIDs.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source stack lifetime references an unmapped instruction"); + } + Range.Begin = Begin->second; + Range.End = End->second; + } + auto UpdateResult = State.Destination.frameInfo().updateObject( + FramePosition->second, std::move(DestinationDescription)); + if (!UpdateResult) + return std::unexpected(UpdateResult.error()); + } + + MachineFrameState FrameState = State.Destination.frameInfo().state(); + auto ClonePoints = + [&State](std::span<const MachineFrameSaveRestorePoint> SourcePoints, + std::vector<MachineFrameSaveRestorePoint> &DestinationPoints) -> Expected<void> { + DestinationPoints.clear(); + DestinationPoints.reserve(SourcePoints.size()); + for (const MachineFrameSaveRestorePoint &Point : SourcePoints) { + const auto Position = State.Result.BlockStableIDs.find(Point.Block); + if (Position == State.Result.BlockStableIDs.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source frame save/restore point references an unmapped block"); + } + DestinationPoints.push_back({Position->second, Point.Registers}); + } + return {}; + }; + auto SaveResult = ClonePoints(State.Source.frameInfo().state().SavePoints, FrameState.SavePoints); + if (!SaveResult) + return SaveResult; + auto RestoreResult = + ClonePoints(State.Source.frameInfo().state().RestorePoints, FrameState.RestorePoints); + if (!RestoreResult) + return RestoreResult; + auto StateResult = State.Destination.frameInfo().setState(std::move(FrameState)); + if (!StateResult) + return std::unexpected(StateResult.error()); + + if (State.Source.frameInfo().isFrozen()) { + const auto StackSize = State.Source.frameInfo().staticStackSize(); + if (!StackSize) { + return makeUnexpected(ErrorCode::InternalError, + "verified frozen source frame has no static stack size"); + } + auto FreezeResult = + State.Destination.frameInfo().freeze(*StackSize, State.Source.frameInfo().stackAlignment()); + if (!FreezeResult) + return std::unexpected(FreezeResult.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> cloneCFG(CloneState &State) { + MachineCFGUpdater Updater = State.Destination.cfgUpdater(); + for (const MachineBasicBlock *SourceBlock = State.Source.firstBlock(); SourceBlock != nullptr; + SourceBlock = SourceBlock->next()) { + const auto DestinationBlockPosition = State.Blocks.find(SourceBlock); + if (DestinationBlockPosition == State.Blocks.end()) { + return makeUnexpected(ErrorCode::InternalError, + "source block was not mapped before CFG cloning"); + } + MachineBasicBlock *DestinationBlock = DestinationBlockPosition->second; + for (const MachineSuccessor &Successor : SourceBlock->successors()) { + const auto Target = State.Blocks.find(Successor.Block); + if (Target == State.Blocks.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "source CFG references a detached block"); + } + auto AddResult = + Updater.addSuccessor(*DestinationBlock, *Target->second, Successor.Probability); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + } + return {}; +} + +} // namespace + +Expected<MachineBasicBlock *> +MachineFunctionCloneContext::mapBlock(const MachineBasicBlock &SourceBlock) const { + const auto Position = Blocks.find(&SourceBlock); + if (Position == Blocks.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "target function state references an unmapped basic block"); + } + return Position->second; +} + +Expected<MachineInstr *> +MachineFunctionCloneContext::mapInstruction(const MachineInstr &SourceInstruction) const { + const auto Position = Instructions.find(&SourceInstruction); + if (Position == Instructions.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "target function state references an unmapped instruction"); + } + return Position->second; +} + +Expected<MachineBlockStableID> +MachineFunctionCloneContext::mapBlockStableID(MachineBlockStableID SourceID) const { + const auto Position = Result.BlockStableIDs.find(SourceID); + if (Position == Result.BlockStableIDs.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "target function state references an unmapped block StableID"); + } + return Position->second; +} + +Expected<MachineInstrStableID> +MachineFunctionCloneContext::mapInstructionStableID(MachineInstrStableID SourceID) const { + const auto Position = Result.InstructionStableIDs.find(SourceID); + if (Position == Result.InstructionStableIDs.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "function state references an unmapped instruction StableID"); + } + return Position->second; +} + +Expected<Register> MachineFunctionCloneContext::mapRegister(Register SourceRegister) const { + if (!SourceRegister.isVirtual()) + return SourceRegister; + const auto Position = Result.VirtualRegisters.find(SourceRegister); + if (Position == Result.VirtualRegisters.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "target function state references an unmapped virtual register"); + } + return Position->second; +} + +Expected<FrameIndex> MachineFunctionCloneContext::mapFrameIndex(FrameIndex SourceIndex) const { + const auto Position = Result.FrameIndices.find(SourceIndex); + if (Position == Result.FrameIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "target function state references an unmapped frame index"); + } + return Position->second; +} + +Expected<ConstantPoolIndex> +MachineFunctionCloneContext::mapConstantPoolIndex(ConstantPoolIndex SourceIndex) const { + const auto Position = Result.ConstantPoolIndices.find(SourceIndex); + if (Position == Result.ConstantPoolIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "target function state references an unmapped constant pool entry"); + } + return Position->second; +} + +Expected<JumpTableIndex> +MachineFunctionCloneContext::mapJumpTableIndex(JumpTableIndex SourceIndex) const { + const auto Position = Result.JumpTableIndices.find(SourceIndex); + if (Position == Result.JumpTableIndices.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "target function state references an unmapped jump table"); + } + return Position->second; +} + +MachineFunctionCloneResult::MachineFunctionCloneResult() = default; +MachineFunctionCloneResult::MachineFunctionCloneResult(MachineFunctionCloneResult &&) noexcept = + default; +MachineFunctionCloneResult & +MachineFunctionCloneResult::operator=(MachineFunctionCloneResult &&) noexcept = default; +MachineFunctionCloneResult::~MachineFunctionCloneResult() = default; + +Expected<MachineFunctionCloneResult> cloneMachineFunction(const MachineFunction &Source, + const TargetDescription &Target, + MachineFunctionCloneOptions Options) { + auto SourceVerification = MachineVerifier::verify(Source, Target); + if (!SourceVerification) + return std::unexpected(SourceVerification.error()); + if (!Options.NewStableID.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, "clone function StableID must be valid"); + } + + std::string Name = Options.NewName ? std::move(*Options.NewName) : std::string(Source.name()); + auto Destination = MachineFunction::create(std::move(Name), Options.NewStableID); + if (!Destination) + return std::unexpected(Destination.error()); + + MachineFunctionCloneResult Result; + Result.ClonedFunction = std::move(*Destination); + CloneState State{Source, *Result.ClonedFunction, Result, {}, {}, {}}; + + auto VRegResult = cloneVirtualRegisters(State); + if (!VRegResult) + return std::unexpected(VRegResult.error()); + auto RegisterResult = cloneRegisterConfiguration(State); + if (!RegisterResult) + return std::unexpected(RegisterResult.error()); + auto FrameResult = cloneFrameObjects(State); + if (!FrameResult) + return std::unexpected(FrameResult.error()); + auto ConstantPoolResult = cloneConstantPool(State); + if (!ConstantPoolResult) + return std::unexpected(ConstantPoolResult.error()); + auto BlockResult = cloneBlocks(State); + if (!BlockResult) + return std::unexpected(BlockResult.error()); + auto JumpTableResult = cloneJumpTables(State); + if (!JumpTableResult) + return std::unexpected(JumpTableResult.error()); + auto InstructionResult = cloneInstructions(State); + if (!InstructionResult) + return std::unexpected(InstructionResult.error()); + auto FrameFinalizationResult = finalizeFrameClone(State); + if (!FrameFinalizationResult) + return std::unexpected(FrameFinalizationResult.error()); + auto CFGResult = cloneCFG(State); + if (!CFGResult) + return std::unexpected(CFGResult.error()); + + const MachineFunctionCloneContext CloneContext(Source, *Result.ClonedFunction, Result, + State.Blocks, State.Instructions); + if (Source.targetState().hasValue()) { + auto TargetState = Source.targetState().clone(CloneContext); + if (!TargetState) + return std::unexpected(TargetState.error()); + auto InstallResult = Result.ClonedFunction->setTargetState(std::move(*TargetState)); + if (!InstallResult) + return std::unexpected(InstallResult.error()); + } + + auto CompilationStatus = Source.compilationStatus().clone(CloneContext); + if (!CompilationStatus) + return std::unexpected(CompilationStatus.error()); + Result.ClonedFunction->compilationStatus() = std::move(*CompilationStatus); + auto Records = Source.records().clone(CloneContext); + if (!Records) + return std::unexpected(Records.error()); + Result.ClonedFunction->records() = std::move(*Records); + auto Metadata = Source.metadata().clone(); + if (!Metadata) + return std::unexpected(Metadata.error()); + Result.ClonedFunction->metadata() = std::move(*Metadata); + auto CFIInfo = Source.cfiInfo().clone(); + if (!CFIInfo) + return std::unexpected(CFIInfo.error()); + Result.ClonedFunction->cfiInfo() = std::move(*CFIInfo); + Result.ClonedFunction->properties() = Source.properties(); + Result.ClonedFunction->attributes() = Source.attributes(); + auto DestinationVerification = MachineVerifier::verify(*Result.ClonedFunction, Target); + if (!DestinationVerification) + return std::unexpected(DestinationVerification.error()); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFunctionProperties.cpp b/backend/lib/CodeGen/MachineFunctionProperties.cpp new file mode 100644 index 0000000..829ad3e --- /dev/null +++ b/backend/lib/CodeGen/MachineFunctionProperties.cpp @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFunctionProperties.h" + +#include <array> +#include <string> + +namespace rcc { +namespace { + +constexpr std::array PropertyNames{ + std::string_view{"IsSSA"}, + std::string_view{"NoPHIs"}, + std::string_view{"TracksLiveness"}, + std::string_view{"Legalized"}, + std::string_view{"RegBankSelected"}, + std::string_view{"Selected"}, + std::string_view{"TiedOpsRewritten"}, + std::string_view{"NoVRegs"}, + std::string_view{"FrameFinalized"}, + std::string_view{"NoFrameIndices"}, + std::string_view{"NoExecutablePseudos"}, + std::string_view{"FailedISel"}, + std::string_view{"FailsVerification"}, +}; + +static_assert(PropertyNames.size() == static_cast<std::size_t>(MachineFunctionProperty::Count)); + +} // namespace + +std::string_view getMachineFunctionPropertyName(MachineFunctionProperty Property) noexcept { + const auto Index = static_cast<std::size_t>(Property); + if (Index >= PropertyNames.size()) + return "Unknown"; + return PropertyNames[Index]; +} + +MachineFunctionProperties::MachineFunctionProperties( + std::initializer_list<MachineFunctionProperty> Properties) noexcept { + for (const MachineFunctionProperty Property : Properties) + set(Property); +} + +Expected<MachineFunctionProperties> MachineFunctionProperties::fromRaw(std::uint64_t RawValue) { + if ((RawValue & ~ValidMask) != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine function property mask contains unknown bits"); + } + return MachineFunctionProperties(RawValue); +} + +Expected<void> MachineFunctionProperties::applyTransition(MachineFunctionProperties Required, + MachineFunctionProperties Sets, + MachineFunctionProperties Clears) { + if (Sets.intersects(Clears)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "a machine function property transition cannot set and clear the " + "same property"); + } + + const MachineFunctionProperties Missing = Required.missingFrom(*this); + if (!Missing.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "missing required machine function properties: " + Missing.toString()); + } + + Mask = (Mask | Sets.Mask) & ~Clears.Mask; + return {}; +} + +std::string MachineFunctionProperties::toString() const { + if (empty()) + return "none"; + + std::string Result; + for (std::size_t Index = 0; Index < PropertyNames.size(); ++Index) { + if ((Mask & (std::uint64_t{1} << Index)) == 0) + continue; + if (!Result.empty()) + Result += ", "; + Result += PropertyNames[Index]; + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineFunctionRecords.cpp b/backend/lib/CodeGen/MachineFunctionRecords.cpp new file mode 100644 index 0000000..18ac596 --- /dev/null +++ b/backend/lib/CodeGen/MachineFunctionRecords.cpp @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFunctionRecords.h" + +#include "rcc/CodeGen/MachineFunctionCloner.h" + +#include <algorithm> +#include <cstddef> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] Expected<void> +verifyCalleeSavedRegisters(std::span<const PhysicalRegister> Registers) { + std::unordered_set<PhysicalRegister> Seen; + for (const PhysicalRegister Reg : Registers) { + if (!Reg.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "callee-saved register list contains noreg"); + } + if (!Seen.insert(Reg).second) { + return makeUnexpected(ErrorCode::VerificationError, + "callee-saved register list contains a duplicate register"); + } + } + return {}; +} + +[[nodiscard]] Expected<void> verifyCallSite(const MachineCallSiteInfo &CallSite) { + if (!CallSite.CallInstruction.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "call-site record has an invalid instruction StableID"); + } + for (const MachineCallSiteArgument &Argument : CallSite.ForwardedArguments) { + if (!Argument.Reg.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "call-site forwarded argument contains noreg"); + } + } + return {}; +} + +[[nodiscard]] Expected<void> verifyCalledGlobal(const MachineCalledGlobal &CalledGlobal) { + if (!CalledGlobal.CallInstruction.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "called-global record has an invalid instruction StableID"); + } + if (CalledGlobal.Callee.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "called-global record has an empty callee name"); + } + return {}; +} + +[[nodiscard]] constexpr std::uint64_t +debugReferenceKey(MachineDebugValueReference Reference) noexcept { + return (static_cast<std::uint64_t>(Reference.InstructionNumber) << 32U) | + static_cast<std::uint64_t>(Reference.OperandNumber); +} + +[[nodiscard]] Expected<void> +verifyDebugValueSubstitution(const MachineDebugValueSubstitution &Substitution) { + if (Substitution.Source.InstructionNumber == 0 || + Substitution.Destination.InstructionNumber == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "debug value substitution has a zero instruction number"); + } + if (Substitution.Source.InstructionNumber == Substitution.Destination.InstructionNumber) { + return makeUnexpected(ErrorCode::VerificationError, + "debug value substitution cannot replace a value within one instruction"); + } + if (Substitution.Source.OperandNumber == MachineDebugMemoryOperandNumber) { + return makeUnexpected(ErrorCode::VerificationError, + "debug value substitution cannot originate from the memory operand"); + } + return {}; +} + +[[nodiscard]] Expected<void> verifyEntryValue(const MachineEntryValueInfo &EntryValue) { + if (!EntryValue.Reg.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "entry-value record requires a physical register"); + } + return EntryValue.DebugInfo.verifyStructure(); +} + +} // namespace + +Expected<void> +MachineFunctionRecords::setCalleeSavedRegisters(std::vector<PhysicalRegister> Registers) { + auto Verification = verifyCalleeSavedRegisters(Registers); + if (!Verification) + return std::unexpected(Verification.error()); + CalleeSavedRegisters = std::move(Registers); + return {}; +} + +Expected<void> MachineFunctionRecords::addCallSite(MachineCallSiteInfo CallSite) { + auto Verification = verifyCallSite(CallSite); + if (!Verification) + return std::unexpected(Verification.error()); + if (std::ranges::any_of(CallSites, [&CallSite](const MachineCallSiteInfo &Existing) { + return Existing.CallInstruction == CallSite.CallInstruction; + })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call instruction already has a call-site record"); + } + CallSites.push_back(std::move(CallSite)); + return {}; +} + +Expected<void> MachineFunctionRecords::addCalledGlobal(MachineCalledGlobal CalledGlobal) { + auto Verification = verifyCalledGlobal(CalledGlobal); + if (!Verification) + return std::unexpected(Verification.error()); + if (std::ranges::any_of(CalledGlobals, [&CalledGlobal](const MachineCalledGlobal &Existing) { + return Existing.CallInstruction == CalledGlobal.CallInstruction; + })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call instruction already has a called-global record"); + } + CalledGlobals.push_back(std::move(CalledGlobal)); + return {}; +} + +Expected<void> +MachineFunctionRecords::replaceInstructionReference(MachineInstrStableID OldInstruction, + MachineInstrStableID NewInstruction) { + if (!OldInstruction.isValid() || !NewInstruction.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction-record replacement requires two valid stable IDs"); + } + if (OldInstruction == NewInstruction) + return {}; + for (const MachineCallSiteInfo &CallSite : CallSites) { + if (CallSite.CallInstruction == NewInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "replacement instruction already owns a call-site record"); + } + } + for (const MachineCalledGlobal &CalledGlobal : CalledGlobals) { + if (CalledGlobal.CallInstruction == NewInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "replacement instruction already owns a called-global record"); + } + } + for (MachineCallSiteInfo &CallSite : CallSites) { + if (CallSite.CallInstruction == OldInstruction) + CallSite.CallInstruction = NewInstruction; + } + for (MachineCalledGlobal &CalledGlobal : CalledGlobals) { + if (CalledGlobal.CallInstruction == OldInstruction) + CalledGlobal.CallInstruction = NewInstruction; + } + return {}; +} + +Expected<void> MachineFunctionRecords::addPrefetchTarget(MachinePrefetchTarget Target) { + if (std::ranges::find(PrefetchTargets, Target) != PrefetchTargets.end()) { + return makeUnexpected(ErrorCode::InvalidArgument, "prefetch target is already recorded"); + } + PrefetchTargets.push_back(Target); + return {}; +} + +Expected<void> +MachineFunctionRecords::addDebugValueSubstitution(MachineDebugValueSubstitution Substitution) { + auto Verification = verifyDebugValueSubstitution(Substitution); + if (!Verification) + return std::unexpected(Verification.error()); + if (std::ranges::any_of(DebugValueSubstitutions, + [&Substitution](const MachineDebugValueSubstitution &Existing) { + return Existing.Source == Substitution.Source; + })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "debug value source already has a substitution"); + } + DebugValueSubstitutions.push_back(Substitution); + return {}; +} + +Expected<void> MachineFunctionRecords::addEntryValue(MachineEntryValueInfo EntryValue) { + auto Verification = verifyEntryValue(EntryValue); + if (!Verification) + return std::unexpected(Verification.error()); + EntryValues.push_back(std::move(EntryValue)); + return {}; +} + +Expected<void> MachineFunctionRecords::verify() const { + if (CalleeSavedRegisters) { + auto CalleeSavedVerification = verifyCalleeSavedRegisters(*CalleeSavedRegisters); + if (!CalleeSavedVerification) + return CalleeSavedVerification; + } + + std::unordered_set<MachineInstrStableID> CallInstructions; + for (const MachineCallSiteInfo &CallSite : CallSites) { + auto CallSiteVerification = verifyCallSite(CallSite); + if (!CallSiteVerification) + return CallSiteVerification; + if (!CallInstructions.insert(CallSite.CallInstruction).second) { + return makeUnexpected(ErrorCode::VerificationError, + "multiple call-site records reference one call instruction"); + } + } + + std::unordered_set<MachineInstrStableID> CalledGlobalInstructions; + for (const MachineCalledGlobal &CalledGlobal : CalledGlobals) { + auto CalledGlobalVerification = verifyCalledGlobal(CalledGlobal); + if (!CalledGlobalVerification) + return CalledGlobalVerification; + if (!CalledGlobalInstructions.insert(CalledGlobal.CallInstruction).second) { + return makeUnexpected(ErrorCode::VerificationError, + "multiple called-global records reference one call instruction"); + } + } + + for (std::size_t Index = 0; Index < PrefetchTargets.size(); ++Index) { + if (std::ranges::find(PrefetchTargets.begin() + static_cast<std::ptrdiff_t>(Index + 1), + PrefetchTargets.end(), PrefetchTargets[Index]) != PrefetchTargets.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "prefetch target list contains a duplicate entry"); + } + } + + std::unordered_map<std::uint64_t, std::uint64_t> DebugSubstitutionEdges; + for (const MachineDebugValueSubstitution &Substitution : DebugValueSubstitutions) { + auto SubstitutionVerification = verifyDebugValueSubstitution(Substitution); + if (!SubstitutionVerification) + return SubstitutionVerification; + const std::uint64_t Source = debugReferenceKey(Substitution.Source); + const std::uint64_t Destination = debugReferenceKey(Substitution.Destination); + if (!DebugSubstitutionEdges.emplace(Source, Destination).second) { + return makeUnexpected(ErrorCode::VerificationError, + "multiple debug substitutions have the same source"); + } + } + + std::unordered_set<std::uint64_t> CompletedDebugReferences; + for (const auto &Edge : DebugSubstitutionEdges) { + const std::uint64_t Start = Edge.first; + std::unordered_set<std::uint64_t> Path; + std::uint64_t Current = Start; + while (!CompletedDebugReferences.contains(Current)) { + const auto Position = DebugSubstitutionEdges.find(Current); + if (Position == DebugSubstitutionEdges.end()) + break; + if (!Path.insert(Current).second) { + return makeUnexpected(ErrorCode::VerificationError, + "debug value substitutions contain a cycle"); + } + Current = Position->second; + } + CompletedDebugReferences.insert(Path.begin(), Path.end()); + } + + for (const MachineEntryValueInfo &EntryValue : EntryValues) { + auto EntryValueVerification = verifyEntryValue(EntryValue); + if (!EntryValueVerification) + return EntryValueVerification; + } + return {}; +} + +Expected<MachineFunctionRecords> +MachineFunctionRecords::clone(const MachineFunctionCloneContext &CloneContext) const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + + MachineFunctionRecords Result; + if (CalleeSavedRegisters) { + auto SetResult = Result.setCalleeSavedRegisters(*CalleeSavedRegisters); + if (!SetResult) + return std::unexpected(SetResult.error()); + } + + for (const MachineCallSiteInfo &Source : CallSites) { + auto Instruction = CloneContext.mapInstructionStableID(Source.CallInstruction); + if (!Instruction) + return std::unexpected(Instruction.error()); + MachineCallSiteInfo Destination = Source; + Destination.CallInstruction = *Instruction; + auto AddResult = Result.addCallSite(std::move(Destination)); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + + for (const MachineCalledGlobal &Source : CalledGlobals) { + auto Instruction = CloneContext.mapInstructionStableID(Source.CallInstruction); + if (!Instruction) + return std::unexpected(Instruction.error()); + MachineCalledGlobal Destination = Source; + Destination.CallInstruction = *Instruction; + auto AddResult = Result.addCalledGlobal(std::move(Destination)); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + + for (const MachinePrefetchTarget Source : PrefetchTargets) { + auto AddResult = Result.addPrefetchTarget(Source); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + for (const MachineDebugValueSubstitution Source : DebugValueSubstitutions) { + auto AddResult = Result.addDebugValueSubstitution(Source); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + for (const MachineEntryValueInfo &Source : EntryValues) { + auto AddResult = Result.addEntryValue(Source); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineIRBuilder.cpp b/backend/lib/CodeGen/MachineIRBuilder.cpp new file mode 100644 index 0000000..d2ac7cc --- /dev/null +++ b/backend/lib/CodeGen/MachineIRBuilder.cpp @@ -0,0 +1,1067 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineIRBuilder.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <algorithm> +#include <array> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { + +Register MachineIRBuilderResult::reg() const noexcept { + assert(ResultKind == Kind::Register && "machine IR builder result is not a register"); + return Reg; +} + +LowLevelType MachineIRBuilderResult::type() const noexcept { + assert(ResultKind == Kind::Type && "machine IR builder result is not a type"); + return Type; +} + +Expected<LowLevelType> MachineIRBuilderResult::type(const MachineRegisterInfo &MRI) const { + if (ResultKind == Kind::Type) + return Type; + return MRI.getType(Reg); +} + +Expected<void> MachineIRBuilderResult::addDef(MachineRegisterInfo &MRI, + MachineInstrBuilder &Builder) const { + Register Definition = Reg; + if (ResultKind == Kind::Type) { + auto Created = MRI.createVirtualRegister(Type); + if (!Created) + return std::unexpected(Created.error()); + Definition = *Created; + } + return Builder.addDef(Definition); +} + +MachineInstrBuilder::MachineInstrBuilder(MachineFunction &Function, const TargetDescription &Target, + const InstructionDescriptor &Descriptor, + MachineBasicBlock &InsertBlock, + MachineInstr *InsertBefore) noexcept + : Function(&Function), Target(&Target), Descriptor(&Descriptor), InsertBlock(&InsertBlock), + InsertBefore(InsertBefore) {} + +MachineInstrBuilder::MachineInstrBuilder(MachineInstrBuilder &&Other) noexcept + : Function(Other.Function), Target(Other.Target), Descriptor(Other.Descriptor), + InsertBlock(Other.InsertBlock), InsertBefore(Other.InsertBefore), + ExplicitOperands(std::move(Other.ExplicitOperands)), + ExtraImplicitOperands(std::move(Other.ExtraImplicitOperands)), + RequestedTies(std::move(Other.RequestedTies)), MemOperands(std::move(Other.MemOperands)), + Flags(Other.Flags), Location(std::move(Other.Location)), + DebugLocation(std::move(Other.DebugLocation)), Origin(Other.Origin), IsBuilt(Other.IsBuilt) { + Other.Function = nullptr; + Other.Target = nullptr; + Other.Descriptor = nullptr; + Other.InsertBlock = nullptr; + Other.InsertBefore = nullptr; + Other.IsBuilt = true; +} + +MachineInstrBuilder &MachineInstrBuilder::operator=(MachineInstrBuilder &&Other) noexcept { + if (this == &Other) + return *this; + + Function = Other.Function; + Target = Other.Target; + Descriptor = Other.Descriptor; + InsertBlock = Other.InsertBlock; + InsertBefore = Other.InsertBefore; + ExplicitOperands = std::move(Other.ExplicitOperands); + ExtraImplicitOperands = std::move(Other.ExtraImplicitOperands); + RequestedTies = std::move(Other.RequestedTies); + MemOperands = std::move(Other.MemOperands); + Flags = Other.Flags; + Location = std::move(Other.Location); + DebugLocation = std::move(Other.DebugLocation); + Origin = Other.Origin; + IsBuilt = Other.IsBuilt; + + Other.Function = nullptr; + Other.Target = nullptr; + Other.Descriptor = nullptr; + Other.InsertBlock = nullptr; + Other.InsertBefore = nullptr; + Other.IsBuilt = true; + return *this; +} + +Expected<void> MachineInstrBuilder::ensurePending() const { + if (IsBuilt) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine instruction builder was already consumed"); + } + return {}; +} + +Expected<void> MachineInstrBuilder::addOperand(MachineOperand &&Operand) { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + if (Operand.parent() != nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder operand already belongs to an instruction"); + } + if (Operand.isImplicit()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "implicit register requires addImplicitOperand"); + } + ExplicitOperands.push_back(std::move(Operand)); + return {}; +} + +Expected<void> MachineInstrBuilder::addImplicitOperand(MachineOperand &&Operand) { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + if (Operand.parent() != nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder operand already belongs to an instruction"); + } + if (!Operand.isRegister() || !Operand.isImplicit()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "dynamic implicit operand must be an implicit register"); + } + ExtraImplicitOperands.push_back(std::move(Operand)); + return {}; +} + +Expected<void> MachineInstrBuilder::appendRegister(Register Reg, RegisterOperandFlags RegisterFlags, + SubRegIndex SubReg, std::uint32_t TargetFlags, + bool IsImplicit) { + auto Operand = MachineOperand::registerOperand(Reg, RegisterFlags, SubReg, TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + if (IsImplicit) + return addImplicitOperand(std::move(*Operand)); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addDef(Register Reg, RegisterOperandFlags AdditionalFlags, + SubRegIndex SubReg, std::uint32_t TargetFlags) { + AdditionalFlags.set(RegisterOperandFlag::Define); + return appendRegister(Reg, AdditionalFlags, SubReg, TargetFlags, false); +} + +Expected<void> MachineInstrBuilder::addUse(Register Reg, RegisterOperandFlags AdditionalFlags, + SubRegIndex SubReg, std::uint32_t TargetFlags) { + AdditionalFlags.set(RegisterOperandFlag::Use); + return appendRegister(Reg, AdditionalFlags, SubReg, TargetFlags, false); +} + +Expected<void> MachineInstrBuilder::addDefUse(Register Reg, RegisterOperandFlags AdditionalFlags, + SubRegIndex SubReg, std::uint32_t TargetFlags) { + AdditionalFlags.set(RegisterOperandFlag::Define); + AdditionalFlags.set(RegisterOperandFlag::Use); + AdditionalFlags.set(RegisterOperandFlag::InternalRead); + return appendRegister(Reg, AdditionalFlags, SubReg, TargetFlags, false); +} + +Expected<void> MachineInstrBuilder::addImplicitDef(Register Reg, + RegisterOperandFlags AdditionalFlags, + SubRegIndex SubReg, std::uint32_t TargetFlags) { + AdditionalFlags.set(RegisterOperandFlag::Define); + AdditionalFlags.set(RegisterOperandFlag::Implicit); + return appendRegister(Reg, AdditionalFlags, SubReg, TargetFlags, true); +} + +Expected<void> MachineInstrBuilder::addImplicitUse(Register Reg, + RegisterOperandFlags AdditionalFlags, + SubRegIndex SubReg, std::uint32_t TargetFlags) { + AdditionalFlags.set(RegisterOperandFlag::Use); + AdditionalFlags.set(RegisterOperandFlag::Implicit); + return appendRegister(Reg, AdditionalFlags, SubReg, TargetFlags, true); +} + +Expected<void> MachineInstrBuilder::addImmediate(std::int64_t Value, std::uint32_t TargetFlags) { + return addOperand(MachineOperand::immediate(Value, TargetFlags)); +} + +Expected<void> MachineInstrBuilder::addIntegerImmediate(APInt Value, std::uint32_t TargetFlags) { + return addOperand(MachineOperand::integerImmediate(std::move(Value), TargetFlags)); +} + +Expected<void> MachineInstrBuilder::addFloatingImmediate(APFloat Bits, std::uint32_t TargetFlags) { + return addOperand(MachineOperand::floatingImmediate(Bits, TargetFlags)); +} + +Expected<void> MachineInstrBuilder::addBasicBlock(MachineBasicBlock &Block, + std::uint32_t TargetFlags) { + return addOperand(MachineOperand::basicBlock(Block, TargetFlags)); +} + +Expected<void> MachineInstrBuilder::addFrameIndex(FrameIndex Index, std::int64_t Offset, + std::uint32_t TargetFlags) { + return addOperand(MachineOperand::frameIndex(Index, Offset, TargetFlags)); +} + +Expected<void> MachineInstrBuilder::addConstantPool(ConstantPoolIndex Index, std::int64_t Offset, + std::uint32_t TargetFlags) { + return addOperand(MachineOperand::constantPool(Index, Offset, TargetFlags)); +} + +Expected<void> MachineInstrBuilder::addJumpTable(JumpTableIndex Index, std::uint32_t TargetFlags) { + return addOperand(MachineOperand::jumpTable(Index, TargetFlags)); +} + +Expected<void> MachineInstrBuilder::addGlobalSymbol(std::string Name, std::int64_t Offset, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::globalSymbol(std::move(Name), Offset, TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addExternalSymbol(std::string Name, std::int64_t Offset, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::externalSymbol(std::move(Name), Offset, TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addBlockAddress(std::string FunctionName, std::string BlockName, + std::int64_t Offset, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::blockAddress(std::move(FunctionName), std::move(BlockName), Offset, + TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addRegisterMask(std::shared_ptr<const RegisterMask> Mask, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::registerMask(std::move(Mask), TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addLiveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::liveOutRegisterMask(std::move(Mask), TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addMetadata(MachineMetadataValue Value, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::metadata(std::move(Value), TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addTargetIndex(TargetIndex Index, std::int64_t Offset, + std::uint32_t TargetFlags) { + return addOperand(MachineOperand::targetIndex(Index, Offset, TargetFlags)); +} + +Expected<void> MachineInstrBuilder::addMCSymbol(std::string Name, std::int64_t Offset, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::mcSymbol(std::move(Name), Offset, TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addCFIInstruction(CFIInstructionIndex Index, + std::uint32_t TargetFlags) { + return addOperand(MachineOperand::cfiIndex(Index, TargetFlags)); +} + +Expected<void> MachineInstrBuilder::addIntrinsic(IntrinsicID ID, std::uint32_t TargetFlags) { + auto Operand = MachineOperand::intrinsic(ID, TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addIntegerPredicate(IntegerPredicate Predicate, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::integerPredicate(Predicate, TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addFloatingPredicate(FloatingPredicate Predicate, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::floatingPredicate(Predicate, TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addShuffleMask(std::vector<std::int32_t> Elements, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::shuffleMask(std::move(Elements), TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addDebugInstrRef(std::uint64_t InstructionNumber, + std::uint64_t OperandNumber, + std::uint32_t TargetFlags) { + auto Operand = MachineOperand::debugInstrRef(InstructionNumber, OperandNumber, TargetFlags); + if (!Operand) + return std::unexpected(Operand.error()); + return addOperand(std::move(*Operand)); +} + +Expected<void> MachineInstrBuilder::addLaneMask(LaneMask Mask, std::uint32_t TargetFlags) { + return addOperand(MachineOperand::laneMask(Mask, TargetFlags)); +} + +Expected<void> MachineInstrBuilder::tieOperands(std::size_t DefOperand, std::size_t UseOperand) { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + if (DefOperand >= ExplicitOperands.size() || UseOperand >= ExplicitOperands.size() || + DefOperand == UseOperand) { + return makeUnexpected(ErrorCode::InvalidArgument, "builder operand tie indices are invalid"); + } + + const MachineOperand &Def = ExplicitOperands[DefOperand]; + const MachineOperand &Use = ExplicitOperands[UseOperand]; + if (!Def.isRegister() || !Use.isRegister() || !Def.isDef() || Def.isUse() || !Use.isUse() || + Use.isDef()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder tie must connect a pure register def and use"); + } + + const MachineOperandTie Tie{DefOperand, UseOperand}; + if (std::ranges::find(RequestedTies, Tie) == RequestedTies.end()) + RequestedTies.push_back(Tie); + return {}; +} + +Expected<void> MachineInstrBuilder::addMemOperand(const MachineMemOperand &MemOperand) { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + if (!Function->ownsMemOperand(MemOperand)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder memory operand belongs to another function"); + } + MemOperands.push_back(&MemOperand); + return {}; +} + +Expected<void> MachineInstrBuilder::setFlags(MachineInstrFlags NewFlags) { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + Flags = NewFlags; + return {}; +} + +Expected<void> MachineInstrBuilder::setSourceLocation(SourceLocation NewLocation) { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + if (!NewLocation.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, "builder source location is incomplete"); + } + Location = std::move(NewLocation); + return {}; +} + +Expected<void> MachineInstrBuilder::clearSourceLocation() { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + Location.reset(); + return {}; +} + +Expected<void> MachineInstrBuilder::setDebugLocation(MachineDebugLocation NewLocation) { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + if (!NewLocation.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, "builder debug location is invalid"); + } + DebugLocation = std::move(NewLocation); + return {}; +} + +Expected<void> MachineInstrBuilder::clearDebugLocation() { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + DebugLocation.reset(); + return {}; +} + +Expected<void> MachineInstrBuilder::setOrigin(OriginID NewOrigin) { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + if (NewOrigin.value() == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, "builder origin ID must be nonzero"); + } + Origin = NewOrigin; + return {}; +} + +Expected<void> MachineInstrBuilder::clearOrigin() { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + Origin.reset(); + return {}; +} + +Expected<void> MachineInstrBuilder::validateInsertionPoint() const { + if (InsertBlock == nullptr || &InsertBlock->function() != Function || !InsertBlock->isLinked()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder insertion block is not linked to its function"); + } + if (InsertBefore != nullptr && InsertBefore->parent() != InsertBlock) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder insertion cursor is no longer in its block"); + } + return {}; +} + +Expected<void> MachineInstrBuilder::validateFrameReference(FrameIndex Index) const { + const StackObject *Object = Function->frameInfo().object(Index); + if (Object == nullptr || Object->description()->IsDead) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder references an unknown or dead stack object"); + } + return {}; +} + +Expected<void> MachineInstrBuilder::validateConstantPoolReference(ConstantPoolIndex Index) const { + if (Function->constantPool().entry(Index) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder references an unknown constant pool entry"); + } + return {}; +} + +Expected<void> MachineInstrBuilder::validateJumpTableReference(JumpTableIndex Index) const { + if (Function->jumpTableInfo().table(Index) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, "builder references an unknown jump table"); + } + return {}; +} + +Expected<void> MachineInstrBuilder::validateCFIReference(CFIInstructionIndex Index) const { + if (Function->cfiInfo().instruction(Index) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder references an unknown CFI instruction"); + } + return {}; +} + +Expected<void> MachineInstrBuilder::validateOperandOwnership(const MachineOperand &Operand) const { + if (Operand.parent() != nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder operand already belongs to an instruction"); + } + if (const auto *Block = Operand.getIf<BasicBlockOperand>(); + Block != nullptr && (Block->Block == nullptr || &Block->Block->function() != Function || + !Block->Block->isLinked())) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder basic block operand is not linked to its function"); + } + if (const Register *Reg = Operand.getIf<Register>(); + Reg != nullptr && Reg->isVirtual() && + !Function->registerInfo().isKnownVirtualRegister(*Reg)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder operand references an unknown virtual register"); + } + if (const auto *Frame = Operand.getIf<FrameIndexOperand>(); Frame != nullptr) { + auto FrameResult = validateFrameReference(Frame->Index); + if (!FrameResult) + return FrameResult; + } + if (const auto *Constant = Operand.getIf<ConstantPoolOperand>(); Constant != nullptr) { + auto ConstantResult = validateConstantPoolReference(Constant->Index); + if (!ConstantResult) + return ConstantResult; + } + if (const auto *JumpTable = Operand.getIf<JumpTableOperand>(); JumpTable != nullptr) { + auto JumpTableResult = validateJumpTableReference(JumpTable->Index); + if (!JumpTableResult) + return JumpTableResult; + } + if (const auto *CFI = Operand.getIf<CFIIndexOperand>(); CFI != nullptr) { + auto CFIResult = validateCFIReference(CFI->Index); + if (!CFIResult) + return CFIResult; + } + return {}; +} + +std::vector<MachineOperandTie> MachineInstrBuilder::collectOperandTies() const { + std::vector<MachineOperandTie> Result; + for (std::size_t Index = 0; Index < Descriptor->Operands.size(); ++Index) { + const OperandDescriptor &Operand = Descriptor->Operands[Index]; + if (!Operand.DefaultTiedOperand || Index > *Operand.DefaultTiedOperand) + continue; + const std::size_t Other = *Operand.DefaultTiedOperand; + const bool IndexIsDef = Operand.RegisterRole == RegisterOperandRole::Def; + Result.push_back(IndexIsDef ? MachineOperandTie{Index, Other} + : MachineOperandTie{Other, Index}); + } + for (const MachineOperandTie Tie : RequestedTies) { + if (std::ranges::find(Result, Tie) == Result.end()) + Result.push_back(Tie); + } + return Result; +} + +Expected<MachineInstr *> MachineInstrBuilder::build() { + auto Pending = ensurePending(); + if (!Pending) + return std::unexpected(Pending.error()); + + auto InsertionValidation = validateInsertionPoint(); + if (!InsertionValidation) + return std::unexpected(InsertionValidation.error()); + + for (const MachineOperand &Operand : ExplicitOperands) { + auto Ownership = validateOperandOwnership(Operand); + if (!Ownership) + return std::unexpected(Ownership.error()); + } + for (const MachineOperand &Operand : ExtraImplicitOperands) { + auto Ownership = validateOperandOwnership(Operand); + if (!Ownership) + return std::unexpected(Ownership.error()); + } + for (const MachineMemOperand *MemOperand : MemOperands) { + const MemoryBase &Base = MemOperand->description().Base; + if (const auto Frame = Base.frameIndex()) { + auto FrameResult = validateFrameReference(*Frame); + if (!FrameResult) + return std::unexpected(FrameResult.error()); + } + if (const auto Constant = Base.constantPoolIndex()) { + auto ConstantResult = validateConstantPoolReference(*Constant); + if (!ConstantResult) + return std::unexpected(ConstantResult.error()); + } + if (const auto JumpTable = Base.jumpTableIndex()) { + auto JumpTableResult = validateJumpTableReference(*JumpTable); + if (!JumpTableResult) + return std::unexpected(JumpTableResult.error()); + } + } + + auto ExplicitValidation = Target->validateExplicitOperands( + Descriptor->InstructionOpcode, ExplicitOperands, Function->registerInfo()); + if (!ExplicitValidation) + return std::unexpected(ExplicitValidation.error()); + + auto DefaultImplicit = Target->createDefaultImplicitOperands(*Descriptor); + if (!DefaultImplicit) + return std::unexpected(DefaultImplicit.error()); + + std::vector<const MachineOperand *> ImplicitOperands; + ImplicitOperands.reserve(DefaultImplicit->size() + ExtraImplicitOperands.size()); + for (const MachineOperand &Operand : *DefaultImplicit) + ImplicitOperands.push_back(&Operand); + for (const MachineOperand &Operand : ExtraImplicitOperands) + ImplicitOperands.push_back(&Operand); + auto ImplicitValidation = Target->validateImplicitOperands( + Descriptor->InstructionOpcode, ImplicitOperands, Function->registerInfo()); + if (!ImplicitValidation) + return std::unexpected(ImplicitValidation.error()); + + const std::vector<MachineOperandTie> Ties = collectOperandTies(); + auto TieValidation = Target->validateOperandTies(Descriptor->InstructionOpcode, ExplicitOperands, + Ties, Function->registerInfo()); + if (!TieValidation) + return std::unexpected(TieValidation.error()); + + auto EarlyClobberValidation = + Target->validateEarlyClobber(ExplicitOperands, ImplicitOperands, Ties); + if (!EarlyClobberValidation) + return std::unexpected(EarlyClobberValidation.error()); + + auto MemoryValidation = Target->validateMemOperands(Descriptor->InstructionOpcode, MemOperands); + if (!MemoryValidation) + return std::unexpected(MemoryValidation.error()); + + auto Instruction = Function->createInstruction(Descriptor->InstructionOpcode); + if (!Instruction) + return std::unexpected(Instruction.error()); + MachineInstr *Result = *Instruction; + Result->reserveOperands(ExplicitOperands.size() + DefaultImplicit->size() + + ExtraImplicitOperands.size()); + + for (MachineOperand &Operand : ExplicitOperands) { + auto AddResult = Result->addOperand(std::move(Operand)); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + for (MachineOperand &Operand : *DefaultImplicit) { + auto AddResult = Result->addOperand(std::move(Operand)); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + for (MachineOperand &Operand : ExtraImplicitOperands) { + auto AddResult = Result->addOperand(std::move(Operand)); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + for (const MachineOperandTie Tie : Ties) { + auto TieResult = Result->tieOperands(Tie.DefOperand, Tie.UseOperand); + if (!TieResult) + return std::unexpected(TieResult.error()); + } + for (const MachineMemOperand *MemOperand : MemOperands) { + auto AddResult = Result->addMemOperand(*MemOperand); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + + Result->setFlags(Flags); + if (Location) + Result->setSourceLocation(std::move(*Location)); + if (DebugLocation) { + auto DebugLocationResult = Result->setDebugLocation(std::move(*DebugLocation)); + if (!DebugLocationResult) + return std::unexpected(DebugLocationResult.error()); + } + if (Origin) + Result->setOrigin(*Origin); + + Expected<void> InsertResult; + if (InsertBefore == nullptr) + InsertResult = InsertBlock->appendInstruction(*Result); + else + InsertResult = InsertBlock->insertInstructionBefore(*InsertBefore, *Result); + if (!InsertResult) + return std::unexpected(InsertResult.error()); + + IsBuilt = true; + return Result; +} + +Expected<void> MachineIRBuilder::validateBlock(const MachineBasicBlock &Block) const { + if (&Block.function() != Function || !Block.isLinked()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder insertion block is not linked to its function"); + } + return {}; +} + +Expected<void> MachineIRBuilder::validateInstruction(const MachineInstr &Instruction) const { + if (&Instruction.function() != Function || Instruction.parent() == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "builder insertion instruction is not linked to its function"); + } + return {}; +} + +Expected<void> MachineIRBuilder::verifyInsertPoint() const { + if (InsertBlock == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point"); + auto BlockValidation = validateBlock(*InsertBlock); + if (!BlockValidation) + return std::unexpected(BlockValidation.error()); + if (InsertBefore != nullptr && InsertBefore->parent() != InsertBlock) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine IR builder insertion cursor is no longer in its block"); + } + return {}; +} + +Expected<void> MachineIRBuilder::setInsertPoint(MachineBasicBlock &Block) { + auto Validation = validateBlock(Block); + if (!Validation) + return std::unexpected(Validation.error()); + InsertBlock = &Block; + InsertBefore = nullptr; + return {}; +} + +Expected<void> MachineIRBuilder::setInsertPointBefore(MachineInstr &Instruction) { + auto Validation = validateInstruction(Instruction); + if (!Validation) + return std::unexpected(Validation.error()); + InsertBlock = Instruction.parent(); + InsertBefore = &Instruction; + return {}; +} + +Expected<void> MachineIRBuilder::setInsertPointAfter(MachineInstr &Instruction) { + auto Validation = validateInstruction(Instruction); + if (!Validation) + return std::unexpected(Validation.error()); + InsertBlock = Instruction.parent(); + InsertBefore = Instruction.next(); + return {}; +} + +Expected<MachineInstrBuilder> MachineIRBuilder::buildInstr(Opcode InstructionOpcode) const { + if (InsertBlock == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point"); + } + const InstructionDescriptor *Descriptor = Target->instructionDescriptor(InstructionOpcode); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine IR builder opcode is not registered"); + } + auto BlockValidation = validateBlock(*InsertBlock); + if (!BlockValidation) + return std::unexpected(BlockValidation.error()); + if (InsertBefore != nullptr && InsertBefore->parent() != InsertBlock) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine IR builder insertion cursor is stale"); + } + return MachineInstrBuilder(*Function, *Target, *Descriptor, *InsertBlock, InsertBefore); +} + +Expected<MachineInstr *> +MachineIRBuilder::buildInstr(Opcode InstructionOpcode, + std::initializer_list<MachineIRBuilderResult> Destinations, + std::initializer_list<Register> Sources) const { + return buildInstr( + InstructionOpcode, + std::span<const MachineIRBuilderResult>(Destinations.begin(), Destinations.size()), + std::span<const Register>(Sources.begin(), Sources.size())); +} + +Expected<MachineInstr *> +MachineIRBuilder::buildInstr(Opcode InstructionOpcode, + std::span<const MachineIRBuilderResult> Destinations, + std::span<const Register> Sources) const { + auto Pending = buildInstr(InstructionOpcode); + if (!Pending) + return std::unexpected(Pending.error()); + for (const MachineIRBuilderResult &Destination : Destinations) { + if (auto Added = Destination.addDef(Function->registerInfo(), *Pending); !Added) + return std::unexpected(Added.error()); + } + for (Register Source : Sources) { + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + } + return Pending->build(); +} + +Expected<LowLevelType> MachineIRBuilder::registerType(Register Reg) const { + if (!Reg.isVirtual()) + return makeUnexpected(ErrorCode::InvalidArgument, + "generic builder operand is not a virtual register"); + auto Info = Function->registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + const std::optional<LowLevelType> &Type = Info->get().Type; + if (!Type.has_value()) + return makeUnexpected(ErrorCode::InvalidArgument, + "generic builder operand has no LowLevelType"); + return Type.value(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildUnary(Opcode InstructionOpcode, + MachineIRBuilderResult Destination, + Register Source) const { + const std::array Destinations{Destination}; + const std::array Sources{Source}; + return buildInstr(InstructionOpcode, std::span<const MachineIRBuilderResult>(Destinations), + std::span<const Register>(Sources)); +} + +Expected<MachineInstr *> MachineIRBuilder::buildBinary(Opcode InstructionOpcode, + MachineIRBuilderResult Destination, + Register Left, Register Right) const { + const std::array Destinations{Destination}; + const std::array Sources{Left, Right}; + return buildInstr(InstructionOpcode, std::span<const MachineIRBuilderResult>(Destinations), + std::span<const Register>(Sources)); +} + +Expected<MachineInstr *> MachineIRBuilder::buildTrunc(MachineIRBuilderResult Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_TRUNC), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildCopy(MachineIRBuilderResult Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::COPY), Destination, Source); +} + +Expected<MachineInstr *> +MachineIRBuilder::buildImplicitDef(MachineIRBuilderResult Destination) const { + const std::array Destinations{Destination}; + return buildInstr(opcode(GenericOpcode::G_IMPLICIT_DEF), + std::span<const MachineIRBuilderResult>(Destinations), + std::span<const Register>()); +} + +Expected<MachineInstr *> MachineIRBuilder::buildAnyExt(MachineIRBuilderResult Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_ANYEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildZExt(MachineIRBuilderResult Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_ZEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildSExt(MachineIRBuilderResult Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_SEXT), Destination, Source); +} + +GenericOpcode MachineIRBuilder::getBoolExtOp(bool IsVector, bool IsFloatingPoint) const noexcept { + switch (Target->getBooleanContents(IsVector, IsFloatingPoint)) { + case BooleanContent::ZeroOrNegativeOne: + return GenericOpcode::G_SEXT; + case BooleanContent::ZeroOrOne: + return GenericOpcode::G_ZEXT; + case BooleanContent::Undefined: + return GenericOpcode::G_ANYEXT; + } + return GenericOpcode::G_ANYEXT; +} + +Expected<MachineInstr *> MachineIRBuilder::buildUndef(MachineIRBuilderResult Destination) const { + return buildImplicitDef(Destination); +} + +Expected<MachineInstr *> MachineIRBuilder::buildExtOrTrunc(Opcode ExtensionOpcode, + MachineIRBuilderResult Destination, + Register Source) const { + auto DestinationType = Destination.type(Function->registerInfo()); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto SourceType = registerType(Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + const std::uint64_t DestinationWidth = DestinationType->minimumBitWidth(); + const std::uint64_t SourceWidth = SourceType->minimumBitWidth(); + if (DestinationWidth > SourceWidth) + return buildUnary(ExtensionOpcode, Destination, Source); + if (DestinationWidth < SourceWidth) + return buildTrunc(Destination, Source); + return buildUnary(opcode(GenericOpcode::COPY), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildAnyExtOrTrunc(MachineIRBuilderResult Destination, + Register Source) const { + return buildExtOrTrunc(opcode(GenericOpcode::G_ANYEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildZExtOrTrunc(MachineIRBuilderResult Destination, + Register Source) const { + return buildExtOrTrunc(opcode(GenericOpcode::G_ZEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildSExtOrTrunc(MachineIRBuilderResult Destination, + Register Source) const { + return buildExtOrTrunc(opcode(GenericOpcode::G_SEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildConstant(MachineIRBuilderResult Destination, + APInt Value) const { + auto Pending = buildInstr(opcode(GenericOpcode::G_CONSTANT)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Destination.addDef(Function->registerInfo(), *Pending); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addIntegerImmediate(std::move(Value)); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildConstant(MachineIRBuilderResult Destination, + std::int64_t Value) const { + auto Type = Destination.type(Function->registerInfo()); + if (!Type) + return std::unexpected(Type.error()); + auto Immediate = APInt::fromSigned(Type->scalarBitWidth(), Value); + if (!Immediate) + return std::unexpected(Immediate.error()); + return buildConstant(Destination, std::move(*Immediate)); +} + +Expected<MachineInstr *> MachineIRBuilder::buildSExtInReg(MachineIRBuilderResult Destination, + Register Source, + std::uint32_t Width) const { + auto Pending = buildInstr(opcode(GenericOpcode::G_SEXT_INREG)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Destination.addDef(Function->registerInfo(), *Pending); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addImmediate(static_cast<std::int64_t>(Width)); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildAnd(MachineIRBuilderResult Destination, + Register Left, Register Right) const { + return buildBinary(opcode(GenericOpcode::G_AND), Destination, Left, Right); +} + +Expected<MachineInstr *> MachineIRBuilder::buildOr(MachineIRBuilderResult Destination, + Register Left, Register Right) const { + return buildBinary(opcode(GenericOpcode::G_OR), Destination, Left, Right); +} + +Expected<MachineInstr *> MachineIRBuilder::buildXor(MachineIRBuilderResult Destination, + Register Left, Register Right) const { + return buildBinary(opcode(GenericOpcode::G_XOR), Destination, Left, Right); +} + +Expected<MachineInstr *> MachineIRBuilder::buildAdd(MachineIRBuilderResult Destination, + Register Left, Register Right) const { + return buildBinary(opcode(GenericOpcode::G_ADD), Destination, Left, Right); +} + +Expected<MachineInstr *> MachineIRBuilder::buildSub(MachineIRBuilderResult Destination, + Register Left, Register Right) const { + return buildBinary(opcode(GenericOpcode::G_SUB), Destination, Left, Right); +} + +Expected<MachineInstr *> MachineIRBuilder::buildShl(MachineIRBuilderResult Destination, + Register Left, Register Right) const { + return buildBinary(opcode(GenericOpcode::G_SHL), Destination, Left, Right); +} + +Expected<MachineInstr *> MachineIRBuilder::buildLShr(MachineIRBuilderResult Destination, + Register Left, Register Right) const { + return buildBinary(opcode(GenericOpcode::G_LSHR), Destination, Left, Right); +} + +Expected<MachineInstr *> MachineIRBuilder::buildAShr(MachineIRBuilderResult Destination, + Register Left, Register Right) const { + return buildBinary(opcode(GenericOpcode::G_ASHR), Destination, Left, Right); +} + +Expected<MachineInstr *> MachineIRBuilder::buildPtrToInt(MachineIRBuilderResult Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_PTRTOINT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildIntToPtr(MachineIRBuilderResult Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_INTTOPTR), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildICmp(IntegerPredicate Predicate, + MachineIRBuilderResult Destination, + Register Left, Register Right) const { + auto Pending = buildInstr(opcode(GenericOpcode::G_ICMP)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Destination.addDef(Function->registerInfo(), *Pending); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addIntegerPredicate(Predicate); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Left); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Right); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> +MachineIRBuilder::buildMergeLikeInstr(MachineIRBuilderResult Destination, + const std::vector<Register> &Sources) const { + auto DestinationType = Destination.type(Function->registerInfo()); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!DestinationType->isVector()) + return buildMergeValues(Destination, Sources); + if (Sources.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, + "merge-like instruction requires source registers"); + auto SourceType = registerType(Sources.front()); + if (!SourceType) + return std::unexpected(SourceType.error()); + const GenericOpcode MergeOpcode = + SourceType->isVector() ? GenericOpcode::G_CONCAT_VECTORS : GenericOpcode::G_BUILD_VECTOR; + const std::array Destinations{Destination}; + return buildInstr(opcode(MergeOpcode), std::span<const MachineIRBuilderResult>(Destinations), + Sources); +} + +Expected<MachineInstr *> +MachineIRBuilder::buildMergeValues(MachineIRBuilderResult Destination, + const std::vector<Register> &Sources) const { + const std::array Destinations{Destination}; + return buildInstr(opcode(GenericOpcode::G_MERGE_VALUES), + std::span<const MachineIRBuilderResult>(Destinations), Sources); +} + +Expected<MachineInstr *> +MachineIRBuilder::buildUnmergeValues(const std::vector<Register> &Destinations, + Register Source) const { + std::vector<MachineIRBuilderResult> Results; + Results.reserve(Destinations.size()); + for (Register Destination : Destinations) + Results.emplace_back(Destination); + const std::array Sources{Source}; + return buildInstr(opcode(GenericOpcode::G_UNMERGE_VALUES), Results, + std::span<const Register>(Sources)); +} + +Expected<MachineInstr *> MachineIRBuilder::buildExtract(MachineIRBuilderResult Destination, + Register Source, + std::uint64_t Offset) const { + if (Offset > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::InvalidArgument, + "generic extract offset does not fit a machine immediate"); + auto Pending = buildInstr(opcode(GenericOpcode::G_EXTRACT)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Destination.addDef(Function->registerInfo(), *Pending); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addImmediate(static_cast<std::int64_t>(Offset)); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildInsert(MachineIRBuilderResult Destination, + Register Source, Register Inserted, + std::uint64_t Offset) const { + if (Offset > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::InvalidArgument, + "generic insert offset does not fit a machine immediate"); + auto Pending = buildInstr(opcode(GenericOpcode::G_INSERT)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Destination.addDef(Function->registerInfo(), *Pending); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Inserted); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addImmediate(static_cast<std::int64_t>(Offset)); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineIRTypes.cpp b/backend/lib/CodeGen/MachineIRTypes.cpp new file mode 100644 index 0000000..2810c92 --- /dev/null +++ b/backend/lib/CodeGen/MachineIRTypes.cpp @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineIRTypes.h" + +#include <algorithm> +#include <bit> +#include <cstdint> + +namespace rcc { + +Expected<Alignment> Alignment::create(std::uint64_t Bytes) { + if (!std::has_single_bit(Bytes)) { + return makeUnexpected(ErrorCode::InvalidArgument, "alignment must be a nonzero power of two"); + } + return Alignment(Bytes); +} + +Expected<BranchProbability> BranchProbability::fromFraction(std::uint32_t Numerator, + std::uint32_t Denominator) { + if (Denominator == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "branch probability denominator must be nonzero"); + } + if (Numerator > Denominator) { + return makeUnexpected(ErrorCode::InvalidArgument, + "branch probability numerator cannot exceed its denominator"); + } + + const std::uint64_t ScaledNumerator = static_cast<std::uint64_t>(Numerator) * MaxRawValue; + const std::uint64_t Rounded = + (ScaledNumerator + static_cast<std::uint64_t>(Denominator / 2)) / Denominator; + return BranchProbability(static_cast<std::uint32_t>(Rounded)); +} + +void BranchProbability::normalizeProbabilities( + std::span<BranchProbability> Probabilities) noexcept { + if (Probabilities.empty()) + return; + + std::uint64_t Sum = 0; + for (const BranchProbability Probability : Probabilities) + Sum += Probability.rawValue(); + + if (Sum == 0) { + const std::uint64_t Count = Probabilities.size(); + const auto Equal = + static_cast<std::uint32_t>((static_cast<std::uint64_t>(MaxRawValue) + Count / 2U) / Count); + std::ranges::fill(Probabilities, BranchProbability(Equal)); + return; + } + + for (BranchProbability &Probability : Probabilities) { + const std::uint64_t Scaled = static_cast<std::uint64_t>(Probability.rawValue()) * MaxRawValue; + Probability = BranchProbability(static_cast<std::uint32_t>((Scaled + Sum / 2U) / Sum)); + } +} + +MachineInstrFlags::MachineInstrFlags(std::initializer_list<MachineInstrFlag> Flags) noexcept { + for (const MachineInstrFlag Flag : Flags) + set(Flag); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineInstr.cpp b/backend/lib/CodeGen/MachineInstr.cpp new file mode 100644 index 0000000..3b200c5 --- /dev/null +++ b/backend/lib/CodeGen/MachineInstr.cpp @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineInstr.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <cstddef> +#include <cstdint> +#include <functional> +#include <limits> +#include <optional> +#include <string> +#include <string_view> +#include <utility> + +namespace rcc { + +bool MachineInstr::isDebugInstr() const noexcept { + return InstructionOpcode == rcc::opcode(GenericOpcode::DBG_VALUE) || + InstructionOpcode == rcc::opcode(GenericOpcode::DBG_VALUE_LIST) || + InstructionOpcode == rcc::opcode(GenericOpcode::DBG_LABEL) || + InstructionOpcode == rcc::opcode(GenericOpcode::DBG_INSTR_REF) || + InstructionOpcode == rcc::opcode(GenericOpcode::DBG_PHI); +} + +MachineInstr::MachineInstr(MachineFunction &OwnerFunction, Opcode InstructionOpcode, + MachineInstrStableID StableID) + : OwnerFunction(&OwnerFunction), InstructionOpcode(InstructionOpcode), StableID(StableID) {} + +MachineInstr::~MachineInstr() { + for (MachineOperand &Operand : Operands) + Operand.detachFromInstruction(); +} + +Expected<void> MachineInstr::setOpcode(Opcode NewOpcode) { + if (!NewOpcode.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "machine opcode must be nonzero"); + InstructionOpcode = NewOpcode; + return {}; +} + +Expected<void> MachineInstr::eraseFromParent() { + if (ParentBlock == nullptr) + return makeUnexpected(ErrorCode::InternalError, "cannot erase a detached machine instruction"); + return ParentBlock->removeInstruction(*this); +} + +Expected<void> MachineInstr::setDebugInstrNumber(std::uint64_t Number) { + if (Number == 0 || Number > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine debug instruction number must fit in nonzero uint32"); + } + if (DebugInstrNumber != 0 && DebugInstrNumber != Number) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine instruction already has a different debug number"); + } + DebugInstrNumber = static_cast<std::uint32_t>(Number); + OwnerFunction->observeDebugInstrNumber(DebugInstrNumber); + return {}; +} + +Expected<std::uint32_t> MachineInstr::assignDebugInstrNumber() { + if (DebugInstrNumber != 0) + return DebugInstrNumber; + auto Number = OwnerFunction->allocateDebugInstrNumber(); + if (!Number) + return std::unexpected(Number.error()); + DebugInstrNumber = *Number; + return DebugInstrNumber; +} + +MachineOperand *MachineInstr::operandAt(std::size_t Index) noexcept { + if (Index >= Operands.size()) + return nullptr; + return &Operands[Index]; +} + +const MachineOperand *MachineInstr::operandAt(std::size_t Index) const noexcept { + if (Index >= Operands.size()) + return nullptr; + return &Operands[Index]; +} + +void MachineInstr::reserveOperands(std::size_t Capacity) { + Operands.reserve(Capacity); +} + +Expected<void> MachineInstr::addOperand(MachineOperand &&Operand) { + return insertOperand(Operands.size(), std::move(Operand)); +} + +Expected<void> MachineInstr::insertOperand(std::size_t Index, MachineOperand &&Operand) { + if (Index > Operands.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "operand insertion index is out of range"); + if (Operand.Parent != nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "operand already belongs to an instruction"); + if (Operand.TiedOperandIndex) + return makeUnexpected(ErrorCode::InvalidArgument, + "detached operand cannot carry a tied operand index"); + if (const auto *BlockOperand = Operand.getIf<BasicBlockOperand>(); + BlockOperand != nullptr && &BlockOperand->Block->function() != OwnerFunction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "basic block operand belongs to another machine function"); + } + if (const Register *Reg = Operand.getIf<Register>(); + Reg != nullptr && Reg->isVirtual() && + !OwnerFunction->registerInfo().isKnownVirtualRegister(*Reg)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register operand references an unknown virtual register"); + } + if (Operand.isUse() && isDebugInstr()) { + RegisterOperandFlags RegisterFlags = Operand.registerFlags(); + RegisterFlags.set(RegisterOperandFlag::Debug); + auto Updated = Operand.setRegisterFlags(RegisterFlags); + if (!Updated) + return std::unexpected(Updated.error()); + } + + Operands.insert(Operands.begin() + static_cast<std::ptrdiff_t>(Index), std::move(Operand)); + adjustTiesForInsertion(Index); + Operands[Index].attachTo(*this, linkedRegisterInfo()); + return {}; +} + +Expected<void> MachineInstr::eraseOperand(std::size_t Index) { + if (Index >= Operands.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "operand erase index is out of range"); + + adjustTiesForErasure(Index); + Operands.erase(Operands.begin() + static_cast<std::ptrdiff_t>(Index)); + return {}; +} + +Expected<void> MachineInstr::tieOperands(std::size_t DefIndex, std::size_t UseIndex) { + if (DefIndex >= Operands.size() || UseIndex >= Operands.size() || DefIndex == UseIndex) { + return makeUnexpected(ErrorCode::InvalidArgument, "tied operand indices are invalid"); + } + + MachineOperand &DefOperand = Operands[DefIndex]; + MachineOperand &UseOperand = Operands[UseIndex]; + if (!DefOperand.isRegister() || !UseOperand.isRegister() || !DefOperand.isDef() || + DefOperand.isUse() || !UseOperand.isUse() || UseOperand.isDef()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "tie must connect a pure register definition and use"); + } + if (DefOperand.TiedOperandIndex || UseOperand.TiedOperandIndex) { + return makeUnexpected(ErrorCode::InvalidArgument, + "an operand cannot participate in more than one tie"); + } + + DefOperand.setTiedOperandIndex(UseIndex); + UseOperand.setTiedOperandIndex(DefIndex); + return {}; +} + +Expected<void> MachineInstr::untieOperand(std::size_t Index) { + if (Index >= Operands.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "operand tie index is out of range"); + const auto OtherIndex = Operands[Index].TiedOperandIndex; + if (!OtherIndex) + return makeUnexpected(ErrorCode::InvalidArgument, "operand is not tied"); + if (*OtherIndex >= Operands.size() || Operands[*OtherIndex].TiedOperandIndex != Index) { + return makeUnexpected(ErrorCode::VerificationError, "operand tie relation is not reciprocal"); + } + + Operands[*OtherIndex].setTiedOperandIndex(std::nullopt); + Operands[Index].setTiedOperandIndex(std::nullopt); + return {}; +} + +Expected<void> MachineInstr::setDebugLocation(MachineDebugLocation NewLocation) { + if (!NewLocation.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine instruction debug location must be valid"); + } + DebugLocation = std::move(NewLocation); + return {}; +} + +std::optional<std::string_view> MachineInstr::preInstrSymbol() const noexcept { + if (!PreInstrSymbol) + return std::nullopt; + return *PreInstrSymbol; +} + +Expected<void> MachineInstr::setPreInstrSymbol(std::string Symbol) { + if (Symbol.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "pre-instruction symbol cannot be empty"); + PreInstrSymbol = std::move(Symbol); + return {}; +} + +std::optional<std::string_view> MachineInstr::postInstrSymbol() const noexcept { + if (!PostInstrSymbol) + return std::nullopt; + return *PostInstrSymbol; +} + +Expected<void> MachineInstr::setPostInstrSymbol(std::string Symbol) { + if (Symbol.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "post-instruction symbol cannot be empty"); + PostInstrSymbol = std::move(Symbol); + return {}; +} + +Expected<void> MachineInstr::setHeapAllocMarker(MachineMetadataReference Marker) { + if (!Marker.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "heap-allocation marker must be valid"); + HeapAllocMarker = Marker; + return {}; +} + +Expected<void> MachineInstr::setPCSections(MachineMetadataReference Sections) { + if (!Sections.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "PC sections metadata must be valid"); + PCSections = Sections; + return {}; +} + +Expected<void> MachineInstr::setMMRA(MachineMetadataReference Metadata) { + if (!Metadata.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "MMRA metadata must be valid"); + MMRA = Metadata; + return {}; +} + +Expected<void> MachineInstr::setCFIType(std::uint64_t Type) { + if (Type == 0 || Type > std::numeric_limits<std::uint32_t>::max()) + return makeUnexpected(ErrorCode::InvalidArgument, "CFI type must fit in nonzero uint32"); + CFIType = static_cast<std::uint32_t>(Type); + return {}; +} + +std::optional<std::string_view> MachineInstr::deactivationSymbol() const noexcept { + if (!DeactivationSymbol) + return std::nullopt; + return *DeactivationSymbol; +} + +Expected<void> MachineInstr::setDeactivationSymbol(std::string Symbol) { + if (Symbol.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "deactivation symbol cannot be empty"); + DeactivationSymbol = std::move(Symbol); + return {}; +} + +Expected<void> MachineInstr::addMemOperand(const MachineMemOperand &MemOperand) { + if (!OwnerFunction->ownsMemOperand(MemOperand)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine memory operand belongs to another function"); + } + MemOperands.push_back(&MemOperand); + return {}; +} + +Expected<void> MachineInstr::replaceMemOperand(std::size_t Index, + const MachineMemOperand &MemOperand) { + if (Index >= MemOperands.size()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine memory operand index is out of range"); + } + if (!OwnerFunction->ownsMemOperand(MemOperand)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "replacement memory operand belongs to another function"); + } + MemOperands[Index] = &MemOperand; + return {}; +} + +Expected<void> MachineInstr::eraseMemOperand(std::size_t Index) { + if (Index >= MemOperands.size()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine memory operand index is out of range"); + } + MemOperands.erase(MemOperands.begin() + static_cast<std::ptrdiff_t>(Index)); + return {}; +} + +MachineRegisterInfo *MachineInstr::linkedRegisterInfo() noexcept { + if (!isLinked()) + return nullptr; + return &OwnerFunction->registerInfo(); +} + +Expected<void> MachineInstr::attachToBlock(MachineBasicBlock &Block) { + if (ParentBlock != nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction is already linked to a basic block"); + } + + MachineRegisterInfo &RegInfo = OwnerFunction->registerInfo(); + for (const MachineOperand &Operand : Operands) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && Reg->isVirtual() && !RegInfo.isKnownVirtualRegister(*Reg)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction references an unknown virtual register"); + } + } + + ParentBlock = &Block; + for (MachineOperand &Operand : Operands) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && !Reg->isNoRegister()) + Operand.linkToRegInfo(RegInfo); + } + return {}; +} + +void MachineInstr::detachFromBlock() noexcept { + for (MachineOperand &Operand : Operands) + Operand.unlinkFromRegInfo(); + ParentBlock = nullptr; + PreviousInstruction = nullptr; + NextInstruction = nullptr; +} + +void MachineInstr::adjustTiesForInsertion(std::size_t Index) noexcept { + for (MachineOperand &Operand : Operands) { + if (Operand.TiedOperandIndex && *Operand.TiedOperandIndex >= Index) + ++*Operand.TiedOperandIndex; + } +} + +void MachineInstr::adjustTiesForErasure(std::size_t Index) noexcept { + for (MachineOperand &Operand : Operands) { + if (!Operand.TiedOperandIndex) + continue; + if (*Operand.TiedOperandIndex == Index) + Operand.TiedOperandIndex.reset(); + else if (*Operand.TiedOperandIndex > Index) + --*Operand.TiedOperandIndex; + } +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineJumpTableInfo.cpp b/backend/lib/CodeGen/MachineJumpTableInfo.cpp new file mode 100644 index 0000000..c832e7c --- /dev/null +++ b/backend/lib/CodeGen/MachineJumpTableInfo.cpp @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineJumpTableInfo.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" + +#include <bit> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <utility> + +namespace rcc { + +Expected<void> +MachineJumpTableInfo::verifyDescription(const MachineJumpTableDesc &Description) const { + if (Description.Encoding >= JumpTableEntryEncoding::Count) + return makeUnexpected(ErrorCode::VerificationError, "jump table encoding is invalid"); + if (Description.EntrySize == 0 || Description.EntrySize > 16 || + !std::has_single_bit(Description.EntrySize)) { + return makeUnexpected(ErrorCode::VerificationError, + "jump table entry size must be a power of two no larger than 16 bytes"); + } + if (Description.Targets.empty()) + return makeUnexpected(ErrorCode::VerificationError, "jump table must have at least one target"); + if (Description.Placement.MaximumDistance && *Description.Placement.MaximumDistance == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "jump table maximum distance must be nonzero"); + } + if ((Description.Encoding == JumpTableEntryEncoding::Target) != + (Description.TargetEncoding != 0)) { + return makeUnexpected(ErrorCode::VerificationError, + "target jump table encoding and target encoding ID are inconsistent"); + } + if ((Description.Encoding == JumpTableEntryEncoding::GPRelative32BlockAddress || + Description.Encoding == JumpTableEntryEncoding::LabelDifference32 || + Description.Encoding == JumpTableEntryEncoding::Custom32) && + Description.EntrySize != 4) { + return makeUnexpected(ErrorCode::VerificationError, + "32-bit jump table encodings require four-byte entries"); + } + if ((Description.Encoding == JumpTableEntryEncoding::GPRelative64BlockAddress || + Description.Encoding == JumpTableEntryEncoding::LabelDifference64) && + Description.EntrySize != 8) { + return makeUnexpected(ErrorCode::VerificationError, + "64-bit jump table encodings require eight-byte entries"); + } + + for (const MachineBasicBlock *Target : Description.Targets) { + if (Target == nullptr || &Target->function() != Owner || !Target->isLinked()) { + return makeUnexpected(ErrorCode::VerificationError, + "jump table target must be linked to its owning function"); + } + } + return {}; +} + +Expected<JumpTableIndex> MachineJumpTableInfo::createTable(MachineJumpTableDesc Description) { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, + "frozen jump table info cannot create tables"); + auto Verification = verifyDescription(Description); + if (!Verification) + return std::unexpected(Verification.error()); + if (Tables.size() > std::numeric_limits<std::uint32_t>::max()) + return makeUnexpected(ErrorCode::Unsupported, "jump table index space is exhausted"); + + const JumpTableIndex Index = JumpTableIndex::fromValue(static_cast<std::uint32_t>(Tables.size())); + Tables.push_back(MachineJumpTableEntry(Index, std::move(Description))); + return Index; +} + +Expected<JumpTableIndex> MachineJumpTableInfo::reserveTable() { + if (Frozen) { + return makeUnexpected(ErrorCode::InvalidArgument, + "frozen jump table info cannot reserve tables"); + } + if (Tables.size() > std::numeric_limits<std::uint32_t>::max()) + return makeUnexpected(ErrorCode::Unsupported, "jump table index space is exhausted"); + + const JumpTableIndex Index = JumpTableIndex::fromValue(static_cast<std::uint32_t>(Tables.size())); + Tables.push_back(MachineJumpTableEntry(Index)); + return Index; +} + +Expected<void> MachineJumpTableInfo::defineReservedTable(JumpTableIndex Index, + MachineJumpTableDesc Description) { + if (Frozen) { + return makeUnexpected(ErrorCode::InvalidArgument, + "frozen jump table info cannot define reserved tables"); + } + if (Index.value() >= Tables.size() || !Tables[Index.value()].isPending()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "jump table index is not an unresolved reservation"); + } + auto Verification = verifyDescription(Description); + if (!Verification) + return std::unexpected(Verification.error()); + Tables[Index.value()].Description.emplace<MachineJumpTableDesc>(std::move(Description)); + return {}; +} + +Expected<MachineJumpTableEntry *> MachineJumpTableInfo::mutableTable(JumpTableIndex Index) { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, + "frozen jump table info cannot modify tables"); + if (Index.value() >= Tables.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "jump table index is out of range"); + MachineJumpTableEntry &Entry = Tables[Index.value()]; + if (Entry.isTombstone() || Entry.isPending()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "jump table index has no defined description"); + } + return &Entry; +} + +Expected<void> MachineJumpTableInfo::updateTable(JumpTableIndex Index, + MachineJumpTableDesc Description) { + auto Verification = verifyDescription(Description); + if (!Verification) + return std::unexpected(Verification.error()); + auto Entry = mutableTable(Index); + if (!Entry) + return std::unexpected(Entry.error()); + (*Entry)->Description.emplace<MachineJumpTableDesc>(std::move(Description)); + return {}; +} + +Expected<void> MachineJumpTableInfo::eraseTable(JumpTableIndex Index) { + auto Entry = mutableTable(Index); + if (!Entry) + return std::unexpected(Entry.error()); + (*Entry)->Description.emplace<std::monostate>(); + return {}; +} + +const MachineJumpTableEntry *MachineJumpTableInfo::table(JumpTableIndex Index) const noexcept { + if (Index.value() >= Tables.size() || Tables[Index.value()].isTombstone()) + return nullptr; + return &Tables[Index.value()]; +} + +std::size_t MachineJumpTableInfo::liveTableCount() const noexcept { + std::size_t Count = 0; + for (const MachineJumpTableEntry &Entry : Tables) { + if (!Entry.isTombstone()) + ++Count; + } + return Count; +} + +Expected<void> MachineJumpTableInfo::freeze() { + if (Frozen) + return makeUnexpected(ErrorCode::InvalidArgument, "jump table info is already frozen"); + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + Frozen = true; + return {}; +} + +Expected<void> MachineJumpTableInfo::verify() const { + if (Owner == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "jump table info has no owner"); + for (std::size_t Position = 0; Position < Tables.size(); ++Position) { + const MachineJumpTableEntry &Entry = Tables[Position]; + if (Entry.index() != JumpTableIndex::fromValue(static_cast<std::uint32_t>(Position))) { + return makeUnexpected(ErrorCode::VerificationError, + "jump table index does not match its stable table position"); + } + const MachineJumpTableDesc *Description = Entry.description(); + if (Entry.isPending()) { + return makeUnexpected(ErrorCode::VerificationError, + "jump table contains an unresolved reservation"); + } + if (Description == nullptr) + continue; + auto Verification = verifyDescription(*Description); + if (!Verification) + return std::unexpected(Verification.error()); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineLICM.cpp b/backend/lib/CodeGen/MachineLICM.cpp new file mode 100644 index 0000000..3863388 --- /dev/null +++ b/backend/lib/CodeGen/MachineLICM.cpp @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineLICM.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &licmContract(MachineCodeMotionStage Stage) { + static const MachinePassContract Selected = [] { + MachinePassContract Value{ + .Name = "machine-licm", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineFunctionAnalysisManager::id<MachineLoopAnalysis>(), + MachineFunctionAnalysisManager::id<MachineBlockFrequencyAnalysis>()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-licm", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineFunctionAnalysisManager::id<MachineLoopAnalysis>(), + MachineFunctionAnalysisManager::id<MachineBlockFrequencyAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCodeMotionStage::SelectedSSA ? Selected : PostRA; +} + +[[nodiscard]] bool containsBlock(const MachineLoop &Loop, const MachineBasicBlock *Block) noexcept { + return std::ranges::find(Loop.Blocks, Block) != Loop.Blocks.end(); +} + +[[nodiscard]] MachineBasicBlock *preheader(const MachineLoop &Loop) noexcept { + if (Loop.Header == nullptr) + return nullptr; + MachineBasicBlock *Result = nullptr; + for (MachineBasicBlock *Predecessor : Loop.Header->predecessors()) { + if (containsBlock(Loop, Predecessor)) + continue; + if (Result != nullptr) + return nullptr; + Result = Predecessor; + } + if (Result == nullptr || Result->successors().size() != 1 || + Result->successors().front().Block != Loop.Header) { + return nullptr; + } + return Result; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +[[nodiscard]] Expected<bool> isSafeToHoist(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target, + MachineCodeMotionStage Stage) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine LICM encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Pseudo) || Flags.has(InstructionFlag::Meta) || + Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Terminator) || + Flags.has(InstructionFlag::Branch) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::MayLoad) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { + return false; + } + if (Flags.has(InstructionFlag::MayRaiseFPException) && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) + return false; + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.kind() == MachineOperand::Kind::RegisterMask || + Operand.kind() == MachineOperand::Kind::LiveOutRegisterMask) { + return false; + } + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr) + continue; + if (Operand.isDebugUse() || Operand.isImplicit() || Operand.tiedOperandIndex() || + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation) { + if (!Reg->isPhysical()) + return false; + HasDefinition |= Operand.isDef(); + continue; + } + if (!Reg->isVirtual()) + return false; + if (Operand.isDef()) + HasDefinition = true; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + } + return HasDefinition; +} + +[[nodiscard]] std::size_t definitionCount(const MachineInstr &Instruction) noexcept { + return static_cast<std::size_t>(std::ranges::count_if( + Instruction.operands(), [](const MachineOperand &Operand) { return Operand.isDef(); })); +} + +[[nodiscard]] bool +operandsAreInvariant(const MachineInstr &Instruction, const MachineLoop &Loop, + const MachineRegisterInfo &Registers, + const std::unordered_set<const MachineInstr *> &Hoisted) noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Operand.isUse()) + continue; + if (!Reg->isVirtual()) + return false; + const MachineOperand *Definition = Registers.getUniqueVRegDef(*Reg); + const MachineInstr *DefiningInstruction = + Definition == nullptr ? nullptr : Definition->parent(); + const MachineBasicBlock *DefiningBlock = + DefiningInstruction == nullptr ? nullptr : DefiningInstruction->parent(); + if (DefiningInstruction == nullptr || DefiningBlock == nullptr) + return false; + if (containsBlock(Loop, DefiningBlock) && !Hoisted.contains(DefiningInstruction)) + return false; + } + return true; +} + +[[nodiscard]] bool postRAOperandsAreInvariant(const MachineInstr &Instruction, + const MachineLoop &Loop, + const MachineFunction &Function, + const TargetDescription &Target) noexcept { + if (Loop.Header == nullptr || Instruction.parent() != Loop.Header) + return false; + + std::vector<PhysicalRegister> Uses; + std::vector<PhysicalRegister> Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if (Operand.isUse()) + Uses.push_back(*Physical); + if (Operand.isDef()) { + if (Target.registers().isReservedRegister(*Physical) || + Function.registerInfo().isReserved(*Physical)) { + return false; + } + Definitions.push_back(*Physical); + } + } + if (Definitions.empty()) + return false; + for (std::size_t Left = 0; Left < Definitions.size(); ++Left) { + if (std::ranges::any_of(Loop.Header->liveIns(), + [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definitions[Left], + LiveIn.PhysReg); + }) || + std::ranges::any_of(Uses, [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(Definitions[Left], Use); + })) { + return false; + } + for (std::size_t Right = Left + 1U; Right < Definitions.size(); ++Right) { + if (Target.registers().regsOverlap(Definitions[Left], Definitions[Right])) + return false; + } + } + + const auto ClobbersCandidate = [&](const RegisterMask &Mask) { + return std::ranges::any_of(Uses, [&](PhysicalRegister Reg) { return Mask.clobbers(Reg); }) || + std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask.clobbers(Reg); }); + }; + for (const MachineBasicBlock *Block : Loop.Blocks) { + if (Block == nullptr) + return false; + for (const MachineInstr *Candidate = Block->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + if (Candidate == &Instruction) + continue; + for (const MachineOperand &Operand : Candidate->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + if (ClobbersCandidate(*Mask->Mask)) + return false; + continue; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical || !Operand.isDef()) + continue; + if (std::ranges::any_of(Uses, + [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(*Physical, Use); + }) || + std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return Target.registers().regsOverlap(*Physical, Definition); + })) { + return false; + } + } + } + } + + for (const MachineInstr *Before = Loop.Header->firstInstruction(); + Before != nullptr && Before != &Instruction; Before = Before->next()) { + for (const MachineOperand &Operand : Before->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + if (std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); })) { + return false; + } + continue; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (Physical && (Operand.isUse() || Operand.isDef()) && + std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return Target.registers().regsOverlap(*Physical, Definition); + })) { + return false; + } + } + } + return true; +} + +[[nodiscard]] Expected<MachineInstr *> firstTerminator(MachineBasicBlock &Block, + const TargetDescription &Target) { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine LICM preheader contains an unknown instruction"); + } + if (Descriptor->Flags.has(InstructionFlag::Terminator)) + return Instruction; + } + return nullptr; +} + +[[nodiscard]] Expected<void> moveToPreheader(MachineInstr &Instruction, + MachineBasicBlock &Preheader, + const TargetDescription &Target) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine LICM candidate is detached"); + auto Anchor = firstTerminator(Preheader, Target); + if (!Anchor) + return std::unexpected(Anchor.error()); + auto Removed = Instruction.eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + if (*Anchor != nullptr) + return Preheader.insertInstructionBefore(**Anchor, Instruction); + return Preheader.appendInstruction(Instruction); +} + +} // namespace + +const MachinePassContract &MachineLICMPass::contract() const noexcept { + return licmContract(Stage); +} + +Expected<MachinePassResult> MachineLICMPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +MachineLICMPass::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Loops = Analyses.getResult<MachineLoopAnalysis>(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + + MachinePassResult Result; + std::size_t TotalHoisted = 0; + for (const MachineLoop &Loop : (*Loops)->loops()) { + MachineBasicBlock *Preheader = preheader(Loop); + if (Preheader == nullptr || Loop.Header == nullptr || + (*Frequencies)->frequency(*Preheader) > (*Frequencies)->frequency(*Loop.Header)) { + continue; + } + std::unordered_set<const MachineInstr *> Hoisted; + std::size_t HoistedDefinitions = 0; + bool LocalChange = true; + while (LocalChange && TotalHoisted < Options.MaximumHoistedInstructions) { + LocalChange = false; + for (MachineBasicBlock *Block : Loop.Blocks) { + if (Block == nullptr) + continue; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + auto Safe = isSafeToHoist(*Instruction, Function, Target, Stage); + if (!Safe) + return std::unexpected(Safe.error()); + const std::size_t Definitions = definitionCount(*Instruction); + const bool Invariant = + Stage == MachineCodeMotionStage::SelectedSSA + ? operandsAreInvariant(*Instruction, Loop, Function.registerInfo(), Hoisted) + : postRAOperandsAreInvariant(*Instruction, Loop, Function, Target); + if (*Safe && Invariant && + Definitions <= + Options.MaximumHoistedDefinitionsPerLoop - + std::min(HoistedDefinitions, Options.MaximumHoistedDefinitionsPerLoop)) { + auto Moved = moveToPreheader(*Instruction, *Preheader, Target); + if (!Moved) + return std::unexpected(Moved.error()); + Hoisted.insert(Instruction); + HoistedDefinitions += Definitions; + ++TotalHoisted; + LocalChange = true; + Result.Changed = true; + if (TotalHoisted >= Options.MaximumHoistedInstructions) + break; + } + Instruction = Next; + } + if (TotalHoisted >= Options.MaximumHoistedInstructions) + break; + } + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions}; + Result.Preserved.preserve<MachineDominatorTreeAnalysis>(); + Result.Preserved.preserve<MachineLoopAnalysis>(); + Result.Preserved.preserve<MachineBranchProbabilityAnalysis>(); + Result.Preserved.preserve<MachineBlockFrequencyAnalysis>(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineMemOperand.cpp b/backend/lib/CodeGen/MachineMemOperand.cpp new file mode 100644 index 0000000..0ae6941 --- /dev/null +++ b/backend/lib/CodeGen/MachineMemOperand.cpp @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineMemOperand.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <ranges> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] std::uint64_t effectiveAlignment(std::uint64_t BaseAlignment, + std::int64_t Offset) noexcept { + if (Offset == 0) + return BaseAlignment; + const auto OffsetBits = static_cast<std::uint64_t>(Offset); + const std::uint64_t OffsetAlignment = std::uint64_t{1} << std::countr_zero(OffsetBits); + return std::min(BaseAlignment, OffsetAlignment); +} + +[[nodiscard]] bool isValidAtomicLoadOrdering(AtomicOrdering Ordering) noexcept { + return Ordering == AtomicOrdering::Unordered || Ordering == AtomicOrdering::Monotonic || + Ordering == AtomicOrdering::Acquire || Ordering == AtomicOrdering::SequentiallyConsistent; +} + +[[nodiscard]] bool isValidAtomicStoreOrdering(AtomicOrdering Ordering) noexcept { + return Ordering == AtomicOrdering::Unordered || Ordering == AtomicOrdering::Monotonic || + Ordering == AtomicOrdering::Release || Ordering == AtomicOrdering::SequentiallyConsistent; +} + +[[nodiscard]] bool isValidAtomicReadModifyWriteOrdering(AtomicOrdering Ordering) noexcept { + return Ordering == AtomicOrdering::Monotonic || Ordering == AtomicOrdering::Acquire || + Ordering == AtomicOrdering::Release || Ordering == AtomicOrdering::AcquireRelease || + Ordering == AtomicOrdering::SequentiallyConsistent; +} + +[[nodiscard]] bool isValidCompareExchangeFailureOrdering(AtomicOrdering Ordering) noexcept { + return Ordering == AtomicOrdering::NotAtomic || Ordering == AtomicOrdering::Monotonic || + Ordering == AtomicOrdering::Acquire || Ordering == AtomicOrdering::SequentiallyConsistent; +} + +[[nodiscard]] bool isFailureOrderingNoStrongerThanSuccess(AtomicOrdering Failure, + AtomicOrdering Success) noexcept { + switch (Failure) { + case AtomicOrdering::NotAtomic: + return true; + case AtomicOrdering::Monotonic: + return isValidAtomicReadModifyWriteOrdering(Success); + case AtomicOrdering::Acquire: + return Success == AtomicOrdering::Acquire || Success == AtomicOrdering::AcquireRelease || + Success == AtomicOrdering::SequentiallyConsistent; + case AtomicOrdering::SequentiallyConsistent: + return Success == AtomicOrdering::SequentiallyConsistent; + case AtomicOrdering::Unordered: + case AtomicOrdering::Release: + case AtomicOrdering::AcquireRelease: + return false; + } + + return false; +} + +[[nodiscard]] Expected<void> verifyAtomicDescription(const MachineMemOperandDesc &Description) { + const bool IsLoad = Description.Accesses.has(MemoryAccessFlag::Load); + const bool IsStore = Description.Accesses.has(MemoryAccessFlag::Store); + + switch (Description.AtomicKind) { + case AtomicOperation::None: + if (Description.SuccessOrdering != AtomicOrdering::NotAtomic || + Description.FailureOrdering != AtomicOrdering::NotAtomic) { + return makeUnexpected(ErrorCode::InvalidArgument, + "non-atomic memory operands cannot carry atomic orderings"); + } + return {}; + case AtomicOperation::Load: + if (!IsLoad || IsStore) { + return makeUnexpected(ErrorCode::InvalidArgument, + "atomic load memory operands must be load-only"); + } + if (!isValidAtomicLoadOrdering(Description.SuccessOrdering) || + Description.FailureOrdering != AtomicOrdering::NotAtomic) { + return makeUnexpected(ErrorCode::InvalidArgument, + "atomic load memory operand has an invalid ordering"); + } + return {}; + case AtomicOperation::Store: + if (IsLoad || !IsStore) { + return makeUnexpected(ErrorCode::InvalidArgument, + "atomic store memory operands must be store-only"); + } + if (!isValidAtomicStoreOrdering(Description.SuccessOrdering) || + Description.FailureOrdering != AtomicOrdering::NotAtomic) { + return makeUnexpected(ErrorCode::InvalidArgument, + "atomic store memory operand has an invalid ordering"); + } + return {}; + case AtomicOperation::ReadModifyWrite: + if (!IsLoad || !IsStore) { + return makeUnexpected(ErrorCode::InvalidArgument, + "atomic read-modify-write memory operands must load and store"); + } + if (!isValidAtomicReadModifyWriteOrdering(Description.SuccessOrdering) || + Description.FailureOrdering != AtomicOrdering::NotAtomic) { + return makeUnexpected(ErrorCode::InvalidArgument, + "atomic read-modify-write memory operand has an invalid ordering"); + } + return {}; + case AtomicOperation::CompareExchange: + if (!IsLoad || !IsStore) { + return makeUnexpected(ErrorCode::InvalidArgument, + "compare-exchange memory operands must load and store"); + } + if (!isValidAtomicReadModifyWriteOrdering(Description.SuccessOrdering) || + !isValidCompareExchangeFailureOrdering(Description.FailureOrdering) || + !isFailureOrderingNoStrongerThanSuccess(Description.FailureOrdering, + Description.SuccessOrdering)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "compare-exchange memory operand has incompatible orderings"); + } + return {}; + } + + return makeUnexpected(ErrorCode::InternalError, "unknown atomic operation"); +} + +} // namespace + +MemoryAccessFlags::MemoryAccessFlags(std::initializer_list<MemoryAccessFlag> Flags) noexcept { + for (const MemoryAccessFlag Flag : Flags) + set(Flag); +} + +Expected<MemorySize> MemorySize::fixed(std::uint64_t Bytes) { + if (Bytes == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "fixed memory size must be nonzero"); + return MemorySize(Kind::Fixed, Bytes); +} + +Expected<MemorySize> MemorySize::scalable(std::uint64_t MinimumBytes) { + if (MinimumBytes == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scalable memory size minimum must be nonzero"); + } + return MemorySize(Kind::Scalable, MinimumBytes); +} + +Expected<MemoryOffsetRange> MemoryOffsetRange::create(std::int64_t Lower, std::int64_t Upper) { + if (Lower > Upper) { + return makeUnexpected(ErrorCode::InvalidArgument, + "memory offset range lower bound cannot exceed its upper bound"); + } + return MemoryOffsetRange{Lower, Upper}; +} + +MemoryBase MemoryBase::frameIndex(FrameIndex Index) { + return MemoryBase(Kind::FrameIndex, Index); +} + +MemoryBase MemoryBase::constantPool(ConstantPoolIndex Index) { + return MemoryBase(Kind::ConstantPool, Index); +} + +MemoryBase MemoryBase::jumpTable(JumpTableIndex Index) { + return MemoryBase(Kind::JumpTable, Index); +} + +MemoryBase MemoryBase::globalSymbol(std::string Name) { + return MemoryBase(Kind::GlobalSymbol, std::move(Name)); +} + +MemoryBase MemoryBase::externalSymbol(std::string Name) { + return MemoryBase(Kind::ExternalSymbol, std::move(Name)); +} + +MemoryBase MemoryBase::irValue(std::string Name) { + return MemoryBase(Kind::IRValue, std::move(Name)); +} + +Expected<MemoryBase> MemoryBase::irConstant(GenericConstantData Constant) { + auto Verification = Constant.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + if (Constant.Type.Kind != GenericConstantTypeKind::Pointer) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine memory IR constant base must have pointer type"); + } + return MemoryBase(Kind::IRConstant, std::move(Constant)); +} + +MemoryBase MemoryBase::stack() { + return MemoryBase(Kind::Stack, std::monostate{}); +} + +MemoryBase MemoryBase::got() { + return MemoryBase(Kind::GOT, std::monostate{}); +} + +MemoryBase MemoryBase::constantPoolPseudo() { + return MemoryBase(Kind::ConstantPoolPseudo, std::monostate{}); +} + +MemoryBase MemoryBase::jumpTablePseudo() { + return MemoryBase(Kind::JumpTablePseudo, std::monostate{}); +} + +MemoryBase MemoryBase::callEntryGlobal(std::string Name) { + return MemoryBase(Kind::CallEntryGlobal, std::move(Name)); +} + +MemoryBase MemoryBase::callEntryExternal(std::string Name) { + return MemoryBase(Kind::CallEntryExternal, std::move(Name)); +} + +MemoryBase MemoryBase::custom(std::string Name) { + return MemoryBase(Kind::Custom, std::move(Name)); +} + +std::optional<FrameIndex> MemoryBase::frameIndex() const noexcept { + if (BaseKind != Kind::FrameIndex) + return std::nullopt; + return std::get<FrameIndex>(Value); +} + +std::optional<ConstantPoolIndex> MemoryBase::constantPoolIndex() const noexcept { + if (BaseKind != Kind::ConstantPool) + return std::nullopt; + return std::get<ConstantPoolIndex>(Value); +} + +std::optional<JumpTableIndex> MemoryBase::jumpTableIndex() const noexcept { + if (BaseKind != Kind::JumpTable) + return std::nullopt; + return std::get<JumpTableIndex>(Value); +} + +std::optional<std::string_view> MemoryBase::symbolName() const noexcept { + if (BaseKind != Kind::GlobalSymbol && BaseKind != Kind::ExternalSymbol && + BaseKind != Kind::IRValue && BaseKind != Kind::CallEntryGlobal && + BaseKind != Kind::CallEntryExternal && BaseKind != Kind::Custom) { + return std::nullopt; + } + return std::get<std::string>(Value); +} + +const GenericConstantData *MemoryBase::constantValue() const noexcept { + if (BaseKind != Kind::IRConstant) + return nullptr; + return std::get_if<GenericConstantData>(&Value); +} + +Expected<MachineMemOperand> MachineMemOperand::create(MachineMemOperandDesc Description) { + const bool IsLoad = Description.Accesses.has(MemoryAccessFlag::Load); + const bool IsStore = Description.Accesses.has(MemoryAccessFlag::Store); + if (!IsLoad && !IsStore) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine memory operand must describe a load or store"); + } + if (Description.AddressSpace > LowLevelType::MaxAddressSpace) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine memory operand address space must fit in 24 bits"); + } + if (Description.BaseAlign) { + const std::uint64_t Effective = + effectiveAlignment(Description.BaseAlign->bytes(), Description.Offset); + if (Effective != Description.Align.bytes()) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "machine memory operand alignment is inconsistent with base alignment and offset"); + } + if (*Description.BaseAlign == Description.Align) + Description.BaseAlign.reset(); + } + if (Description.Accesses.has(MemoryAccessFlag::Invariant) && (!IsLoad || IsStore)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "invariant memory operands must be load-only"); + } + + const auto AtomicResult = verifyAtomicDescription(Description); + if (!AtomicResult) + return std::unexpected(AtomicResult.error()); + + const auto SymbolName = Description.Base.symbolName(); + if (SymbolName && SymbolName->empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, "memory base symbol name cannot be empty"); + } + const std::array MetadataReferences{ + Description.TBAA, Description.AliasScope, + Description.NoAlias, Description.NoAliasAddressSpace, + Description.RangeMetadata, + }; + if (std::ranges::any_of(MetadataReferences, [](const auto &Reference) { + return Reference.has_value() && !Reference->isValid(); + })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine memory operand contains an invalid metadata reference"); + } + + return MachineMemOperand(std::move(Description)); +} + +std::string_view getAtomicOrderingName(AtomicOrdering Ordering) noexcept { + switch (Ordering) { + case AtomicOrdering::NotAtomic: + return "not-atomic"; + case AtomicOrdering::Unordered: + return "unordered"; + case AtomicOrdering::Monotonic: + return "monotonic"; + case AtomicOrdering::Acquire: + return "acquire"; + case AtomicOrdering::Release: + return "release"; + case AtomicOrdering::AcquireRelease: + return "acquire-release"; + case AtomicOrdering::SequentiallyConsistent: + return "sequentially-consistent"; + } + + return "unknown"; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineMemoryFolding.cpp b/backend/lib/CodeGen/MachineMemoryFolding.cpp new file mode 100644 index 0000000..fb4b86d --- /dev/null +++ b/backend/lib/CodeGen/MachineMemoryFolding.cpp @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineMemoryFolding.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetMemoryFolding.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &memoryFoldingContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-memory-folding", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +struct FoldCandidate final { + MachineInstr *MemoryInstruction = nullptr; + MachineInstr *Consumer = nullptr; + Register DefinedRegister; + std::size_t ConsumerOperand = 0; +}; + +[[nodiscard]] std::optional<FoldCandidate> findCandidate(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr || !Descriptor->Flags.has(InstructionFlag::MayLoad) || + Descriptor->Flags.has(InstructionFlag::MayStore) || + Descriptor->Flags.has(InstructionFlag::Call) || + Descriptor->Flags.has(InstructionFlag::Barrier) || + Descriptor->Flags.has(InstructionFlag::UnmodeledSideEffects) || + hasProtectedAttachment(Instruction) || Instruction.memOperands().empty()) { + return std::nullopt; + } + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory == nullptr || Memory->isStore() || Memory->isAtomic() || Memory->isVolatile()) { + return std::nullopt; + } + } + + MachineOperand *Definition = nullptr; + Register DefinedRegister; + for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { + MachineOperand *Operand = Instruction.operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Operand->isDef()) + continue; + if (Definition != nullptr || !Reg->isVirtual()) + return std::nullopt; + Definition = Operand; + DefinedRegister = *Reg; + } + if (Definition == nullptr || !Function.registerInfo().hasOneUse(DefinedRegister, true)) { + return std::nullopt; + } + MachineOperand *Use = nullptr; + for (MachineOperand &Candidate : Function.registerInfo().useOperands(DefinedRegister, false)) { + Use = &Candidate; + break; + } + MachineInstr *Consumer = Use == nullptr ? nullptr : Use->parent(); + if (Consumer == nullptr || Consumer->parent() != Instruction.parent() || + Instruction.next() != Consumer || hasProtectedAttachment(*Consumer) || Use->isImplicit() || + Use->tiedOperandIndex()) { + return std::nullopt; + } + for (std::size_t Index = 0; Index < Consumer->operandCount(); ++Index) { + if (Consumer->operandAt(Index) == Use) + return FoldCandidate{&Instruction, Consumer, DefinedRegister, Index}; + } + return std::nullopt; +} + +[[nodiscard]] bool containsMemoryDescription(const MachineInstr &Instruction, + const MachineMemOperandDesc &Description) noexcept { + return std::ranges::any_of(Instruction.memOperands(), + [&Description](const MachineMemOperand *Memory) { + return Memory != nullptr && Memory->description() == Description; + }); +} + +[[nodiscard]] Expected<bool> applyFold(FoldCandidate Candidate, MachineFunction &Function, + const TargetDescription &Target, + const TargetMemoryFoldingInfo &Info) { + auto CanFold = Info.canFoldMemoryOperand(*Candidate.Consumer, Candidate.ConsumerOperand, + *Candidate.MemoryInstruction, Function, Target); + if (!CanFold) + return std::unexpected(CanFold.error()); + if (!*CanFold) + return false; + std::vector<MachineMemOperandDesc> MemoryDescriptions; + for (const MachineMemOperand *Memory : Candidate.MemoryInstruction->memOperands()) { + if (Memory == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory folding encountered a null memory operand"); + } + MemoryDescriptions.push_back(Memory->description()); + } + auto Folded = Info.foldMemoryOperand(*Candidate.Consumer, Candidate.ConsumerOperand, + *Candidate.MemoryInstruction, Function, Target); + if (!Folded) + return std::unexpected(Folded.error()); + if (*Folded == nullptr || (*Folded)->parent() == nullptr || + Candidate.MemoryInstruction->parent() != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold did not replace the consumer and erase the producer"); + } + for (const MachineMemOperandDesc &Description : MemoryDescriptions) { + if (!containsMemoryDescription(**Folded, Description)) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold did not migrate every memory operand"); + } + } + auto FoldedSize = Info.getFoldedMemorySize(**Folded, Function, Target); + if (!FoldedSize) + return std::unexpected(FoldedSize.error()); + if (!FoldedSize->has_value() || !FoldedSize->value().isKnown()) { + return makeUnexpected(ErrorCode::VerificationError, + "target folded instruction has no known folded memory size"); + } + if (MemoryDescriptions.size() == 1 && MemoryDescriptions.front().Size.isKnown() && + FoldedSize->value() != MemoryDescriptions.front().Size) { + return makeUnexpected(ErrorCode::VerificationError, + "target folded memory size disagrees with the migrated MMO"); + } + if (!Function.registerInfo().useEmpty(Candidate.DefinedRegister, true) || + Function.registerInfo().getUniqueVRegDef(Candidate.DefinedRegister) != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "target memory fold retained the load result register"); + } + auto Erased = Function.registerInfo().eraseVirtualRegister(Candidate.DefinedRegister); + if (!Erased) + return std::unexpected(Erased.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineMemoryFoldingPass::contract() const noexcept { + return memoryFoldingContract(); +} + +Expected<MachinePassResult> MachineMemoryFoldingPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + return runImplementation(Function, Target); +} + +Expected<MachinePassResult> +MachineMemoryFoldingPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t FoldCount = 0; + bool LocalChange = true; + while (LocalChange && FoldCount < Options.MaximumFolds) { + LocalChange = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !LocalChange; + Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + std::optional<FoldCandidate> Candidate = findCandidate(*Instruction, Function, Target); + if (!Candidate) + continue; + auto Folded = applyFold(*Candidate, Function, Target, *Info); + if (!Folded) + return std::unexpected(Folded.error()); + if (!*Folded) + continue; + ++FoldCount; + Result.Changed = true; + LocalChange = true; + break; + } + } + } + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineMetadata.cpp b/backend/lib/CodeGen/MachineMetadata.cpp new file mode 100644 index 0000000..756124c --- /dev/null +++ b/backend/lib/CodeGen/MachineMetadata.cpp @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineMetadata.h" + +#include <cstddef> +#include <string> +#include <unordered_set> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] Expected<void> verifyNodeElements(const MachineMetadataNode &Node) { + for (const MachineMetadataElement &Element : Node.Elements) { + if (!Element.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine metadata tuple contains an invalid element"); + } + const MachineMetadataReference *Reference = Element.reference(); + if (Reference != nullptr && !Reference->isValid()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine metadata tuple contains an invalid reference"); + } + } + return {}; +} + +} // namespace + +MachineMetadataElement MachineMetadataElement::null() { + return MachineMetadataElement(NullValue{}); +} + +Expected<MachineMetadataElement> +MachineMetadataElement::reference(MachineMetadataReference Reference) { + if (!Reference.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine metadata element reference must be valid"); + } + return MachineMetadataElement(Reference); +} + +MachineMetadataElement MachineMetadataElement::string(std::string Value) { + return MachineMetadataElement(std::move(Value)); +} + +Expected<void> MachineMetadataInfo::addNode(MachineMetadataNode Node) { + auto Verification = verifyNodeElements(Node); + if (!Verification) + return std::unexpected(Verification.error()); + if (SlotToIndex.contains(Node.Slot)) { + return makeUnexpected(ErrorCode::InvalidArgument, "machine metadata slot is already defined"); + } + + const std::size_t Index = Nodes.size(); + SlotToIndex.emplace(Node.Slot, Index); + Nodes.push_back(std::move(Node)); + return {}; +} + +const MachineMetadataNode *MachineMetadataInfo::node(std::uint32_t Slot) const noexcept { + const auto Position = SlotToIndex.find(Slot); + if (Position == SlotToIndex.end()) + return nullptr; + return &Nodes[Position->second]; +} + +Expected<void> MachineMetadataInfo::verify() const { + if (SlotToIndex.size() != Nodes.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine metadata slot index has an inconsistent size"); + } + + std::unordered_set<std::uint32_t> Slots; + for (std::size_t Index = 0; Index < Nodes.size(); ++Index) { + const MachineMetadataNode &Node = Nodes[Index]; + if (!Slots.insert(Node.Slot).second) { + return makeUnexpected(ErrorCode::VerificationError, + "machine metadata contains duplicate slot definitions"); + } + const auto Position = SlotToIndex.find(Node.Slot); + if (Position == SlotToIndex.end() || Position->second != Index) { + return makeUnexpected(ErrorCode::VerificationError, + "machine metadata slot index is inconsistent"); + } + auto ElementVerification = verifyNodeElements(Node); + if (!ElementVerification) + return ElementVerification; + } + + for (const MachineMetadataNode &Node : Nodes) { + for (const MachineMetadataElement &Element : Node.Elements) { + const MachineMetadataReference *Reference = Element.reference(); + if (Reference != nullptr && Reference->scope() == MachineMetadataScope::MachineFunction && + !Slots.contains(Reference->slot())) { + return makeUnexpected(ErrorCode::VerificationError, + "machine metadata tuple references an undefined function slot"); + } + } + } + return {}; +} + +Expected<MachineMetadataInfo> MachineMetadataInfo::clone() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + + MachineMetadataInfo Result; + for (const MachineMetadataNode &Node : Nodes) { + auto AddResult = Result.addNode(Node); + if (!AddResult) + return std::unexpected(AddResult.error()); + } + return Result; +} + +Expected<void> verifyMachineMetadataReference(MachineMetadataReference Reference, + const MachineMetadataInfo &Metadata) { + if (!Reference.isValid()) { + return makeUnexpected(ErrorCode::VerificationError, "machine metadata reference is invalid"); + } + if (Reference.scope() == MachineMetadataScope::MachineFunction && + Metadata.node(Reference.slot()) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine metadata reference uses an undefined function slot"); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineOperand.cpp b/backend/lib/CodeGen/MachineOperand.cpp new file mode 100644 index 0000000..3994286 --- /dev/null +++ b/backend/lib/CodeGen/MachineOperand.cpp @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineOperand.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <algorithm> +#include <cstdint> +#include <limits> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { + +RegisterOperandFlags::RegisterOperandFlags( + std::initializer_list<RegisterOperandFlag> Flags) noexcept { + for (const RegisterOperandFlag Flag : Flags) + set(Flag); +} + +Expected<void> PredicateOperand::verify() const { + if (const auto *Integer = integerPredicate(); + Integer != nullptr && + static_cast<std::uint8_t>(*Integer) < static_cast<std::uint8_t>(IntegerPredicate::Count)) { + return {}; + } + if (const auto *Floating = floatingPredicate(); + Floating != nullptr && + static_cast<std::uint8_t>(*Floating) < static_cast<std::uint8_t>(FloatingPredicate::Count)) { + return {}; + } + return makeUnexpected(ErrorCode::VerificationError, "predicate operand contains an invalid kind"); +} + +Expected<void> MachineOperand::verifyRegisterFlags(RegisterOperandFlags Flags) { + const bool IsDef = Flags.has(RegisterOperandFlag::Define); + const bool IsUse = Flags.has(RegisterOperandFlag::Use); + if (!IsDef && !IsUse) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register operand must be a definition or use"); + } + if (Flags.has(RegisterOperandFlag::Kill) && !IsUse) { + return makeUnexpected(ErrorCode::InvalidArgument, "kill flag requires a register use"); + } + if (Flags.has(RegisterOperandFlag::Dead) && !IsDef) { + return makeUnexpected(ErrorCode::InvalidArgument, "dead flag requires a register definition"); + } + if (Flags.has(RegisterOperandFlag::Undef) && !IsUse) { + return makeUnexpected(ErrorCode::InvalidArgument, "undef flag requires a register use"); + } + if (Flags.has(RegisterOperandFlag::EarlyClobber) && !IsDef) { + return makeUnexpected(ErrorCode::InvalidArgument, + "early-clobber flag requires a register definition"); + } + if (Flags.has(RegisterOperandFlag::InternalRead) && (!IsDef || !IsUse)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "internal-read flag requires a register def-use operand"); + } + if (IsDef && IsUse && !Flags.has(RegisterOperandFlag::InternalRead)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register def-use operand requires internal-read"); + } + if (Flags.has(RegisterOperandFlag::Debug) && (!IsUse || IsDef)) { + return makeUnexpected(ErrorCode::InvalidArgument, "debug register operand must be a use"); + } + return {}; +} + +Expected<MachineOperand> MachineOperand::registerOperand(Register Reg, RegisterOperandFlags Flags, + SubRegIndex SubReg, + std::uint32_t TargetFlags) { + const auto Validation = verifyRegisterFlags(Flags); + if (!Validation) + return std::unexpected(Validation.error()); + + MachineOperand Operand(Kind::Register, Reg, TargetFlags); + Operand.RegFlags = Flags; + Operand.RegSubReg = SubReg; + return Operand; +} + +MachineOperand MachineOperand::immediate(std::int64_t Value, std::uint32_t TargetFlags) { + return MachineOperand(Kind::Immediate, Value, TargetFlags); +} + +MachineOperand MachineOperand::integerImmediate(APInt Value, std::uint32_t TargetFlags) { + return MachineOperand(Kind::IntegerImmediate, IntegerImmediateOperand{std::move(Value)}, + TargetFlags); +} + +MachineOperand MachineOperand::floatingImmediate(APFloat Bits, std::uint32_t TargetFlags) { + return MachineOperand(Kind::FloatingImmediate, FloatingImmediateOperand{Bits}, TargetFlags); +} + +MachineOperand MachineOperand::basicBlock(MachineBasicBlock &Block, std::uint32_t TargetFlags) { + return MachineOperand(Kind::BasicBlock, BasicBlockOperand{&Block}, TargetFlags); +} + +MachineOperand MachineOperand::frameIndex(FrameIndex Index, std::int64_t Offset, + std::uint32_t TargetFlags) { + return MachineOperand(Kind::FrameIndex, FrameIndexOperand{Index, Offset}, TargetFlags); +} + +MachineOperand MachineOperand::constantPool(ConstantPoolIndex Index, std::int64_t Offset, + std::uint32_t TargetFlags) { + return MachineOperand(Kind::ConstantPoolIndex, ConstantPoolOperand{Index, Offset}, TargetFlags); +} + +MachineOperand MachineOperand::jumpTable(JumpTableIndex Index, std::uint32_t TargetFlags) { + return MachineOperand(Kind::JumpTableIndex, JumpTableOperand{Index}, TargetFlags); +} + +Expected<MachineOperand> MachineOperand::globalSymbol(std::string Name, std::int64_t Offset, + std::uint32_t TargetFlags, + GlobalValueProperties Properties) { + if (Name.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, "global symbol operand name cannot be empty"); + } + return MachineOperand(Kind::GlobalSymbol, + GlobalSymbolOperand{std::move(Name), Offset, Properties}, TargetFlags); +} + +Expected<MachineOperand> MachineOperand::externalSymbol(std::string Name, std::int64_t Offset, + std::uint32_t TargetFlags) { + return MachineOperand(Kind::ExternalSymbol, ExternalSymbolOperand{std::move(Name), Offset}, + TargetFlags); +} + +Expected<MachineOperand> MachineOperand::blockAddress(std::string FunctionName, + std::string BlockName, std::int64_t Offset, + std::uint32_t TargetFlags) { + if (FunctionName.empty() || BlockName.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "block address operand requires function and block names"); + } + return MachineOperand(Kind::BlockAddress, + BlockAddressOperand{std::move(FunctionName), std::move(BlockName), Offset}, + TargetFlags); +} + +Expected<MachineOperand> MachineOperand::registerMask(std::shared_ptr<const RegisterMask> Mask, + std::uint32_t TargetFlags) { + if (!Mask) { + return makeUnexpected(ErrorCode::InvalidArgument, "register mask operand cannot be null"); + } + return MachineOperand(Kind::RegisterMask, RegisterMaskOperand{std::move(Mask)}, TargetFlags); +} + +Expected<MachineOperand> +MachineOperand::liveOutRegisterMask(std::shared_ptr<const RegisterMask> Mask, + std::uint32_t TargetFlags) { + if (!Mask) { + return makeUnexpected(ErrorCode::InvalidArgument, + "live-out register-mask operand cannot be null"); + } + return MachineOperand(Kind::LiveOutRegisterMask, LiveOutRegisterMaskOperand{std::move(Mask)}, + TargetFlags); +} + +Expected<MachineOperand> MachineOperand::metadata(MachineMetadataValue Value, + std::uint32_t TargetFlags) { + if (!Value.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, "metadata operand value must be valid"); + } + return MachineOperand(Kind::Metadata, MetadataOperand{std::move(Value)}, TargetFlags); +} + +MachineOperand MachineOperand::targetIndex(TargetIndex Index, std::int64_t Offset, + std::uint32_t TargetFlags) { + return MachineOperand(Kind::TargetIndex, TargetIndexOperand{Index, Offset}, TargetFlags); +} + +Expected<MachineOperand> MachineOperand::mcSymbol(std::string Name, std::int64_t Offset, + std::uint32_t TargetFlags) { + if (Name.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, "MC symbol operand name cannot be empty"); + } + return MachineOperand(Kind::MCSymbol, MCSymbolOperand{std::move(Name), Offset}, TargetFlags); +} + +MachineOperand MachineOperand::cfiIndex(CFIInstructionIndex Index, std::uint32_t TargetFlags) { + return MachineOperand(Kind::CFIIndex, CFIIndexOperand{Index}, TargetFlags); +} + +Expected<MachineOperand> MachineOperand::intrinsic(IntrinsicID ID, std::uint32_t TargetFlags) { + if (ID.value() == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, "intrinsic operand ID must be nonzero"); + } + return MachineOperand(Kind::Intrinsic, IntrinsicOperand{ID}, TargetFlags); +} + +Expected<MachineOperand> MachineOperand::integerPredicate(IntegerPredicate Predicate, + std::uint32_t TargetFlags) { + PredicateOperand Value(Predicate); + auto Verification = Value.verify(); + if (!Verification) + return makeUnexpected(ErrorCode::InvalidArgument, "integer predicate operand is invalid"); + return MachineOperand(Kind::Predicate, Value, TargetFlags); +} + +Expected<MachineOperand> MachineOperand::floatingPredicate(FloatingPredicate Predicate, + std::uint32_t TargetFlags) { + PredicateOperand Value(Predicate); + auto Verification = Value.verify(); + if (!Verification) + return makeUnexpected(ErrorCode::InvalidArgument, "floating predicate operand is invalid"); + return MachineOperand(Kind::Predicate, Value, TargetFlags); +} + +Expected<MachineOperand> MachineOperand::shuffleMask(std::vector<std::int32_t> Elements, + std::uint32_t TargetFlags) { + if (Elements.size() < 2) { + return makeUnexpected(ErrorCode::InvalidArgument, + "shuffle-mask operand requires at least two elements"); + } + return MachineOperand(Kind::ShuffleMask, ShuffleMaskOperand{std::move(Elements)}, TargetFlags); +} + +Expected<MachineOperand> MachineOperand::debugInstrRef(std::uint64_t InstructionNumber, + std::uint64_t OperandNumber, + std::uint32_t TargetFlags) { + if (InstructionNumber > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "debug instruction reference instruction number must fit in uint32"); + } + if (OperandNumber > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "debug instruction reference operand number must fit in uint32"); + } + return MachineOperand(Kind::DebugInstrRef, + DebugInstrRefOperand{static_cast<std::uint32_t>(InstructionNumber), + static_cast<std::uint32_t>(OperandNumber)}, + TargetFlags); +} + +MachineOperand MachineOperand::laneMask(LaneMask Mask, std::uint32_t TargetFlags) { + return MachineOperand(Kind::LaneMask, LaneMaskOperand{Mask}, TargetFlags); +} + +MachineOperand::MachineOperand(MachineOperand &&Other) noexcept + : OperandKind(Other.OperandKind), Storage(std::move(Other.Storage)), RegFlags(Other.RegFlags), + RegSubReg(Other.RegSubReg), TiedOperandIndex(Other.TiedOperandIndex), + TargetFlags(Other.TargetFlags), Parent(Other.Parent) { + takeRegLinkFrom(Other); + Other.Parent = nullptr; + Other.TiedOperandIndex.reset(); +} + +MachineOperand &MachineOperand::operator=(MachineOperand &&Other) noexcept { + if (this == &Other) + return *this; + + unlinkFromRegInfo(); + OperandKind = Other.OperandKind; + Storage = std::move(Other.Storage); + RegFlags = Other.RegFlags; + RegSubReg = Other.RegSubReg; + TiedOperandIndex = Other.TiedOperandIndex; + TargetFlags = Other.TargetFlags; + Parent = Other.Parent; + takeRegLinkFrom(Other); + Other.Parent = nullptr; + Other.TiedOperandIndex.reset(); + return *this; +} + +MachineOperand::~MachineOperand() { + unlinkFromRegInfo(); +} + +std::optional<Register> MachineOperand::registerValue() const noexcept { + if (!isRegister()) + return std::nullopt; + return std::get<Register>(Storage); +} + +bool MachineOperand::isDef() const noexcept { + return isRegister() && RegFlags.has(RegisterOperandFlag::Define); +} + +bool MachineOperand::isUse() const noexcept { + return isRegister() && RegFlags.has(RegisterOperandFlag::Use); +} + +bool MachineOperand::isDebugUse() const noexcept { + return isUse() && RegFlags.has(RegisterOperandFlag::Debug); +} + +bool MachineOperand::isImplicit() const noexcept { + return isRegister() && RegFlags.has(RegisterOperandFlag::Implicit); +} + +Expected<void> MachineOperand::setRegister(Register Reg) { + if (!isRegister()) { + return makeUnexpected(ErrorCode::InvalidArgument, "setRegister requires a register operand"); + } + + if (std::get<Register>(Storage) == Reg) + return {}; + + RegFlags.clear(RegisterOperandFlag::Renamable); + + MachineRegisterInfo *RegInfo = LinkedRegInfo; + if (RegInfo == nullptr && Parent != nullptr) + RegInfo = &Parent->function().registerInfo(); + if (RegInfo != nullptr && Reg.isVirtual() && !RegInfo->isKnownVirtualRegister(Reg)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register operand references an unknown virtual register"); + } + + const bool WasLinked = LinkedRegInfo != nullptr; + unlinkFromRegInfo(); + Storage = Reg; + if (WasLinked && !Reg.isNoRegister()) + linkToRegInfo(*RegInfo); + return {}; +} + +Expected<void> MachineOperand::changeToImmediate(std::int64_t Value) { + if (!isRegister() || isImplicit() || isDef() || tiedOperandIndex()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "only an untied explicit register use can become an immediate"); + } + unlinkFromRegInfo(); + OperandKind = Kind::Immediate; + Storage = Value; + RegFlags = {}; + RegSubReg = {}; + return {}; +} + +Expected<void> MachineOperand::setImmediate(std::int64_t Value) { + if (OperandKind != Kind::Immediate) + return makeUnexpected(ErrorCode::InvalidArgument, "setImmediate requires an immediate operand"); + Storage = Value; + return {}; +} + +Expected<void> MachineOperand::setIntegerImmediate(APInt Value) { + if (OperandKind != Kind::IntegerImmediate) { + return makeUnexpected(ErrorCode::InvalidArgument, + "setIntegerImmediate requires an integer-immediate operand"); + } + Storage = IntegerImmediateOperand{std::move(Value)}; + return {}; +} + +Expected<void> MachineOperand::setRegisterFlags(RegisterOperandFlags Flags) { + if (!isRegister()) { + return makeUnexpected(ErrorCode::InvalidArgument, "register flags require a register operand"); + } + const auto Validation = verifyRegisterFlags(Flags); + if (!Validation) + return std::unexpected(Validation.error()); + RegFlags = Flags; + return {}; +} + +Expected<void> MachineOperand::setSubReg(SubRegIndex SubReg) { + if (!isRegister()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "subregister index requires a register operand"); + } + RegSubReg = SubReg; + return {}; +} + +Expected<void> MachineOperand::setBasicBlock(MachineBasicBlock &Block) { + if (OperandKind != Kind::BasicBlock) + return makeUnexpected(ErrorCode::InvalidArgument, + "setBasicBlock requires a basic-block operand"); + if (Parent != nullptr && &Block.function() != &Parent->function()) + return makeUnexpected(ErrorCode::InvalidArgument, + "basic-block operand references another machine function"); + std::get<BasicBlockOperand>(Storage).Block = &Block; + return {}; +} + +Expected<void> MachineOperand::setFrameIndex(FrameIndex Index) { + if (OperandKind != Kind::FrameIndex) + return makeUnexpected(ErrorCode::InvalidArgument, + "setFrameIndex requires a frame-index operand"); + if (Parent != nullptr && Parent->function().frameInfo().object(Index) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "frame-index operand references an unknown stack object"); + } + std::get<FrameIndexOperand>(Storage).Index = Index; + return {}; +} + +MachineOperand MachineOperand::cloneDetached() const { + MachineOperand Result(OperandKind, Storage, TargetFlags); + Result.RegFlags = RegFlags; + Result.RegSubReg = RegSubReg; + return Result; +} + +void MachineOperand::attachTo(MachineInstr &Instruction, MachineRegisterInfo *RegInfo) { + Parent = &Instruction; + if (RegInfo != nullptr && isRegister() && !std::get<Register>(Storage).isNoRegister()) { + linkToRegInfo(*RegInfo); + } +} + +void MachineOperand::detachFromInstruction() noexcept { + unlinkFromRegInfo(); + Parent = nullptr; +} + +void MachineOperand::linkToRegInfo(MachineRegisterInfo &RegInfo) { + RegInfo.linkOperand(*this); +} + +void MachineOperand::unlinkFromRegInfo() noexcept { + if (PreviousRegLink == nullptr) + return; + + *PreviousRegLink = NextRegOperand; + if (NextRegOperand != nullptr) + NextRegOperand->PreviousRegLink = PreviousRegLink; + + PreviousRegLink = nullptr; + NextRegOperand = nullptr; + LinkedRegInfo = nullptr; +} + +void MachineOperand::takeRegLinkFrom(MachineOperand &Other) noexcept { + LinkedRegInfo = Other.LinkedRegInfo; + PreviousRegLink = Other.PreviousRegLink; + NextRegOperand = Other.NextRegOperand; + + if (PreviousRegLink != nullptr) + *PreviousRegLink = this; + if (NextRegOperand != nullptr) + NextRegOperand->PreviousRegLink = &NextRegOperand; + + Other.LinkedRegInfo = nullptr; + Other.PreviousRegLink = nullptr; + Other.NextRegOperand = nullptr; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineOptimizationPipeline.cpp b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp new file mode 100644 index 0000000..6fb6550 --- /dev/null +++ b/backend/lib/CodeGen/MachineOptimizationPipeline.cpp @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineOptimizationPipeline.h" + +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBlockPlacement.h" +#include "rcc/CodeGen/MachineBranchOptimization.h" +#include "rcc/CodeGen/MachineBranchRelaxation.h" +#include "rcc/CodeGen/MachineCSE.h" +#include "rcc/CodeGen/MachineCombiner.h" +#include "rcc/CodeGen/MachineCopyPropagation.h" +#include "rcc/CodeGen/MachineDCE.h" +#include "rcc/CodeGen/MachineFinalization.h" +#include "rcc/CodeGen/MachineLICM.h" +#include "rcc/CodeGen/MachineMemoryFolding.h" +#include "rcc/CodeGen/MachinePeepholeOptimizer.h" +#include "rcc/CodeGen/MachinePseudoExpansion.h" +#include "rcc/CodeGen/MachineScheduler.h" +#include "rcc/CodeGen/MachineShrinkWrapping.h" +#include "rcc/CodeGen/MachineSink.h" +#include "rcc/CodeGen/MachineStackSlotColoring.h" +#include "rcc/CodeGen/MachineTailDuplication.h" +#include "rcc/CodeGen/PostRAScheduler.h" +#include "rcc/CodeGen/PrologueEpilogueInsertion.h" + +#include <memory> +#include <utility> + +namespace rcc { +namespace { + +template <typename Pass, typename... Arguments> +[[nodiscard]] Expected<void> addPass(MachineFunctionPassManager &Pipeline, Arguments &&...Values) { + return Pipeline.addPass(std::make_unique<Pass>(std::forward<Arguments>(Values)...)); +} + +} // namespace + +Expected<void> registerMachineOptimizationAnalyses(MachineFunctionAnalysisManager &Analyses) { + auto Registered = registerStage8MachineAnalyses(Analyses); + if (!Registered) + return std::unexpected(Registered.error()); + return registerStage11MachineAnalyses(Analyses); +} + +Expected<void> +addSelectedSSAOptimizationPipeline(MachineFunctionPassManager &Pipeline, + const MachineOptimizationPipelineTargetInfo &TargetInfo) { + auto Added = addPass<MachineCombinerPass>(Pipeline, TargetInfo.Combiner); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineMemoryFoldingPass>(Pipeline, TargetInfo.MemoryFolding); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineCSEPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineDCEPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineLICMPass>(Pipeline, MachineCodeMotionStage::SelectedSSA); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineSinkPass>(Pipeline, MachineCodeMotionStage::SelectedSSA); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachinePeepholeOptimizerPass>(Pipeline, TargetInfo.InstrInfo); + if (!Added) + return std::unexpected(Added.error()); + return addPass<MachineDCEPass>(Pipeline); +} + +Expected<void> addPreRegisterAllocationOptimizationPipeline( + MachineFunctionPassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo) { + return addPass<MachineSchedulerPass>(Pipeline, TargetInfo.SchedulingModel, + TargetInfo.SchedulerOptions); +} + +Expected<void> addPostRegisterAllocationPipeline( + MachineFunctionPassManager &Pipeline, const MachineOptimizationPipelineTargetInfo &TargetInfo, + MachineOptimizationPipelineOptions Options, + MachineFunctionPassManager::PassList PostRAPseudoExpansionTargetPasses) { + auto Added = addPass<MachineStackSlotColoringPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineLICMPass>(Pipeline, MachineCodeMotionStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineSinkPass>(Pipeline, MachineCodeMotionStage::PostRegisterAllocation); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineShrinkWrappingPass>(Pipeline, TargetInfo.FrameLowering); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<PrologueEpilogueInsertionPass>(Pipeline, TargetInfo.FrameLowering, + TargetInfo.FrameOptions); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineBranchOptimizationPass>(Pipeline, TargetInfo.InstrInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineTailDuplicationPass>(Pipeline, TargetInfo.InstrInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineCopyPropagationPass>( + Pipeline, TargetInfo.InstrInfo, + MachineCopyPropagationOptions{.UseTargetCopyInstructions = + Options.UseTargetCopyInstructions}); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<ExpandPostRAPseudosPass>(Pipeline, TargetInfo.PseudoExpansion); + if (!Added) + return std::unexpected(Added.error()); + for (std::unique_ptr<MachineFunctionPass> &TargetPass : PostRAPseudoExpansionTargetPasses) { + Added = Pipeline.addPass(std::move(TargetPass)); + if (!Added) + return std::unexpected(Added.error()); + } + Added = addPass<PostRASchedulerPass>(Pipeline, TargetInfo.SchedulingModel, + TargetInfo.SchedulerOptions); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineBlockPlacementPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<MachineBranchRelaxationPass>(Pipeline, TargetInfo.InstrInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<TargetLatePseudoExpansionPass>(Pipeline, TargetInfo.PseudoExpansion, + TargetInfo.InstrInfo); + if (!Added) + return std::unexpected(Added.error()); + Added = addPass<FinalizeEmissionPseudosPass>(Pipeline); + if (!Added) + return std::unexpected(Added.error()); + return addPass<FinalEncodabilityVerifierPass>(Pipeline, TargetInfo.InstrInfo); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineOutliner.cpp b/backend/lib/CodeGen/MachineOutliner.cpp new file mode 100644 index 0000000..a36a740 --- /dev/null +++ b/backend/lib/CodeGen/MachineOutliner.cpp @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineOutliner.h" + +#include "rcc/CodeGen/CompilationUnit.h" +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFIInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionAttributes.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <limits> +#include <memory> +#include <ranges> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct InstructionSegment final { + MachineFunction *Function = nullptr; + MachineBasicBlock *Block = nullptr; + std::vector<MachineInstr *> Instructions; +}; + +struct SequenceGroup final { + std::vector<MachineInstr *> Representative; + std::vector<MachineOutlinerCandidate> Candidates; + std::size_t DiscoveryOrder = 0; + std::uint32_t SequenceSize = 0; + std::int64_t Benefit = 0; +}; + +[[nodiscard]] RegisterOperandFlags normalizedRegisterFlags(RegisterOperandFlags Flags) noexcept { + Flags.clear(RegisterOperandFlag::Kill); + Flags.clear(RegisterOperandFlag::Dead); + Flags.clear(RegisterOperandFlag::Renamable); + return Flags; +} + +template <typename PayloadT> +[[nodiscard]] bool payloadsEqual(const MachineOperand &Left, const MachineOperand &Right) noexcept { + const PayloadT *LeftValue = Left.getIf<PayloadT>(); + const PayloadT *RightValue = Right.getIf<PayloadT>(); + return LeftValue != nullptr && RightValue != nullptr && *LeftValue == *RightValue; +} + +[[nodiscard]] bool operandsEqual(const MachineOperand &Left, const MachineOperand &Right, + const MachineFunction &LeftFunction, + const MachineFunction &RightFunction) noexcept { + if (Left.kind() != Right.kind() || Left.targetFlags() != Right.targetFlags() || + Left.subReg() != Right.subReg() || Left.tiedOperandIndex() != Right.tiedOperandIndex()) { + return false; + } + if (Left.isRegister() && normalizedRegisterFlags(Left.registerFlags()) != + normalizedRegisterFlags(Right.registerFlags())) { + return false; + } + + switch (Left.kind()) { + case MachineOperand::Kind::Register: + return payloadsEqual<Register>(Left, Right); + case MachineOperand::Kind::Immediate: + return payloadsEqual<std::int64_t>(Left, Right); + case MachineOperand::Kind::IntegerImmediate: + return payloadsEqual<IntegerImmediateOperand>(Left, Right); + case MachineOperand::Kind::FloatingImmediate: + return payloadsEqual<FloatingImmediateOperand>(Left, Right); + case MachineOperand::Kind::BasicBlock: + return payloadsEqual<BasicBlockOperand>(Left, Right); + case MachineOperand::Kind::FrameIndex: + return payloadsEqual<FrameIndexOperand>(Left, Right); + case MachineOperand::Kind::ConstantPoolIndex: + return payloadsEqual<ConstantPoolOperand>(Left, Right); + case MachineOperand::Kind::JumpTableIndex: + return payloadsEqual<JumpTableOperand>(Left, Right); + case MachineOperand::Kind::GlobalSymbol: + return payloadsEqual<GlobalSymbolOperand>(Left, Right); + case MachineOperand::Kind::ExternalSymbol: + return payloadsEqual<ExternalSymbolOperand>(Left, Right); + case MachineOperand::Kind::BlockAddress: + return payloadsEqual<BlockAddressOperand>(Left, Right); + case MachineOperand::Kind::RegisterMask: + return payloadsEqual<RegisterMaskOperand>(Left, Right); + case MachineOperand::Kind::LiveOutRegisterMask: + return payloadsEqual<LiveOutRegisterMaskOperand>(Left, Right); + case MachineOperand::Kind::Metadata: + return payloadsEqual<MetadataOperand>(Left, Right); + case MachineOperand::Kind::TargetIndex: + return payloadsEqual<TargetIndexOperand>(Left, Right); + case MachineOperand::Kind::MCSymbol: + return payloadsEqual<MCSymbolOperand>(Left, Right); + case MachineOperand::Kind::CFIIndex: { + const CFIIndexOperand *LeftIndex = Left.getIf<CFIIndexOperand>(); + const CFIIndexOperand *RightIndex = Right.getIf<CFIIndexOperand>(); + if (LeftIndex == nullptr || RightIndex == nullptr) + return false; + const MachineCFIInstruction *LeftCFI = LeftFunction.cfiInfo().instruction(LeftIndex->Index); + const MachineCFIInstruction *RightCFI = RightFunction.cfiInfo().instruction(RightIndex->Index); + return LeftCFI != nullptr && RightCFI != nullptr && *LeftCFI == *RightCFI; + } + case MachineOperand::Kind::Intrinsic: + return payloadsEqual<IntrinsicOperand>(Left, Right); + case MachineOperand::Kind::Predicate: + return payloadsEqual<PredicateOperand>(Left, Right); + case MachineOperand::Kind::ShuffleMask: + return payloadsEqual<ShuffleMaskOperand>(Left, Right); + case MachineOperand::Kind::DebugInstrRef: + return payloadsEqual<DebugInstrRefOperand>(Left, Right); + case MachineOperand::Kind::LaneMask: + return payloadsEqual<LaneMaskOperand>(Left, Right); + case MachineOperand::Kind::Count: + return false; + } + return false; +} + +[[nodiscard]] bool instructionsEqual(const MachineInstr &Left, const MachineInstr &Right) noexcept { + if (Left.opcode() != Right.opcode() || Left.flags() != Right.flags() || + Left.operandCount() != Right.operandCount() || + Left.memOperands().size() != Right.memOperands().size()) { + return false; + } + for (std::size_t Index = 0; Index < Left.operandCount(); ++Index) { + if (!operandsEqual(Left.operands()[Index], Right.operands()[Index], Left.function(), + Right.function())) { + return false; + } + } + for (std::size_t Index = 0; Index < Left.memOperands().size(); ++Index) { + const MachineMemOperand *LeftMemory = Left.memOperands()[Index]; + const MachineMemOperand *RightMemory = Right.memOperands()[Index]; + if (LeftMemory == nullptr || RightMemory == nullptr || *LeftMemory != *RightMemory) + return false; + } + return true; +} + +[[nodiscard]] bool sequencesEqual(const std::vector<MachineInstr *> &Left, + const std::vector<MachineInstr *> &Right) noexcept { + return Left.size() == Right.size() && + std::ranges::equal( + Left, Right, + [](const MachineInstr *LeftInstruction, const MachineInstr *RightInstruction) { + return LeftInstruction != nullptr && RightInstruction != nullptr && + instructionsEqual(*LeftInstruction, *RightInstruction); + }); +} + +[[nodiscard]] bool overlaps(PhysicalRegister Left, PhysicalRegister Right, + const TargetDescription &Target) noexcept { + return Target.registers().regsOverlap(Left, Right); +} + +[[nodiscard]] bool containsRegister(std::span<const PhysicalRegister> Registers, + PhysicalRegister Reg, + const TargetDescription &Target) noexcept { + return std::ranges::any_of(Registers, [Reg, &Target](PhysicalRegister Existing) { + return overlaps(Existing, Reg, Target); + }); +} + +void appendUnique(std::vector<PhysicalRegister> &Registers, PhysicalRegister Reg, + const TargetDescription &Target) { + if (!containsRegister(Registers, Reg, Target)) + Registers.push_back(Reg); +} + +void computeCandidateRegisters(MachineOutlinerCandidate &Candidate, + const TargetDescription &Target) { + std::vector<PhysicalRegister> Defined; + for (const MachineInstr *Instruction : Candidate.Instructions) { + for (const MachineOperand &Operand : Instruction->operands()) { + const auto RegisterValue = Operand.registerValue(); + const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt; + if (!Physical || Operand.isDebugUse()) + continue; + if (Operand.isUse() && !containsRegister(Defined, *Physical, Target)) + appendUnique(Candidate.LiveIns, *Physical, Target); + } + for (const MachineOperand &Operand : Instruction->operands()) { + const auto RegisterValue = Operand.registerValue(); + const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt; + if (!Physical || Operand.isDebugUse()) + continue; + if (Operand.isDef()) { + appendUnique(Defined, *Physical, Target); + appendUnique(Candidate.DefinedRegisters, *Physical, Target); + } + } + } + std::ranges::sort(Candidate.DefinedRegisters, {}, &PhysicalRegister::number); +} + +[[nodiscard]] const MachineOutlinerFunctionProperties & +propertiesFor(const MachineOutlinerOptions &Options, const MachineFunction &Function) noexcept { + static const MachineOutlinerFunctionProperties Empty; + const auto Position = Options.FunctionProperties.find(std::string(Function.name())); + return Position == Options.FunctionProperties.end() ? Empty : Position->second; +} + +[[nodiscard]] std::vector<InstructionSegment> +collectSegments(CompilationUnit &Unit, const TargetMachineOutlinerInfo &TargetInfo, + const MachineOutlinerOptions &Options, const TargetDescription &Target) { + std::vector<InstructionSegment> Segments; + for (std::size_t FunctionIndex = 0; FunctionIndex < Unit.functionCount(); ++FunctionIndex) { + MachineFunction *Function = Unit.functionAt(FunctionIndex); + if (Function == nullptr || Function->attributes().has(MachineFunctionAttribute::IsOutlined) || + !TargetInfo.isFunctionSafeToOutlineFrom(*Function, propertiesFor(Options, *Function))) { + continue; + } + for (MachineBasicBlock *Block = Function->firstBlock(); Block != nullptr; + Block = Block->next()) { + InstructionSegment Current{ + .Function = Function, + .Block = Block, + .Instructions = {}, + }; + const auto Flush = [&]() { + if (Current.Instructions.size() >= 2) + Segments.push_back(Current); + Current.Instructions.clear(); + }; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const MachineOutlinerInstrType Type = + TargetInfo.getOutliningType(*Instruction, propertiesFor(Options, *Function), Target); + if (Type == MachineOutlinerInstrType::Illegal || + Type == MachineOutlinerInstrType::Invisible) { + Flush(); + continue; + } + Current.Instructions.push_back(Instruction); + if (Type == MachineOutlinerInstrType::LegalTerminator) + Flush(); + } + Flush(); + } + } + return Segments; +} + +[[nodiscard]] Expected<std::vector<SequenceGroup>> +buildSequenceGroups(std::vector<InstructionSegment> &Segments, + const TargetMachineOutlinerInfo &TargetInfo, const TargetDescription &Target) { + std::vector<SequenceGroup> Groups; + std::size_t DiscoveryOrder = 0; + for (InstructionSegment &Segment : Segments) { + for (std::size_t Begin = 0; Begin + 1 < Segment.Instructions.size(); ++Begin) { + for (std::size_t Length = 2; Begin + Length <= Segment.Instructions.size(); ++Length) { + std::vector<MachineInstr *> Sequence( + Segment.Instructions.begin() + static_cast<std::ptrdiff_t>(Begin), + Segment.Instructions.begin() + static_cast<std::ptrdiff_t>(Begin + Length)); + auto Position = std::ranges::find_if(Groups, [&Sequence](const SequenceGroup &Group) { + return sequencesEqual(Group.Representative, Sequence); + }); + if (Position == Groups.end()) { + Groups.push_back({ + .Representative = Sequence, + .Candidates = {}, + .DiscoveryOrder = DiscoveryOrder++, + .SequenceSize = 0, + .Benefit = 0, + }); + Position = std::prev(Groups.end()); + } + MachineOutlinerCandidate Candidate{ + .Function = Segment.Function, + .Block = Segment.Block, + .Instructions = std::move(Sequence), + .LiveIns = {}, + .DefinedRegisters = {}, + .CallConstructionID = 0, + .CallOverhead = 0, + .SaveRegister = {}, + }; + computeCandidateRegisters(Candidate, Target); + Position->Candidates.push_back(std::move(Candidate)); + } + } + } + + for (SequenceGroup &Group : Groups) { + std::vector<MachineOutlinerCandidate> Viable; + for (MachineOutlinerCandidate &Candidate : Group.Candidates) { + auto Accepted = TargetInfo.analyzeCandidate(Candidate, Target); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (*Accepted) + Viable.push_back(std::move(Candidate)); + } + Group.Candidates = std::move(Viable); + } + std::erase_if(Groups, [](const SequenceGroup &Group) { return Group.Candidates.size() < 2; }); + + std::unordered_map<const MachineInstr *, std::size_t> LongestAtStart; + for (const SequenceGroup &Group : Groups) { + for (const MachineOutlinerCandidate &Candidate : Group.Candidates) { + const MachineInstr *Start = Candidate.Instructions.front(); + LongestAtStart[Start] = std::max(LongestAtStart[Start], Candidate.Instructions.size()); + } + } + for (SequenceGroup &Group : Groups) { + std::erase_if(Group.Candidates, [&LongestAtStart](const MachineOutlinerCandidate &Candidate) { + return Candidate.Instructions.size() < LongestAtStart[Candidate.Instructions.front()]; + }); + std::unordered_set<const MachineInstr *> Claimed; + std::vector<MachineOutlinerCandidate> NonOverlapping; + for (MachineOutlinerCandidate &Candidate : Group.Candidates) { + const bool HasOverlap = std::ranges::any_of( + Candidate.Instructions, [&Claimed](const auto *I) { return Claimed.contains(I); }); + if (HasOverlap) + continue; + Claimed.insert(Candidate.Instructions.begin(), Candidate.Instructions.end()); + NonOverlapping.push_back(std::move(Candidate)); + } + Group.Candidates = std::move(NonOverlapping); + } + std::erase_if(Groups, [](const SequenceGroup &Group) { return Group.Candidates.size() < 2; }); + return Groups; +} + +[[nodiscard]] Expected<void> calculateBenefit(SequenceGroup &Group, + const TargetMachineOutlinerInfo &TargetInfo, + const TargetDescription &Target) { + std::uint64_t SequenceSize = 0; + for (const MachineInstr *Instruction : Group.Representative) { + auto Size = TargetInfo.instructionSize(*Instruction, Instruction->function(), Target); + if (!Size) + return std::unexpected(Size.error()); + SequenceSize += *Size; + } + if (SequenceSize > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::Unsupported, + "machine outliner sequence size exceeds the supported range"); + } + Group.SequenceSize = static_cast<std::uint32_t>(SequenceSize); + std::uint64_t Cost = SequenceSize + TargetInfo.frameOverhead(Group.Candidates.front()); + for (const MachineOutlinerCandidate &Candidate : Group.Candidates) + Cost += Candidate.CallOverhead; + const std::uint64_t Original = SequenceSize * Group.Candidates.size(); + Group.Benefit = static_cast<std::int64_t>(Original) - static_cast<std::int64_t>(Cost); + return {}; +} + +[[nodiscard]] Expected<MachineOperand> cloneOperand(const MachineOperand &Operand, + const MachineFunction &Source, + MachineFunction &Destination) { + if (Operand.kind() == MachineOperand::Kind::CFIIndex) { + const CFIIndexOperand *Index = Operand.getIf<CFIIndexOperand>(); + const MachineCFIInstruction *Instruction = + Index == nullptr ? nullptr : Source.cfiInfo().instruction(Index->Index); + if (Instruction == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine outliner encountered an invalid CFI reference"); + } + auto NewIndex = Destination.cfiInfo().addInstruction(*Instruction); + if (!NewIndex) + return std::unexpected(NewIndex.error()); + return MachineOperand::cfiIndex(*NewIndex, Operand.targetFlags()); + } + return Operand.cloneDetached(); +} + +[[nodiscard]] Expected<MachineInstr *> cloneInstruction(const MachineInstr &Source, + MachineFunction &Destination, + MachineBasicBlock &DestinationBlock) { + auto Created = Destination.createInstruction(Source.opcode()); + if (!Created) + return std::unexpected(Created.error()); + MachineInstr *Clone = *Created; + Clone->reserveOperands(Source.operandCount()); + for (const MachineOperand &Operand : Source.operands()) { + auto ClonedOperand = cloneOperand(Operand, Source.function(), Destination); + if (!ClonedOperand) + return std::unexpected(ClonedOperand.error()); + auto Added = Clone->addOperand(std::move(*ClonedOperand)); + if (!Added) + return std::unexpected(Added.error()); + } + for (std::size_t Index = 0; Index < Source.operandCount(); ++Index) { + const std::optional<std::size_t> Tied = Source.operands()[Index].tiedOperandIndex(); + if (!Tied || Index > *Tied) + continue; + const std::size_t DefIndex = Source.operands()[Index].isDef() ? Index : *Tied; + const std::size_t UseIndex = Source.operands()[Index].isDef() ? *Tied : Index; + auto TiedResult = Clone->tieOperands(DefIndex, UseIndex); + if (!TiedResult) + return std::unexpected(TiedResult.error()); + } + for (const MachineMemOperand *Memory : Source.memOperands()) { + if (Memory == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine outliner encountered a null memory operand"); + } + auto ClonedMemory = Destination.createMemOperand(Memory->description()); + if (!ClonedMemory) + return std::unexpected(ClonedMemory.error()); + auto Added = Clone->addMemOperand(**ClonedMemory); + if (!Added) + return std::unexpected(Added.error()); + } + Clone->setFlags(Source.flags()); + auto Appended = DestinationBlock.appendInstruction(*Clone); + if (!Appended) + return std::unexpected(Appended.error()); + return Clone; +} + +[[nodiscard]] Expected<std::string> nextOutlinedFunctionName(const CompilationUnit &Unit) { + for (std::uint64_t Index = 0; Index < std::numeric_limits<std::uint32_t>::max(); ++Index) { + std::string Name = "OUTLINED_FUNCTION_" + std::to_string(Index); + if (Unit.function(Name) == nullptr && Unit.globalObject(Name) == nullptr) + return Name; + } + return makeUnexpected(ErrorCode::Unsupported, + "machine outliner exhausted outlined function names"); +} + +[[nodiscard]] Expected<void> outlineGroup(SequenceGroup &Group, CompilationUnit &Unit, + const TargetMachineOutlinerInfo &TargetInfo, + const TargetDescription &Target) { + auto Name = nextOutlinedFunctionName(Unit); + if (!Name) + return std::unexpected(Name.error()); + auto Created = Unit.createMachineFunction(*Name); + if (!Created) + return std::unexpected(Created.error()); + MachineFunction &Outlined = **Created; + Outlined.properties() = Group.Candidates.front().Function->properties(); + Outlined.attributes().set(MachineFunctionAttribute::IsOutlined); + + auto CreatedBlock = Outlined.createBasicBlock(); + if (!CreatedBlock) + return std::unexpected(CreatedBlock.error()); + MachineBasicBlock &OutlinedBlock = **CreatedBlock; + auto Appended = Outlined.appendBlock(OutlinedBlock); + if (!Appended) + return std::unexpected(Appended.error()); + std::vector<PhysicalRegister> OutlinedLiveIns = Group.Candidates.front().LiveIns; + std::ranges::sort(OutlinedLiveIns, std::greater{}, &PhysicalRegister::number); + for (const PhysicalRegister LiveIn : OutlinedLiveIns) { + auto Added = OutlinedBlock.addLiveIn(LiveIn); + if (!Added) + return std::unexpected(Added.error()); + } + for (const MachineInstr *Instruction : Group.Representative) { + auto Cloned = cloneInstruction(*Instruction, Outlined, OutlinedBlock); + if (!Cloned) + return std::unexpected(Cloned.error()); + } + auto Frame = + TargetInfo.buildOutlinedFrame(Outlined, OutlinedBlock, Group.Candidates.front(), Target); + if (!Frame) + return std::unexpected(Frame.error()); + + for (MachineOutlinerCandidate &Candidate : Group.Candidates) { + auto Inserted = TargetInfo.insertOutlinedCall(Candidate, *Name, Target); + if (!Inserted) + return std::unexpected(Inserted.error()); + for (MachineInstr *Instruction : Candidate.Instructions) { + auto Removed = Instruction->eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + } + } + return {}; +} + +class MachineOutlinerPass final : public CompilationUnitPass { +public: + MachineOutlinerPass(std::unique_ptr<TargetMachineOutlinerInfo> TargetInfo, + MachineOutlinerOptions Options) + : TargetInfo(std::move(TargetInfo)), Options(std::move(Options)) {} + + [[nodiscard]] const CompilationUnitPassContract &contract() const noexcept override { + static const CompilationUnitPassContract Contract{ + .Name = "machine-outliner", + .RequiredAnalyses = {}, + .Parameters = {}, + .DeclaredChanges = MachinePassChanges::all(), + }; + return Contract; + } + + [[nodiscard]] Expected<CompilationUnitPassResult> run(CompilationUnit &Unit, + const TargetDescription &Target, + CompilationUnitAnalysisManager &, + MachineFunctionAnalysisManager &) override { + if (TargetInfo == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine outliner requires target-specific hooks"); + } + bool Changed = false; + while (true) { + std::vector<InstructionSegment> Segments = + collectSegments(Unit, *TargetInfo, Options, Target); + auto Groups = buildSequenceGroups(Segments, *TargetInfo, Target); + if (!Groups) + return std::unexpected(Groups.error()); + for (SequenceGroup &Group : *Groups) { + auto Calculated = calculateBenefit(Group, *TargetInfo, Target); + if (!Calculated) + return std::unexpected(Calculated.error()); + } + auto Best = std::ranges::max_element( + *Groups, [](const SequenceGroup &Left, const SequenceGroup &Right) { + if (Left.Benefit != Right.Benefit) + return Left.Benefit < Right.Benefit; + if (Left.Representative.size() != Right.Representative.size()) + return Left.Representative.size() < Right.Representative.size(); + return Left.DiscoveryOrder > Right.DiscoveryOrder; + }); + if (Best == Groups->end() || Best->Benefit <= 0) + break; + auto Outlined = outlineGroup(*Best, Unit, *TargetInfo, Target); + if (!Outlined) + return std::unexpected(Outlined.error()); + Changed = true; + } + return CompilationUnitPassResult{ + .Changed = Changed, + .Changes = Changed ? MachinePassChanges::all() : MachinePassChanges{}, + .PreservedUnitAnalyses = PreservedAnalyses::none(), + .PreservedFunctionAnalyses = PreservedAnalyses::none(), + }; + } + +private: + std::unique_ptr<TargetMachineOutlinerInfo> TargetInfo; + MachineOutlinerOptions Options; +}; + +} // namespace + +MachineInstr &MachineOutlinerCandidate::front() const noexcept { + return *Instructions.front(); +} + +MachineInstr &MachineOutlinerCandidate::back() const noexcept { + return *Instructions.back(); +} + +bool MachineOutlinerCandidate::containsRegister(PhysicalRegister Reg, + const TargetDescription &Target) const noexcept { + for (const MachineInstr *Instruction : Instructions) { + for (const MachineOperand &Operand : Instruction->operands()) { + const auto RegisterValue = Operand.registerValue(); + const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt; + if (Physical && !Operand.isDebugUse() && overlaps(*Physical, Reg, Target)) + return true; + } + } + return false; +} + +bool MachineOutlinerCandidate::isRegisterLiveAfter(PhysicalRegister Reg, + const TargetDescription &Target) const noexcept { + bool Live = false; + for (const MachineSuccessor &Successor : Block->successors()) { + if (Successor.Block == nullptr) + continue; + for (const MachineBasicBlockLiveIn &LiveIn : Successor.Block->liveIns()) { + if (overlaps(LiveIn.PhysReg, Reg, Target)) + Live = true; + } + } + for (const MachineInstr *Instruction = Block->lastInstruction(); + Instruction != nullptr && Instruction != &back(); Instruction = Instruction->previous()) { + for (const MachineOperand &Operand : Instruction->operands()) { + const auto RegisterValue = Operand.registerValue(); + const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt; + if (Physical && Operand.isDef() && overlaps(*Physical, Reg, Target)) + Live = false; + } + for (const MachineOperand &Operand : Instruction->operands()) { + const auto RegisterValue = Operand.registerValue(); + const auto Physical = RegisterValue ? RegisterValue->physicalRegister() : std::nullopt; + if (Physical && Operand.isUse() && !Operand.isDebugUse() && overlaps(*Physical, Reg, Target)) + Live = true; + } + } + return Live; +} + +bool MachineOutlinerCandidate::isRegisterAvailable(PhysicalRegister Reg, + const TargetDescription &Target) const noexcept { + return !containsRegister(Reg, Target) && !isRegisterLiveAfter(Reg, Target) && + !Function->registerInfo().isReserved(Reg) && !Target.registers().isReservedRegister(Reg); +} + +TargetMachineOutlinerInfo::~TargetMachineOutlinerInfo() = default; + +std::unique_ptr<CompilationUnitPass> +createMachineOutlinerPass(std::unique_ptr<TargetMachineOutlinerInfo> TargetInfo, + MachineOutlinerOptions Options) { + return std::make_unique<MachineOutlinerPass>(std::move(TargetInfo), std::move(Options)); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachinePassInstrumentation.cpp b/backend/lib/CodeGen/MachinePassInstrumentation.cpp new file mode 100644 index 0000000..ce8c62c --- /dev/null +++ b/backend/lib/CodeGen/MachinePassInstrumentation.cpp @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePassInstrumentation.h" + +#include "rcc/CodeGen/CompilationUnit.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachinePassManager.h" + +#include <algorithm> +#include <ranges> +#include <utility> + +namespace rcc { + +void AnalysisManagerInstrumentation<MachineFunction, MachinePassInstrumentation>::beforeAnalysis( + MachinePassInstrumentation *Instrumentation, std::string_view Name, + const MachineFunction &Function) { + Instrumentation->beforeAnalysis(Name, Function); +} + +void AnalysisManagerInstrumentation<MachineFunction, MachinePassInstrumentation>::afterAnalysis( + MachinePassInstrumentation *Instrumentation, std::string_view Name, + const MachineFunction &Function, bool CacheHit, std::uint64_t Nanoseconds) { + Instrumentation->afterAnalysis(Name, Function, CacheHit, Nanoseconds); +} + +void AnalysisManagerInstrumentation<MachineFunction, MachinePassInstrumentation>::analysisFailure( + MachinePassInstrumentation *Instrumentation, std::string_view Name, + const MachineFunction &Function, const Error &Failure, std::uint64_t Nanoseconds) { + Instrumentation->analysisFailure(Name, Function, Failure, Nanoseconds); +} + +Expected<bool> +AnalysisManagerInstrumentation<MachineFunction, MachinePassInstrumentation>::requestAction( + MachinePassInstrumentation *Instrumentation, std::string_view Pass, std::string_view Rule, + const MachineFunction &Function, MachineInstrStableID Instruction, std::uint64_t Cost) { + return Instrumentation->requestAction(Pass, Rule, Function, Instruction, Cost); +} + +Expected<bool> +AnalysisManagerInstrumentation<MachineFunction, MachinePassInstrumentation>::requestAction( + MachinePassInstrumentation *Instrumentation, std::string_view Pass, std::string_view Rule, + const MachineFunction &Function, std::nullptr_t, std::uint64_t Cost) { + return Instrumentation->requestAction(Pass, Rule, Function, nullptr, Cost); +} + +void AnalysisManagerInstrumentation<CompilationUnit, MachinePassInstrumentation>::beforeAnalysis( + MachinePassInstrumentation *Instrumentation, std::string_view Name, + const CompilationUnit &Unit) { + Instrumentation->beforeAnalysis(Name, Unit); +} + +void AnalysisManagerInstrumentation<CompilationUnit, MachinePassInstrumentation>::afterAnalysis( + MachinePassInstrumentation *Instrumentation, std::string_view Name, const CompilationUnit &Unit, + bool CacheHit, std::uint64_t Nanoseconds) { + Instrumentation->afterAnalysis(Name, Unit, CacheHit, Nanoseconds); +} + +void AnalysisManagerInstrumentation<CompilationUnit, MachinePassInstrumentation>::analysisFailure( + MachinePassInstrumentation *Instrumentation, std::string_view Name, const CompilationUnit &Unit, + const Error &Failure, std::uint64_t Nanoseconds) { + Instrumentation->analysisFailure(Name, Unit, Failure, Nanoseconds); +} + +std::string MachineActionID::toString() const { + std::string Result; + Result.reserve(Pass.size() + Rule.size() + 48U); + Result.append(Pass); + Result.push_back(':'); + Result.append(Rule); + Result.append(":f"); + Result.append(std::to_string(Function.value())); + Result.append(":i"); + Result.append(std::to_string(Instruction.value())); + return Result; +} + +void MachinePassInstrumentation::addBeforeCallback(BeforeCallback Callback) { + FunctionCallbacks.addBeforeCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addAfterCallback(AfterCallback Callback) { + FunctionCallbacks.addAfterCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addFailureCallback(FailureCallback Callback) { + FunctionCallbacks.addFailureCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addBeforeAnalysisCallback(BeforeAnalysisCallback Callback) { + FunctionCallbacks.addBeforeAnalysisCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addAfterAnalysisCallback(AfterAnalysisCallback Callback) { + FunctionCallbacks.addAfterAnalysisCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addAnalysisFailureCallback(AnalysisFailureCallback Callback) { + FunctionCallbacks.addAnalysisFailureCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addIRPrintCallback(IRPrintCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + IRPrint.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::addChangeCallback(ChangeCallback Callback) { + if (!Callback) + return; + std::scoped_lock Lock(Mutex); + Change.push_back(std::move(Callback)); +} + +void MachinePassInstrumentation::setActionFilter(ActionFilter NewFilter) { + std::scoped_lock Lock(Mutex); + Filter = std::move(NewFilter); +} + +void MachinePassInstrumentation::setPerFunctionFuel(std::optional<std::uint64_t> Fuel) { + std::scoped_lock Lock(Mutex); + PerFunctionFuel = Fuel; + ConsumedFuel.clear(); + ActionEvents.clear(); +} + +void MachinePassInstrumentation::addBeforeUnitCallback(BeforeUnitCallback Callback) { + UnitCallbacks.addBeforeCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addAfterUnitCallback(AfterUnitCallback Callback) { + UnitCallbacks.addAfterCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addUnitFailureCallback(UnitFailureCallback Callback) { + UnitCallbacks.addFailureCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addBeforeUnitAnalysisCallback( + BeforeUnitAnalysisCallback Callback) { + UnitCallbacks.addBeforeAnalysisCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addAfterUnitAnalysisCallback(AfterUnitAnalysisCallback Callback) { + UnitCallbacks.addAfterAnalysisCallback(std::move(Callback)); +} + +void MachinePassInstrumentation::addUnitAnalysisFailureCallback( + UnitAnalysisFailureCallback Callback) { + UnitCallbacks.addAnalysisFailureCallback(std::move(Callback)); +} + +InstrumentationStatistic &MachinePassInstrumentation::statistic(std::string_view Name, + bool IsAnalysis) const { + const std::string Key = std::string(IsAnalysis ? "analysis:" : "pass:") + std::string(Name); + auto [Position, Inserted] = Statistics.try_emplace(Key); + if (Inserted) { + Position->second.Name = std::string(Name); + Position->second.IsAnalysis = IsAnalysis; + } + return Position->second; +} + +void MachinePassInstrumentation::before(std::string_view Name, + const MachineFunction &Function) const { + FunctionCallbacks.runBefore(Name, Function); + std::vector<IRPrintCallback> Printers; + { + std::scoped_lock Lock(Mutex); + Printers = IRPrint; + } + for (const IRPrintCallback &Printer : Printers) + Printer(InstrumentationPhase::Before, Name, Function); +} + +void MachinePassInstrumentation::after(std::string_view Name, const MachineFunction &Function, + const MachinePassResult &Result, + std::uint64_t Nanoseconds) const { + std::vector<IRPrintCallback> Printers; + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic(Name, false); + ++Record.Invocations; + Record.Changes += Result.Changed ? 1U : 0U; + Record.TotalNanoseconds += Nanoseconds; + Printers = IRPrint; + } + FunctionCallbacks.runAfter(Name, Function, Result); + for (const IRPrintCallback &Printer : Printers) + Printer(InstrumentationPhase::After, Name, Function); +} + +void MachinePassInstrumentation::failure(std::string_view Name, const MachineFunction &Function, + const Error &ErrorValue, std::uint64_t Nanoseconds) const { + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic(Name, false); + ++Record.Invocations; + ++Record.Failures; + Record.TotalNanoseconds += Nanoseconds; + } + FunctionCallbacks.runFailure(Name, Function, ErrorValue); +} + +void MachinePassInstrumentation::beforeAnalysis(std::string_view Name, + const MachineFunction &Function) const { + FunctionCallbacks.runBeforeAnalysis(Name, Function); +} + +void MachinePassInstrumentation::afterAnalysis(std::string_view Name, + const MachineFunction &Function, bool CacheHit, + std::uint64_t Nanoseconds) const { + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic(Name, true); + ++Record.Invocations; + Record.CacheHits += CacheHit ? 1U : 0U; + Record.TotalNanoseconds += Nanoseconds; + } + FunctionCallbacks.runAfterAnalysis(Name, Function, CacheHit, Nanoseconds); +} + +void MachinePassInstrumentation::analysisFailure(std::string_view Name, + const MachineFunction &Function, + const Error &ErrorValue, + std::uint64_t Nanoseconds) const { + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic(Name, true); + ++Record.Invocations; + ++Record.Failures; + Record.TotalNanoseconds += Nanoseconds; + } + FunctionCallbacks.runAnalysisFailure(Name, Function, ErrorValue); +} + +void MachinePassInstrumentation::change(std::string_view Name, const MachineFunction &Function, + MachinePassChanges Changes, bool FingerprintChanged) const { + std::vector<ChangeCallback> Callbacks; + { + std::scoped_lock Lock(Mutex); + Callbacks = Change; + } + for (const ChangeCallback &Callback : Callbacks) + Callback(Name, Function, Changes, FingerprintChanged); +} + +Expected<bool> MachinePassInstrumentation::requestAction(std::string_view Pass, + std::string_view Rule, + const MachineFunction &Function, + MachineInstrStableID Instruction, + std::uint64_t Cost) { + MachineActionID Action{ + .Pass = std::string(Pass), + .Rule = std::string(Rule), + .Function = Function.stableID(), + .Instruction = Instruction, + }; + if (Action.Pass.empty() || Action.Rule.empty() || Action.Function != Function.stableID() || + Cost == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, "machine action identity is invalid"); + } + std::scoped_lock ActionLock(ActionMutex); + ActionFilter CurrentFilter; + { + std::scoped_lock Lock(Mutex); + CurrentFilter = Filter; + } + if (CurrentFilter && !CurrentFilter(Action, Cost)) { + std::scoped_lock Lock(Mutex); + ActionEvents.push_back({Action, Cost, false}); + return false; + } + + std::scoped_lock Lock(Mutex); + if (!PerFunctionFuel) { + ActionEvents.push_back({Action, Cost, true}); + return true; + } + std::uint64_t &Consumed = ConsumedFuel[Function.stableID().value()]; + if (Consumed > *PerFunctionFuel || Cost > *PerFunctionFuel - Consumed) { + ActionEvents.push_back({Action, Cost, false}); + return false; + } + Consumed += Cost; + ActionEvents.push_back({Action, Cost, true}); + return true; +} + +Expected<bool> MachinePassInstrumentation::requestAction(std::string_view Pass, + std::string_view Rule, + const MachineFunction &Function, + std::nullptr_t, std::uint64_t Cost) { + return requestAction(Pass, Rule, Function, MachineInstrStableID{}, Cost); +} + +void MachinePassInstrumentation::beforeUnit(std::string_view Name, + const CompilationUnit &Unit) const { + UnitCallbacks.runBefore(Name, Unit); +} + +void MachinePassInstrumentation::afterUnit(std::string_view Name, const CompilationUnit &Unit, + const CompilationUnitPassResult &Result, + std::uint64_t Nanoseconds) const { + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), false); + ++Record.Invocations; + Record.Changes += Result.Changed ? 1U : 0U; + Record.TotalNanoseconds += Nanoseconds; + } + UnitCallbacks.runAfter(Name, Unit, Result); +} + +void MachinePassInstrumentation::unitFailure(std::string_view Name, const CompilationUnit &Unit, + const Error &ErrorValue, + std::uint64_t Nanoseconds) const { + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), false); + ++Record.Invocations; + ++Record.Failures; + Record.TotalNanoseconds += Nanoseconds; + } + UnitCallbacks.runFailure(Name, Unit, ErrorValue); +} + +void MachinePassInstrumentation::beforeAnalysis(std::string_view Name, + const CompilationUnit &Unit) const { + UnitCallbacks.runBeforeAnalysis(Name, Unit); +} + +void MachinePassInstrumentation::afterAnalysis(std::string_view Name, const CompilationUnit &Unit, + bool CacheHit, std::uint64_t Nanoseconds) const { + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), true); + ++Record.Invocations; + Record.CacheHits += CacheHit ? 1U : 0U; + Record.TotalNanoseconds += Nanoseconds; + } + UnitCallbacks.runAfterAnalysis(Name, Unit, CacheHit, Nanoseconds); +} + +void MachinePassInstrumentation::analysisFailure(std::string_view Name, const CompilationUnit &Unit, + const Error &ErrorValue, + std::uint64_t Nanoseconds) const { + { + std::scoped_lock Lock(Mutex); + InstrumentationStatistic &Record = statistic("unit:" + std::string(Name), true); + ++Record.Invocations; + ++Record.Failures; + Record.TotalNanoseconds += Nanoseconds; + } + UnitCallbacks.runAnalysisFailure(Name, Unit, ErrorValue); +} + +std::vector<InstrumentationStatistic> MachinePassInstrumentation::statistics() const { + std::vector<InstrumentationStatistic> Result; + { + std::scoped_lock Lock(Mutex); + Result.reserve(Statistics.size()); + for (const auto &[Key, Record] : Statistics) { + (void)Key; + Result.push_back(Record); + } + } + std::ranges::sort( + Result, [](const InstrumentationStatistic &Left, const InstrumentationStatistic &Right) { + if (Left.IsAnalysis != Right.IsAnalysis) + return Left.IsAnalysis < Right.IsAnalysis; + return Left.Name < Right.Name; + }); + return Result; +} + +std::vector<MachineActionEvent> MachinePassInstrumentation::actionEvents() const { + std::vector<MachineActionEvent> Result; + { + std::scoped_lock Lock(Mutex); + Result = ActionEvents; + } + std::ranges::sort(Result, [](const MachineActionEvent &Left, const MachineActionEvent &Right) { + if (Left.ID.Function.value() != Right.ID.Function.value()) + return Left.ID.Function.value() < Right.ID.Function.value(); + if (Left.ID.Instruction.value() != Right.ID.Instruction.value()) + return Left.ID.Instruction.value() < Right.ID.Instruction.value(); + if (Left.ID.Pass != Right.ID.Pass) + return Left.ID.Pass < Right.ID.Pass; + if (Left.ID.Rule != Right.ID.Rule) + return Left.ID.Rule < Right.ID.Rule; + if (Left.Cost != Right.Cost) + return Left.Cost < Right.Cost; + return Left.Accepted < Right.Accepted; + }); + return Result; +} + +bool MachinePassInstrumentation::requiresSerialFunctionExecution() const { + if (!FunctionCallbacks.empty()) + return true; + std::scoped_lock Lock(Mutex); + return !IRPrint.empty() || !Change.empty(); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachinePassManager.cpp b/backend/lib/CodeGen/MachinePassManager.cpp new file mode 100644 index 0000000..3d56357 --- /dev/null +++ b/backend/lib/CodeGen/MachinePassManager.cpp @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePassManager.h" + +#include "rcc/CodeGen/CompilationUnit.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineVerifier.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <atomic> +#include <chrono> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <string_view> +#include <thread> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { + +namespace { + +using Clock = std::chrono::steady_clock; + +[[nodiscard]] std::uint64_t elapsedNanoseconds(Clock::time_point Start) noexcept { + const auto Elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(Clock::now() - Start); + if (Elapsed.count() <= 0) + return 0; + return static_cast<std::uint64_t>(Elapsed.count()); +} + +[[nodiscard]] bool hasDuplicateAnalysisIDs(std::span<const AnalysisID> IDs) { + std::unordered_set<AnalysisID> Unique; + for (const AnalysisID ID : IDs) { + if (ID == nullptr || !Unique.insert(ID).second) + return true; + } + return false; +} + +[[nodiscard]] Expected<void> validateParameters(std::span<const PassParameter> Parameters) { + std::unordered_set<std::string> Names; + for (const PassParameter &Parameter : Parameters) { + if (Parameter.Name.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "pass parameter name cannot be empty"); + if (!Names.insert(Parameter.Name).second) + return makeUnexpected(ErrorCode::InvalidArgument, "pass parameter names must be unique"); + if (Parameter.Required && Parameter.DefaultValue) { + return makeUnexpected(ErrorCode::InvalidArgument, + "a required pass parameter cannot have a default value"); + } + } + return {}; +} + +void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept { + State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U); +} + +[[nodiscard]] std::uint64_t machineFunctionFingerprint(const MachineFunction &Function) noexcept { + std::uint64_t Result = 0xcbf29ce484222325ULL; + combineFingerprint(Result, Function.stableID().value()); + combineFingerprint(Result, Function.properties().rawValue()); + combineFingerprint(Result, Function.blockCount()); + combineFingerprint(Result, Function.allInstructions().size()); + combineFingerprint(Result, Function.allMemOperands().size()); + + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + combineFingerprint(Result, Block->stableID().value()); + combineFingerprint(Result, Block->instructionCount()); + combineFingerprint(Result, Block->successors().size()); + for (const MachineSuccessor &Successor : Block->successors()) { + combineFingerprint(Result, + Successor.Block == nullptr ? 0 : Successor.Block->stableID().value()); + combineFingerprint(Result, Successor.Probability.rawValue()); + } + for (const MachineBasicBlockLiveIn &LiveIn : Block->liveIns()) { + combineFingerprint(Result, LiveIn.PhysReg.number()); + combineFingerprint(Result, LiveIn.Lanes.rawValue()); + } + + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + combineFingerprint(Result, Instruction->stableID().value()); + combineFingerprint(Result, Instruction->opcode().value()); + combineFingerprint(Result, Instruction->flags().rawValue()); + combineFingerprint(Result, Instruction->operandCount()); + combineFingerprint(Result, Instruction->memOperands().size()); + for (const MachineOperand &Operand : Instruction->operands()) { + combineFingerprint(Result, static_cast<std::uint64_t>(Operand.kind())); + combineFingerprint(Result, Operand.targetFlags()); + combineFingerprint(Result, Operand.registerFlags().rawValue()); + combineFingerprint(Result, Operand.subReg().value()); + if (const Register *Reg = Operand.getIf<Register>()) + combineFingerprint(Result, Reg->opaqueValue()); + else if (const std::int64_t *Immediate = Operand.getIf<std::int64_t>()) + combineFingerprint(Result, static_cast<std::uint64_t>(*Immediate)); + } + } + } + return Result; +} + +[[nodiscard]] Expected<void> verifyCompilationUnit(const CompilationUnit &Unit, + const TargetDescription &Target) { + auto UnitVerification = Unit.verify(); + if (!UnitVerification) + return std::unexpected(UnitVerification.error()); + for (std::size_t Index = 0; Index < Unit.functionCount(); ++Index) { + const MachineFunction *Function = Unit.functionAt(Index); + if (Function == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "compilation unit function order contains a null entry"); + auto FunctionVerification = MachineVerifier::verify(*Function, Target); + if (!FunctionVerification) + return std::unexpected(FunctionVerification.error()); + } + return {}; +} + +} // namespace + +Expected<void> +PassManagerCallbacks<MachineFunction, MachineFunctionPass, MachinePassManagerOptions>::validate( + const MachineFunctionPass &Pass) { + const MachinePassContract &Contract = Pass.contract(); + if (Contract.Name.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "machine pass must have a non-empty name"); + if (Contract.Sets.intersects(Contract.Clears)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine pass cannot set and clear the same property"); + } + if (hasDuplicateAnalysisIDs(Contract.RequiredAnalyses)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine pass required analyses must be non-null and unique"); + } + return validateParameters(Contract.Parameters); +} + +Expected<void> +PassManagerCallbacks<MachineFunction, MachineFunctionPass, MachinePassManagerOptions>::run( + const PassList &Passes, const MachinePassManagerOptions &Options, MachineFunction &Function, + const TargetDescription &Target, MachineFunctionAnalysisManager &Analyses, + MachinePassInstrumentation *Instrumentation) { + Analyses.setInstrumentation(Instrumentation); + for (const std::unique_ptr<MachineFunctionPass> &Pass : Passes) { + const MachinePassContract &Contract = Pass->contract(); + const std::string_view Name = Contract.Name; + + if (const MachineFunctionProperties Missing = + Contract.Requires.missingFrom(Function.properties()); + !Missing.empty()) { + Error Failure(ErrorCode::VerificationError, + "pass '" + std::string(Name) + "' requires properties: " + Missing.toString()); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + + if (Options.VerifyBefore) { + auto Verification = MachineVerifier::verify(Function, Target); + if (!Verification) { + Error Failure = withErrorContext( + Verification.error(), "pass '" + std::string(Name) + "' received invalid MachineIR: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + + for (const AnalysisID Required : Contract.RequiredAnalyses) { + auto Result = Analyses.getResultByID(Required, Function, Target); + if (!Result) { + Error Failure = + withErrorContext(Result.error(), "pass '" + std::string(Name) + + "' could not materialize a required analysis: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + + const std::uint64_t FingerprintBefore = + Options.DetectUnreportedChanges ? machineFunctionFingerprint(Function) : 0; + if (Instrumentation != nullptr) + Instrumentation->before(Name, Function); + const Clock::time_point Start = Clock::now(); + Analyses.setActivePass(std::string(Name)); + auto PassResult = Pass->run(Function, Target, Analyses); + Analyses.setActivePass({}); + const std::uint64_t Duration = elapsedNanoseconds(Start); + if (!PassResult) { + Error Failure = + withErrorContext(PassResult.error(), "pass '" + std::string(Name) + "' failed: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + + if (PassResult->Changed != !PassResult->Changes.empty()) { + Error Failure(ErrorCode::VerificationError, + "pass '" + std::string(Name) + + "' reported inconsistent changed state and modification categories"); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + if (!Contract.DeclaredChanges.contains(PassResult->Changes)) { + Error Failure(ErrorCode::VerificationError, + "pass '" + std::string(Name) + + "' reported a modification category absent from its registration"); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + + bool FingerprintChanged = PassResult->Changed; + if (Options.DetectUnreportedChanges) { + FingerprintChanged = FingerprintBefore != machineFunctionFingerprint(Function); + if (FingerprintChanged && !PassResult->Changed) { + Error Failure(ErrorCode::VerificationError, + "pass '" + std::string(Name) + + "' changed MachineIR without reporting a modification"); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + if (Instrumentation != nullptr) + Instrumentation->change(Name, Function, PassResult->Changes, FingerprintChanged); + + auto Transition = + Function.properties().applyTransition(Contract.Requires, Contract.Sets, Contract.Clears); + if (!Transition) { + Error Failure = + withErrorContext(Transition.error(), "pass '" + std::string(Name) + + "' produced an invalid property transition: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + Analyses.invalidate(Function, PassResult->Preserved, PassResult->Changes); + + if (Options.VerifyEach) { + MachineVerifierOptions VerifyOptions; + VerifyOptions.Stage = Contract.VerificationStage; + auto Verification = MachineVerifier::verify(Function, Target, VerifyOptions); + if (!Verification) { + Error Failure = withErrorContext(Verification.error(), "pass '" + std::string(Name) + + "' left invalid MachineIR: "); + if (Instrumentation != nullptr) + Instrumentation->failure(Name, Function, Failure, Duration); + Analyses.clear(Function); + Analyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + if (Instrumentation != nullptr) + Instrumentation->after(Name, Function, *PassResult, Duration); + } + Analyses.setInstrumentation(nullptr); + return {}; +} + +Expected<void> +PassManagerCallbacks<MachineFunction, MachineFunctionPass, MachinePassManagerOptions>::runFunctions( + const PassList &Passes, const MachinePassManagerOptions &Options, CompilationUnit &Unit, + const TargetDescription &Target, const MachineFunctionAnalysisManager &AnalysisPrototype, + std::size_t ThreadCount, MachinePassInstrumentation *Instrumentation) { + if (ThreadCount == 0) + return makeUnexpected(ErrorCode::InvalidArgument, "machine pass thread count must be nonzero"); + auto UnitVerification = verifyCompilationUnit(Unit, Target); + if (!UnitVerification) + return std::unexpected(UnitVerification.error()); + const std::size_t FunctionCount = Unit.functionCount(); + if (FunctionCount == 0) + return {}; + const bool RequiresSerialInstrumentation = + Instrumentation != nullptr && Instrumentation->requiresSerialFunctionExecution(); + const std::size_t WorkerCount = + RequiresSerialInstrumentation ? 1 : std::min(ThreadCount, FunctionCount); + if (WorkerCount > 1) { + for (const std::unique_ptr<MachineFunctionPass> &Pass : Passes) { + if (Pass->contract().ThreadSafety != PassThreadSafety::ThreadSafe) { + return makeUnexpected(ErrorCode::InvalidArgument, + "parallel machine pipeline contains a pass not marked thread-safe"); + } + auto AnalysisSafety = + AnalysisPrototype.validateParallelRequirements(Pass->contract().RequiredAnalyses); + if (!AnalysisSafety) + return std::unexpected(AnalysisSafety.error()); + } + } + std::vector<std::unique_ptr<MachineFunctionAnalysisManager>> Managers; + Managers.reserve(FunctionCount); + for (std::size_t Index = 0; Index < FunctionCount; ++Index) { + auto Manager = AnalysisPrototype.forkEmpty(); + if (!Manager) + return std::unexpected(Manager.error()); + Managers.push_back(std::move(*Manager)); + } + + std::vector<std::optional<Error>> Failures(FunctionCount); + std::atomic<std::size_t> NextFunction = 0; + const auto Worker = [&]() { + while (true) { + const std::size_t Index = NextFunction.fetch_add(1, std::memory_order_relaxed); + if (Index >= FunctionCount) + return; + MachineFunction *Function = Unit.functionAt(Index); + if (Function == nullptr) { + Failures[Index] = Error(ErrorCode::InternalError, + "compilation unit contains a null function during parallel run"); + continue; + } + auto Result = PassManagerCallbacks::run(Passes, Options, *Function, Target, *Managers[Index], + Instrumentation); + if (!Result) + Failures[Index] = Result.error(); + } + }; + + std::vector<std::thread> Workers; + Workers.reserve(WorkerCount); + for (std::size_t Index = 0; Index < WorkerCount; ++Index) + Workers.emplace_back(Worker); + for (std::thread &WorkerThread : Workers) + WorkerThread.join(); + + for (std::size_t Index = 0; Index < FunctionCount; ++Index) { + const auto &Failure = Failures[Index]; + if (Failure) { + return std::unexpected(withErrorContext( + *Failure, "function '" + std::string(Unit.functionAt(Index)->name()) + "' failed: ")); + } + } + return {}; +} + +Expected<void> +PassManagerCallbacks<CompilationUnit, CompilationUnitPass, + CompilationUnitPassManagerOptions>::validate(const CompilationUnitPass &Pass) { + const CompilationUnitPassContract &Contract = Pass.contract(); + if (Contract.Name.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "compilation-unit pass must have a non-empty name"); + } + if (hasDuplicateAnalysisIDs(Contract.RequiredAnalyses)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "compilation-unit pass required analyses must be non-null and unique"); + } + auto Parameters = validateParameters(Contract.Parameters); + if (!Parameters) + return std::unexpected(Parameters.error()); + return {}; +} + +Expected<void> +PassManagerCallbacks<CompilationUnit, CompilationUnitPass, CompilationUnitPassManagerOptions>::run( + const PassList &Passes, const CompilationUnitPassManagerOptions &Options, CompilationUnit &Unit, + const TargetDescription &Target, CompilationUnitAnalysisManager &UnitAnalyses, + MachineFunctionAnalysisManager &FunctionAnalyses, MachinePassInstrumentation *Instrumentation) { + UnitAnalyses.setInstrumentation(Instrumentation); + FunctionAnalyses.setInstrumentation(Instrumentation); + for (const std::unique_ptr<CompilationUnitPass> &Pass : Passes) { + const CompilationUnitPassContract &Contract = Pass->contract(); + const std::string_view Name = Contract.Name; + + if (Options.VerifyBefore) { + auto Verification = verifyCompilationUnit(Unit, Target); + if (!Verification) { + Error Failure = + withErrorContext(Verification.error(), "compilation-unit pass '" + std::string(Name) + + "' received invalid MachineIR: "); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + + for (const AnalysisID Required : Contract.RequiredAnalyses) { + auto Result = UnitAnalyses.getResultByID(Required, Unit, Target); + if (!Result) { + Error Failure = + withErrorContext(Result.error(), "compilation-unit pass '" + std::string(Name) + + "' could not materialize a required analysis: "); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + + if (Instrumentation != nullptr) + Instrumentation->beforeUnit(Name, Unit); + const Clock::time_point Start = Clock::now(); + auto Result = Pass->run(Unit, Target, UnitAnalyses, FunctionAnalyses); + const std::uint64_t Duration = elapsedNanoseconds(Start); + if (!Result) { + Error Failure = withErrorContext(Result.error(), "compilation-unit pass '" + + std::string(Name) + "' failed: "); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure, Duration); + UnitAnalyses.clearAll(); + FunctionAnalyses.clearAll(); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + if (Result->Changed != !Result->Changes.empty() || + !Contract.DeclaredChanges.contains(Result->Changes)) { + Error Failure(ErrorCode::VerificationError, + "compilation-unit pass '" + std::string(Name) + + "' reported inconsistent or undeclared modifications"); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure, Duration); + UnitAnalyses.clearAll(); + FunctionAnalyses.clearAll(); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + + UnitAnalyses.invalidate(Unit, Result->PreservedUnitAnalyses, Result->Changes); + for (std::size_t Index = 0; Index < Unit.functionCount(); ++Index) { + MachineFunction *Function = Unit.functionAt(Index); + if (Function != nullptr) { + FunctionAnalyses.invalidate(*Function, Result->PreservedFunctionAnalyses, Result->Changes); + } + } + if (Options.VerifyEach) { + auto Verification = verifyCompilationUnit(Unit, Target); + if (!Verification) { + Error Failure = + withErrorContext(Verification.error(), "compilation-unit pass '" + std::string(Name) + + "' left invalid MachineIR: "); + if (Instrumentation != nullptr) + Instrumentation->unitFailure(Name, Unit, Failure, Duration); + UnitAnalyses.clearAll(); + FunctionAnalyses.clearAll(); + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return std::unexpected(std::move(Failure)); + } + } + if (Instrumentation != nullptr) + Instrumentation->afterUnit(Name, Unit, *Result, Duration); + } + UnitAnalyses.setInstrumentation(nullptr); + FunctionAnalyses.setInstrumentation(nullptr); + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachinePeepholeOptimizer.cpp b/backend/lib/CodeGen/MachinePeepholeOptimizer.cpp new file mode 100644 index 0000000..ae39334 --- /dev/null +++ b/backend/lib/CodeGen/MachinePeepholeOptimizer.cpp @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePeepholeOptimizer.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetInstrInfo.h" + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &peepholeOptimizerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "peephole-opt", + .Requires = {MachineFunctionProperty::IsSSA}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + return Contract; +} + +} // namespace + +const MachinePassContract &MachinePeepholeOptimizerPass::contract() const noexcept { + return peepholeOptimizerContract(); +} + +Expected<MachinePassResult> MachinePeepholeOptimizerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + return runImplementation(Function, Target); +} + +Expected<MachinePassResult> +MachinePeepholeOptimizerPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + if (!Instruction->isDebugInstr()) { + auto Changed = Info->optimizeCondBranch(*Instruction, Function, Target); + if (!Changed) + return std::unexpected(Changed.error()); + Result.Changed |= *Changed; + } + Instruction = Next; + } + } + if (Result.Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachinePseudoExpansion.cpp b/backend/lib/CodeGen/MachinePseudoExpansion.cpp new file mode 100644 index 0000000..1115cd3 --- /dev/null +++ b/backend/lib/CodeGen/MachinePseudoExpansion.cpp @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePseudoExpansion.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetInstrInfo.h" + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <string> +#include <tuple> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +using CFGEdge = std::tuple<std::uint32_t, std::uint32_t, std::uint32_t>; + +struct ResourceSnapshot final { + std::size_t BlockCount = 0; + std::size_t VirtualRegisterCount = 0; + std::size_t FrameObjectCount = 0; + bool FrameFrozen = false; + std::vector<CFGEdge> Edges; +}; + +using BlockSizeSnapshot = std::vector<std::pair<MachineBlockStableID, std::uint64_t>>; + +[[nodiscard]] Expected<std::uint64_t> blockSize(const MachineBasicBlock &Block, + const MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &BranchInfo) { + std::uint64_t Size = 0; + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto InstructionSize = BranchInfo.instructionSize(*Instruction, Function, Target); + if (!InstructionSize) + return std::unexpected(InstructionSize.error()); + if (Size > std::numeric_limits<std::uint64_t>::max() - *InstructionSize) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, "target-late pseudo size overflowed"); + } + Size += *InstructionSize; + } + return Size; +} + +[[nodiscard]] Expected<BlockSizeSnapshot> snapshotBlockSizes(const MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &BranchInfo) { + BlockSizeSnapshot Result; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Size = blockSize(*Block, Function, Target, BranchInfo); + if (!Size) + return std::unexpected(Size.error()); + Result.emplace_back(Block->stableID(), *Size); + } + return Result; +} + +[[nodiscard]] ResourceSnapshot snapshotResources(const MachineFunction &Function) { + ResourceSnapshot Result; + Result.BlockCount = Function.blockCount(); + Result.VirtualRegisterCount = Function.registerInfo().virtualRegisters().size(); + Result.FrameObjectCount = Function.frameInfo().objects().size(); + Result.FrameFrozen = Function.frameInfo().isFrozen(); + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineSuccessor &Successor : Block->successors()) { + Result.Edges.emplace_back(Block->stableID().value(), Successor.Block->stableID().value(), + Successor.Probability.rawValue()); + } + } + return Result; +} + +[[nodiscard]] Expected<void> verifyResourceContract(const ResourceSnapshot &Before, + const MachineFunction &Function, + MachinePseudoExpansionPhase Phase) { + const ResourceSnapshot After = snapshotResources(Function); + if (Before.BlockCount != After.BlockCount || Before.Edges != After.Edges) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "pseudo expansion changed CFG resources outside its owner contract"); + } + if (Phase != MachinePseudoExpansionPhase::PreRA && + Before.VirtualRegisterCount != After.VirtualRegisterCount) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "post-selection pseudo expansion created a virtual register"); + } + if (Before.FrameObjectCount != After.FrameObjectCount || + Before.FrameFrozen != After.FrameFrozen) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "post-PEI pseudo expansion changed frozen frame resources"); + } + return {}; +} + +[[nodiscard]] PseudoOwner ownerFor(MachinePseudoExpansionPhase Phase) noexcept { + switch (Phase) { + case MachinePseudoExpansionPhase::PreRA: + return PseudoOwner::PreRAExpansion; + case MachinePseudoExpansionPhase::PostRA: + return PseudoOwner::PostRAExpansion; + case MachinePseudoExpansionPhase::TargetLate: + return PseudoOwner::TargetLateExpansion; + } + return PseudoOwner::None; +} + +[[nodiscard]] const MachinePassContract &preparationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "prepare-post-ra-pseudos", + .Requires = {MachineFunctionProperty::Selected}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] const MachinePassContract &phaseContract(MachinePseudoExpansionPhase Phase) { + static const MachinePassContract PreRA = [] { + MachinePassContract Value{ + .Name = "pre-ra-target-pseudo-expansion", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "postrapseudos", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + static const MachinePassContract TargetLate = [] { + MachinePassContract Value{ + .Name = "target-late-pseudo-expansion", + .Requires = {MachineFunctionProperty::NoPHIs, MachineFunctionProperty::NoVRegs, + MachineFunctionProperty::FrameFinalized, + MachineFunctionProperty::NoFrameIndices}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::PostRAFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + switch (Phase) { + case MachinePseudoExpansionPhase::PreRA: + return PreRA; + case MachinePseudoExpansionPhase::PostRA: + return PostRA; + case MachinePseudoExpansionPhase::TargetLate: + return TargetLate; + } + return TargetLate; +} + +[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) { + std::vector<MachineInstr *> Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) + Result.push_back(Instruction); + return Result; +} + +[[nodiscard]] Expected<MachinePassResult> +prepareImplementation(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info, + MachineFunctionAnalysisManager &Analyses) { + bool Changed = false; + for (MachineInstr *Instruction : layoutInstructions(Function)) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "pseudo preparation encountered an unknown opcode"); + if (Descriptor->Owner != PseudoOwner::PostRAExpansion) + continue; + std::string Rule = "prepare-pseudo."; + Rule.append(Descriptor->Name); + auto Accepted = Analyses.requestAction(Rule, Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "required pseudo preparation action was rejected"); + } + auto Prepared = Info.preparePostRAPseudo(*Instruction, Function, Target); + if (!Prepared) + return std::unexpected(Prepared.error()); + Changed |= *Prepared; + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +[[nodiscard]] Expected<MachinePassResult> +expandImplementation(MachineFunction &Function, const TargetDescription &Target, + const TargetPseudoExpansionInfo &Info, MachinePseudoExpansionPhase Phase, + const TargetInstrInfo *BranchInfo, MachineFunctionAnalysisManager &Analyses) { + const ResourceSnapshot Before = snapshotResources(Function); + std::optional<BlockSizeSnapshot> SizesBefore; + if (Phase == MachinePseudoExpansionPhase::TargetLate) { + if (BranchInfo == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target-late pseudo expansion requires branch sizing"); + } + auto Snapshot = snapshotBlockSizes(Function, Target, *BranchInfo); + if (!Snapshot) + return std::unexpected(Snapshot.error()); + SizesBefore = std::move(*Snapshot); + } + std::vector<MachineInstr *> Worklist; + Worklist = layoutInstructions(Function); + bool Changed = false; + for (MachineInstr *Instruction : Worklist) { + if (Instruction->parent() == nullptr) + continue; + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "pseudo expansion encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::Pseudo) || Descriptor->Owner != ownerFor(Phase)) { + continue; + } + std::string Rule = "expand-pseudo."; + Rule.append(Descriptor->Name); + auto Accepted = Analyses.requestAction(Rule, Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "required pseudo expansion action was rejected"); + } + MachineBasicBlock *ExpansionBlock = Instruction->parent(); + std::optional<std::uint64_t> ExpansionSizeBefore; + if (Phase == MachinePseudoExpansionPhase::TargetLate) { + auto Size = blockSize(*ExpansionBlock, Function, Target, *BranchInfo); + if (!Size) + return std::unexpected(Size.error()); + ExpansionSizeBefore = *Size; + } + auto Expanded = Info.expandPseudo(*Instruction, Phase, Function, Target); + if (!Expanded) + return std::unexpected(Expanded.error()); + if (!*Expanded) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target has no expansion for an owned executable pseudo"); + } + if (ExpansionSizeBefore.has_value()) { + auto SizeAfter = blockSize(*ExpansionBlock, Function, Target, *BranchInfo); + if (!SizeAfter) + return std::unexpected(SizeAfter.error()); + if (*SizeAfter > *ExpansionSizeBefore) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "one target-late pseudo expansion increased its encoded block size"); + } + } + Changed = true; + } + + auto Resources = verifyResourceContract(Before, Function, Phase); + if (!Resources) + return std::unexpected(Resources.error()); + if (SizesBefore.has_value()) { + auto SizesAfter = snapshotBlockSizes(Function, Target, *BranchInfo); + if (!SizesAfter) + return std::unexpected(SizesAfter.error()); + if (SizesAfter->size() != SizesBefore->size()) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target-late pseudo expansion changed block layout"); + } + for (std::size_t Index = 0; Index < SizesAfter->size(); ++Index) { + if ((*SizesAfter)[Index].first != (*SizesBefore)[Index].first || + (*SizesAfter)[Index].second > (*SizesBefore)[Index].second) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "target-late pseudo expansion increased an encoded block size"); + } + } + } + for (MachineInstr *Instruction : layoutInstructions(Function)) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Pseudo) && + Descriptor->Owner == ownerFor(Phase)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "pseudo expansion left an instruction owned by the completed phase"); + } + } + if (Changed && Phase != MachinePseudoExpansionPhase::PostRA && + Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + MachinePassResult Result; + Result.Changed = Changed; + if (Changed) { + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace + +const MachinePassContract &PreRATargetPseudoExpansionPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::PreRA); +} + +Expected<MachinePassResult> +PreRATargetPseudoExpansionPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return expandImplementation(Function, Target, *Info, MachinePseudoExpansionPhase::PreRA, nullptr, + Analyses); +} + +const MachinePassContract &PreparePostRAPseudosPass::contract() const noexcept { + return preparationContract(); +} + +Expected<MachinePassResult> +PreparePostRAPseudosPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return prepareImplementation(Function, Target, *Info, Analyses); +} + +const MachinePassContract &ExpandPostRAPseudosPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::PostRA); +} + +Expected<MachinePassResult> ExpandPostRAPseudosPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return expandImplementation(Function, Target, *Info, MachinePseudoExpansionPhase::PostRA, nullptr, + Analyses); +} + +const MachinePassContract &TargetLatePseudoExpansionPass::contract() const noexcept { + return phaseContract(MachinePseudoExpansionPhase::TargetLate); +} + +Expected<MachinePassResult> +TargetLatePseudoExpansionPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return expandImplementation(Function, Target, *Info, MachinePseudoExpansionPhase::TargetLate, + BranchInfo, Analyses); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineRegisterInfo.cpp b/backend/lib/CodeGen/MachineRegisterInfo.cpp new file mode 100644 index 0000000..6147717 --- /dev/null +++ b/backend/lib/CodeGen/MachineRegisterInfo.cpp @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" + +#include <algorithm> +#include <functional> +#include <memory> +#include <optional> +#include <unordered_set> + +namespace rcc { + +RegOperandIterator::RegOperandIterator(MachineOperand *Current, bool RequireUse, bool RequireDef, + bool SkipDebug) noexcept + : Current(Current), RequireUse(RequireUse), RequireDef(RequireDef), SkipDebug(SkipDebug) { + skipFiltered(); +} + +RegOperandIterator::reference RegOperandIterator::operator*() const noexcept { + return *Current; +} + +RegOperandIterator::pointer RegOperandIterator::operator->() const noexcept { + return Current; +} + +RegOperandIterator &RegOperandIterator::operator++() noexcept { + Current = Current->NextRegOperand; + skipFiltered(); + return *this; +} + +RegOperandIterator RegOperandIterator::operator++(int) noexcept { + RegOperandIterator Previous = *this; + ++*this; + return Previous; +} + +void RegOperandIterator::skipFiltered() noexcept { + while (Current != nullptr) { + if (RequireUse && !Current->isUse()) { + Current = Current->NextRegOperand; + continue; + } + if (RequireDef && !Current->isDef()) { + Current = Current->NextRegOperand; + continue; + } + if (SkipDebug && Current->isDebugUse()) { + Current = Current->NextRegOperand; + continue; + } + break; + } +} + +MachineRegisterInfo::~MachineRegisterInfo() = default; + +Expected<Register> MachineRegisterInfo::createVirtualRegister() { + auto Reg = Register::makeVirtual(VirtualRegisters.size()); + if (!Reg) + return std::unexpected(Reg.error()); + VirtualRegisters.emplace_back(); + return *Reg; +} + +Expected<Register> MachineRegisterInfo::createVirtualRegister(LowLevelType Type) { + if (!Type.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "virtual register type must be a valid LowLevelType"); + } + + auto Reg = createVirtualRegister(); + if (!Reg) + return std::unexpected(Reg.error()); + VirtualRegisters.back().Type = Type; + return *Reg; +} + +Expected<Register> MachineRegisterInfo::cloneVirtualRegister(Register Reg) { + auto Source = virtualRegisterInfo(Reg); + if (!Source) + return std::unexpected(Source.error()); + auto Clone = createVirtualRegister(); + if (!Clone) + return std::unexpected(Clone.error()); + VirtualRegisters.back() = Source->get(); + VirtualRegisters.back().Name.reset(); + VirtualRegisters.back().AllocationHints.clear(); + VirtualRegisters.back().IsTombstone = false; + return *Clone; +} + +Expected<void> MachineRegisterInfo::eraseVirtualRegister(Register Reg) { + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (headFor(Reg) != nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "cannot erase a virtual register with operands"); + } + + Info->get().IsTombstone = true; + Info->get().Name.reset(); + Info->get().Type.reset(); + Info->get().Bank.reset(); + Info->get().RegClass.reset(); + Info->get().AllocationHints.clear(); + Info->get().TargetFlags = 0; + return {}; +} + +bool MachineRegisterInfo::isKnownVirtualRegister(Register Reg) const noexcept { + const auto Index = Reg.virtualIndex(); + if (!Index || *Index >= VirtualRegisters.size()) + return false; + return !VirtualRegisters[*Index].IsTombstone; +} + +Expected<std::reference_wrapper<VirtualRegisterInfo>> +MachineRegisterInfo::virtualRegisterInfo(Register Reg) { + const auto Index = Reg.virtualIndex(); + if (!Index || *Index >= VirtualRegisters.size() || VirtualRegisters[*Index].IsTombstone) { + return makeUnexpected(ErrorCode::InvalidArgument, "register is not a live virtual register"); + } + return std::ref(VirtualRegisters[*Index]); +} + +Expected<std::reference_wrapper<const VirtualRegisterInfo>> +MachineRegisterInfo::virtualRegisterInfo(Register Reg) const { + const auto Index = Reg.virtualIndex(); + if (!Index || *Index >= VirtualRegisters.size() || VirtualRegisters[*Index].IsTombstone) { + return makeUnexpected(ErrorCode::InvalidArgument, "register is not a live virtual register"); + } + return std::cref(VirtualRegisters[*Index]); +} + +Expected<void> MachineRegisterInfo::setType(Register Reg, LowLevelType Type) { + if (!Type.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "virtual register type must be a valid LowLevelType"); + } + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + Info->get().Type = Type; + return {}; +} + +Expected<void> MachineRegisterInfo::clearType(Register Reg) { + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + Info->get().Type.reset(); + return {}; +} + +Expected<void> MachineRegisterInfo::setRegisterBank(Register Reg, RegisterBankID Bank) { + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + Info->get().Bank = Bank; + return {}; +} + +Expected<void> MachineRegisterInfo::clearRegisterBank(Register Reg) { + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + Info->get().Bank.reset(); + return {}; +} + +Expected<void> MachineRegisterInfo::setRegisterClass(Register Reg, RegisterClassID RegClass) { + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + Info->get().RegClass = RegClass; + return {}; +} + +Expected<void> MachineRegisterInfo::clearRegisterClass(Register Reg) { + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + Info->get().RegClass.reset(); + return {}; +} + +Expected<void> MachineRegisterInfo::addAllocationHint(Register Reg, PhysicalRegister Hint) { + if (!Hint.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "allocation hint must name a physical register"); + } + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + + auto &Hints = Info->get().AllocationHints; + if (std::ranges::find(Hints, Hint) == Hints.end()) + Hints.push_back(Hint); + return {}; +} + +Expected<void> MachineRegisterInfo::setTargetRegisterFlags(Register Reg, std::uint8_t Flags) { + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + Info->get().TargetFlags = Flags; + return {}; +} + +Expected<void> MachineRegisterInfo::clearTargetRegisterFlags(Register Reg) { + return setTargetRegisterFlags(Reg, 0); +} + +void MachineRegisterInfo::clearAllVirtualRegisterTypes() noexcept { + for (VirtualRegisterInfo &Info : VirtualRegisters) { + if (!Info.IsTombstone) + Info.Type.reset(); + } +} + +Expected<void> MachineRegisterInfo::addLiveIn(PhysicalRegister PhysReg, + std::optional<Register> InitialVirtualRegister) { + if (!PhysReg.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "function live-in must name a physical register"); + } + if (InitialVirtualRegister && !isKnownVirtualRegister(*InitialVirtualRegister)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "function live-in initial value must be a live virtual register"); + } + if (std::ranges::any_of(LiveIns, [PhysReg](const PhysicalRegisterLiveIn &LiveIn) { + return LiveIn.PhysReg == PhysReg; + })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "function physical live-in is already present"); + } + + LiveIns.push_back(PhysicalRegisterLiveIn{PhysReg, InitialVirtualRegister}); + return {}; +} + +void MachineRegisterInfo::clearLiveInVirtualRegisters() noexcept { + for (PhysicalRegisterLiveIn &LiveIn : LiveIns) + LiveIn.InitialVirtualRegister.reset(); +} + +Expected<void> MachineRegisterInfo::reservePhysicalRegister(PhysicalRegister PhysReg) { + if (!PhysReg.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "reserved register must name a physical register"); + } + if (isReserved(PhysReg)) { + return makeUnexpected(ErrorCode::InvalidArgument, "physical register is already reserved"); + } + ReservedRegisters.push_back(PhysReg); + return {}; +} + +bool MachineRegisterInfo::isReserved(PhysicalRegister PhysReg) const noexcept { + return std::ranges::find(ReservedRegisters, PhysReg) != ReservedRegisters.end(); +} + +RegOperandRange MachineRegisterInfo::regOperands(Register Reg) noexcept { + return RegOperandRange(headFor(Reg), false, false, false); +} + +RegOperandRange MachineRegisterInfo::regOperands(Register Reg) const noexcept { + return RegOperandRange(headFor(Reg), false, false, false); +} + +RegOperandRange MachineRegisterInfo::regNoDebugOperands(Register Reg) noexcept { + return RegOperandRange(headFor(Reg), false, false, true); +} + +RegOperandRange MachineRegisterInfo::useOperands(Register Reg, bool IncludeDebug) noexcept { + return RegOperandRange(headFor(Reg), true, false, !IncludeDebug); +} + +RegOperandRange MachineRegisterInfo::defOperands(Register Reg) noexcept { + return RegOperandRange(headFor(Reg), false, true, false); +} + +bool MachineRegisterInfo::useEmpty(Register Reg, bool IncludeDebug) const noexcept { + for (MachineOperand *Operand = headFor(Reg); Operand != nullptr; + Operand = Operand->NextRegOperand) { + if (Operand->isUse() && (IncludeDebug || !Operand->isDebugUse())) + return false; + } + return true; +} + +bool MachineRegisterInfo::hasOneUse(Register Reg, bool IncludeDebug) const noexcept { + bool FoundUse = false; + for (MachineOperand *Operand = headFor(Reg); Operand != nullptr; + Operand = Operand->NextRegOperand) { + if (!Operand->isUse() || (!IncludeDebug && Operand->isDebugUse())) + continue; + if (FoundUse) + return false; + FoundUse = true; + } + return FoundUse; +} + +const MachineOperand *MachineRegisterInfo::getUniqueVRegDef(Register Reg) const noexcept { + if (!isKnownVirtualRegister(Reg)) + return nullptr; + + const MachineOperand *UniqueDef = nullptr; + for (MachineOperand *Operand = headFor(Reg); Operand != nullptr; + Operand = Operand->NextRegOperand) { + if (!Operand->isDef()) + continue; + if (UniqueDef != nullptr) + return nullptr; + UniqueDef = Operand; + } + return UniqueDef; +} + +MachineInstr *MachineRegisterInfo::getVRegDef(Register Reg) noexcept { + const MachineOperand *Definition = getUniqueVRegDef(Reg); + return Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); +} + +const MachineInstr *MachineRegisterInfo::getVRegDef(Register Reg) const noexcept { + const MachineOperand *Definition = getUniqueVRegDef(Reg); + return Definition == nullptr ? nullptr : Definition->parent(); +} + +Expected<LowLevelType> MachineRegisterInfo::getType(Register Reg) const { + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + const std::optional<LowLevelType> &Type = Info->get().Type; + if (!Type) + return makeUnexpected(ErrorCode::InvalidArgument, "virtual register has no LowLevelType"); + return *Type; +} + +Expected<std::size_t> MachineRegisterInfo::replaceRegWith(Register OldReg, Register NewReg) { + if (OldReg.isNoRegister() || NewReg.isNoRegister()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register replacement requires two nonzero registers"); + } + if ((OldReg.isVirtual() && !isKnownVirtualRegister(OldReg)) || + (NewReg.isVirtual() && !isKnownVirtualRegister(NewReg))) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register replacement references an unknown virtual register"); + } + + std::size_t ReplacementCount = 0; + MachineOperand *Operand = headFor(OldReg); + while (Operand != nullptr) { + MachineOperand *Next = Operand->NextRegOperand; + const auto Result = Operand->setRegister(NewReg); + if (!Result) + return std::unexpected(Result.error()); + ++ReplacementCount; + Operand = Next; + } + return ReplacementCount; +} + +Expected<void> MachineRegisterInfo::verifyDefUseChains() const { + std::unordered_set<const MachineOperand *> SeenOperands; + + for (const auto &[Reg, HeadStorage] : DefUseHeads) { + if (!HeadStorage) { + return makeUnexpected(ErrorCode::VerificationError, + "register def-use table contains null head storage"); + } + if (HeadStorage->Head == nullptr) + continue; + if (Reg.isNoRegister() || (Reg.isVirtual() && !isKnownVirtualRegister(Reg))) { + return makeUnexpected(ErrorCode::VerificationError, + "register def-use chain is keyed by an invalid register"); + } + MachineOperand *const *ExpectedPreviousLink = &HeadStorage->Head; + for (MachineOperand *Operand = HeadStorage->Head; Operand != nullptr; + Operand = Operand->NextRegOperand) { + if (!SeenOperands.insert(Operand).second) { + return makeUnexpected(ErrorCode::VerificationError, + "register def-use chains contain a cycle or duplicate operand"); + } + if (!Operand->isRegister() || Operand->registerValue() != Reg) { + return makeUnexpected(ErrorCode::VerificationError, + "register def-use chain contains an operand for another register"); + } + if (Operand->LinkedRegInfo != this || Operand->PreviousRegLink != ExpectedPreviousLink) { + return makeUnexpected(ErrorCode::VerificationError, + "register def-use chain back-link is corrupt"); + } + if (Operand->Parent == nullptr || !Operand->Parent->isLinked() || + &Operand->Parent->function().registerInfo() != this) { + return makeUnexpected( + ErrorCode::VerificationError, + "register def-use chain contains an operand outside its function layout"); + } + ExpectedPreviousLink = &Operand->NextRegOperand; + } + } + + return {}; +} + +bool MachineRegisterInfo::isLinkedOperand(const MachineOperand &Operand) const noexcept { + return Operand.LinkedRegInfo == this && Operand.PreviousRegLink != nullptr; +} + +MachineOperand *MachineRegisterInfo::headFor(Register Reg) const noexcept { + const auto Position = DefUseHeads.find(Reg); + if (Position == DefUseHeads.end()) + return nullptr; + return Position->second->Head; +} + +void MachineRegisterInfo::linkOperand(MachineOperand &Operand) { + const auto Reg = Operand.registerValue(); + if (!Reg || Reg->isNoRegister()) + return; + + auto &HeadStorage = DefUseHeads[*Reg]; + if (!HeadStorage) + HeadStorage = std::make_unique<RegOperandListHead>(); + + Operand.unlinkFromRegInfo(); + Operand.LinkedRegInfo = this; + Operand.PreviousRegLink = &HeadStorage->Head; + Operand.NextRegOperand = HeadStorage->Head; + if (HeadStorage->Head != nullptr) + HeadStorage->Head->PreviousRegLink = &Operand.NextRegOperand; + HeadStorage->Head = &Operand; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineScheduler.cpp b/backend/lib/CodeGen/MachineScheduler.cpp new file mode 100644 index 0000000..65f1e01 --- /dev/null +++ b/backend/lib/CodeGen/MachineScheduler.cpp @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineScheduler.h" + +#include "MachineSchedulerInternal.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/RegisterPressure.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/ScheduleHazardRecognizer.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <iostream> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &schedulerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-scheduler", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<LiveVariablesAnalysis>()}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + Value.RequiredAnalyses.push_back( + MachineFunctionAnalysisManager::id<MachineBlockFrequencyAnalysis>()); + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits<std::uint64_t>::max() - Left < Right + ? std::numeric_limits<std::uint64_t>::max() + : Left + Right; +} + +[[nodiscard]] Expected<bool> isSchedulingBoundary(const MachineInstr &Instruction, + const TargetDescription &Target, + const TargetSchedulingModel &Model) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered an unknown instruction"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Model.hooks().isSchedulingBoundary(Instruction, Target)) + return true; + if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Phi)) { + return true; + } + if (Model.schedulingClass(Descriptor->SchedulingClass) == nullptr) { + if (Flags.has(InstructionFlag::Pseudo) || !Flags.has(InstructionFlag::ProducesCode)) + return true; + return makeUnexpected(ErrorCode::VerificationError, + "executable instruction has no scheduling class"); + } + return false; +} + +struct ScheduleCandidate final { + std::size_t Unit = 0; + bool Bottom = false; + std::uint64_t Cycle = 0; + RegisterPressureCost Pressure; + std::uint64_t CriticalPath = 0; + std::uint32_t Latency = 0; + std::int64_t TargetPreference = 0; + bool ContinuesCluster = false; + bool OptimizeLatency = false; + std::size_t StableOrder = 0; +}; + +[[nodiscard]] bool isBetterCandidate(const ScheduleCandidate &Left, + const ScheduleCandidate &Right) noexcept { + if (Left.Cycle != Right.Cycle) + return Left.Cycle < Right.Cycle; + if (Left.ContinuesCluster != Right.ContinuesCluster) + return Left.ContinuesCluster; + if (Left.TargetPreference != Right.TargetPreference) + return Left.TargetPreference > Right.TargetPreference; + if (Left.OptimizeLatency && Right.OptimizeLatency) { + if (Left.CriticalPath != Right.CriticalPath) + return Left.CriticalPath > Right.CriticalPath; + if (Left.Latency != Right.Latency) + return Left.Latency > Right.Latency; + } + if (Left.Pressure.PeakIncrease != Right.Pressure.PeakIncrease) + return Left.Pressure.PeakIncrease < Right.Pressure.PeakIncrease; + if (Left.Pressure.PeakExcess != Right.Pressure.PeakExcess) + return Left.Pressure.PeakExcess < Right.Pressure.PeakExcess; + if (Left.Pressure.CurrentExcess != Right.Pressure.CurrentExcess) + return Left.Pressure.CurrentExcess < Right.Pressure.CurrentExcess; + if (Left.CriticalPath != Right.CriticalPath) + return Left.CriticalPath > Right.CriticalPath; + if (Left.Latency != Right.Latency) + return Left.Latency > Right.Latency; + if (Left.StableOrder != Right.StableOrder) + return Left.StableOrder < Right.StableOrder; + return !Left.Bottom && Right.Bottom; +} + +[[nodiscard]] Expected<std::vector<std::size_t>> +scheduleDAG(const ScheduleDAG &DAG, const MachineFunction &Function, + const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const MachineBlockLiveness &Liveness, + std::uint64_t BlockFrequency, bool EnableBidirectionalScheduling, + MachineFunctionAnalysisManager &Analyses) { + auto TopPressure = RegisterPressureTracker::create(DAG, Function, Target, Model, Mode, Liveness, + RegisterPressureDirection::TopDown); + if (!TopPressure) + return std::unexpected(TopPressure.error()); + auto BottomPressure = RegisterPressureTracker::create( + DAG, Function, Target, Model, Mode, Liveness, RegisterPressureDirection::BottomUp); + if (!BottomPressure) + return std::unexpected(BottomPressure.error()); + auto TopHazards = ScheduleHazardRecognizer::create(Model, ScheduleDirection::TopDown); + if (!TopHazards) + return std::unexpected(TopHazards.error()); + auto BottomHazards = ScheduleHazardRecognizer::create(Model, ScheduleDirection::BottomUp); + if (!BottomHazards) + return std::unexpected(BottomHazards.error()); + + const std::size_t UnitCount = DAG.units().size(); + std::vector<std::size_t> TopPredecessors(UnitCount); + std::vector<std::size_t> BottomSuccessors(UnitCount); + std::vector<std::uint64_t> TopReadyCycle(UnitCount, 0); + std::vector<std::uint64_t> BottomReadyCycle(UnitCount, 0); + std::vector<bool> Scheduled(UnitCount, false); + for (const ScheduleUnit &Unit : DAG.units()) { + TopPredecessors[Unit.ID] = Unit.Predecessors.size(); + BottomSuccessors[Unit.ID] = Unit.Successors.size(); + } + + std::vector<std::size_t> TopOrder; + std::vector<std::size_t> BottomOrder; + TopOrder.reserve(UnitCount); + BottomOrder.reserve(UnitCount); + for (std::size_t ScheduledCount = 0; ScheduledCount < UnitCount; ++ScheduledCount) { + std::optional<ScheduleCandidate> Best; + for (const ScheduleUnit &Unit : DAG.units()) { + if (Scheduled[Unit.ID]) + continue; + if (TopPredecessors[Unit.ID] == 0) { + auto Cycle = TopHazards->earliestIssueCycle(Unit, TopReadyCycle[Unit.ID]); + if (!Cycle) + return std::unexpected(Cycle.error()); + auto Pressure = TopPressure->evaluate(Unit.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + const ScheduleCandidate Candidate{ + Unit.ID, + false, + *Cycle, + *Pressure, + Unit.Height, + Unit.Latency, + Model.hooks().candidatePreference(Unit, ScheduleDirection::TopDown, DAG.block(), + BlockFrequency), + Unit.ClusterPredecessor && Scheduled[*Unit.ClusterPredecessor], + BlockFrequency >= MachineBlockFrequencyAnalysis::EntryFrequency, + Unit.ID}; + if (!Best || isBetterCandidate(Candidate, *Best)) + Best = Candidate; + } + if (EnableBidirectionalScheduling && BottomSuccessors[Unit.ID] == 0) { + auto Cycle = BottomHazards->earliestIssueCycle(Unit, BottomReadyCycle[Unit.ID]); + if (!Cycle) + return std::unexpected(Cycle.error()); + auto Pressure = BottomPressure->evaluate(Unit.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + const ScheduleCandidate Candidate{ + Unit.ID, + true, + *Cycle, + *Pressure, + Unit.Depth, + Unit.Latency, + Model.hooks().candidatePreference(Unit, ScheduleDirection::BottomUp, DAG.block(), + BlockFrequency), + Unit.ClusterSuccessor && Scheduled[*Unit.ClusterSuccessor], + BlockFrequency >= MachineBlockFrequencyAnalysis::EntryFrequency, + Unit.ID}; + if (!Best || isBetterCandidate(Candidate, *Best)) + Best = Candidate; + } + } + if (!Best) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "schedule DAG has no legal top or bottom ready unit"); + } + + const ScheduleUnit &Selected = DAG.units()[Best->Unit]; + auto Accepted = Analyses.requestAction(Best->Bottom ? "schedule-bottom" : "schedule-top", + Function, Selected.StableID); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + return std::vector<std::size_t>{}; + Scheduled[Best->Unit] = true; + if (Best->Bottom) { + auto Issued = BottomHazards->issue(Selected, Best->Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + auto Pressure = BottomPressure->schedule(Selected.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + BottomOrder.push_back(Selected.ID); + for (const ScheduleDependency &Dependency : Selected.Predecessors) { + if (BottomSuccessors[Dependency.Source] == 0) { + return makeUnexpected(ErrorCode::InternalError, + "bottom scheduler successor count underflowed"); + } + --BottomSuccessors[Dependency.Source]; + BottomReadyCycle[Dependency.Source] = std::max( + BottomReadyCycle[Dependency.Source], saturatingAdd(Best->Cycle, Dependency.Latency)); + } + } else { + auto Issued = TopHazards->issue(Selected, Best->Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + auto Pressure = TopPressure->schedule(Selected.ID); + if (!Pressure) + return std::unexpected(Pressure.error()); + TopOrder.push_back(Selected.ID); + for (const ScheduleDependency &Dependency : Selected.Successors) { + if (TopPredecessors[Dependency.Destination] == 0) { + return makeUnexpected(ErrorCode::InternalError, + "top scheduler predecessor count underflowed"); + } + --TopPredecessors[Dependency.Destination]; + TopReadyCycle[Dependency.Destination] = std::max( + TopReadyCycle[Dependency.Destination], saturatingAdd(Best->Cycle, Dependency.Latency)); + } + } + } + + TopOrder.insert(TopOrder.end(), BottomOrder.rbegin(), BottomOrder.rend()); + if (TopOrder.size() != UnitCount) + return makeUnexpected(ErrorCode::InternalError, "scheduler lost a schedule unit"); + return TopOrder; +} + +[[nodiscard]] Expected<bool> +scheduleRegion(MachineBasicBlock &Block, std::span<MachineInstr *const> Instructions, + MachineFunction &Function, const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode, + const MachineBlockLiveness &Liveness, std::uint64_t BlockFrequency, + MachineSchedulerOptions Options, MachineFunctionAnalysisManager &Analyses) { + if (Instructions.size() < 2 || Instructions.size() > Options.MaximumRegionSize) + return false; + if (Options.EmitDebugOutput) { + std::cerr << Function.name() << ":%bb." << Block.denseNumber().value_or(0) << '\n'; + } + auto DAG = ScheduleDAG::build(Block, Instructions, Function, Target, Model, Mode, + Options.EmitDebugOutput); + if (!DAG) + return std::unexpected(DAG.error()); + if (DAG->units().size() < 2) + return false; + auto UnitOrder = scheduleDAG(*DAG, Function, Target, Model, Mode, Liveness, BlockFrequency, + Options.EnableBidirectionalScheduling, Analyses); + if (!UnitOrder) + return std::unexpected(UnitOrder.error()); + if (UnitOrder->empty()) + return false; + + std::vector<MachineInstr *> NewOrder; + NewOrder.reserve(Instructions.size()); + for (const std::size_t UnitID : *UnitOrder) { + const ScheduleUnit *Unit = DAG->unit(UnitID); + if (Unit == nullptr) + return makeUnexpected(ErrorCode::InternalError, "scheduler produced an unknown unit ID"); + NewOrder.insert(NewOrder.end(), Unit->Instructions.begin(), Unit->Instructions.end()); + } + if (std::ranges::equal(NewOrder, Instructions)) + return false; + + MachineInstr *Anchor = Instructions.back()->next(); + for (MachineInstr *Instruction : Instructions) { + auto Removed = Instruction->eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + } + for (MachineInstr *Instruction : NewOrder) { + Expected<void> Inserted = Anchor == nullptr + ? Block.appendInstruction(*Instruction) + : Block.insertInstructionBefore(*Anchor, *Instruction); + if (!Inserted) + return std::unexpected(Inserted.error()); + } + return true; +} + +} // namespace + +Expected<MachineSchedulingRunResult> +runMachineScheduler(MachineFunction &Function, const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode, + const LiveVariablesAnalysis::Result &Liveness, + const MachineBlockFrequencyAnalysis::Result &Frequencies, + MachineSchedulerOptions Options, MachineFunctionAnalysisManager &Analyses) { + if (Options.MaximumRegionSize == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machine scheduler region budget must be nonzero"); + } + MachineSchedulingRunResult Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + const MachineBlockLiveness *BlockLiveness = Liveness.block(*Block); + if (BlockLiveness == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "machine scheduler has no block liveness boundary"); + } + bool BlockChanged = false; + std::vector<MachineInstr *> Region; + auto FlushRegion = [&]() -> Expected<void> { + auto Changed = scheduleRegion(*Block, Region, Function, Target, Model, Mode, *BlockLiveness, + Frequencies.frequency(*Block), Options, Analyses); + if (!Changed) + return std::unexpected(Changed.error()); + BlockChanged |= *Changed; + Region.clear(); + return {}; + }; + + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + if (Instruction->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered a broken bundle header"); + } + std::vector<MachineInstr *> Group; + bool Boundary = false; + for (;;) { + Group.push_back(Instruction); + auto IsBoundary = isSchedulingBoundary(*Instruction, Target, Model); + if (!IsBoundary) + return std::unexpected(IsBoundary.error()); + Boundary |= *IsBoundary; + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + if (!HasSuccessor) + break; + MachineInstr *Next = Instruction->next(); + if (Next == nullptr || !Next->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine scheduler encountered a truncated bundle"); + } + Instruction = Next; + } + MachineInstr *NextGroup = Instruction->next(); + if (Boundary) { + auto Flushed = FlushRegion(); + if (!Flushed) + return std::unexpected(Flushed.error()); + } else { + Region.insert(Region.end(), Group.begin(), Group.end()); + } + Instruction = NextGroup; + } + auto Flushed = FlushRegion(); + if (!Flushed) + return std::unexpected(Flushed.error()); + if (BlockChanged) { + Result.Changed = true; + Result.ChangedBlocks.push_back(Block); + } + } + return Result; +} + +const MachinePassContract &MachineSchedulerPass::contract() const noexcept { + return schedulerContract(); +} + +Expected<MachinePassResult> MachineSchedulerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Liveness = Analyses.getResult<LiveVariablesAnalysis>(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Run = runMachineScheduler(Function, Target, *Model, ScheduleDAGMode::PreRegisterAllocation, + **Liveness, **Frequencies, Options, Analyses); + if (!Run) + return std::unexpected(Run.error()); + + MachinePassResult Result; + Result.Changed = Run->Changed; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineSchedulerInternal.h b/backend/lib/CodeGen/MachineSchedulerInternal.h new file mode 100644 index 0000000..a2815ad --- /dev/null +++ b/backend/lib/CodeGen/MachineSchedulerInternal.h @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) +#ifndef RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H +#define RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H + +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineScheduler.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/Support/Error.h" + +#include <vector> + +namespace rcc { + +class MachineBasicBlock; +class MachineFunction; +class TargetDescription; +class TargetSchedulingModel; +struct MachineSchedulingRunResult final { + bool Changed = false; + std::vector<MachineBasicBlock *> ChangedBlocks; +}; + +[[nodiscard]] Expected<MachineSchedulingRunResult> +runMachineScheduler(MachineFunction &Function, const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode, + const LiveVariablesAnalysis::Result &Liveness, + const MachineBlockFrequencyAnalysis::Result &Frequencies, + MachineSchedulerOptions Options, MachineFunctionAnalysisManager &Analyses); + +} // namespace rcc + +#endif // RCC_LIB_CODEGEN_MACHINESCHEDULERINTERNAL_H diff --git a/backend/lib/CodeGen/MachineShrinkWrapping.cpp b/backend/lib/CodeGen/MachineShrinkWrapping.cpp new file mode 100644 index 0000000..9c7e589 --- /dev/null +++ b/backend/lib/CodeGen/MachineShrinkWrapping.cpp @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineShrinkWrapping.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetFrameLowering.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <unordered_set> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &shrinkWrappingContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-shrink-wrapping", + .Requires = {MachineFunctionProperty::NoVRegs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineFunctionAnalysisManager::id<MachineLoopAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool requiresEntryFrame(const MachineFrameState &State) noexcept { + return State.IsFrameAddressTaken || State.IsReturnAddressTaken || State.HasStackMap || + State.HasPatchPoint || State.HasOpaqueSPAdjustment || State.HasVAStart || + State.HasMustTailInVarArgFunc || State.HasTailCall; +} + +[[nodiscard]] bool containsEpilogueTerminator(const MachineBasicBlock &Block, + const TargetFrameLowering &Info, + const TargetDescription &Target) noexcept { + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Info.isEpilogueTerminator(*Instruction, Target)) + return true; + } + return false; +} + +} // namespace + +const MachinePassContract &MachineShrinkWrappingPass::contract() const noexcept { + return shrinkWrappingContract(); +} + +Expected<MachinePassResult> +MachineShrinkWrappingPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + if (Function.frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::FrameLayoutFailed, + "shrink wrapping must run before frame layout freeze"); + } + if (Function.blockCount() > Options.MaximumAnalyzedBlocks) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + const MachineFrameState &OriginalState = Function.frameInfo().state(); + if (!OriginalState.SavePoints.empty() || !OriginalState.RestorePoints.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + MachineBasicBlock *Entry = Function.firstBlock(); + if (Entry == nullptr || requiresEntryFrame(OriginalState)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + auto Loops = Analyses.getResult<MachineLoopAnalysis>(Function, Target); + if (!Loops) + return std::unexpected(Loops.error()); + + std::vector<MachineBasicBlock *> RequiredBlocks; + std::unordered_set<FrameIndex> ReferencedObjects; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + bool Required = false; + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "shrink wrapping encountered an unknown opcode"); + } + Required |= Descriptor->Flags.has(InstructionFlag::Call); + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Frame = Operand.getIf<FrameIndexOperand>()) { + ReferencedObjects.insert(Frame->Index); + Required = true; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (Physical && Operand.isDef() && + Info->isCalleeSavedRegister(*Physical, Function, Target)) { + Required = true; + } + } + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + const std::optional<FrameIndex> Frame = + Memory == nullptr ? std::nullopt : Memory->description().Base.frameIndex(); + if (Frame) { + ReferencedObjects.insert(*Frame); + Required = true; + } + } + } + if (Required) + RequiredBlocks.push_back(Block); + } + + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + if (Description == nullptr || Description->IsDead) + continue; + if (!ReferencedObjects.contains(Object.index())) { + RequiredBlocks.push_back(Entry); + break; + } + } + if (RequiredBlocks.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + const MachineBasicBlock *Save = RequiredBlocks.front(); + if (!(*Dominators)->isReachable(*Save)) + Save = Entry; + for (MachineBasicBlock *Block : RequiredBlocks) { + if (!(*Dominators)->isReachable(*Block)) { + Save = Entry; + break; + } + const MachineBasicBlock *Common = (*Dominators)->nearestCommonDominator(*Save, *Block); + if (Common == nullptr) { + Save = Entry; + break; + } + Save = Common; + } + if (Save == Entry || Save->predecessors().size() != 1 || (*Loops)->loopDepth(*Save) != 0 || + (*Loops)->isIrreducible(*Save)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + std::vector<MachineBasicBlock *> RestoreBlocks; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!(*Dominators)->dominates(*Save, *Block)) + continue; + for (const MachineSuccessor &Successor : Block->successors()) { + if (Successor.Block == nullptr || !(*Dominators)->dominates(*Save, *Successor.Block)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + } + if (!Block->successors().empty()) + continue; + if (!containsEpilogueTerminator(*Block, *Info, Target)) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + RestoreBlocks.push_back(Block); + } + if (RestoreBlocks.empty()) { + MachinePassResult Result; + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + + MachineFrameState Planned = OriginalState; + Planned.SavePoints.push_back({Save->stableID(), {}}); + std::ranges::sort(RestoreBlocks, {}, + [](const MachineBasicBlock *Block) { return Block->stableID().value(); }); + for (const MachineBasicBlock *Block : RestoreBlocks) + Planned.RestorePoints.push_back({Block->stableID(), {}}); + auto Updated = Function.frameInfo().setState(std::move(Planned)); + if (!Updated) + return std::unexpected(Updated.error()); + + MachinePassResult Result; + Result.Changed = true; + Result.Changes = {MachinePassChange::Frame}; + Result.Preserved.preserve<MachineDominatorTreeAnalysis>(); + Result.Preserved.preserve<MachineLoopAnalysis>(); + Result.Preserved.preserve<MachineBranchProbabilityAnalysis>(); + Result.Preserved.preserve<MachineBlockFrequencyAnalysis>(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineSink.cpp b/backend/lib/CodeGen/MachineSink.cpp new file mode 100644 index 0000000..f82f3a7 --- /dev/null +++ b/backend/lib/CodeGen/MachineSink.cpp @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineSink.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &sinkContract(MachineCodeMotionStage Stage) { + static const MachinePassContract Selected = [] { + MachinePassContract Value{ + .Name = "machine-sink", + .Requires = {MachineFunctionProperty::IsSSA}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineFunctionAnalysisManager::id<MachineBlockFrequencyAnalysis>()}, + .VerificationStage = PipelineStage::Selected, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions}; + return Value; + }(); + static const MachinePassContract PostRA = [] { + MachinePassContract Value{ + .Name = "post-ra-machine-sink", + .Requires = {MachineFunctionProperty::NoVRegs}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<MachineDominatorTreeAnalysis>(), + MachineFunctionAnalysisManager::id<MachineBlockFrequencyAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Stage == MachineCodeMotionStage::SelectedSSA ? Selected : PostRA; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value() || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc); +} + +[[nodiscard]] Expected<bool> isSafeToSink(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target, + MachineCodeMotionStage Stage) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "machine sink encountered an instruction without a descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::Pseudo) || Flags.has(InstructionFlag::Meta) || + Flags.has(InstructionFlag::Phi) || Flags.has(InstructionFlag::Terminator) || + Flags.has(InstructionFlag::Branch) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::MayLoad) || Flags.has(InstructionFlag::MayStore) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + hasProtectedAttachment(Instruction) || !Instruction.memOperands().empty()) { + return false; + } + if (Flags.has(InstructionFlag::MayRaiseFPException) && + !Instruction.flags().has(MachineInstrFlag::NoFPExcept)) + return false; + + bool HasDefinition = false; + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.kind() == MachineOperand::Kind::RegisterMask || + Operand.kind() == MachineOperand::Kind::LiveOutRegisterMask) { + return false; + } + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr) + continue; + if (Operand.isDebugUse() || Operand.isImplicit() || Operand.tiedOperandIndex() || + Operand.registerFlags().has(RegisterOperandFlag::EarlyClobber) || + (Operand.isDef() && Operand.isUse())) { + return false; + } + if (Stage == MachineCodeMotionStage::PostRegisterAllocation) { + if (!Reg->isPhysical()) + return false; + HasDefinition |= Operand.isDef(); + continue; + } + if (!Reg->isVirtual()) + return false; + if (Operand.isDef()) + HasDefinition = true; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + } + return HasDefinition; +} + +struct SinkDestination final { + MachineBasicBlock *Block = nullptr; + MachineInstr *FirstUse = nullptr; +}; + +[[nodiscard]] std::optional<SinkDestination> findDestination(MachineInstr &Instruction, + MachineRegisterInfo &Registers) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr) + return std::nullopt; + std::unordered_set<Register> Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && Operand.isDef()) + Definitions.insert(*Reg); + } + if (Definitions.empty()) + return std::nullopt; + + MachineBasicBlock *Destination = nullptr; + for (const Register Reg : Definitions) { + for (MachineOperand &Use : Registers.useOperands(Reg, false)) { + MachineInstr *UseInstruction = Use.parent(); + MachineBasicBlock *UseBlock = UseInstruction == nullptr ? nullptr : UseInstruction->parent(); + if (UseBlock == nullptr || UseBlock == Source) + return std::nullopt; + if (Destination == nullptr) + Destination = UseBlock; + else if (Destination != UseBlock) + return std::nullopt; + } + } + if (Destination == nullptr || Destination->predecessors().size() != 1 || + Destination->predecessors().front() != Source || + std::ranges::find_if(Source->successors(), [Destination](const MachineSuccessor &Successor) { + return Successor.Block == Destination; + }) == Source->successors().end()) { + return std::nullopt; + } + + MachineInstr *FirstUse = nullptr; + for (MachineInstr *Candidate = Destination->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + const bool UsesDefinition = + std::ranges::any_of(Candidate->operands(), [&Definitions](const MachineOperand &Operand) { + const Register *Reg = Operand.getIf<Register>(); + return Reg != nullptr && Operand.isUse() && !Operand.isDebugUse() && + Definitions.contains(*Reg); + }); + if (UsesDefinition) { + FirstUse = Candidate; + break; + } + } + return FirstUse == nullptr ? std::nullopt + : std::optional<SinkDestination>{{Destination, FirstUse}}; +} + +[[nodiscard]] std::optional<SinkDestination> +findPostRADestination(MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr || Source->successors().empty()) + return std::nullopt; + + std::vector<PhysicalRegister> Uses; + std::vector<PhysicalRegister> Definitions; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if (Operand.isUse()) + Uses.push_back(*Physical); + if (Operand.isDef()) { + if (Target.registers().isReservedRegister(*Physical) || + Function.registerInfo().isReserved(*Physical)) { + return std::nullopt; + } + Definitions.push_back(*Physical); + } + } + if (Definitions.empty()) + return std::nullopt; + MachineBasicBlock *Destination = nullptr; + for (const MachineSuccessor &Successor : Source->successors()) { + if (Successor.Block == nullptr) + return std::nullopt; + const bool CarriesAny = std::ranges::any_of(Definitions, [&](PhysicalRegister Definition) { + return std::ranges::any_of( + Successor.Block->liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definition, LiveIn.PhysReg); + }); + }); + const bool CarriesAll = std::ranges::all_of(Definitions, [&](PhysicalRegister Definition) { + return std::ranges::any_of( + Successor.Block->liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definition, LiveIn.PhysReg); + }); + }); + if (CarriesAny != CarriesAll || (CarriesAll && Destination != nullptr)) + return std::nullopt; + if (CarriesAll) + Destination = Successor.Block; + } + if (Destination == nullptr || Destination->predecessors().size() != 1 || + Destination->predecessors().front() != Source) { + return std::nullopt; + } + for (std::size_t Left = 0; Left < Definitions.size(); ++Left) { + if (!std::ranges::any_of(Destination->liveIns(), + [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Definitions[Left], + LiveIn.PhysReg); + }) || + std::ranges::any_of(Uses, [&](PhysicalRegister Use) { + return Target.registers().regsOverlap(Definitions[Left], Use); + })) { + return std::nullopt; + } + for (std::size_t Right = Left + 1U; Right < Definitions.size(); ++Right) { + if (Target.registers().regsOverlap(Definitions[Left], Definitions[Right])) + return std::nullopt; + } + } + + const auto OverlapsAny = [&](PhysicalRegister Reg, std::span<const PhysicalRegister> Values) { + return std::ranges::any_of( + Values, [&](PhysicalRegister Value) { return Target.registers().regsOverlap(Reg, Value); }); + }; + for (MachineInstr *After = Instruction.next(); After != nullptr; After = After->next()) { + for (const MachineOperand &Operand : After->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + if (std::ranges::any_of(Uses, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); }) || + std::ranges::any_of(Definitions, + [&](PhysicalRegister Reg) { return Mask->Mask->clobbers(Reg); })) { + return std::nullopt; + } + continue; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + if ((Operand.isUse() || Operand.isDef()) && OverlapsAny(*Physical, Definitions)) { + return std::nullopt; + } + if (Operand.isDef() && OverlapsAny(*Physical, Uses)) + return std::nullopt; + } + } + + std::vector<bool> SeenUse(Definitions.size()); + MachineInstr *FirstUse = nullptr; + for (MachineInstr *Candidate = Destination->firstInstruction(); Candidate != nullptr; + Candidate = Candidate->next()) { + for (const MachineOperand &Operand : Candidate->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && Mask->Mask != nullptr) { + for (std::size_t Index = 0; Index < Definitions.size(); ++Index) { + if (!SeenUse[Index] && Mask->Mask->clobbers(Definitions[Index])) + return std::nullopt; + } + if (FirstUse == nullptr && std::ranges::any_of(Uses, [&](PhysicalRegister Reg) { + return Mask->Mask->clobbers(Reg); + })) { + return std::nullopt; + } + continue; + } + const Register *Reg = Operand.getIf<Register>(); + const std::optional<PhysicalRegister> Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + if (!Physical) + continue; + for (std::size_t Index = 0; Index < Definitions.size(); ++Index) { + if (!Target.registers().regsOverlap(*Physical, Definitions[Index])) + continue; + if (Operand.isUse() && !Operand.isDebugUse()) { + SeenUse[Index] = true; + if (FirstUse == nullptr) + FirstUse = Candidate; + } + if (Operand.isDef() && !SeenUse[Index]) + return std::nullopt; + } + if (FirstUse == nullptr && Operand.isDef() && OverlapsAny(*Physical, Uses)) { + return std::nullopt; + } + } + } + if (FirstUse == nullptr || !std::ranges::all_of(SeenUse, [](bool Value) { return Value; })) + return std::nullopt; + return SinkDestination{Destination, FirstUse}; +} + +[[nodiscard]] Expected<void> moveBefore(MachineInstr &Instruction, + const SinkDestination &Destination) { + MachineBasicBlock *Source = Instruction.parent(); + if (Source == nullptr || Destination.Block == nullptr || Destination.FirstUse == nullptr) + return makeUnexpected(ErrorCode::InternalError, "machine sink has an invalid move plan"); + auto Removed = Instruction.eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + return Destination.Block->insertInstructionBefore(*Destination.FirstUse, Instruction); +} + +} // namespace + +const MachinePassContract &MachineSinkPass::contract() const noexcept { + return sinkContract(Stage); +} + +Expected<MachinePassResult> MachineSinkPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +MachineSinkPass::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Dominators = Analyses.getResult<MachineDominatorTreeAnalysis>(Function, Target); + if (!Dominators) + return std::unexpected(Dominators.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + + MachinePassResult Result; + std::size_t Sunk = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); + Block != nullptr && Sunk < Options.MaximumSunkInstructions; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); + Instruction != nullptr && Sunk < Options.MaximumSunkInstructions;) { + MachineInstr *Next = Instruction->next(); + auto Safe = isSafeToSink(*Instruction, Function, Target, Stage); + if (!Safe) + return std::unexpected(Safe.error()); + if (!*Safe) { + Instruction = Next; + continue; + } + const std::optional<SinkDestination> Destination = + Stage == MachineCodeMotionStage::SelectedSSA + ? findDestination(*Instruction, Function.registerInfo()) + : findPostRADestination(*Instruction, Function, Target); + if (!Destination || !(*Dominators)->dominates(*Block, *Destination->Block) || + (*Frequencies)->frequency(*Destination->Block) > (*Frequencies)->frequency(*Block)) { + Instruction = Next; + continue; + } + auto Moved = moveBefore(*Instruction, *Destination); + if (!Moved) + return std::unexpected(Moved.error()); + ++Sunk; + Result.Changed = true; + Instruction = Next; + } + } + + if (Result.Changed) { + if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + } + Result.Changes = {MachinePassChange::Instructions}; + Result.Preserved.preserve<MachineDominatorTreeAnalysis>(); + Result.Preserved.preserve<MachineBranchProbabilityAnalysis>(); + Result.Preserved.preserve<MachineBlockFrequencyAnalysis>(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineStackSlotColoring.cpp b/backend/lib/CodeGen/MachineStackSlotColoring.cpp new file mode 100644 index 0000000..1717a20 --- /dev/null +++ b/backend/lib/CodeGen/MachineStackSlotColoring.cpp @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineStackSlotColoring.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <string> +#include <unordered_map> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &stackSlotColoringContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "stack-slot-coloring", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Operands, MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +struct SlotInterval final { + MachineBasicBlock *Block = nullptr; + std::size_t First = 0; + std::size_t Last = 0; + bool FirstStores = false; + bool LastLoads = false; + bool IsValid = true; +}; + +struct InstructionReference final { + MachineInstr *Instruction = nullptr; + std::size_t Position = 0; + bool Loads = false; + bool Stores = false; + bool HasMemoryReference = false; + bool HasFrameOperand = false; +}; + +struct SlotUseInfo final { + FrameIndex Slot; + SlotInterval Interval; + std::vector<InstructionReference> References; +}; + +[[nodiscard]] InstructionReference &getReference(SlotUseInfo &Info, MachineInstr &Instruction, + std::size_t Position) { + const auto Existing = + std::ranges::find_if(Info.References, [&Instruction](const InstructionReference &Reference) { + return Reference.Instruction == &Instruction; + }); + if (Existing != Info.References.end()) + return *Existing; + Info.References.push_back({.Instruction = &Instruction, .Position = Position}); + return Info.References.back(); +} + +[[nodiscard]] std::unordered_map<FrameIndex, SlotUseInfo> +collectSlotUses(MachineFunction &Function) { + std::unordered_map<FrameIndex, SlotUseInfo> Result; + std::unordered_map<MachineInstrStableID, std::pair<MachineBasicBlock *, std::size_t>> Locations; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + std::size_t Position = 0; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next(), ++Position) { + Locations.emplace(Instruction->stableID(), std::pair{Block, Position}); + for (const MachineOperand &Operand : Instruction->operands()) { + const FrameIndexOperand *Frame = Operand.getIf<FrameIndexOperand>(); + if (Frame == nullptr) + continue; + SlotUseInfo &Info = + Result + .try_emplace(Frame->Index, + SlotUseInfo{.Slot = Frame->Index, .Interval = {}, .References = {}}) + .first->second; + getReference(Info, *Instruction, Position).HasFrameOperand = true; + } + for (const MachineMemOperand *Memory : Instruction->memOperands()) { + if (Memory == nullptr) + continue; + const std::optional<FrameIndex> Frame = Memory->description().Base.frameIndex(); + if (!Frame) + continue; + SlotUseInfo &Info = + Result + .try_emplace(*Frame, SlotUseInfo{.Slot = *Frame, .Interval = {}, .References = {}}) + .first->second; + InstructionReference &Reference = getReference(Info, *Instruction, Position); + Reference.HasMemoryReference = true; + Reference.Loads |= Memory->isLoad(); + Reference.Stores |= Memory->isStore(); + } + } + } + + for (auto &[Slot, Info] : Result) { + (void)Slot; + if (Info.References.empty()) { + Info.Interval.IsValid = false; + continue; + } + Info.Interval.Block = Info.References.front().Instruction->parent(); + Info.Interval.First = Info.References.front().Position; + Info.Interval.Last = Info.References.back().Position; + Info.Interval.FirstStores = Info.References.front().Stores; + Info.Interval.LastLoads = Info.References.back().Loads; + for (const InstructionReference &Reference : Info.References) { + if (Reference.Instruction->parent() != Info.Interval.Block || !Reference.HasMemoryReference || + (Reference.HasFrameOperand && !Reference.HasMemoryReference)) { + Info.Interval.IsValid = false; + } + } + Info.Interval.IsValid &= + Info.Interval.Block != nullptr && Info.Interval.FirstStores && Info.Interval.LastLoads; + const StackObject *Object = Function.frameInfo().object(Info.Slot); + const StackObjectDesc *Description = Object == nullptr ? nullptr : Object->description(); + if (Description == nullptr) { + Info.Interval.IsValid = false; + continue; + } + for (const StackObjectLifetimeRange &Range : Description->Lifetime) { + const auto Begin = Locations.find(Range.Begin); + const auto End = Locations.find(Range.End); + if (Begin == Locations.end() || End == Locations.end() || + Begin->second.first != Info.Interval.Block || End->second.first != Info.Interval.Block || + Begin->second.second > End->second.second) { + Info.Interval.IsValid = false; + break; + } + Info.Interval.First = std::min(Info.Interval.First, Begin->second.second); + Info.Interval.Last = std::max(Info.Interval.Last, End->second.second); + } + } + return Result; +} + +[[nodiscard]] bool isEligibleSlot(const StackObjectDesc &Description) noexcept { + return Description.Kind == StackObjectKind::SpillSlot && !Description.IsDead && + !Description.IsImmutable && !Description.IsAliased && !Description.AddressTaken && + !Description.AllocaName && !Description.Offset && !Description.LocalOffset && + !Description.CalleeSavedRegister && !Description.DebugInfo && + Description.TargetFlags == 0 && Description.Size.has_value(); +} + +[[nodiscard]] bool compatibleSlots(const StackObjectDesc &Left, + const StackObjectDesc &Right) noexcept { + return isEligibleSlot(Left) && isEligibleSlot(Right) && Left.Kind == Right.Kind && + Left.Size == Right.Size && Left.Align == Right.Align && Left.Stack == Right.Stack; +} + +[[nodiscard]] bool intervalsOverlap(const SlotInterval &Left, const SlotInterval &Right) noexcept { + if (Left.Block != Right.Block) + return true; + return Left.Last >= Right.First && Right.Last >= Left.First; +} + +struct StackColor final { + FrameIndex Canonical; + const StackObjectDesc *Description = nullptr; + std::vector<SlotInterval> Intervals; +}; + +[[nodiscard]] Expected<void> rewriteSlot(MachineFunction &Function, FrameIndex OldSlot, + FrameIndex NewSlot) { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const FrameIndexOperand *Frame = + Operand == nullptr ? nullptr : Operand->getIf<FrameIndexOperand>(); + if (Frame == nullptr || Frame->Index != OldSlot) + continue; + auto Rewritten = Operand->setFrameIndex(NewSlot); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + } + for (std::size_t Index = 0; Index < Instruction->memOperands().size(); ++Index) { + const MachineMemOperand *Memory = Instruction->memOperands()[Index]; + if (Memory == nullptr || Memory->description().Base.frameIndex() != OldSlot) + continue; + MachineMemOperandDesc Description = Memory->description(); + Description.Base = MemoryBase::frameIndex(NewSlot); + auto Replacement = Function.createMemOperand(std::move(Description)); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Replaced = Instruction->replaceMemOperand(Index, **Replacement); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + } + } + return {}; +} + +} // namespace + +const MachinePassContract &MachineStackSlotColoringPass::contract() const noexcept { + return stackSlotColoringContract(); +} + +Expected<MachinePassResult> MachineStackSlotColoringPass::run(MachineFunction &Function, + const TargetDescription &, + MachineFunctionAnalysisManager &) { + if (Function.frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::FrameLayoutFailed, + "stack-slot coloring must run before frame layout is frozen"); + } + return runImplementation(Function); +} + +Expected<MachinePassResult> +MachineStackSlotColoringPass::runImplementation(MachineFunction &Function) const { + std::unordered_map<FrameIndex, SlotUseInfo> Uses = collectSlotUses(Function); + std::vector<StackColor> Colors; + std::vector<std::pair<FrameIndex, FrameIndex>> Rewrites; + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + const auto Use = Uses.find(Object.index()); + if (Description == nullptr || Use == Uses.end() || !Use->second.Interval.IsValid || + !isEligibleSlot(*Description)) { + continue; + } + auto Color = std::ranges::find_if(Colors, [&](const StackColor &Candidate) { + return Candidate.Description != nullptr && + compatibleSlots(*Candidate.Description, *Description) && + std::ranges::none_of(Candidate.Intervals, [&](const SlotInterval &Interval) { + return intervalsOverlap(Interval, Use->second.Interval); + }); + }); + if (Color == Colors.end()) { + Colors.push_back({Object.index(), Description, {Use->second.Interval}}); + continue; + } + if (Rewrites.size() == Options.MaximumColoredSlots) + break; + Rewrites.emplace_back(Object.index(), Color->Canonical); + Color->Intervals.push_back(Use->second.Interval); + } + + for (const auto &[OldSlot, NewSlot] : Rewrites) { + const StackObject *OldObject = Function.frameInfo().object(OldSlot); + const StackObject *NewObject = Function.frameInfo().object(NewSlot); + if (OldObject == nullptr || NewObject == nullptr || OldObject->description() == nullptr || + NewObject->description() == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "stack-slot coloring lost a planned stack object"); + } + auto Rewritten = rewriteSlot(Function, OldSlot, NewSlot); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + StackObjectDesc Combined = *NewObject->description(); + Combined.Lifetime.insert(Combined.Lifetime.end(), OldObject->description()->Lifetime.begin(), + OldObject->description()->Lifetime.end()); + auto Updated = Function.frameInfo().updateObject(NewSlot, std::move(Combined)); + if (!Updated) + return std::unexpected(Updated.error()); + auto Dead = Function.frameInfo().markObjectDead(OldSlot); + if (!Dead) + return std::unexpected(Dead.error()); + } + + MachinePassResult Result; + Result.Changed = !Rewrites.empty(); + if (Result.Changed) { + Result.Changes = {MachinePassChange::Operands, MachinePassChange::Frame}; + Result.Preserved = PreservedAnalyses::none(); + } else { + Result.Preserved = PreservedAnalyses::all(); + } + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineStripDebug.cpp b/backend/lib/CodeGen/MachineStripDebug.cpp new file mode 100644 index 0000000..46d92d1 --- /dev/null +++ b/backend/lib/CodeGen/MachineStripDebug.cpp @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineStripDebug.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" + +namespace rcc { + +Expected<bool> stripDebugMachineFunction(MachineFunction &Function) { + bool Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) { + MachineInstr *Next = Instruction->next(); + if (Instruction->isDebugInstr() && Instruction->operandCount() > 1) { + auto Removed = Instruction->eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + Changed = true; + Instruction = Next; + continue; + } + if (Instruction->debugLocation()) { + Instruction->clearDebugLocation(); + Changed = true; + } + Instruction = Next; + } + } + return Changed; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineTailDuplication.cpp b/backend/lib/CodeGen/MachineTailDuplication.cpp new file mode 100644 index 0000000..53c67c6 --- /dev/null +++ b/backend/lib/CodeGen/MachineTailDuplication.cpp @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineTailDuplication.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineBasicBlockAttributes.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineJumpTableInfo.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetInstrInfo.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &tailDuplicationContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "machine-tail-duplication", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::FrameFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + return Value; + }(); + return Contract; +} + +[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept { + return Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() || + Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() || + Instruction.pcSections().has_value() || Instruction.mmra().has_value() || + Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value(); +} + +[[nodiscard]] bool isJumpTableTarget(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + for (const MachineJumpTableEntry &Entry : Function.jumpTableInfo().tables()) { + const MachineJumpTableDesc *Description = Entry.description(); + if (Description != nullptr && + std::ranges::find(Description->Targets, &Block) != Description->Targets.end()) { + return true; + } + } + return false; +} + +[[nodiscard]] bool hasProtectedIdentity(const MachineBasicBlock &Block, + const MachineFunction &Function) noexcept { + const MachineBasicBlockAttributes &Attributes = Block.attributes(); + return Attributes.rawFlags() != 0 || !Attributes.sectionID().isDefault() || + Attributes.profileID().has_value() || isJumpTableTarget(Block, Function); +} + +[[nodiscard]] Expected<bool> isDuplicableInstruction(const MachineInstr &Instruction, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication encountered an unknown instruction"); + } + const InstructionFlags Flags = Descriptor->Flags; + if (Flags.has(InstructionFlag::NotDuplicable) || Flags.has(InstructionFlag::Call) || + Flags.has(InstructionFlag::Return) || Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::Meta) || Flags.has(InstructionFlag::Phi) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || Flags.has(InstructionFlag::Convergent) || + Instruction.flags().has(MachineInstrFlag::NoMerge) || + Instruction.flags().has(MachineInstrFlag::BundledWithPred) || + Instruction.flags().has(MachineInstrFlag::BundledWithSucc) || + hasProtectedAttachment(Instruction)) { + return false; + } + return std::ranges::none_of(Instruction.operands(), [](const MachineOperand &Operand) { + return Operand.kind() == MachineOperand::Kind::BasicBlock || + Operand.kind() == MachineOperand::Kind::DebugInstrRef; + }); +} + +[[nodiscard]] Expected<MachineInstr *> +cloneInstructionBefore(MachineInstr &Source, MachineBasicBlock &Destination, MachineInstr *Before) { + MachineFunction &Function = Destination.function(); + auto Created = Function.createInstruction(Source.opcode()); + if (!Created) + return std::unexpected(Created.error()); + MachineInstr *Clone = *Created; + Clone->reserveOperands(Source.operandCount()); + for (const MachineOperand &Operand : Source.operands()) { + auto Added = Clone->addOperand(Operand.cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + } + for (std::size_t Index = 0; Index < Source.operandCount(); ++Index) { + const MachineOperand *Operand = Source.operandAt(Index); + const std::optional<std::size_t> Tied = + Operand == nullptr ? std::nullopt : Operand->tiedOperandIndex(); + if (!Tied || Index > *Tied) + continue; + const std::size_t DefIndex = Operand->isDef() ? Index : *Tied; + const std::size_t UseIndex = Operand->isDef() ? *Tied : Index; + auto TiedResult = Clone->tieOperands(DefIndex, UseIndex); + if (!TiedResult) + return std::unexpected(TiedResult.error()); + } + for (const MachineMemOperand *Memory : Source.memOperands()) { + if (Memory == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "tail duplication encountered a null memory operand"); + auto Added = Clone->addMemOperand(*Memory); + if (!Added) + return std::unexpected(Added.error()); + } + Clone->setFlags(Source.flags()); + if (Source.sourceLocation()) + Clone->setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Clone->setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Clone->setOrigin(*Source.origin()); + Expected<void> Inserted = Before == nullptr + ? Destination.appendInstruction(*Clone) + : Destination.insertInstructionBefore(*Before, *Clone); + if (!Inserted) + return std::unexpected(Inserted.error()); + return Clone; +} + +struct TailCandidate final { + MachineBasicBlock *Successor = nullptr; + MachineInstr *Branch = nullptr; + std::vector<MachineInstr *> Body; +}; + +[[nodiscard]] Expected<std::optional<TailCandidate>> +analyzeTail(MachineBasicBlock &Tail, MachineFunction &Function, const TargetDescription &Target, + const TargetInstrInfo &Info, std::size_t MaximumInstructions) { + if (&Tail == Function.firstBlock() || Tail.predecessors().size() < 2 || + Tail.successors().size() != 1 || Tail.successors().front().Block == nullptr || + hasProtectedIdentity(Tail, Function)) { + return std::nullopt; + } + auto Branch = Info.analyzeBranch(Tail, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + MachineBasicBlock *Successor = Tail.successors().front().Block; + if (Branch->HasUnanalyzableTerminator || Branch->Conditional != nullptr || + Branch->Unconditional == nullptr || Branch->TrueTarget != Successor || + Branch->Unconditional != Tail.lastInstruction() || + hasProtectedAttachment(*Branch->Unconditional)) { + return std::nullopt; + } + + TailCandidate Candidate{.Successor = Successor, .Branch = Branch->Unconditional, .Body = {}}; + for (MachineInstr *Instruction = Tail.firstInstruction(); Instruction != Candidate.Branch; + Instruction = Instruction->next()) { + if (Instruction == nullptr) + return makeUnexpected(ErrorCode::CFGOptimizationFailed, + "tail branch is not reachable from its block body"); + auto Duplicable = isDuplicableInstruction(*Instruction, Target); + if (!Duplicable) + return std::unexpected(Duplicable.error()); + if (!*Duplicable) + return std::nullopt; + Candidate.Body.push_back(Instruction); + if (Candidate.Body.size() > MaximumInstructions) + return std::nullopt; + } + return Candidate; +} + +[[nodiscard]] Expected<bool> +duplicateTailIntoPredecessor(const TailCandidate &Candidate, MachineBasicBlock &Tail, + MachineBasicBlock &Predecessor, MachineFunction &Function, + const TargetDescription &Target, const TargetInstrInfo &Info) { + if (Predecessor.successors().size() != 1 || Predecessor.successors().front().Block != &Tail) { + return false; + } + auto Branch = Info.analyzeBranch(Predecessor, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + if (Branch->HasUnanalyzableTerminator || Branch->Conditional != nullptr) + return false; + MachineInstr *OldBranch = Branch->Unconditional; + if (OldBranch != nullptr) { + if (Branch->TrueTarget != &Tail || OldBranch != Predecessor.lastInstruction() || + hasProtectedAttachment(*OldBranch)) { + return false; + } + } else if (Predecessor.next() != &Tail) { + return false; + } + + for (MachineInstr *Instruction : Candidate.Body) { + auto Cloned = cloneInstructionBefore(*Instruction, Predecessor, OldBranch); + if (!Cloned) + return std::unexpected(Cloned.error()); + } + if (OldBranch != nullptr) { + auto Removed = Info.removeBranch(*OldBranch, Function, Target); + if (!Removed) + return std::unexpected(Removed.error()); + } + auto NewBranch = Info.insertBranch(Predecessor, *Candidate.Successor, Function, Target); + if (!NewBranch) + return std::unexpected(NewBranch.error()); + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Redirected = CFG.replaceSuccessor(Predecessor, Tail, *Candidate.Successor); + if (!Redirected) + return std::unexpected(Redirected.error()); + return true; +} + +} // namespace + +const MachinePassContract &MachineTailDuplicationPass::contract() const noexcept { + return tailDuplicationContract(); +} + +Expected<MachinePassResult> MachineTailDuplicationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + return runImplementation(Function, Target); +} + +Expected<MachinePassResult> +MachineTailDuplicationPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target) const { + MachinePassResult Result; + std::size_t DuplicatedInstructions = 0; + for (MachineBasicBlock *Tail = Function.firstBlock(); Tail != nullptr;) { + MachineBasicBlock *Next = Tail->next(); + auto Candidate = analyzeTail(*Tail, Function, Target, *Info, Options.MaximumTailInstructions); + if (!Candidate) + return std::unexpected(Candidate.error()); + if (!Candidate->has_value()) { + Tail = Next; + continue; + } + const auto &TailCandidate = Candidate->value(); + const std::size_t Cost = TailCandidate.Body.size() + 1U; + const std::vector<MachineBasicBlock *> Predecessors(Tail->predecessors().begin(), + Tail->predecessors().end()); + for (MachineBasicBlock *Predecessor : Predecessors) { + if (Predecessor == nullptr || + DuplicatedInstructions > Options.MaximumDuplicatedInstructions || + Cost > Options.MaximumDuplicatedInstructions - DuplicatedInstructions) { + break; + } + auto Duplicated = + duplicateTailIntoPredecessor(TailCandidate, *Tail, *Predecessor, Function, Target, *Info); + if (!Duplicated) + return std::unexpected(Duplicated.error()); + if (!*Duplicated) + continue; + DuplicatedInstructions += Cost; + Result.Changed = true; + } + if (Tail->predecessors().empty()) { + MachineCFGUpdater CFG = Function.cfgUpdater(); + auto Deleted = CFG.deleteBlock(*Tail); + if (!Deleted) + return std::unexpected(Deleted.error()); + Result.Changed = true; + } + Tail = Next; + } + + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::CFG, MachinePassChange::BlockLayout}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MachineVerifier.cpp b/backend/lib/CodeGen/MachineVerifier.cpp new file mode 100644 index 0000000..085d7ac --- /dev/null +++ b/backend/lib/CodeGen/MachineVerifier.cpp @@ -0,0 +1,987 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineVerifier.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionProperties.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineMetadata.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <algorithm> +#include <array> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <string> +#include <string_view> +#include <unordered_map> +#include <unordered_set> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] std::string formatDebugInstruction(const MachineInstr &Instruction, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + std::string Result = Descriptor == nullptr ? "<unknown>" : Descriptor->Name; + for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { + if (Index == 0) + Result.push_back(' '); + else + Result.append(", "); + const MachineOperand &Operand = Instruction.operands()[Index]; + if (const auto *Immediate = Operand.getIf<std::int64_t>()) { + Result.append(std::to_string(*Immediate)); + continue; + } + if (const auto *RegisterValue = Operand.getIf<Register>()) { + if (RegisterValue->isNoRegister()) { + Result.append("$noreg"); + continue; + } + if (RegisterValue->isVirtual()) { + Result.append(RegisterValue->toString()); + continue; + } + const auto Physical = RegisterValue->physicalRegister(); + const PhysicalRegisterDesc *Register = + Physical ? Target.registers().physicalRegister(*Physical) : nullptr; + Result.push_back('$'); + Result.append(Register == nullptr ? RegisterValue->toString() : Register->Name); + continue; + } + Result.append("<operand>"); + } + return Result; +} + +class FunctionVerifier final { +public: + FunctionVerifier(const MachineFunction &Function, const TargetDescription &Target, + MachineVerifierOptions Options) noexcept + : Function(Function), Target(Target), Options(Options) {} + + [[nodiscard]] Expected<void> run() { + if (Function.properties().has(MachineFunctionProperty::FailedISel)) + return {}; + if (!Function.stableID().isValid()) + return fail("function identity is invalid"); + + auto AttributeResult = Function.attributes().verify(); + if (!AttributeResult) + return failWith("function attributes", AttributeResult.error()); + auto MetadataResult = Function.metadata().verify(); + if (!MetadataResult) + return failWith("machine metadata", MetadataResult.error()); + auto CFIResult = Function.cfiInfo().verify(); + if (!CFIResult) + return failWith("CFI information", CFIResult.error()); + auto TargetCFIResult = Target.validateCFIInfo(Function.cfiInfo()); + if (!TargetCFIResult) + return failWith("target CFI information", TargetCFIResult.error()); + if (Function.targetState().hasValue()) { + auto TargetStateResult = Function.targetState().verify(); + if (!TargetStateResult) + return failWith("target function state", TargetStateResult.error()); + const auto RequiredType = Target.mirText().functionStateType(); + if (RequiredType && Function.targetState().typeID() != RequiredType) { + return fail("target function state does not match the target MIR schema"); + } + } + auto CatalogResult = catalogArenaObjects(); + if (!CatalogResult) + return CatalogResult; + auto FrameResult = Function.frameInfo().verify(); + if (!FrameResult) + return failWith("frame info", FrameResult.error()); + auto TargetFrameResult = Target.validateFrameInfo(Function.frameInfo()); + if (!TargetFrameResult) + return failWith("target frame info", TargetFrameResult.error()); + auto ConstantPoolResult = Function.constantPool().verify(); + if (!ConstantPoolResult) + return failWith("constant pool", ConstantPoolResult.error()); + auto LayoutResult = verifyBlockLayout(); + if (!LayoutResult) + return LayoutResult; + auto RecordResult = verifyFunctionRecords(); + if (!RecordResult) + return RecordResult; + auto CompilationStatusResult = verifyCompilationStatus(); + if (!CompilationStatusResult) + return CompilationStatusResult; + auto JumpTableResult = Function.jumpTableInfo().verify(); + if (!JumpTableResult) + return failWith("jump table info", JumpTableResult.error()); + auto FrameLifetimeResult = verifyFrameLifetimes(); + if (!FrameLifetimeResult) + return FrameLifetimeResult; + auto CFGResult = verifyCFG(); + if (!CFGResult) + return CFGResult; + auto PHIResult = verifyPHIs(); + if (!PHIResult) + return PHIResult; + auto RegisterResult = verifyRegisterInfo(); + if (!RegisterResult) + return RegisterResult; + auto DefUseResult = Function.registerInfo().verifyDefUseChains(); + if (!DefUseResult) + return failWith("register info", DefUseResult.error()); + if (Options.VerifyProperties) { + auto PropertyResult = verifyProperties(); + if (!PropertyResult) + return PropertyResult; + } + return {}; + } + +private: + [[nodiscard]] Expected<void> fail(std::string Message) const { + return makeUnexpected(ErrorCode::VerificationError, "machine function '" + + std::string(Function.name()) + + "': " + std::move(Message)); + } + + [[nodiscard]] Expected<void> failWith(std::string_view Context, const Error &Nested) const { + return fail(std::string(Context) + ": " + std::string(Nested.message())); + } + + [[nodiscard]] static bool hasMatchingStorage(const MachineOperand &Operand) noexcept { + switch (Operand.kind()) { + case MachineOperand::Kind::Register: + return Operand.getIf<Register>() != nullptr; + case MachineOperand::Kind::Immediate: + return Operand.getIf<std::int64_t>() != nullptr; + case MachineOperand::Kind::IntegerImmediate: + return Operand.getIf<IntegerImmediateOperand>() != nullptr; + case MachineOperand::Kind::FloatingImmediate: + return Operand.getIf<FloatingImmediateOperand>() != nullptr; + case MachineOperand::Kind::BasicBlock: + return Operand.getIf<BasicBlockOperand>() != nullptr; + case MachineOperand::Kind::FrameIndex: + return Operand.getIf<FrameIndexOperand>() != nullptr; + case MachineOperand::Kind::ConstantPoolIndex: + return Operand.getIf<ConstantPoolOperand>() != nullptr; + case MachineOperand::Kind::JumpTableIndex: + return Operand.getIf<JumpTableOperand>() != nullptr; + case MachineOperand::Kind::GlobalSymbol: + return Operand.getIf<GlobalSymbolOperand>() != nullptr; + case MachineOperand::Kind::ExternalSymbol: + return Operand.getIf<ExternalSymbolOperand>() != nullptr; + case MachineOperand::Kind::BlockAddress: + return Operand.getIf<BlockAddressOperand>() != nullptr; + case MachineOperand::Kind::RegisterMask: + return Operand.getIf<RegisterMaskOperand>() != nullptr; + case MachineOperand::Kind::LiveOutRegisterMask: + return Operand.getIf<LiveOutRegisterMaskOperand>() != nullptr; + case MachineOperand::Kind::Metadata: + return Operand.getIf<MetadataOperand>() != nullptr; + case MachineOperand::Kind::TargetIndex: + return Operand.getIf<TargetIndexOperand>() != nullptr; + case MachineOperand::Kind::MCSymbol: + return Operand.getIf<MCSymbolOperand>() != nullptr; + case MachineOperand::Kind::CFIIndex: + return Operand.getIf<CFIIndexOperand>() != nullptr; + case MachineOperand::Kind::Intrinsic: + return Operand.getIf<IntrinsicOperand>() != nullptr; + case MachineOperand::Kind::Predicate: + return Operand.getIf<PredicateOperand>() != nullptr; + case MachineOperand::Kind::ShuffleMask: + return Operand.getIf<ShuffleMaskOperand>() != nullptr; + case MachineOperand::Kind::DebugInstrRef: + return Operand.getIf<DebugInstrRefOperand>() != nullptr; + case MachineOperand::Kind::LaneMask: + return Operand.getIf<LaneMaskOperand>() != nullptr; + case MachineOperand::Kind::Count: + return false; + } + return false; + } + + [[nodiscard]] Expected<void> catalogArenaObjects() { + std::unordered_set<MachineBlockStableID> BlockIDs; + for (const MachineBasicBlock *Block : Function.allBlocks()) { + if (Block == nullptr || !RegisteredBlocks.insert(Block).second) + return fail("arena block registry contains a null or duplicate pointer"); + if (&Block->function() != &Function) + return fail("arena block registry contains a foreign block"); + if (!Block->stableID().isValid() || !BlockIDs.insert(Block->stableID()).second) { + return fail("arena block registry contains an invalid or duplicate StableID"); + } + auto AttributeResult = Block->attributes().verify(); + if (!AttributeResult) + return failWith("basic block attributes", AttributeResult.error()); + } + + std::unordered_set<MachineInstrStableID> InstructionIDs; + for (const MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction == nullptr || !RegisteredInstructions.insert(Instruction).second) { + return fail("arena instruction registry contains a null or duplicate pointer"); + } + if (&Instruction->function() != &Function) + return fail("arena instruction registry contains a foreign instruction"); + if (!Instruction->stableID().isValid() || + !InstructionIDs.insert(Instruction->stableID()).second) { + return fail("arena instruction registry contains an invalid or duplicate StableID"); + } + } + + for (const MachineMemOperand *MemOperand : Function.allMemOperands()) { + if (MemOperand == nullptr || !RegisteredMemOperands.insert(MemOperand).second) { + return fail("arena MMO registry contains a null or duplicate pointer"); + } + } + return {}; + } + + [[nodiscard]] Expected<void> verifyBlockLayout() { + const MachineBasicBlock *Previous = nullptr; + const MachineBasicBlock *Block = Function.firstBlock(); + std::uint32_t LayoutIndex = 0; + while (Block != nullptr) { + if (!RegisteredBlocks.contains(Block)) + return fail("block layout references an unregistered block"); + if (!LinkedBlocks.insert(Block).second) + return fail("block layout contains a cycle or duplicate block"); + if (!Block->isLinked() || Block->previous() != Previous) + return fail("block layout ownership or previous link is inconsistent"); + LinkedBlockIDs.insert(Block->stableID()); + if (Block->denseNumber() != LayoutIndex || Block->layoutIndex() != LayoutIndex) { + return fail("block dense number or layout index is stale"); + } + if (const auto &ProfileID = Block->attributes().profileID()) { + const std::uint64_t Key = (static_cast<std::uint64_t>(ProfileID->BaseID) << 32U) | + static_cast<std::uint64_t>(ProfileID->CloneID); + if (!LinkedBlockProfileIDs.insert(Key).second) + return fail("multiple linked blocks have the same bb_id"); + } + + auto InstructionResult = verifyInstructionLayout(*Block); + if (!InstructionResult) + return InstructionResult; + Previous = Block; + Block = Block->next(); + ++LayoutIndex; + } + + if (Previous != Function.lastBlock()) + return fail("function last-block pointer is inconsistent"); + if ((Function.firstBlock() == nullptr) != (Function.lastBlock() == nullptr)) { + return fail("function has only one of its first/last block pointers"); + } + + for (const MachineBasicBlock *Registered : RegisteredBlocks) { + if (LinkedBlocks.contains(Registered)) { + if (!Registered->isLinked()) + return fail("reachable block is not marked linked"); + continue; + } + if (Registered->isLinked()) + return fail("unreachable arena block is marked linked"); + if (Registered->previous() != nullptr || Registered->next() != nullptr || + Registered->firstInstruction() != nullptr || Registered->lastInstruction() != nullptr || + !Registered->predecessors().empty() || !Registered->successors().empty() || + Registered->denseNumber() || Registered->layoutIndex()) { + return fail("detached block retains layout, instruction, or CFG state"); + } + } + + for (const MachineInstr *Instruction : RegisteredInstructions) { + auto LocalResult = verifyInstructionLocalState(*Instruction); + if (!LocalResult) + return LocalResult; + if (Instruction->parent() != nullptr) { + if (!LinkedInstructions.contains(Instruction)) + return fail("instruction has a parent but is absent from block layout"); + } else { + if (Instruction->previous() != nullptr || Instruction->next() != nullptr) { + return fail("detached instruction retains layout links"); + } + for (const MachineOperand &Operand : Instruction->operands()) { + if (Function.registerInfo().isLinkedOperand(Operand)) + return fail("detached instruction operand remains in a def-use chain"); + } + } + } + return {}; + } + + [[nodiscard]] Expected<void> verifyInstructionLayout(const MachineBasicBlock &Block) { + const MachineInstr *Previous = nullptr; + const MachineInstr *Instruction = Block.firstInstruction(); + while (Instruction != nullptr) { + if (!RegisteredInstructions.contains(Instruction)) + return fail("block layout references an unregistered instruction"); + if (!LinkedInstructions.insert(Instruction).second) + return fail("instruction layout contains a cycle or duplicate instruction"); + LinkedInstructionIDs.insert(Instruction->stableID()); + LinkedInstructionsByID.emplace(Instruction->stableID(), Instruction); + if (Instruction->parent() != &Block || Instruction->previous() != Previous) { + return fail("instruction parent or previous link is inconsistent"); + } + + auto LiveResult = verifyLiveInstruction(*Instruction); + if (!LiveResult) + return LiveResult; + Previous = Instruction; + Instruction = Instruction->next(); + } + if (Previous != Block.lastInstruction()) + return fail("block last-instruction pointer is inconsistent"); + if ((Block.firstInstruction() == nullptr) != (Block.lastInstruction() == nullptr)) { + return fail("block has only one of its first/last instruction pointers"); + } + return {}; + } + + [[nodiscard]] Expected<void> verifyInstructionLocalState(const MachineInstr &Instruction) { + if (Instruction.sourceLocation() && !Instruction.sourceLocation()->isValid()) { + return fail("instruction has an invalid source location"); + } + const Opcode OpcodeValue = Instruction.opcode(); + const bool IsDebugValue = OpcodeValue == opcode(GenericOpcode::DBG_VALUE) || + OpcodeValue == opcode(GenericOpcode::DBG_VALUE_LIST); + if (IsDebugValue && Instruction.operandCount() == 4 && !Instruction.debugLocation()) { + return fail("Bad machine code: Missing DebugLoc for debug instruction\n" + "- instruction: " + + formatDebugInstruction(Instruction, Target)); + } + if (Instruction.debugLocation()) { + auto DebugLocationResult = Instruction.debugLocation()->verify(Function.metadata()); + if (!DebugLocationResult) + return failWith("instruction debug location", DebugLocationResult.error()); + } + const auto VerifyAttachment = [this](const std::optional<MachineMetadataReference> &Reference, + std::string_view Name) -> Expected<void> { + if (!Reference) + return {}; + auto Verification = verifyMachineMetadataReference(*Reference, Function.metadata()); + if (!Verification) + return failWith(Name, Verification.error()); + return {}; + }; + if (auto Verification = + VerifyAttachment(Instruction.heapAllocMarker(), "instruction heap-allocation marker"); + !Verification) { + return Verification; + } + if (auto Verification = + VerifyAttachment(Instruction.pcSections(), "instruction PC sections metadata"); + !Verification) { + return Verification; + } + if (auto Verification = VerifyAttachment(Instruction.mmra(), "instruction MMRA metadata"); + !Verification) { + return Verification; + } + if (Instruction.origin() && Instruction.origin()->value() == 0) + return fail("instruction has an invalid origin ID"); + + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.parent() != &Instruction || !hasMatchingStorage(Operand)) + return fail("instruction operand parent or tagged storage is inconsistent"); + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); Mask != nullptr && !Mask->Mask) { + return fail("register-mask operand contains a null mask"); + } + if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); + Mask != nullptr && !Mask->Mask) { + return fail("live-out register-mask operand contains a null mask"); + } + if (const auto *Integer = Operand.getIf<IntegerImmediateOperand>(); Integer != nullptr) { + auto IntegerResult = Integer->Value.verify(); + if (!IntegerResult) + return failWith("integer immediate operand", IntegerResult.error()); + } + if (const auto *Global = Operand.getIf<GlobalSymbolOperand>(); + Global != nullptr && Global->Name.empty()) { + return fail("global symbol operand has an empty name"); + } + if (const auto *Address = Operand.getIf<BlockAddressOperand>(); + Address != nullptr && (Address->FunctionName.empty() || Address->BlockName.empty())) { + return fail("block-address operand has an incomplete name"); + } + if (const auto *Metadata = Operand.getIf<MetadataOperand>(); Metadata != nullptr) { + auto MetadataResult = Metadata->Value.verify(Function.metadata()); + if (!MetadataResult) + return failWith("metadata operand", MetadataResult.error()); + } + if (const auto *Symbol = Operand.getIf<MCSymbolOperand>(); + Symbol != nullptr && Symbol->Name.empty()) { + return fail("MC symbol operand has an empty name"); + } + if (const auto *Intrinsic = Operand.getIf<IntrinsicOperand>(); + Intrinsic != nullptr && Intrinsic->ID.value() == 0) { + return fail("intrinsic operand has an invalid ID"); + } + if (const auto *Predicate = Operand.getIf<PredicateOperand>(); Predicate != nullptr) { + auto PredicateResult = Predicate->verify(); + if (!PredicateResult) + return failWith("predicate operand", PredicateResult.error()); + } + if (const auto *Shuffle = Operand.getIf<ShuffleMaskOperand>(); + Shuffle != nullptr && Shuffle->Elements.size() < 2) { + return fail("shuffle-mask operand has fewer than two elements"); + } + } + + for (const MachineMemOperand *MemOperand : Instruction.memOperands()) { + if (MemOperand == nullptr || !RegisteredMemOperands.contains(MemOperand)) { + return fail("instruction references a foreign or null MMO"); + } + const MachineMemOperandDesc &Description = MemOperand->description(); + const std::array MetadataReferences{ + Description.TBAA, Description.AliasScope, + Description.NoAlias, Description.NoAliasAddressSpace, + Description.RangeMetadata, + }; + for (const auto &Reference : MetadataReferences) { + if (!Reference) + continue; + auto Verification = verifyMachineMetadataReference(*Reference, Function.metadata()); + if (!Verification) + return failWith("machine memory operand metadata", Verification.error()); + } + } + return {}; + } + + [[nodiscard]] Expected<void> verifyFrameIndex(FrameIndex Index) const { + const StackObject *Object = Function.frameInfo().object(Index); + const StackObjectDesc *Description = Object == nullptr ? nullptr : Object->description(); + if (Description == nullptr || Description->IsDead) + return fail("live instruction references an unknown or dead stack object"); + return {}; + } + + [[nodiscard]] Expected<void> verifyConstantPoolIndex(ConstantPoolIndex Index) const { + if (Function.constantPool().entry(Index) == nullptr) + return fail("live instruction references an unknown constant pool entry"); + return {}; + } + + [[nodiscard]] Expected<void> verifyJumpTableIndex(JumpTableIndex Index) const { + if (Function.jumpTableInfo().table(Index) == nullptr) + return fail("live instruction references an unknown jump table"); + return {}; + } + + [[nodiscard]] Expected<void> verifyFrameLifetimes() const { + auto VerifyPoints = [this](std::span<const MachineFrameSaveRestorePoint> Points, + std::string_view Kind) -> Expected<void> { + for (const MachineFrameSaveRestorePoint &Point : Points) { + if (!LinkedBlockIDs.contains(Point.Block)) { + return fail(std::string("frame ") + std::string(Kind) + + " point references a detached or unknown block"); + } + for (const PhysicalRegister Reg : Point.Registers) { + if (!Target.registers().isKnownPhysicalRegister(Reg)) { + return fail(std::string("frame ") + std::string(Kind) + + " point references an unknown physical register"); + } + } + } + return {}; + }; + auto SaveResult = VerifyPoints(Function.frameInfo().state().SavePoints, "save"); + if (!SaveResult) + return SaveResult; + auto RestoreResult = VerifyPoints(Function.frameInfo().state().RestorePoints, "restore"); + if (!RestoreResult) + return RestoreResult; + + for (const StackObject &Object : Function.frameInfo().objects()) { + const StackObjectDesc *Description = Object.description(); + if (Description == nullptr) + continue; + for (const StackObjectLifetimeRange &Range : Description->Lifetime) { + if (!LinkedInstructionIDs.contains(Range.Begin) || + !LinkedInstructionIDs.contains(Range.End)) { + return fail("stack object lifetime references a detached or unknown instruction"); + } + } + if (Description->DebugInfo) { + auto DebugInfoResult = Description->DebugInfo->verify(Function.metadata()); + if (!DebugInfoResult) + return failWith("stack object debug information", DebugInfoResult.error()); + } + if (Description->CalleeSavedRegister && + !Target.registers().isKnownPhysicalRegister(*Description->CalleeSavedRegister)) { + return fail("callee-saved stack object references an unknown physical register"); + } + } + return {}; + } + + [[nodiscard]] Expected<void> verifyFunctionRecords() const { + const MachineFunctionRecords &Records = Function.records(); + auto Verification = Records.verify(); + if (!Verification) + return failWith("function records", Verification.error()); + + for (const PhysicalRegister Reg : Records.calleeSavedRegisters()) { + if (!Target.registers().isKnownPhysicalRegister(Reg)) + return fail("callee-saved register list references an unknown physical register"); + } + for (const MachineEntryValueInfo &EntryValue : Records.entryValues()) { + if (!Target.registers().isKnownPhysicalRegister(EntryValue.Reg)) { + return fail("entry-value record references an unknown physical register"); + } + auto DebugInfoResult = EntryValue.DebugInfo.verify(Function.metadata()); + if (!DebugInfoResult) + return failWith("entry-value debug information", DebugInfoResult.error()); + } + + auto VerifyCall = [this](MachineInstrStableID ID, + std::string_view Kind) -> Expected<const MachineInstr *> { + const auto Position = LinkedInstructionsByID.find(ID); + if (Position == LinkedInstructionsByID.end()) { + return makeUnexpected(ErrorCode::VerificationError, + "machine function '" + std::string(Function.name()) + + "': " + std::string(Kind) + + " references a detached or unknown instruction"); + } + const InstructionDescriptor *Descriptor = + Target.instructionDescriptor(Position->second->opcode()); + if (Descriptor == nullptr || !Descriptor->Flags.has(InstructionFlag::Call)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine function '" + std::string(Function.name()) + + "': " + std::string(Kind) + " references a non-call instruction"); + } + return Position->second; + }; + + for (const MachineCallSiteInfo &CallSite : Records.callSites()) { + auto Call = VerifyCall(CallSite.CallInstruction, "call-site record"); + if (!Call) + return std::unexpected(Call.error()); + for (const MachineCallSiteArgument &Argument : CallSite.ForwardedArguments) { + if (!Target.registers().isKnownPhysicalRegister(Argument.Reg)) { + return fail("call-site record references an unknown forwarded physical register"); + } + } + } + + for (const MachineCalledGlobal &CalledGlobal : Records.calledGlobals()) { + auto Call = VerifyCall(CalledGlobal.CallInstruction, "called-global record"); + if (!Call) + return std::unexpected(Call.error()); + } + + if (!Records.debugValueSubstitutions().empty() && + !Function.attributes().has(MachineFunctionAttribute::DebugInstrRef)) { + return fail("debug value substitutions require DebugInstrRef"); + } + for (const MachineDebugValueSubstitution &Substitution : Records.debugValueSubstitutions()) { + if (Substitution.Subregister.value() != 0 && + Target.registers().subRegIndex(Substitution.Subregister) == nullptr) { + return fail("debug value substitution references an unknown subregister index"); + } + } + return {}; + } + + [[nodiscard]] Expected<void> verifyCompilationStatus() const { + const MachineCompilationStatus &Status = Function.compilationStatus(); + auto Verification = Status.verify(); + if (!Verification) + return failWith("compilation status", Verification.error()); + + const MachineCompilationFailure *Failure = Status.failure(); + if (Failure != nullptr && Failure->FailureInstruction && + !LinkedInstructionIDs.contains(*Failure->FailureInstruction)) { + return fail("compilation failure references a detached or unknown instruction"); + } + return {}; + } + + [[nodiscard]] Expected<void> verifyLiveInstruction(const MachineInstr &Instruction) { + auto LocalResult = verifyInstructionLocalState(Instruction); + if (!LocalResult) + return LocalResult; + + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return fail("live instruction uses an unregistered opcode"); + if (Instruction.debugInstrNumber() != 0) { + if (Descriptor->Flags.has(InstructionFlag::Meta)) { + return fail("metadata instruction has a debug value tracking number"); + } + if (!DebugInstructionNumbers.insert(Instruction.debugInstrNumber()).second) { + return fail("multiple instructions have the same debug value tracking number"); + } + } + auto DescriptorResult = Target.validateInstruction(Instruction, Function.registerInfo()); + if (!DescriptorResult) + return failWith("instruction descriptor", DescriptorResult.error()); + + if (Options.Stage && Descriptor->Flags.has(InstructionFlag::Pseudo) && + static_cast<std::uint8_t>(*Options.Stage) > + static_cast<std::uint8_t>(Descriptor->Deadline)) { + return fail("pseudo instruction survives beyond its expansion deadline"); + } + + HasPhi |= Descriptor->Flags.has(InstructionFlag::Phi); + HasStageTenPseudo |= Instruction.opcode() == opcode(GenericOpcode::REG_SEQUENCE) || + Instruction.opcode() == opcode(GenericOpcode::INSERT_SUBREG); + HasGenericInstruction |= Descriptor->Flags.has(InstructionFlag::Generic); + HasExecutablePseudo |= Descriptor->Flags.has(InstructionFlag::Pseudo) && + !Descriptor->Flags.has(InstructionFlag::Meta) && + !Descriptor->Flags.has(InstructionFlag::TerminalEmission); + + const auto Operands = Instruction.operands(); + for (std::size_t Index = 0; Index < Operands.size(); ++Index) { + const MachineOperand &Operand = Operands[Index]; + if (const auto *Frame = Operand.getIf<FrameIndexOperand>(); Frame != nullptr) { + HasFrameIndex = true; + auto FrameResult = verifyFrameIndex(Frame->Index); + if (!FrameResult) + return FrameResult; + } + if (const auto *Constant = Operand.getIf<ConstantPoolOperand>(); Constant != nullptr) { + auto ConstantResult = verifyConstantPoolIndex(Constant->Index); + if (!ConstantResult) + return ConstantResult; + } + if (const auto *JumpTable = Operand.getIf<JumpTableOperand>(); JumpTable != nullptr) { + auto JumpTableResult = verifyJumpTableIndex(JumpTable->Index); + if (!JumpTableResult) + return JumpTableResult; + } + if (const auto *CFI = Operand.getIf<CFIIndexOperand>(); + CFI != nullptr && Function.cfiInfo().instruction(CFI->Index) == nullptr) { + return fail("live instruction references an unknown CFI instruction"); + } + + if (const auto *Block = Operand.getIf<BasicBlockOperand>(); + Block != nullptr && + (Block->Block == nullptr || &Block->Block->function() != &Function || + !RegisteredBlocks.contains(Block->Block) || !Block->Block->isLinked())) { + return fail("basic-block operand references a foreign or detached block"); + } + + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr) + continue; + const bool ShouldBeLinked = !Reg->isNoRegister(); + if (Function.registerInfo().isLinkedOperand(Operand) != ShouldBeLinked) { + return fail("live register operand has incorrect def-use linkage"); + } + if (Reg->isVirtual()) { + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) + return fail("instruction references an unknown virtual register"); + if (Operand.isDef()) + ++VirtualDefCounts[*Reg]; + } + + if (Operand.tiedOperandIndex() && Index < *Operand.tiedOperandIndex()) { + const MachineOperand *Other = Instruction.operandAt(*Operand.tiedOperandIndex()); + const Register *OtherReg = Other == nullptr ? nullptr : Other->getIf<Register>(); + if (OtherReg == nullptr || *OtherReg != *Reg) + HasUnrewrittenTie = true; + } + } + for (const MachineMemOperand *MemOperand : Instruction.memOperands()) { + const MemoryBase &Base = MemOperand->description().Base; + if (const auto Frame = Base.frameIndex()) { + auto FrameResult = verifyFrameIndex(*Frame); + if (!FrameResult) + return FrameResult; + } + if (const auto Constant = Base.constantPoolIndex()) { + auto ConstantResult = verifyConstantPoolIndex(*Constant); + if (!ConstantResult) + return ConstantResult; + } + if (const auto JumpTable = Base.jumpTableIndex()) { + auto JumpTableResult = verifyJumpTableIndex(*JumpTable); + if (!JumpTableResult) + return JumpTableResult; + } + } + return {}; + } + + [[nodiscard]] Expected<void> verifyCFG() { + for (const MachineBasicBlock *Block : LinkedBlocks) { + std::unordered_set<const MachineBasicBlock *> Successors; + for (const MachineSuccessor &Successor : Block->successors()) { + if (Successor.Block == nullptr || !LinkedBlocks.contains(Successor.Block) || + !Successors.insert(Successor.Block).second) { + return fail("CFG successor list contains a null, detached, or duplicate block"); + } + if (std::ranges::count(Successor.Block->predecessors(), Block) != 1) + return fail("CFG successor lacks one reciprocal predecessor"); + } + + std::unordered_set<const MachineBasicBlock *> Predecessors; + for (const MachineBasicBlock *Predecessor : Block->predecessors()) { + if (Predecessor == nullptr || !LinkedBlocks.contains(Predecessor) || + !Predecessors.insert(Predecessor).second) { + return fail("CFG predecessor list contains a null, detached, or duplicate block"); + } + const auto Matches = std::ranges::count_if( + Predecessor->successors(), + [Block](const MachineSuccessor &Successor) { return Successor.Block == Block; }); + if (Matches != 1) + return fail("CFG predecessor lacks one reciprocal successor"); + } + + std::unordered_set<PhysicalRegister> LiveIns; + for (const MachineBasicBlockLiveIn &LiveIn : Block->liveIns()) { + if (!LiveIn.PhysReg.isValid() || LiveIn.Lanes.empty() || + !LiveIns.insert(LiveIn.PhysReg).second) { + return fail("block live-in list contains invalid or duplicate entries"); + } + } + auto TargetLiveInResult = Target.validateBlockLiveIns(*Block); + if (!TargetLiveInResult) + return failWith("block live-ins", TargetLiveInResult.error()); + } + return {}; + } + + [[nodiscard]] Expected<void> verifyPHIs() { + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + bool SawNonMetaInstruction = false; + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = + Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return fail("PHI verification encountered an unregistered instruction"); + + if (!Descriptor->Flags.has(InstructionFlag::Phi)) { + SawNonMetaInstruction |= !Descriptor->Flags.has(InstructionFlag::Meta); + continue; + } + if (SawNonMetaInstruction) + return fail("PHI instruction appears after a non-meta instruction"); + + const std::span<const MachineOperand> Operands = Instruction->operands(); + if (Operands.size() < 3 || (Operands.size() - 1U) % 2U != 0) { + return fail("PHI must contain a destination and at least one incoming pair"); + } + const Register *Destination = Operands.front().getIf<Register>(); + if (Destination == nullptr || !Destination->isVirtual() || !Operands.front().isDef() || + Operands.front().isUse() || Operands.front().isImplicit()) { + return fail("PHI destination must be an explicit virtual-register definition"); + } + + std::unordered_set<const MachineBasicBlock *> IncomingBlocks; + for (std::size_t Index = 1; Index < Operands.size(); Index += 2U) { + const MachineOperand &ValueOperand = Operands[Index]; + const MachineOperand &BlockOperand = Operands[Index + 1U]; + const Register *Value = ValueOperand.getIf<Register>(); + const BasicBlockOperand *Incoming = BlockOperand.getIf<BasicBlockOperand>(); + if (Value == nullptr || !ValueOperand.isUse() || ValueOperand.isDef() || + ValueOperand.isImplicit() || Incoming == nullptr || Incoming->Block == nullptr || + BlockOperand.isImplicit()) { + return fail("PHI incoming operands must be explicit register/block pairs"); + } + if (std::ranges::find(Block->predecessors(), Incoming->Block) == + Block->predecessors().end()) { + return fail("PHI incoming block is not a CFG predecessor"); + } + if (!IncomingBlocks.insert(Incoming->Block).second) + return fail("PHI contains duplicate incoming blocks"); + + if (!Value->isNoRegister() && Destination->isVirtual() && Value->isVirtual() && + Operands.front().subReg().value() == 0 && ValueOperand.subReg().value() == 0) { + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); + auto ValueInfo = Function.registerInfo().virtualRegisterInfo(*Value); + if (!DestinationInfo || !ValueInfo) + return fail("PHI references an unknown virtual register"); + if (DestinationInfo->get().Type && ValueInfo->get().Type && + DestinationInfo->get().Type != ValueInfo->get().Type) { + return fail("PHI destination and incoming value types do not match"); + } + } + auto Compatibility = Target.validateTieCompatibility(Operands.front(), ValueOperand, + Function.registerInfo()); + if (!Compatibility) + return failWith("PHI register compatibility", Compatibility.error()); + } + if (IncomingBlocks.size() != Block->predecessors().size()) + return fail("PHI incoming blocks do not exactly cover CFG predecessors"); + } + } + return {}; + } + + [[nodiscard]] Expected<void> verifyRegisterInfo() { + const MachineRegisterInfo &RegInfo = Function.registerInfo(); + std::unordered_set<PhysicalRegister> FunctionLiveIns; + for (const PhysicalRegisterLiveIn &LiveIn : RegInfo.liveIns()) { + if (!LiveIn.PhysReg.isValid() || !FunctionLiveIns.insert(LiveIn.PhysReg).second) { + return fail("function live-in list contains invalid or duplicate entries"); + } + if (LiveIn.InitialVirtualRegister && + !RegInfo.isKnownVirtualRegister(*LiveIn.InitialVirtualRegister)) { + return fail("function live-in references an unknown virtual register"); + } + } + + std::unordered_set<PhysicalRegister> Reserved; + for (const PhysicalRegister Reg : RegInfo.reservedRegisters()) { + if (!Reg.isValid() || !Reserved.insert(Reg).second) + return fail("reserved-register list contains invalid or duplicate entries"); + } + + const auto VirtualRegisters = RegInfo.virtualRegisters(); + for (std::size_t Index = 0; Index < VirtualRegisters.size(); ++Index) { + const VirtualRegisterInfo &Info = VirtualRegisters[Index]; + if (Info.IsTombstone) { + if (Info.Type || Info.Bank || Info.RegClass || !Info.AllocationHints.empty() || + Info.TargetFlags != 0) { + return fail("virtual-register tombstone retains semantic state"); + } + continue; + } + auto Reg = Register::makeVirtual(Index); + if (!Reg) + return failWith("virtual register", Reg.error()); + const auto Operands = RegInfo.regOperands(*Reg); + if (Operands.begin() != Operands.end()) { + HasLiveVirtualRegister = true; + MissingVirtualType |= !Info.Type; + MissingVirtualBank |= !Info.Bank; + MissingVirtualClass |= !Info.RegClass; + } + if (Info.Type && !Info.Type->isValid()) + return fail("virtual register has an invalid LowLevelType"); + + std::unordered_set<PhysicalRegister> Hints; + for (const PhysicalRegister Hint : Info.AllocationHints) { + if (!Hint.isValid() || !Hints.insert(Hint).second) + return fail("virtual register has an invalid or duplicate allocation hint"); + } + } + auto TargetResult = Target.validateRegisterInfo(RegInfo); + if (!TargetResult) + return failWith("target register info", TargetResult.error()); + return {}; + } + + [[nodiscard]] Expected<void> verifyProperties() { + const MachineFunctionProperties &Properties = Function.properties(); + auto Has = [&Properties](MachineFunctionProperty Property) { return Properties.has(Property); }; + + if (Has(MachineFunctionProperty::RegBankSelected) && !Has(MachineFunctionProperty::Legalized)) { + return fail("RegBankSelected requires Legalized"); + } + if (Has(MachineFunctionProperty::Selected) && !Has(MachineFunctionProperty::RegBankSelected)) { + return fail("Selected requires RegBankSelected"); + } + if (Has(MachineFunctionProperty::TiedOpsRewritten) && !Has(MachineFunctionProperty::Selected)) { + return fail("TiedOpsRewritten requires Selected"); + } + if (Has(MachineFunctionProperty::TiedOpsRewritten) && !Has(MachineFunctionProperty::NoPHIs)) { + return fail("TiedOpsRewritten requires NoPHIs"); + } + if (Has(MachineFunctionProperty::FrameFinalized) && + (!Has(MachineFunctionProperty::NoVRegs) || !Has(MachineFunctionProperty::NoFrameIndices))) { + return fail("FrameFinalized requires NoVRegs and NoFrameIndices"); + } + if (Has(MachineFunctionProperty::FrameFinalized) && !Function.frameInfo().isFrozen()) + return fail("FrameFinalized requires a frozen frame layout"); + + if (Has(MachineFunctionProperty::IsSSA)) { + for (const auto &[Reg, DefinitionCount] : VirtualDefCounts) { + if (DefinitionCount > 1) + return fail("IsSSA function has a multiply-defined virtual register"); + } + } + if (Has(MachineFunctionProperty::NoPHIs) && HasPhi) + return fail("NoPHIs function contains a PHI instruction"); + if (Has(MachineFunctionProperty::NoVRegs) && HasLiveVirtualRegister) { + return fail("NoVRegs function retains a live virtual register"); + } + if (Has(MachineFunctionProperty::NoFrameIndices) && HasFrameIndex) + return fail("NoFrameIndices function contains a frame-index operand"); + if (Has(MachineFunctionProperty::NoExecutablePseudos) && HasExecutablePseudo) { + return fail("NoExecutablePseudos function contains an executable pseudo"); + } + if (Has(MachineFunctionProperty::RegBankSelected) && MissingVirtualBank) { + return fail("RegBankSelected function has a virtual register without a bank"); + } + if (Has(MachineFunctionProperty::Selected) && (HasGenericInstruction || MissingVirtualClass)) { + return fail("Selected function contains generic code or an unclassified virtual register"); + } + if (Has(MachineFunctionProperty::TiedOpsRewritten) && HasUnrewrittenTie) { + return fail("TiedOpsRewritten function has unequal tied registers"); + } + if (Has(MachineFunctionProperty::TiedOpsRewritten) && HasStageTenPseudo) { + return fail("TiedOpsRewritten function contains a Stage 10 pseudo"); + } + if (Has(MachineFunctionProperty::TracksLiveness)) { + auto Liveness = verifyTrackedMachineLiveness(Function, Target); + if (!Liveness) + return failWith("tracked machine liveness", Liveness.error()); + } + + const bool NeedsTypes = Options.Mode == MachineVerifierMode::PreLegalizer || + (!Has(MachineFunctionProperty::Selected) && + (Has(MachineFunctionProperty::Legalized) || + Has(MachineFunctionProperty::RegBankSelected))); + if (NeedsTypes && MissingVirtualType) + return fail("typed pipeline stage has a virtual register without LowLevelType"); + + if (Options.Mode == MachineVerifierMode::PreLegalizer) { + if (!Has(MachineFunctionProperty::IsSSA) || !Has(MachineFunctionProperty::TracksLiveness)) { + return fail("pre-legalizer input requires IsSSA and TracksLiveness"); + } + if (Has(MachineFunctionProperty::Legalized) || + Has(MachineFunctionProperty::RegBankSelected) || Has(MachineFunctionProperty::Selected) || + Has(MachineFunctionProperty::TiedOpsRewritten) || Has(MachineFunctionProperty::NoVRegs) || + Has(MachineFunctionProperty::FrameFinalized)) { + return fail("pre-legalizer input carries a later-stage property"); + } + } + return {}; + } + + const MachineFunction &Function; + const TargetDescription &Target; + MachineVerifierOptions Options; + std::unordered_set<const MachineBasicBlock *> RegisteredBlocks; + std::unordered_set<const MachineBasicBlock *> LinkedBlocks; + std::unordered_set<MachineBlockStableID> LinkedBlockIDs; + std::unordered_set<const MachineInstr *> RegisteredInstructions; + std::unordered_set<const MachineInstr *> LinkedInstructions; + std::unordered_set<MachineInstrStableID> LinkedInstructionIDs; + std::unordered_map<MachineInstrStableID, const MachineInstr *> LinkedInstructionsByID; + std::unordered_set<std::uint64_t> LinkedBlockProfileIDs; + std::unordered_set<std::uint32_t> DebugInstructionNumbers; + std::unordered_set<const MachineMemOperand *> RegisteredMemOperands; + std::unordered_map<Register, std::size_t> VirtualDefCounts; + bool HasPhi = false; + bool HasGenericInstruction = false; + bool HasExecutablePseudo = false; + bool HasFrameIndex = false; + bool HasUnrewrittenTie = false; + bool HasStageTenPseudo = false; + bool HasLiveVirtualRegister = false; + bool MissingVirtualType = false; + bool MissingVirtualBank = false; + bool MissingVirtualClass = false; +}; + +} // namespace + +Expected<void> MachineVerifier::verify(const MachineFunction &Function, + const TargetDescription &Target, + MachineVerifierOptions Options) { + return FunctionVerifier(Function, Target, Options).run(); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/MacroFusion.cpp b/backend/lib/CodeGen/MacroFusion.cpp new file mode 100644 index 0000000..726ca59 --- /dev/null +++ b/backend/lib/CodeGen/MacroFusion.cpp @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MacroFusion.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <iostream> +#include <optional> +#include <ranges> +#include <span> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachineInstr *instructionFor(MacroFusionInstruction Side, + const MachineInstr *First, + const MachineInstr &Second) noexcept { + return Side == MacroFusionInstruction::First ? First : &Second; +} + +[[nodiscard]] std::optional<Register> registerAt(const MachineInstr &Instruction, + std::uint32_t OperandIndex) noexcept { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + return Operand == nullptr ? std::nullopt : Operand->registerValue(); +} + +[[nodiscard]] const std::int64_t *immediateAt(const MachineInstr &Instruction, + std::uint32_t OperandIndex) noexcept { + const MachineOperand *Operand = Instruction.operandAt(OperandIndex); + return Operand == nullptr ? nullptr : Operand->getIf<std::int64_t>(); +} + +[[nodiscard]] bool containsOpcode(std::span<const Opcode> Opcodes, Opcode Value) noexcept { + return std::ranges::contains(Opcodes, Value); +} + +[[nodiscard]] bool matchesLink(const MacroFusionRule &Rule, const MachineInstr &First, + const MachineInstr &Second) noexcept { + const std::optional<Register> FirstDefinition = registerAt(First, Rule.FirstDefinitionOperand); + const std::optional<Register> SecondUse = registerAt(Second, Rule.SecondUseOperand); + const std::optional<Register> AlternateSecondUse = + Rule.CommutableSecondUseOperand == MacroFusionRule::NoOperand + ? std::nullopt + : registerAt(Second, Rule.CommutableSecondUseOperand); + if (!FirstDefinition || + (*FirstDefinition != SecondUse && *FirstDefinition != AlternateSecondUse)) { + return false; + } + + if (Rule.RequireSingleNonDebugUse && FirstDefinition->isVirtual() && + !Second.function().registerInfo().hasOneUse(*FirstDefinition, false)) { + return false; + } + + if (!Rule.RequireSecondDestinationTie) + return true; + const std::optional<Register> SecondDefinition = registerAt(Second, Rule.SecondDefinitionOperand); + if (!SecondDefinition || SecondDefinition->isVirtual()) + return SecondDefinition.has_value(); + return *SecondDefinition == SecondUse || *SecondDefinition == AlternateSecondUse; +} + +[[nodiscard]] bool isFusionCandidateDependency(ScheduleDependencyKind Kind) noexcept { + return Kind != ScheduleDependencyKind::Anti && Kind != ScheduleDependencyKind::Output && + Kind != ScheduleDependencyKind::Cluster; +} + +[[nodiscard]] std::string_view opcodeName(const TargetDescription &Target, Opcode Value) noexcept { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Value); + return Descriptor == nullptr ? std::string_view{"<unknown>"} : Descriptor->Name; +} + +} // namespace + +bool matchesMacroFusion(const MacroFusionRule &Rule, const MachineInstr *First, + const MachineInstr &Second) noexcept { + if (!containsOpcode(Rule.SecondOpcodes, Second.opcode())) + return false; + if (First == nullptr) + return true; + if (!containsOpcode(Rule.FirstOpcodes, First->opcode())) + return false; + if (!Rule.OpcodePairs.empty() && + std::ranges::none_of(Rule.OpcodePairs, [First, &Second](MacroFusionOpcodePair Pair) { + return Pair.First == First->opcode() && Pair.Second == Second.opcode(); + })) { + return false; + } + + for (const MacroFusionImmediatePredicate &Predicate : Rule.ImmediatePredicates) { + const MachineInstr *Instruction = instructionFor(Predicate.Instruction, First, Second); + const std::int64_t *Value = + Instruction == nullptr ? nullptr : immediateAt(*Instruction, Predicate.OperandIndex); + if (Value == nullptr || *Value < Predicate.Minimum || *Value > Predicate.Maximum) + return false; + } + for (const MacroFusionRegisterInequality &Predicate : Rule.RegisterInequalities) { + const MachineInstr *Instruction = instructionFor(Predicate.Instruction, First, Second); + if (Instruction == nullptr) + return false; + const std::optional<Register> Left = registerAt(*Instruction, Predicate.LeftOperand); + const std::optional<Register> Right = registerAt(*Instruction, Predicate.RightOperand); + if (!Left || !Right || *Left == *Right) + return false; + } + for (const MacroFusionImmediateRelation &Predicate : Rule.ImmediateRelations) { + const std::int64_t *FirstValue = immediateAt(*First, Predicate.FirstOperand); + const std::int64_t *SecondValue = immediateAt(Second, Predicate.SecondOperand); + if (FirstValue == nullptr || SecondValue == nullptr || *FirstValue > *SecondValue) + return false; + } + return matchesLink(Rule, *First, Second); +} + +Expected<void> applyMacroFusion(ScheduleDAG &DAG, std::span<const MacroFusionRule> Rules, + const TargetDescription &Target, bool EmitDebugOutput) { + if (Rules.empty()) + return {}; + for (ScheduleUnit &Anchor : DAG.units()) { + if (Anchor.Instructions.size() != 1 || Anchor.ClusterPredecessor || Anchor.ClusterSuccessor) + continue; + const MachineInstr &Second = *Anchor.Instructions.front(); + if (std::ranges::none_of(Rules, [&Second](const MacroFusionRule &Rule) { + return matchesMacroFusion(Rule, nullptr, Second); + })) { + continue; + } + const std::vector<ScheduleDependency> Predecessors = Anchor.Predecessors; + for (const ScheduleDependency &Dependency : Predecessors) { + if (!isFusionCandidateDependency(Dependency.Kind)) + continue; + const ScheduleUnit *FirstUnit = DAG.unit(Dependency.Source); + if (FirstUnit == nullptr || FirstUnit->Instructions.size() != 1 || + FirstUnit->ClusterPredecessor || FirstUnit->ClusterSuccessor) { + continue; + } + const MachineInstr &First = *FirstUnit->Instructions.front(); + if (std::ranges::none_of(Rules, [&First, &Second](const MacroFusionRule &Rule) { + return matchesMacroFusion(Rule, &First, Second); + })) { + continue; + } + auto Fused = DAG.fuseInstructionPair(FirstUnit->ID, Anchor.ID); + if (!Fused) + return std::unexpected(Fused.error()); + if (!*Fused) + continue; + if (EmitDebugOutput) { + std::cerr << "Macro fuse: SU(" << FirstUnit->ID << ") - SU(" << Anchor.ID << ") / " + << opcodeName(Target, First.opcode()) << " - " + << opcodeName(Target, Second.opcode()) << '\n'; + } + break; + } + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/PHIElimination.cpp b/backend/lib/CodeGen/PHIElimination.cpp new file mode 100644 index 0000000..8ad785e --- /dev/null +++ b/backend/lib/CodeGen/PHIElimination.cpp @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/PHIElimination.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFGUpdater.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetInstrInfo.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct EdgeKey final { + const MachineBasicBlock *From = nullptr; + const MachineBasicBlock *To = nullptr; + + friend bool operator==(const EdgeKey &, const EdgeKey &) noexcept = default; +}; + +struct EdgeKeyHash final { + [[nodiscard]] std::size_t operator()(const EdgeKey &Key) const noexcept { + const std::size_t FromHash = std::hash<const MachineBasicBlock *>{}(Key.From); + const std::size_t ToHash = std::hash<const MachineBasicBlock *>{}(Key.To); + return FromHash ^ (ToHash << 1U); + } +}; + +struct RegisterValue final { + Register Reg; + SubRegIndex SubReg; + RegisterOperandFlags Flags; + std::uint32_t TargetFlags = 0; +}; + +struct PhiIncoming final { + RegisterValue Value; + MachineBasicBlock *Predecessor = nullptr; + bool IsUndef = false; +}; + +struct PhiPlan final { + MachineInstr *Instruction = nullptr; + RegisterValue Destination; + Register FreshIncoming; + std::vector<PhiIncoming> IncomingValues; +}; + +[[nodiscard]] bool isPhi(const MachineInstr &Instruction, const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + return Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Phi); +} + +[[nodiscard]] bool isMeta(const MachineInstr &Instruction, const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + return Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Meta); +} + +[[nodiscard]] MachineInstr *firstTerminator(MachineBasicBlock &Block, + const TargetDescription &Target) { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Terminator)) + return Instruction; + } + return nullptr; +} + +[[nodiscard]] MachineInstr *firstCodeAfterPhiRegion(MachineBasicBlock &Block, + const TargetDescription &Target) { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (!isPhi(*Instruction, Target) && !isMeta(*Instruction, Target)) + return Instruction; + } + return nullptr; +} + +[[nodiscard]] RegisterOperandFlags generatedDefFlags(RegisterOperandFlags Original) noexcept { + RegisterOperandFlags Result; + if (Original.has(RegisterOperandFlag::Undef)) + Result.set(RegisterOperandFlag::Undef); + return Result; +} + +[[nodiscard]] Expected<void> copyDebugLocation(MachineInstrBuilder &Builder, + const MachineInstr *Source) { + if (Source == nullptr || !Source->debugLocation()) + return {}; + return Builder.setDebugLocation(*Source->debugLocation()); +} + +[[nodiscard]] Expected<Register> cloneRegisterConstraints(MachineFunction &Function, + Register Source) { + if (!Source.isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "SSA deconstruction requires a virtual PHI destination"); + } + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo) + return std::unexpected(SourceInfo.error()); + const VirtualRegisterInfo Info = SourceInfo->get(); + + auto Fresh = Function.registerInfo().createVirtualRegister(); + if (!Fresh) + return std::unexpected(Fresh.error()); + if (Info.Type) { + auto SetType = Function.registerInfo().setType(*Fresh, *Info.Type); + if (!SetType) + return std::unexpected(SetType.error()); + } + if (Info.Bank) { + auto SetBank = Function.registerInfo().setRegisterBank(*Fresh, *Info.Bank); + if (!SetBank) + return std::unexpected(SetBank.error()); + } + if (Info.RegClass) { + auto SetClass = Function.registerInfo().setRegisterClass(*Fresh, *Info.RegClass); + if (!SetClass) + return std::unexpected(SetClass.error()); + } + for (const PhysicalRegister Hint : Info.AllocationHints) { + auto AddedHint = Function.registerInfo().addAllocationHint(*Fresh, Hint); + if (!AddedHint) + return std::unexpected(AddedHint.error()); + } + if (Info.TargetFlags != 0) { + auto SetFlags = Function.registerInfo().setTargetRegisterFlags(*Fresh, Info.TargetFlags); + if (!SetFlags) + return std::unexpected(SetFlags.error()); + } + return *Fresh; +} + +[[nodiscard]] Expected<MachineInstr *> +buildCopy(MachineFunction &Function, const TargetDescription &Target, + const TargetInstrInfo &TargetInfo, MachineBasicBlock &Block, MachineInstr *Before, + RegisterValue Destination, RegisterValue Source, const MachineInstr *DebugSource) { + auto Legality = TargetInfo.validateCopy(Destination.Reg, Destination.SubReg, Source.Reg, + Source.SubReg, Function, Target); + if (!Legality) + return std::unexpected(withErrorContext(Legality.error(), "illegal COPY: ")); + + MachineIRBuilder Builder(Function, Target); + auto Point = + Before == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Before); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::COPY)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Destination.Reg, generatedDefFlags(Destination.Flags), + Destination.SubReg, Destination.TargetFlags); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Pending->addUse(Source.Reg, {}, Source.SubReg, Source.TargetFlags); + if (!Use) + return std::unexpected(Use.error()); + auto Debug = copyDebugLocation(*Pending, DebugSource); + if (!Debug) + return std::unexpected(Debug.error()); + return Pending->build(); +} + +[[nodiscard]] Expected<MachineInstr *> +buildImplicitDef(MachineFunction &Function, const TargetDescription &Target, + MachineBasicBlock &Block, MachineInstr *Before, RegisterValue Destination, + const MachineInstr *DebugSource) { + MachineIRBuilder Builder(Function, Target); + auto Point = + Before == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Before); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::IMPLICIT_DEF)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Destination.Reg, generatedDefFlags(Destination.Flags), + Destination.SubReg, Destination.TargetFlags); + if (!Def) + return std::unexpected(Def.error()); + auto Debug = copyDebugLocation(*Pending, DebugSource); + if (!Debug) + return std::unexpected(Debug.error()); + return Pending->build(); +} + +[[nodiscard]] bool isImplicitlyUndef(const MachineFunction &Function, + const MachineOperand &Operand) { + if (Operand.registerFlags().has(RegisterOperandFlag::Undef)) + return true; + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return true; + if (!Reg->isVirtual()) + return false; + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg); + if (Definition == nullptr || Definition->parent() == nullptr) + return false; + if (Definition->subReg().value() != 0 && Definition->subReg() != Operand.subReg()) + return false; + const Opcode DefinitionOpcode = Definition->parent()->opcode(); + return DefinitionOpcode == opcode(GenericOpcode::IMPLICIT_DEF) || + DefinitionOpcode == opcode(GenericOpcode::G_IMPLICIT_DEF) || + Definition->registerFlags().has(RegisterOperandFlag::Undef); +} + +[[nodiscard]] Expected<void> validatePhiValue(const MachineFunction &Function, + const TargetDescription &Target, + const MachineOperand &Destination, + const MachineOperand &Source) { + const Register *DestinationReg = Destination.getIf<Register>(); + const Register *SourceReg = Source.getIf<Register>(); + if (DestinationReg == nullptr || SourceReg == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "PHI operands must name registers"); + if (SourceReg->isNoRegister()) + return {}; + + if (DestinationReg->isVirtual() && SourceReg->isVirtual()) { + auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*DestinationReg); + if (!DestinationInfo) + return std::unexpected(DestinationInfo.error()); + auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*SourceReg); + if (!SourceInfo) + return std::unexpected(SourceInfo.error()); + if (DestinationInfo->get().Type && SourceInfo->get().Type && + DestinationInfo->get().Type != SourceInfo->get().Type) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI destination and incoming value types do not match"); + } + } + return Target.validateTieCompatibility(Destination, Source, Function.registerInfo()); +} + +[[nodiscard]] Expected<std::vector<PhiPlan>> collectPhiPlans(MachineBasicBlock &Block, + MachineFunction &Function, + const TargetDescription &Target) { + std::vector<PhiPlan> Plans; + bool SawCode = false; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI elimination encountered an unknown opcode"); + } + if (Instruction->opcode() == opcode(GenericOpcode::G_PHI)) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI elimination requires G_PHI to be selected to PHI"); + } + if (!Descriptor->Flags.has(InstructionFlag::Phi)) { + SawCode |= !Descriptor->Flags.has(InstructionFlag::Meta); + continue; + } + if (SawCode) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI instruction appears after the leading PHI/meta region"); + } + if (Instruction->operandCount() < 3 || (Instruction->operandCount() - 1U) % 2U != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI must contain a destination and incoming value/block pairs"); + } + + MachineOperand *DestinationOperand = Instruction->operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr || !Destination->isVirtual() || !DestinationOperand->isDef() || + DestinationOperand->isUse() || DestinationOperand->isImplicit()) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI destination must be an explicit virtual-register definition"); + } + + PhiPlan Plan; + Plan.Instruction = Instruction; + Plan.Destination = + RegisterValue{*Destination, DestinationOperand->subReg(), + DestinationOperand->registerFlags(), DestinationOperand->targetFlags()}; + std::unordered_set<MachineBasicBlock *> SeenPredecessors; + for (std::size_t Index = 1; Index < Instruction->operandCount(); Index += 2U) { + MachineOperand *SourceOperand = Instruction->operandAt(Index); + MachineOperand *BlockOperand = Instruction->operandAt(Index + 1U); + const Register *Source = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + const BasicBlockOperand *IncomingBlock = + BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>(); + if (Source == nullptr || SourceOperand == nullptr || !SourceOperand->isUse() || + SourceOperand->isDef() || SourceOperand->isImplicit() || IncomingBlock == nullptr || + IncomingBlock->Block == nullptr || BlockOperand->isImplicit()) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI incoming operands must be explicit register/block pairs"); + } + if (std::ranges::find(Block.predecessors(), IncomingBlock->Block) == + Block.predecessors().end()) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI incoming block is not a CFG predecessor"); + } + if (!SeenPredecessors.insert(IncomingBlock->Block).second) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI contains duplicate incoming predecessors"); + } + auto Compatibility = validatePhiValue(Function, Target, *DestinationOperand, *SourceOperand); + if (!Compatibility) + return std::unexpected(Compatibility.error()); + Plan.IncomingValues.push_back( + PhiIncoming{RegisterValue{*Source, SourceOperand->subReg(), + SourceOperand->registerFlags(), SourceOperand->targetFlags()}, + IncomingBlock->Block, isImplicitlyUndef(Function, *SourceOperand)}); + } + if (SeenPredecessors.size() != Block.predecessors().size()) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI incoming blocks do not exactly cover CFG predecessors"); + } + Plans.push_back(std::move(Plan)); + } + return Plans; +} + +[[nodiscard]] Expected<void> copyLiveIns(const MachineBasicBlock &Source, + MachineBasicBlock &Destination) { + for (const MachineBasicBlockLiveIn &LiveIn : Source.liveIns()) { + auto Added = Destination.addLiveIn(LiveIn.PhysReg, LiveIn.Lanes); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> ensurePhysicalLiveIn(MachineBasicBlock &Block, PhysicalRegister Reg, + const TargetDescription &Target) { + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(Reg); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI incoming value references an unknown physical register"); + } + LaneMask Required; + for (const RegisterUnitLane &Unit : Description->Units) + Required = Required | Unit.Lanes; + if (Required.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI incoming physical register has no addressable lanes"); + } + + for (const MachineBasicBlockLiveIn &LiveIn : Block.liveIns()) { + if (LiveIn.PhysReg != Reg) + continue; + if (LiveIn.Lanes.contains(Required)) + return {}; + const LaneMask Combined = LiveIn.Lanes | Required; + auto Removed = Block.removeLiveIn(Reg); + if (!Removed) + return std::unexpected(Removed.error()); + return Block.addLiveIn(Reg, Combined); + } + return Block.addLiveIn(Reg, Required); +} + +[[nodiscard]] Expected<MachineBasicBlock *> +placePhiCopiesOnEdge(MachineFunction &Function, const TargetDescription &Target, + const TargetInstrInfo &TargetInfo, MachineBasicBlock &From, + MachineBasicBlock &To, + std::unordered_map<EdgeKey, MachineBasicBlock *, EdgeKeyHash> &Cache, + MachinePassResult &PassResult) { + if (From.successors().size() <= 1) + return &From; + + const EdgeKey Key{&From, &To}; + if (const auto Existing = Cache.find(Key); Existing != Cache.end()) + return Existing->second; + if (std::ranges::none_of(From.successors(), [&To](const MachineSuccessor &Successor) { + return Successor.Block == &To; + })) { + return makeUnexpected(ErrorCode::VerificationError, + "PHI incoming predecessor has no edge to its successor"); + } + + auto SourceFallthrough = TargetInfo.isFallthroughEdge(From, To, Function, Target); + if (!SourceFallthrough) + return std::unexpected(SourceFallthrough.error()); + + MachineBasicBlock *LayoutPredecessor = To.previous(); + if (LayoutPredecessor != nullptr && LayoutPredecessor != &From) { + auto OtherFallthrough = TargetInfo.isFallthroughEdge(*LayoutPredecessor, To, Function, Target); + if (!OtherFallthrough) + return std::unexpected(OtherFallthrough.error()); + if (*OtherFallthrough) { + auto Branch = TargetInfo.insertBranch(*LayoutPredecessor, To, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + PassResult.Changed = true; + PassResult.Changes.add(MachinePassChange::Instructions); + } + } + + auto EdgeBlockResult = Function.createBasicBlock(); + if (!EdgeBlockResult) + return std::unexpected(EdgeBlockResult.error()); + MachineBasicBlock *EdgeBlock = *EdgeBlockResult; + auto Inserted = Function.insertBlockBefore(To, *EdgeBlock); + if (!Inserted) + return std::unexpected(Inserted.error()); + + auto Rewritten = TargetInfo.rewriteBranchTarget(From, To, *EdgeBlock, Function, Target); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + if (!*Rewritten && !*SourceFallthrough) { + return makeUnexpected(ErrorCode::Unsupported, + "PHI elimination cannot split an indirect or unmodeled CFG edge"); + } + + auto Split = Function.cfgUpdater().splitEdge(From, To, *EdgeBlock); + if (!Split) + return std::unexpected(Split.error()); + auto LiveIns = copyLiveIns(To, *EdgeBlock); + if (!LiveIns) + return std::unexpected(LiveIns.error()); + auto Branch = TargetInfo.insertBranch(*EdgeBlock, To, Function, Target); + if (!Branch) + return std::unexpected(Branch.error()); + + Cache.emplace(Key, EdgeBlock); + PassResult.Changed = true; + PassResult.Changes.add(MachinePassChange::CFG); + PassResult.Changes.add(MachinePassChange::BlockLayout); + PassResult.Changes.add(MachinePassChange::Instructions); + if (*Rewritten) + PassResult.Changes.add(MachinePassChange::Operands); + return EdgeBlock; +} + +[[nodiscard]] Expected<void> repairTrackedLiveness(MachineFunction &Function, + const TargetDescription &Target, + MachinePassResult &Result) { + if (!Result.Changed || !Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + return {}; + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + if (*Rebuilt) + Result.Changes.add(MachinePassChange::Operands); + return {}; +} + +} // namespace + +const MachinePassContract &PHIEliminationPass::contract() const noexcept { + static const MachinePassContract Contract{ + .Name = "phi-node-elimination", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::Selected, + MachineFunctionProperty::RegBankSelected}, + .Sets = {MachineFunctionProperty::NoPHIs}, + .Clears = {MachineFunctionProperty::IsSSA}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::SSAEliminated, + }; + return Contract; +} + +Expected<MachinePassResult> PHIEliminationPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +PHIEliminationPass::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + MachinePassResult Result; + std::unordered_map<EdgeKey, MachineBasicBlock *, EdgeKeyHash> SplitEdges; + const std::vector<MachineBasicBlock *> Blocks(Function.allBlocks().begin(), + Function.allBlocks().end()); + + for (MachineBasicBlock *Block : Blocks) { + auto Plans = collectPhiPlans(*Block, Function, Target); + if (!Plans) + return std::unexpected(Plans.error()); + if (Plans->empty()) + continue; + + for (PhiPlan &Plan : *Plans) { + auto Accepted = + Analyses.requestAction("eliminate-phi", Function, Plan.Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return makeUnexpected(ErrorCode::VerificationError, + "required PHI elimination action was rejected"); + } + auto Fresh = cloneRegisterConstraints(Function, Plan.Destination.Reg); + if (!Fresh) + return std::unexpected(Fresh.error()); + Plan.FreshIncoming = *Fresh; + } + + for (PhiPlan &Plan : *Plans) { + for (const PhiIncoming &Incoming : Plan.IncomingValues) { + auto EdgeBlock = placePhiCopiesOnEdge(Function, Target, Info, *Incoming.Predecessor, *Block, + SplitEdges, Result); + if (!EdgeBlock) + return std::unexpected(EdgeBlock.error()); + if (*EdgeBlock != Incoming.Predecessor) { + const std::optional<PhysicalRegister> Physical = Incoming.Value.Reg.physicalRegister(); + if (Physical) { + auto LiveIn = ensurePhysicalLiveIn(**EdgeBlock, *Physical, Target); + if (!LiveIn) + return std::unexpected(LiveIn.error()); + } + } + MachineInstr *Terminator = firstTerminator(**EdgeBlock, Target); + RegisterValue Fresh{Plan.FreshIncoming, Plan.Destination.SubReg, Plan.Destination.Flags, + Plan.Destination.TargetFlags}; + Expected<MachineInstr *> Transfer = + Incoming.IsUndef ? buildImplicitDef(Function, Target, **EdgeBlock, Terminator, Fresh, + Plan.Instruction) + : buildCopy(Function, Target, Info, **EdgeBlock, Terminator, Fresh, + Incoming.Value, Plan.Instruction); + if (!Transfer) + return std::unexpected(Transfer.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + } + } + + MachineInstr *InsertionPoint = firstCodeAfterPhiRegion(*Block, Target); + for (PhiPlan &Plan : *Plans) { + RegisterValue Fresh{ + Plan.FreshIncoming, Plan.Destination.SubReg, {}, Plan.Destination.TargetFlags}; + auto Copy = buildCopy(Function, Target, Info, *Block, InsertionPoint, Plan.Destination, Fresh, + Plan.Instruction); + if (!Copy) + return std::unexpected(Copy.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + } + + for (PhiPlan &Plan : *Plans) { + auto Removed = Plan.Instruction->eraseFromParent(); + if (!Removed) + return std::unexpected(Removed.error()); + } + Result.Changes.add(MachinePassChange::Registers); + } + + if (Result.Changed && Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Repaired = repairTrackedLiveness(Function, Target, Result); + if (!Repaired) + return std::unexpected( + withErrorContext(Repaired.error(), "PHI-elimination liveness repair failed: ")); + } + Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/PostRAScheduler.cpp b/backend/lib/CodeGen/PostRAScheduler.cpp new file mode 100644 index 0000000..d8c6a36 --- /dev/null +++ b/backend/lib/CodeGen/PostRAScheduler.cpp @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/PostRAScheduler.h" + +#include "MachineSchedulerInternal.h" +#include "rcc/CodeGen/LivePhysRegs.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <ranges> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &postRASchedulerContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "postmisched", + .Requires = {}, + .Sets = {}, + .Clears = {}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<LiveVariablesAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.RequiredAnalyses.push_back( + MachineFunctionAnalysisManager::id<MachineBlockFrequencyAnalysis>()); + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands}; + return Value; + }(); + return Contract; +} + +} // namespace + +const MachinePassContract &PostRASchedulerPass::contract() const noexcept { + return postRASchedulerContract(); +} + +Expected<MachinePassResult> PostRASchedulerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Liveness = Analyses.getResult<LiveVariablesAnalysis>(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + auto Frequencies = Analyses.getResult<MachineBlockFrequencyAnalysis>(Function, Target); + if (!Frequencies) + return std::unexpected(Frequencies.error()); + auto Run = runMachineScheduler(Function, Target, *Model, ScheduleDAGMode::PostRegisterAllocation, + **Liveness, **Frequencies, Options, Analyses); + if (!Run) + return std::unexpected(Run.error()); + auto Repaired = repairPostRAMachineBlockLiveness(Run->ChangedBlocks, Target, **Liveness); + if (!Repaired) + return std::unexpected(Repaired.error()); + + MachinePassResult Result; + Result.Changed = Run->Changed || *Repaired; + if (!Result.Changed) { + Result.Preserved = PreservedAnalyses::all(); + return Result; + } + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands}; + Result.Preserved.preserve<LiveVariablesAnalysis>(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/PrologueEpilogueInsertion.cpp b/backend/lib/CodeGen/PrologueEpilogueInsertion.cpp new file mode 100644 index 0000000..2469039 --- /dev/null +++ b/backend/lib/CodeGen/PrologueEpilogueInsertion.cpp @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/PrologueEpilogueInsertion.h" + +#include "rcc/CodeGen/FrameIndexElimination.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterScavenger.h" +#include "rcc/CodeGen/TargetFrameLowering.h" + +#include <utility> +#include <vector> + +namespace rcc { +namespace {} // namespace + +const MachinePassContract &PrologueEpilogueInsertionPass::contract() const noexcept { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "prologepilog", + .Requires = {MachineFunctionProperty::NoPHIs, MachineFunctionProperty::NoVRegs}, + .Sets = {MachineFunctionProperty::FrameFinalized, MachineFunctionProperty::NoFrameIndices}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::FrameFinalized, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + Value.DeclaredChanges = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers, MachinePassChange::Frame}; + return Value; + }(); + return Contract; +} + +Expected<MachinePassResult> PrologueEpilogueInsertionPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + if (Info == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, + "PrologueEpilogueInsertion requires target frame lowering"); + return runImplementation(Function, Target, Options); +} + +Expected<MachinePassResult> +PrologueEpilogueInsertionPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + FrameLoweringOptions RunOptions) const { + auto Prepared = Info->prepareFrame(Function, Target, RunOptions); + if (!Prepared) + return std::unexpected(Prepared.error()); + + RegisterScavenger Scavenger(Function, Target); + std::vector<MachineInstr *> CallFramePseudos; + std::vector<std::pair<MachineBasicBlock *, MachineInstr *>> Epilogues; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Info->isCallFramePseudo(*Instruction)) + CallFramePseudos.push_back(Instruction); + if (Info->isEpilogueTerminator(*Instruction, Target)) + Epilogues.emplace_back(Block, Instruction); + } + } + for (MachineInstr *Pseudo : CallFramePseudos) { + auto Eliminated = Info->eliminateCallFramePseudo(*Pseudo, Function, Target, Scavenger); + if (!Eliminated) + return std::unexpected(Eliminated.error()); + } + + auto CalleeSaved = Info->insertCalleeSavedSpillsAndRestores(Function, Target); + if (!CalleeSaved) + return std::unexpected(CalleeSaved.error()); + + auto PreFinalized = Info->processFunctionBeforeFrameFinalized(Function, Target); + if (!PreFinalized) + return std::unexpected(PreFinalized.error()); + + auto Finalized = Info->finalizeFrame(Function, Target); + if (!Finalized) + return std::unexpected(Finalized.error()); + if (!Function.frameInfo().isFrozen()) { + return makeUnexpected(ErrorCode::InternalError, + "target frame lowering did not freeze the frame layout"); + } + + auto Prologue = Info->emitPrologue(Function, Target, Scavenger); + if (!Prologue) + return std::unexpected(Prologue.error()); + for (const auto &[Block, Terminator] : Epilogues) { + auto Epilogue = Info->emitEpilogue(*Block, *Terminator, Function, Target, Scavenger); + if (!Epilogue) + return std::unexpected(Epilogue.error()); + } + + auto PreFrameIndices = Info->processFunctionBeforeFrameIndexElimination(Function, Target); + if (!PreFrameIndices) + return std::unexpected(PreFrameIndices.error()); + + auto FrameIndices = eliminateFrameIndices(Function, Target, *Info, Scavenger); + if (!FrameIndices) + return std::unexpected(FrameIndices.error()); + + MachinePassResult Result; + Result.Changed = true; + Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands, + MachinePassChange::Registers, MachinePassChange::Frame}; + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/ReachingDefAnalysis.cpp b/backend/lib/CodeGen/ReachingDefAnalysis.cpp new file mode 100644 index 0000000..cac69c4 --- /dev/null +++ b/backend/lib/CodeGen/ReachingDefAnalysis.cpp @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/ReachingDefAnalysis.h" + +#include "rcc/CodeGen/LiveRegUnits.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <cstddef> +#include <ranges> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +using DefinitionSet = ReachingDefAnalysis::DefinitionSet; + +struct ReachingDefState final { + std::unordered_map<RegisterUnitID, DefinitionSet> Registers; + std::unordered_map<FrameIndex, DefinitionSet> FrameObjects; + + friend bool operator==(const ReachingDefState &, const ReachingDefState &) = default; +}; + +void sortAndUnique(DefinitionSet &Definitions, + const std::unordered_map<const MachineInstr *, std::size_t> &Numbers) { + std::ranges::sort(Definitions, [&Numbers](const MachineInstr *Left, const MachineInstr *Right) { + return Numbers.at(Left) < Numbers.at(Right); + }); + Definitions.erase(std::ranges::unique(Definitions).begin(), Definitions.end()); +} + +void mergeDefinitions(DefinitionSet &Destination, const DefinitionSet &Source, + const std::unordered_map<const MachineInstr *, std::size_t> &Numbers) { + Destination.insert(Destination.end(), Source.begin(), Source.end()); + sortAndUnique(Destination, Numbers); +} + +void mergeState(ReachingDefState &Destination, const ReachingDefState &Source, + const std::unordered_map<const MachineInstr *, std::size_t> &Numbers) { + for (const auto &[Unit, Definitions] : Source.Registers) + mergeDefinitions(Destination.Registers[Unit], Definitions, Numbers); + for (const auto &[Index, Definitions] : Source.FrameObjects) + mergeDefinitions(Destination.FrameObjects[Index], Definitions, Numbers); +} + +[[nodiscard]] Expected<void> definePhysicalOperand(ReachingDefState &State, + const MachineOperand &Operand, + const MachineInstr &Instruction, + const TargetDescription &Target) { + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + for (const PhysicalUnitAccess &Access : *Units) + State.Registers[Access.Unit] = {&Instruction}; + return {}; +} + +void defineRegisterMask(ReachingDefState &State, const RegisterMask &Mask, + const MachineInstr &Instruction, const TargetDescription &Target) { + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask.clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + State.Registers[Unit.Unit] = {&Instruction}; + } +} + +[[nodiscard]] bool instructionMayStore(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory != nullptr && Memory->isStore(); + }); +} + +[[nodiscard]] std::unordered_set<FrameIndex> storedFrameObjects(const MachineInstr &Instruction) { + std::unordered_set<FrameIndex> Result; + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + if (Memory == nullptr || !Memory->isStore()) + continue; + const std::optional<FrameIndex> Index = Memory->description().Base.frameIndex(); + if (Index) + Result.insert(*Index); + } + return Result; +} + +[[nodiscard]] Expected<void> transferInstruction(ReachingDefState &State, + const MachineInstr &Instruction, + const TargetDescription &Target) { + if (Instruction.isDebugInstr()) + return {}; + + for (const MachineOperand &Operand : Instruction.operands()) { + if (Operand.isRegister()) { + const std::optional<Register> Reg = Operand.registerValue(); + if (!Operand.isDef() || !Reg || Reg->isNoRegister()) + continue; + if (!Reg->isPhysical()) { + return makeUnexpected(ErrorCode::VerificationError, + "reaching-def analysis requires physical registers"); + } + auto Defined = definePhysicalOperand(State, Operand, Instruction, Target); + if (!Defined) + return std::unexpected(Defined.error()); + continue; + } + const auto *MaskOperand = Operand.getIf<RegisterMaskOperand>(); + if (MaskOperand == nullptr) + continue; + if (!MaskOperand->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "reaching-def analysis encountered a null register mask"); + } + defineRegisterMask(State, *MaskOperand->Mask, Instruction, Target); + } + + if (!instructionMayStore(Instruction)) + return {}; + const std::unordered_set<FrameIndex> ExplicitStores = storedFrameObjects(Instruction); + for (const MachineOperand &Operand : Instruction.operands()) { + const auto *Frame = Operand.getIf<FrameIndexOperand>(); + if (Frame == nullptr || (!ExplicitStores.empty() && !ExplicitStores.contains(Frame->Index))) + continue; + State.FrameObjects[Frame->Index] = {&Instruction}; + } + return {}; +} + +[[nodiscard]] Expected<DefinitionSet> +registerDefinitions(const ReachingDefState &State, const MachineOperand &Operand, + const TargetDescription &Target, + const std::unordered_map<const MachineInstr *, std::size_t> &Numbers) { + DefinitionSet Result; + auto Units = physicalOperandUnits(Operand, Target); + if (!Units) + return std::unexpected(Units.error()); + for (const PhysicalUnitAccess &Access : *Units) { + const auto Position = State.Registers.find(Access.Unit); + if (Position != State.Registers.end()) + mergeDefinitions(Result, Position->second, Numbers); + } + return Result; +} + +} // namespace + +std::span<const MachineInstr *const> +ReachingDefAnalysis::Result::reachingDefinitions(const MachineOperand &Operand) const noexcept { + const auto Position = OperandDefinitions.find(&Operand); + if (Position == OperandDefinitions.end()) + return {}; + return Position->second; +} + +std::size_t +ReachingDefAnalysis::Result::instructionNumber(const MachineInstr &Instruction) const noexcept { + const auto Position = InstructionNumbers.find(&Instruction); + return Position == InstructionNumbers.end() ? 0 : Position->second; +} + +bool ReachingDefAnalysis::Result::invalidate(const MachineFunction &, + const PreservedAnalyses &Preserved) const noexcept { + return !Preserved.preserves(MachineFunctionAnalysisManager::id<ReachingDefAnalysis>()); +} + +Expected<ReachingDefAnalysis::Result> ReachingDefAnalysis::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &) { + Result Analysis; + std::size_t Number = 0; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Analysis.InstructionNumbers.emplace(Instruction, Number++); + } + } + + std::unordered_map<const MachineBasicBlock *, ReachingDefState> Inputs; + std::unordered_map<const MachineBasicBlock *, ReachingDefState> Outputs; + bool Changed = false; + do { + Changed = false; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + ReachingDefState Input; + for (MachineBasicBlock *Predecessor : Block->predecessors()) { + const auto Position = Outputs.find(Predecessor); + if (Position != Outputs.end()) + mergeState(Input, Position->second, Analysis.InstructionNumbers); + } + + ReachingDefState Output = Input; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + auto Transferred = transferInstruction(Output, *Instruction, Target); + if (!Transferred) + return std::unexpected(Transferred.error()); + } + if (Inputs[Block] != Input || Outputs[Block] != Output) { + Inputs[Block] = std::move(Input); + Outputs[Block] = std::move(Output); + Changed = true; + } + } + } while (Changed); + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + ReachingDefState State = Inputs[Block]; + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + if (Operand.isRegister()) { + const std::optional<Register> Reg = Operand.registerValue(); + if (Operand.isDef() || !Reg || Reg->isNoRegister()) + continue; + if (!Reg->isPhysical()) { + return makeUnexpected(ErrorCode::VerificationError, + "reaching-def analysis requires physical registers"); + } + auto Definitions = + registerDefinitions(State, Operand, Target, Analysis.InstructionNumbers); + if (!Definitions) + return std::unexpected(Definitions.error()); + Analysis.OperandDefinitions.emplace(&Operand, std::move(*Definitions)); + continue; + } + const auto *Frame = Operand.getIf<FrameIndexOperand>(); + if (Frame == nullptr) + continue; + const auto Position = State.FrameObjects.find(Frame->Index); + Analysis.OperandDefinitions.emplace( + &Operand, Position == State.FrameObjects.end() ? DefinitionSet{} : Position->second); + } + auto Transferred = transferInstruction(State, *Instruction, Target); + if (!Transferred) + return std::unexpected(Transferred.error()); + } + } + return Analysis; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegAllocBase.cpp b/backend/lib/CodeGen/RegAllocBase.cpp new file mode 100644 index 0000000..291fc29 --- /dev/null +++ b/backend/lib/CodeGen/RegAllocBase.cpp @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RegAllocBase.h" + +#include "rcc/CodeGen/LiveIntervals.h" +#include "rcc/CodeGen/Spiller.h" + +#include <algorithm> +#include <cstdint> +#include <unordered_set> +#include <utility> + +namespace rcc { + +std::vector<const LiveInterval *> +orderedAllocationIntervals(const LiveIntervalsAnalysis::Result &Intervals, + const MachineFunction &Function) { + std::vector<const LiveInterval *> Result; + for (std::size_t Index = 0; Index < Function.registerInfo().virtualRegisters().size(); ++Index) { + auto Reg = Register::makeVirtual(Index); + if (!Reg) + continue; + const LiveInterval *Interval = Intervals.interval(*Reg); + if (Interval != nullptr && !Interval->MainRange.empty()) + Result.push_back(Interval); + } + std::ranges::sort(Result, [](const LiveInterval *Left, const LiveInterval *Right) { + const LiveSegment &LeftSegment = Left->MainRange.segments().front(); + const LiveSegment &RightSegment = Right->MainRange.segments().front(); + if (LeftSegment.Start != RightSegment.Start) + return LeftSegment.Start < RightSegment.Start; + if (LeftSegment.End != RightSegment.End) + return LeftSegment.End < RightSegment.End; + return Left->Reg.opaqueValue() < Right->Reg.opaqueValue(); + }); + return Result; +} + +Expected<regalloc_detail::AllocationRunResult> RegAllocBase::run(VirtRegMap &Mapping) { + const std::uint64_t InitialFingerprint = regalloc_detail::allocationFingerprint(Function); + const std::size_t InitialFrameObjects = Function.frameInfo().liveObjectCount(); + Mapping = VirtRegMap(Function); + auto Initialized = regalloc_detail::initializeMapping(Function, Mapping); + if (!Initialized) + return std::unexpected(Initialized.error()); + + std::unordered_set<std::uint64_t> SeenFingerprints; + while (true) { + const std::uint64_t Fingerprint = regalloc_detail::allocationFingerprint(Function); + if (!SeenFingerprints.insert(Fingerprint).second) { + return makeUnexpected(ErrorCode::InternalError, + "register allocation spill/rebuild cycle made no progress"); + } + auto Intervals = Analyses.getResult<LiveIntervalsAnalysis>(Function, Target); + if (!Intervals) + return std::unexpected(Intervals.error()); + auto Round = allocateRound(**Intervals, Mapping); + if (!Round) + return std::unexpected(Round.error()); + if (Round->Spilled.empty()) + break; + + std::ranges::sort(Round->Spilled, [](Register Left, Register Right) { + return Left.opaqueValue() < Right.opaqueValue(); + }); + Spiller ValueSpiller(Function, Target, TargetInfo, Mapping, Options.EnableRematerialization, + &Analyses); + for (const Register VirtualReg : Round->Spilled) { + VirtRegMapEntry *Entry = Mapping.entry(VirtualReg); + if (Entry != nullptr) { + Entry->PhysicalReg.reset(); + Entry->State = VirtRegAllocationState::Unassigned; + } + auto NewRegisters = ValueSpiller.spill(VirtualReg); + if (!NewRegisters) + return std::unexpected(NewRegisters.error()); + } + PreservedAnalyses Preserved; + Preserved.preserve<VirtRegMapAnalysis>(); + Analyses.invalidate(Function, Preserved, MachinePassChanges::all()); + } + const std::uint64_t FinalFingerprint = regalloc_detail::allocationFingerprint(Function); + regalloc_detail::AllocationRunResult Result; + Result.Changed = InitialFingerprint != FinalFingerprint; + Result.ChangedInstructions = InitialFingerprint != FinalFingerprint; + Result.ChangedFrame = InitialFrameObjects != Function.frameInfo().liveObjectCount(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegAllocBase.h b/backend/lib/CodeGen/RegAllocBase.h new file mode 100644 index 0000000..10759f1 --- /dev/null +++ b/backend/lib/CodeGen/RegAllocBase.h @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) +#ifndef RCC_LIB_CODEGEN_REGALLOCBASE_H +#define RCC_LIB_CODEGEN_REGALLOCBASE_H + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/LiveIntervals.h" +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" +#include "rcc/CodeGen/VirtRegMap.h" + +#include <vector> + +namespace rcc { + +struct AllocationRoundResult final { + std::vector<Register> Spilled; +}; + +class RegAllocBase { +public: + virtual ~RegAllocBase() = default; + + [[nodiscard]] Expected<regalloc_detail::AllocationRunResult> run(VirtRegMap &Mapping); + +protected: + RegAllocBase(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, + MachineFunctionAnalysisManager &Analyses, RegisterAllocationOptions Options) noexcept + : Function(Function), Target(Target), TargetInfo(TargetInfo), Analyses(Analyses), + Options(Options) {} + + [[nodiscard]] virtual Expected<AllocationRoundResult> + allocateRound(const LiveIntervalsAnalysis::Result &Intervals, VirtRegMap &Mapping) = 0; + + MachineFunction &Function; + const TargetDescription &Target; + const TargetRegisterAllocationInfo &TargetInfo; + MachineFunctionAnalysisManager &Analyses; + RegisterAllocationOptions Options; +}; + +[[nodiscard]] std::vector<const LiveInterval *> +orderedAllocationIntervals(const LiveIntervalsAnalysis::Result &Intervals, + const MachineFunction &Function); + +} // namespace rcc + +#endif // RCC_LIB_CODEGEN_REGALLOCBASE_H diff --git a/backend/lib/CodeGen/RegAllocBasic.cpp b/backend/lib/CodeGen/RegAllocBasic.cpp new file mode 100644 index 0000000..8acfa5b --- /dev/null +++ b/backend/lib/CodeGen/RegAllocBasic.cpp @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RegAllocBasic.h" + +#include "RegAllocBase.h" +#include "rcc/CodeGen/LiveIntervals.h" +#include "rcc/CodeGen/LiveRegMatrix.h" + +#include <algorithm> +#include <memory> +#include <ranges> + +namespace rcc { +namespace { + +class BasicAllocatorEngine final : public RegAllocBase { +public: + BasicAllocatorEngine(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, + MachineFunctionAnalysisManager &Analyses, RegisterAllocationOptions Options) + : RegAllocBase(Function, Target, TargetInfo, Analyses, Options) {} + +private: + [[nodiscard]] Expected<AllocationRoundResult> + allocateRound(const LiveIntervalsAnalysis::Result &Intervals, VirtRegMap &Mapping) override { + LiveRegMatrix Matrix(Function, Target, Intervals); + AllocationRoundResult Result; + std::vector<const LiveInterval *> Worklist = orderedAllocationIntervals(Intervals, Function); + std::ranges::stable_sort(Worklist, [&](const LiveInterval *Left, const LiveInterval *Right) { + const bool LeftSpillable = Mapping.isSpillable(Left->Reg); + const bool RightSpillable = Mapping.isSpillable(Right->Reg); + if (LeftSpillable != RightSpillable) + return !LeftSpillable; + if (Left->SpillWeight != Right->SpillWeight) + return Left->SpillWeight > Right->SpillWeight; + return false; + }); + for (const LiveInterval *Interval : Worklist) { + const Register VirtualReg = Interval->Reg; + if (regalloc_detail::hasTerminatorDefinition(Function, Target, VirtualReg)) { + auto Marked = Mapping.setSpillable(VirtualReg, false); + if (!Marked) + return std::unexpected(Marked.error()); + } + auto Candidates = regalloc_detail::allocationOrder(Function, Target, TargetInfo, VirtualReg); + if (!Candidates) + return std::unexpected(Candidates.error()); + bool Assigned = false; + for (const PhysicalRegister Candidate : *Candidates) { + if (Matrix.checkInterference(VirtualReg, Candidate).Kind != RegisterInterferenceKind::Free) + continue; + auto MatrixAssignment = Matrix.assign(VirtualReg, Candidate); + if (!MatrixAssignment) + continue; + auto MapAssignment = Mapping.assign(VirtualReg, Candidate); + if (!MapAssignment) + return std::unexpected(MapAssignment.error()); + Assigned = true; + break; + } + if (!Assigned) { + if (!Mapping.isSpillable(VirtualReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "BasicAllocator cannot assign a non-spillable virtual register"); + } + Result.Spilled.push_back(VirtualReg); + } + } + return Result; + } +}; + +[[nodiscard]] const MachinePassContract &basicContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "BasicAllocator", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, + MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {MachineFunctionProperty::IsSSA}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<LiveIntervalsAnalysis>(), + MachineFunctionAnalysisManager::id<VirtRegMapAnalysis>()}, + .VerificationStage = std::nullopt, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + return Value; + }(); + return Contract; +} + +} // namespace + +const MachinePassContract &BasicAllocatorPass::contract() const noexcept { + return basicContract(); +} + +Expected<MachinePassResult> BasicAllocatorPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + if (Info == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "BasicAllocator requires target behavior"); + const std::vector<PhysicalRegister> FunctionReserved = + regalloc_detail::functionReservedRegisters(Function, Target, *Info); + auto Reserved = regalloc_detail::reserveFunctionRegisters(Function, FunctionReserved); + if (!Reserved) + return std::unexpected(Reserved.error()); + BasicAllocatorEngine Allocator(Function, Target, *Info, Analyses, Options); + auto Mapping = Analyses.getResult<VirtRegMapAnalysis>(Function, Target); + if (!Mapping) + return std::unexpected(Mapping.error()); + auto Allocated = Allocator.run(**Mapping); + if (!Allocated) + return std::unexpected(Allocated.error()); + Allocated->Changed |= *Reserved; + MachinePassResult Result = regalloc_detail::passResult(*Allocated); + Result.Preserved.preserve<VirtRegMapAnalysis>(); + return Result; +} + +std::unique_ptr<MachineFunctionPass> +createBasicAllocatorPass(const TargetRegisterAllocationInfo &Info, + RegisterAllocationOptions Options) { + return std::make_unique<BasicAllocatorPass>(Info, Options); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegAllocBasic.h b/backend/lib/CodeGen/RegAllocBasic.h new file mode 100644 index 0000000..f88c977 --- /dev/null +++ b/backend/lib/CodeGen/RegAllocBasic.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) +#ifndef RCC_LIB_CODEGEN_REGALLOCBASIC_H +#define RCC_LIB_CODEGEN_REGALLOCBASIC_H + +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" + +namespace rcc { + +class BasicAllocatorPass final : public MachineFunctionPass { +public: + explicit BasicAllocatorPass(const TargetRegisterAllocationInfo &Info, + RegisterAllocationOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected<MachinePassResult> run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) override; + +private: + const TargetRegisterAllocationInfo *Info; + RegisterAllocationOptions Options; +}; + +} // namespace rcc + +#endif // RCC_LIB_CODEGEN_REGALLOCBASIC_H diff --git a/backend/lib/CodeGen/RegAllocFast.cpp b/backend/lib/CodeGen/RegAllocFast.cpp new file mode 100644 index 0000000..eca9494 --- /dev/null +++ b/backend/lib/CodeGen/RegAllocFast.cpp @@ -0,0 +1,589 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegAllocFast.h" + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/LiveRangeEdit.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <algorithm> +#include <cstddef> +#include <map> +#include <optional> +#include <ranges> +#include <span> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +class FastAllocatorEngine final { +public: + FastAllocatorEngine(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, + const LiveVariablesAnalysis::Result &Liveness) + : Function(Function), Target(Target), TargetInfo(TargetInfo), Liveness(Liveness), + Mapping(Function), Edit(Function, Target, Mapping) {} + + [[nodiscard]] Expected<regalloc_detail::AllocationRunResult> run() { + const std::uint64_t InitialFingerprint = regalloc_detail::allocationFingerprint(Function); + const std::size_t InitialFrameObjects = Function.frameInfo().liveObjectCount(); + auto Initialized = regalloc_detail::initializeMapping(Function, Mapping); + if (!Initialized) + return std::unexpected(Initialized.error()); + + bool IsEntry = true; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + auto Allocated = allocateBlock(*Block, IsEntry); + if (!Allocated) + return std::unexpected(Allocated.error()); + IsEntry = false; + } + + Function.registerInfo().clearLiveInVirtualRegisters(); + for (std::size_t Index = 0; Index < Function.registerInfo().virtualRegisters().size(); + ++Index) { + auto Reg = regalloc_detail::virtualRegisterAt(Index); + if (!Reg) + return std::unexpected(Reg.error()); + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) + continue; + auto Erased = Function.registerInfo().eraseVirtualRegister(*Reg); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + + regalloc_detail::AllocationRunResult Result; + const std::uint64_t FinalFingerprint = regalloc_detail::allocationFingerprint(Function); + Result.Changed = InitialFingerprint != FinalFingerprint; + Result.ChangedInstructions = Result.Changed; + Result.ChangedFrame = InitialFrameObjects != Function.frameInfo().liveObjectCount(); + return Result; + } + +private: + struct Location final { + PhysicalRegister PhysicalReg; + bool Dirty = false; + }; + + struct InstructionRegister final { + Register VirtualReg; + std::vector<std::size_t> Operands; + bool HasUse = false; + bool HasDef = false; + std::size_t FirstOperand = 0; + }; + + struct BlockLiveInState final { + PhysicalRegister Reg; + std::size_t RemainingUses = 0; + }; + + [[nodiscard]] bool liveOut(const MachineBasicBlock &Block, Register Reg) const noexcept { + return Liveness.isLiveOut(Block, Reg); + } + + [[nodiscard]] bool neededLater(const MachineBasicBlock &Block, Register Reg) const noexcept { + const auto Position = RemainingUses.find(Reg); + return (Position != RemainingUses.end() && Position->second != 0) || liveOut(Block, Reg); + } + + [[nodiscard]] std::optional<Register> ownerOf(PhysicalRegister Reg) const noexcept { + for (const auto &[VirtualReg, Value] : Locations) { + if (Target.registers().regsOverlap(Value.PhysicalReg, Reg)) + return VirtualReg; + } + return std::nullopt; + } + + [[nodiscard]] bool physicalLiveOut(const MachineBasicBlock &Block, + PhysicalRegister Reg) const noexcept { + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(Reg); + if (Description == nullptr) + return true; + return std::ranges::any_of(Description->Units, [&](const RegisterUnitLane &Unit) { + return Liveness.isUnitLiveOut(Block, Unit.Unit, Unit.Lanes); + }); + } + + [[nodiscard]] bool blockedByBlockLiveIn(PhysicalRegister Reg) const noexcept { + return std::ranges::any_of(BlockLiveIns, [&](const BlockLiveInState &LiveIn) { + return Target.registers().regsOverlap(LiveIn.Reg, Reg); + }); + } + + void initializeBlockLiveIns(const MachineBasicBlock &Block) { + BlockLiveIns.clear(); + for (const MachineBasicBlockLiveIn &LiveIn : Block.liveIns()) { + if (LiveIn.Lanes.empty() || + std::ranges::any_of(BlockLiveIns, [&](const BlockLiveInState &Existing) { + return Existing.Reg == LiveIn.PhysReg; + })) + continue; + BlockLiveIns.push_back({LiveIn.PhysReg, 0}); + } + + for (const MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isPhysical() || !Operand.isUse()) + continue; + const auto PhysicalValue = Reg->physicalRegister(); + if (!PhysicalValue) + continue; + const PhysicalRegister Physical = PhysicalValue.value(); + for (BlockLiveInState &LiveIn : BlockLiveIns) { + if (Target.registers().regsOverlap(LiveIn.Reg, Physical)) + ++LiveIn.RemainingUses; + } + } + } + } + + void updateBlockLiveIns(const MachineBasicBlock &Block, const MachineInstr &Instruction) { + std::vector<PhysicalRegister> Defined; + std::vector<const RegisterMask *> Masks; + for (const MachineOperand &Operand : Instruction.operands()) { + if (const Register *Reg = Operand.getIf<Register>(); Reg != nullptr && Reg->isPhysical()) { + const auto PhysicalValue = Reg->physicalRegister(); + if (Operand.isDef() && PhysicalValue) + Defined.push_back(PhysicalValue.value()); + } + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); Mask != nullptr && Mask->Mask) + Masks.push_back(Mask->Mask.get()); + } + + for (auto Position = BlockLiveIns.begin(); Position != BlockLiveIns.end();) { + BlockLiveInState &LiveIn = *Position; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isPhysical() || !Operand.isUse()) + continue; + const auto PhysicalValue = Reg->physicalRegister(); + if (!PhysicalValue || !Target.registers().regsOverlap(LiveIn.Reg, PhysicalValue.value())) + continue; + if (LiveIn.RemainingUses != 0) + --LiveIn.RemainingUses; + } + + const bool DefinedHere = std::ranges::any_of(Defined, [&](PhysicalRegister Reg) { + return Target.registers().regsOverlap(LiveIn.Reg, Reg); + }); + const bool MaskClobbered = std::ranges::any_of( + Masks, [&](const RegisterMask *Mask) { return Mask->clobbers(LiveIn.Reg); }); + const bool KeepForOutput = physicalLiveOut(Block, LiveIn.Reg); + if (!KeepForOutput && (DefinedHere || MaskClobbered || LiveIn.RemainingUses == 0)) + Position = BlockLiveIns.erase(Position); + else + ++Position; + } + } + + [[nodiscard]] bool overlapsAny(PhysicalRegister Candidate, + std::span<const PhysicalRegister> Values) const noexcept { + return std::ranges::any_of(Values, [&](PhysicalRegister Value) { + return Target.registers().regsOverlap(Candidate, Value); + }); + } + + [[nodiscard]] Expected<FrameIndex> spillSlot(Register Reg) { + auto ClassID = regalloc_detail::registerClassFor(Function, Reg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Validated = TargetInfo.validateSpillReload(*ClassID, Target); + if (!Validated) { + return makeUnexpected( + ErrorCode::InternalError, + "target declares an allocatable class without FastAllocator spill hooks"); + } + return Edit.getOrCreateSpillSlot(Reg); + } + + [[nodiscard]] Expected<void> emitSpillBefore(Register Reg, PhysicalRegister PhysicalReg, + MachineInstr *Before, MachineBasicBlock &Block) { + auto ClassID = regalloc_detail::registerClassFor(Function, Reg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Slot = spillSlot(Reg); + if (!Slot) + return std::unexpected(Slot.error()); + MachineIRBuilder Builder(Function, Target); + Expected<void> Positioned = + Before != nullptr ? Builder.setInsertPointBefore(*Before) : Builder.setInsertPoint(Block); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Spill = TargetInfo.emitSpill(Builder, Register::fromPhysical(PhysicalReg), *ClassID, *Slot, + LaneMask::all()); + if (!Spill) + return std::unexpected(Spill.error()); + VirtRegMapEntry *Entry = Mapping.entry(Reg); + if (Entry != nullptr) + ++Entry->Spill.SpillCount; + return {}; + } + + [[nodiscard]] Expected<void> evict(Register Reg, MachineInstr &Before, MachineBasicBlock &Block, + bool Preserve) { + const auto Position = Locations.find(Reg); + if (Position == Locations.end()) + return {}; + if (Position->second.Dirty && Preserve) { + auto Stored = emitSpillBefore(Reg, Position->second.PhysicalReg, &Before, Block); + if (!Stored) + return std::unexpected(Stored.error()); + } + Locations.erase(Position); + return {}; + } + + [[nodiscard]] Expected<PhysicalRegister> + ensureLocation(Register Reg, bool NeedsValue, MachineInstr &Instruction, MachineBasicBlock &Block, + std::span<const PhysicalRegister> Fixed, std::span<const PhysicalRegister> Locked, + const std::unordered_set<Register> &CurrentRegisters) { + const auto Existing = Locations.find(Reg); + if (Existing != Locations.end() && !overlapsAny(Existing->second.PhysicalReg, Fixed) && + !overlapsAny(Existing->second.PhysicalReg, Locked)) + return Existing->second.PhysicalReg; + if (Existing != Locations.end()) { + auto Evicted = evict(Reg, Instruction, Block, NeedsValue || neededLater(Block, Reg)); + if (!Evicted) + return std::unexpected(Evicted.error()); + } + + auto Candidates = regalloc_detail::allocationOrder(Function, Target, TargetInfo, Reg); + if (!Candidates) + return std::unexpected(Candidates.error()); + for (const PhysicalRegister Candidate : *Candidates) { + if (overlapsAny(Candidate, Fixed) || overlapsAny(Candidate, Locked)) + continue; + if (blockedByBlockLiveIn(Candidate)) + continue; + if (const auto Owner = ownerOf(Candidate)) { + if (CurrentRegisters.contains(*Owner)) + continue; + auto Evicted = evict(*Owner, Instruction, Block, neededLater(Block, *Owner)); + if (!Evicted) + return std::unexpected(Evicted.error()); + } + if (ownerOf(Candidate)) + continue; + + if (NeedsValue) { + auto ClassID = regalloc_detail::registerClassFor(Function, Reg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Slot = spillSlot(Reg); + if (!Slot) + return std::unexpected(Slot.error()); + MachineIRBuilder Builder(Function, Target); + auto Positioned = Builder.setInsertPointBefore(Instruction); + if (!Positioned) + return std::unexpected(Positioned.error()); + auto Reload = TargetInfo.emitReload(Builder, Register::fromPhysical(Candidate), *ClassID, + *Slot, LaneMask::all()); + if (!Reload) + return std::unexpected(Reload.error()); + VirtRegMapEntry *Entry = Mapping.entry(Reg); + if (Entry != nullptr) + ++Entry->Spill.ReloadCount; + } + Locations.emplace(Reg, Location{Candidate, false}); + return Candidate; + } + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "FastAllocator cannot satisfy simultaneous register constraints"); + } + + [[nodiscard]] Expected<void> installEntryLiveIns(MachineBasicBlock &Block) { + if (&Block != Function.firstBlock()) + return {}; + for (const PhysicalRegisterLiveIn &LiveIn : Function.registerInfo().liveIns()) { + if (!LiveIn.InitialVirtualRegister) + continue; + const Register Reg = *LiveIn.InitialVirtualRegister; + if (!Function.registerInfo().isKnownVirtualRegister(Reg)) + continue; + auto Candidates = regalloc_detail::allocationOrder(Function, Target, TargetInfo, Reg); + if (!Candidates) + return std::unexpected(Candidates.error()); + if (std::ranges::find(*Candidates, LiveIn.PhysReg) == Candidates->end()) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "function live-in is outside its allocation order"); + } + if (ownerOf(LiveIn.PhysReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "overlapping function live-ins require the same physical register"); + } + Locations.emplace(Reg, Location{LiveIn.PhysReg, true}); + } + return {}; + } + + [[nodiscard]] Expected<void> allocateInstruction(MachineInstr &Instruction, + MachineBasicBlock &Block) { + std::vector<PhysicalRegister> Fixed; + std::vector<const RegisterMask *> Masks; + std::map<std::uint32_t, InstructionRegister> Registers; + std::vector<std::size_t> DebugOperands; + for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { + MachineOperand *Operand = Instruction.operandAt(Index); + if (const auto *Mask = Operand->getIf<RegisterMaskOperand>(); Mask != nullptr && Mask->Mask) + Masks.push_back(Mask->Mask.get()); + const Register *Reg = Operand->getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + if (Reg->isPhysical()) { + const auto PhysicalValue = Reg->physicalRegister(); + if (!PhysicalValue) { + return makeUnexpected(ErrorCode::InternalError, + "physical register operand lacks a physical register value"); + } + Fixed.push_back(PhysicalValue.value()); + continue; + } + if (Operand->isDebugUse()) { + DebugOperands.push_back(Index); + continue; + } + auto [Position, Inserted] = Registers.try_emplace( + Reg->opaqueValue(), InstructionRegister{*Reg, {}, false, false, Index}); + if (!Inserted) + Position->second.FirstOperand = std::min(Position->second.FirstOperand, Index); + Position->second.Operands.push_back(Index); + Position->second.HasUse |= Operand->isUse(); + Position->second.HasDef |= Operand->isDef(); + } + std::ranges::sort(Fixed, {}, &PhysicalRegister::number); + Fixed.erase(std::unique(Fixed.begin(), Fixed.end()), Fixed.end()); + + std::unordered_set<Register> CurrentRegisters; + std::unordered_set<Register> DefinedRegisters; + for (const auto &[Key, Value] : Registers) { + (void)Key; + CurrentRegisters.insert(Value.VirtualReg); + if (Value.HasDef) + DefinedRegisters.insert(Value.VirtualReg); + } + for (const auto &[Reg, Value] : + std::vector<std::pair<Register, Location>>(Locations.begin(), Locations.end())) { + bool Clobbered = false; + for (const RegisterMask *Mask : Masks) + Clobbered |= Mask->clobbers(Value.PhysicalReg); + if (Clobbered) { + auto Evicted = evict(Reg, Instruction, Block, neededLater(Block, Reg)); + if (!Evicted) + return std::unexpected(Evicted.error()); + } + } + + std::vector<InstructionRegister *> Ordered; + Ordered.reserve(Registers.size()); + for (auto &[Key, Value] : Registers) { + (void)Key; + Ordered.push_back(&Value); + } + std::ranges::sort(Ordered, + [](const InstructionRegister *Left, const InstructionRegister *Right) { + if (Left->HasUse != Right->HasUse) + return Left->HasUse > Right->HasUse; + return Left->FirstOperand < Right->FirstOperand; + }); + + std::vector<PhysicalRegister> Locked; + for (InstructionRegister *Value : Ordered) { + auto Physical = ensureLocation(Value->VirtualReg, Value->HasUse, Instruction, Block, Fixed, + Locked, CurrentRegisters); + if (!Physical) + return std::unexpected(Physical.error()); + Locked.push_back(*Physical); + for (const std::size_t Index : Value->Operands) { + MachineOperand *Operand = Instruction.operandAt(Index); + PhysicalRegister Effective = *Physical; + if (Operand->subReg().value() != 0) { + const auto Sub = Target.registers().subRegister(*Physical, Operand->subReg()); + if (!Sub) + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "FastAllocator assignment lacks a required subregister"); + Effective = *Sub; + } + auto Set = Operand->setRegister(Register::fromPhysical(Effective)); + if (!Set) + return std::unexpected(Set.error()); + if (Operand->subReg().value() != 0) { + auto Cleared = Operand->setSubReg(SubRegIndex{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + } + if (Value->HasDef) + Locations[Value->VirtualReg].Dirty = true; + } + + for (const std::size_t Index : DebugOperands) { + MachineOperand *Operand = Instruction.operandAt(Index); + const Register *Reg = Operand->getIf<Register>(); + const auto Position = Reg == nullptr ? Locations.end() : Locations.find(*Reg); + Register Replacement; + if (Position != Locations.end()) + Replacement = Register::fromPhysical(Position->second.PhysicalReg); + auto Set = Operand->setRegister(Replacement); + if (!Set) + return std::unexpected(Set.error()); + if (Operand->subReg().value() != 0) { + auto Cleared = Operand->setSubReg(SubRegIndex{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + if (Replacement.isNoRegister()) { + RegisterOperandFlags Flags = Operand->registerFlags(); + Flags.set(RegisterOperandFlag::Undef); + auto Updated = Operand->setRegisterFlags(Flags); + if (!Updated) + return std::unexpected(Updated.error()); + } + } + + for (InstructionRegister *Value : Ordered) { + if (Value->HasUse) { + auto Position = RemainingUses.find(Value->VirtualReg); + if (Position != RemainingUses.end() && Position->second != 0) + --Position->second; + } + if (!neededLater(Block, Value->VirtualReg)) + Locations.erase(Value->VirtualReg); + } + for (const RegisterMask *Mask : Masks) { + for (auto Position = Locations.begin(); Position != Locations.end();) { + if (Mask->clobbers(Position->second.PhysicalReg) && + !DefinedRegisters.contains(Position->first)) + Position = Locations.erase(Position); + else + ++Position; + } + } + return {}; + } + + [[nodiscard]] Expected<void> flushLiveOuts(MachineBasicBlock &Block) { + MachineInstr *Terminator = regalloc_detail::firstTerminator(Block, Target); + for (const auto &[Reg, Value] : + std::vector<std::pair<Register, Location>>(Locations.begin(), Locations.end())) { + if (!Value.Dirty || !liveOut(Block, Reg)) + continue; + if (Terminator != nullptr) { + for (const MachineOperand &Operand : Terminator->operands()) { + const Register *OperandReg = Operand.getIf<Register>(); + if (OperandReg != nullptr && *OperandReg == Reg && Operand.isDef()) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "FastAllocator cannot spill a terminator-defined live-out"); + } + } + } + auto Stored = emitSpillBefore(Reg, Value.PhysicalReg, Terminator, Block); + if (!Stored) + return std::unexpected(Stored.error()); + } + Locations.clear(); + return {}; + } + + [[nodiscard]] Expected<void> allocateBlock(MachineBasicBlock &Block, bool IsEntry) { + (void)IsEntry; + Locations.clear(); + RemainingUses.clear(); + initializeBlockLiveIns(Block); + std::vector<MachineInstr *> Instructions; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Instructions.push_back(Instruction); + std::unordered_set<Register> UsesInInstruction; + for (const MachineOperand &Operand : Instruction->operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && Reg->isVirtual() && Operand.isUse() && !Operand.isDebugUse()) + UsesInInstruction.insert(*Reg); + } + for (const Register Reg : UsesInInstruction) + ++RemainingUses[Reg]; + } + auto EntryValues = installEntryLiveIns(Block); + if (!EntryValues) + return std::unexpected(EntryValues.error()); + for (MachineInstr *Instruction : Instructions) { + auto Allocated = allocateInstruction(*Instruction, Block); + if (!Allocated) + return std::unexpected(Allocated.error()); + updateBlockLiveIns(Block, *Instruction); + } + auto Flushed = flushLiveOuts(Block); + BlockLiveIns.clear(); + return Flushed; + } + + MachineFunction &Function; + const TargetDescription &Target; + const TargetRegisterAllocationInfo &TargetInfo; + const LiveVariablesAnalysis::Result &Liveness; + VirtRegMap Mapping; + LiveRangeEdit Edit; + std::unordered_map<Register, Location> Locations; + std::unordered_map<Register, std::size_t> RemainingUses; + std::vector<BlockLiveInState> BlockLiveIns; +}; + +} // namespace + +const MachinePassContract &FastAllocatorPass::contract() const noexcept { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "FastAllocator", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs, + MachineFunctionProperty::TiedOpsRewritten, + MachineFunctionProperty::TracksLiveness}, + .Sets = {MachineFunctionProperty::NoVRegs}, + .Clears = {}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<LiveVariablesAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + return Value; + }(); + return Contract; +} + +Expected<MachinePassResult> FastAllocatorPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + if (Info == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "FastAllocator requires target behavior"); + const std::vector<PhysicalRegister> FunctionReserved = + regalloc_detail::functionReservedRegisters(Function, Target, *Info); + auto Reserved = regalloc_detail::reserveFunctionRegisters(Function, FunctionReserved); + if (!Reserved) + return std::unexpected(Reserved.error()); + auto Liveness = Analyses.getResult<LiveVariablesAnalysis>(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + FastAllocatorEngine Allocator(Function, Target, *Info, **Liveness); + auto Allocated = Allocator.run(); + if (!Allocated) + return std::unexpected(Allocated.error()); + Allocated->Changed |= *Reserved; + return regalloc_detail::passResult(*Allocated); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegAllocGreedy.cpp b/backend/lib/CodeGen/RegAllocGreedy.cpp new file mode 100644 index 0000000..c298c78 --- /dev/null +++ b/backend/lib/CodeGen/RegAllocGreedy.cpp @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RegAllocGreedy.h" + +#include "RegAllocBase.h" +#include "rcc/CodeGen/LiveIntervals.h" +#include "rcc/CodeGen/LiveRegMatrix.h" + +#include <algorithm> +#include <cstdint> +#include <limits> +#include <memory> +#include <optional> +#include <ranges> +#include <unordered_set> +#include <vector> + +namespace rcc { +namespace { + +struct EvictionCandidate final { + PhysicalRegister PhysicalReg; + std::vector<Register> Victims; + std::uint64_t Cost = 0; +}; + +[[nodiscard]] std::uint64_t saturatedAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + if (Right > std::numeric_limits<std::uint64_t>::max() - Left) + return std::numeric_limits<std::uint64_t>::max(); + return Left + Right; +} + +[[nodiscard]] std::uint64_t normalizedSpillCost(const LiveInterval &Interval) noexcept { + std::uint64_t Length = 0; + for (const LiveSegment &Segment : Interval.MainRange.segments()) + Length = saturatedAdd(Length, Segment.End.value() - Segment.Start.value()); + Length = std::max<std::uint64_t>(1, Length); + constexpr std::uint64_t Scale = std::uint64_t{1} << 20U; + const std::uint64_t Weight = std::max<std::uint64_t>(1, Interval.SpillWeight); + const std::uint64_t ScaledWeight = Weight > std::numeric_limits<std::uint64_t>::max() / Scale + ? std::numeric_limits<std::uint64_t>::max() + : Weight * Scale; + return std::max<std::uint64_t>(1, ScaledWeight / Length); +} + +[[nodiscard]] bool isBetterEviction(const EvictionCandidate &Left, + const EvictionCandidate &Right) noexcept { + if (Left.Cost != Right.Cost) + return Left.Cost < Right.Cost; + if (Left.Victims.size() != Right.Victims.size()) + return Left.Victims.size() < Right.Victims.size(); + return Left.PhysicalReg.number() < Right.PhysicalReg.number(); +} + +class GreedyAllocatorEngine final : public RegAllocBase { +public: + GreedyAllocatorEngine(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, + MachineFunctionAnalysisManager &Analyses, RegisterAllocationOptions Options) + : RegAllocBase(Function, Target, TargetInfo, Analyses, Options) {} + +private: + [[nodiscard]] Expected<std::optional<EvictionCandidate>> + evictionCandidate(Register VirtualReg, PhysicalRegister Candidate, + const LiveIntervalsAnalysis::Result &Intervals, const LiveRegMatrix &Matrix, + const VirtRegMap &Mapping) const { + const LiveInterval *Current = Intervals.interval(VirtualReg); + if (Current == nullptr) + return std::optional<EvictionCandidate>{}; + + EvictionCandidate Result{.PhysicalReg = Candidate, .Victims = {}, .Cost = 0}; + for (const RegisterInterference &Interference : + Matrix.collectInterferences(VirtualReg, Candidate)) { + if (Interference.Kind != RegisterInterferenceKind::VirtualRegister || + !Interference.VirtualReg) { + return std::optional<EvictionCandidate>{}; + } + const Register Victim = *Interference.VirtualReg; + if (std::ranges::find(Result.Victims, Victim) != Result.Victims.end()) + continue; + if (!Mapping.isSpillable(Victim)) + return std::optional<EvictionCandidate>{}; + const LiveInterval *VictimInterval = Intervals.interval(Victim); + if (VictimInterval == nullptr) + return std::optional<EvictionCandidate>{}; + Result.Victims.push_back(Victim); + Result.Cost = std::max(Result.Cost, normalizedSpillCost(*VictimInterval)); + } + if (Result.Victims.empty()) + return std::optional<EvictionCandidate>{}; + + const std::uint64_t CurrentCost = normalizedSpillCost(*Current); + if (Mapping.isSpillable(VirtualReg) && Result.Cost >= CurrentCost) + return std::optional<EvictionCandidate>{}; + std::ranges::sort(Result.Victims, [](Register Left, Register Right) { + return Left.opaqueValue() < Right.opaqueValue(); + }); + return std::optional<EvictionCandidate>{std::move(Result)}; + } + + [[nodiscard]] Expected<AllocationRoundResult> + allocateRound(const LiveIntervalsAnalysis::Result &Intervals, VirtRegMap &Mapping) override { + LiveRegMatrix Matrix(Function, Target, Intervals); + AllocationRoundResult Result; + std::unordered_set<Register> Spilled; + std::vector<const LiveInterval *> Worklist = orderedAllocationIntervals(Intervals, Function); + std::ranges::stable_sort(Worklist, [&](const LiveInterval *Left, const LiveInterval *Right) { + const bool LeftSpillable = Mapping.isSpillable(Left->Reg); + const bool RightSpillable = Mapping.isSpillable(Right->Reg); + if (LeftSpillable != RightSpillable) + return !LeftSpillable; + if (Left->SpillWeight != Right->SpillWeight) + return Left->SpillWeight > Right->SpillWeight; + return false; + }); + + for (const LiveInterval *Interval : Worklist) { + const Register VirtualReg = Interval->Reg; + if (regalloc_detail::hasTerminatorDefinition(Function, Target, VirtualReg)) { + auto Marked = Mapping.setSpillable(VirtualReg, false); + if (!Marked) + return std::unexpected(Marked.error()); + } + auto Candidates = regalloc_detail::allocationOrder(Function, Target, TargetInfo, VirtualReg); + if (!Candidates) + return std::unexpected(Candidates.error()); + + std::optional<PhysicalRegister> Assignment; + for (const PhysicalRegister Candidate : *Candidates) { + if (Matrix.checkInterference(VirtualReg, Candidate).Kind == + RegisterInterferenceKind::Free) { + Assignment = Candidate; + break; + } + } + + std::optional<EvictionCandidate> BestEviction; + if (!Assignment) { + for (const PhysicalRegister Candidate : *Candidates) { + auto CandidateEviction = + evictionCandidate(VirtualReg, Candidate, Intervals, Matrix, Mapping); + if (!CandidateEviction) + return std::unexpected(CandidateEviction.error()); + if (!*CandidateEviction) + continue; + if (!BestEviction || isBetterEviction(**CandidateEviction, *BestEviction)) + BestEviction = std::move(**CandidateEviction); + } + } + + if (BestEviction) { + for (const Register Victim : BestEviction->Victims) { + auto Unassigned = Matrix.unassign(Victim); + if (!Unassigned) + return std::unexpected(Unassigned.error()); + VirtRegMapEntry *Entry = Mapping.entry(Victim); + if (Entry == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "greedy eviction found an unknown virtual register"); + Entry->PhysicalReg.reset(); + Entry->State = VirtRegAllocationState::Unassigned; + if (Spilled.insert(Victim).second) + Result.Spilled.push_back(Victim); + } + Assignment = BestEviction->PhysicalReg; + } + + if (!Assignment) { + if (!Mapping.isSpillable(VirtualReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "greedy allocator cannot assign a non-spillable virtual register"); + } + if (Spilled.insert(VirtualReg).second) + Result.Spilled.push_back(VirtualReg); + continue; + } + + auto MatrixAssignment = Matrix.assign(VirtualReg, *Assignment); + if (!MatrixAssignment) + return std::unexpected(MatrixAssignment.error()); + auto MapAssignment = Mapping.assign(VirtualReg, *Assignment); + if (!MapAssignment) + return std::unexpected(MapAssignment.error()); + } + return Result; + } +}; + +[[nodiscard]] const MachinePassContract &greedyContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "greedy", + .Requires = {MachineFunctionProperty::NoPHIs, MachineFunctionProperty::TracksLiveness}, + .Sets = {}, + .Clears = {MachineFunctionProperty::IsSSA}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<LiveIntervalsAnalysis>(), + MachineFunctionAnalysisManager::id<VirtRegMapAnalysis>()}, + .VerificationStage = std::nullopt, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + return Value; + }(); + return Contract; +} + +} // namespace + +const MachinePassContract &GreedyAllocatorPass::contract() const noexcept { + return greedyContract(); +} + +Expected<MachinePassResult> GreedyAllocatorPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + if (Info == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "greedy allocator requires target behavior"); + const std::vector<PhysicalRegister> FunctionReserved = + regalloc_detail::functionReservedRegisters(Function, Target, *Info); + auto Reserved = regalloc_detail::reserveFunctionRegisters(Function, FunctionReserved); + if (!Reserved) + return std::unexpected(Reserved.error()); + auto Mapping = Analyses.getResult<VirtRegMapAnalysis>(Function, Target); + if (!Mapping) + return std::unexpected(Mapping.error()); + GreedyAllocatorEngine Allocator(Function, Target, *Info, Analyses, Options); + auto Allocated = Allocator.run(**Mapping); + if (!Allocated) + return std::unexpected(Allocated.error()); + Allocated->Changed |= *Reserved; + MachinePassResult Result = regalloc_detail::passResult(*Allocated); + Result.Preserved.preserve<VirtRegMapAnalysis>(); + return Result; +} + +std::unique_ptr<MachineFunctionPass> +createGreedyAllocatorPass(const TargetRegisterAllocationInfo &Info, + RegisterAllocationOptions Options) { + return std::make_unique<GreedyAllocatorPass>(Info, Options); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegAllocGreedy.h b/backend/lib/CodeGen/RegAllocGreedy.h new file mode 100644 index 0000000..1f25a5c --- /dev/null +++ b/backend/lib/CodeGen/RegAllocGreedy.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) +#ifndef RCC_LIB_CODEGEN_REGALLOCGREEDY_H +#define RCC_LIB_CODEGEN_REGALLOCGREEDY_H + +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" + +namespace rcc { + +class GreedyAllocatorPass final : public MachineFunctionPass { +public: + explicit GreedyAllocatorPass(const TargetRegisterAllocationInfo &Info, + RegisterAllocationOptions Options = {}) noexcept + : Info(&Info), Options(Options) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected<MachinePassResult> run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) override; + +private: + const TargetRegisterAllocationInfo *Info; + RegisterAllocationOptions Options; +}; + +} // namespace rcc + +#endif // RCC_LIB_CODEGEN_REGALLOCGREEDY_H diff --git a/backend/lib/CodeGen/RegAllocInternal.h b/backend/lib/CodeGen/RegAllocInternal.h new file mode 100644 index 0000000..8d7af5a --- /dev/null +++ b/backend/lib/CodeGen/RegAllocInternal.h @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// The private header has no stable include-root-relative guard name. +// NOLINTNEXTLINE(llvm-header-guard) +#ifndef RCC_LIB_CODEGEN_REGALLOCINTERNAL_H +#define RCC_LIB_CODEGEN_REGALLOCINTERNAL_H + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/LiveIntervals.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegAllocCommon.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterAllocationInfo.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/VirtRegMap.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <ranges> +#include <span> +#include <vector> + +namespace rcc::regalloc_detail { + +struct AllocationRunResult final { + bool Changed = false; + bool ChangedInstructions = false; + bool ChangedFrame = false; +}; + +[[nodiscard]] inline Expected<Register> virtualRegisterAt(std::size_t Index) { + auto Reg = Register::makeVirtual(Index); + if (!Reg) + return std::unexpected(Reg.error()); + return *Reg; +} + +[[nodiscard]] inline Expected<RegisterClassID> registerClassFor(const MachineFunction &Function, + Register VirtualReg) { + auto Info = Function.registerInfo().virtualRegisterInfo(VirtualReg); + if (!Info) + return std::unexpected(Info.error()); + const auto ClassID = Info->get().RegClass; + if (!ClassID) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "virtual register has no register class after instruction selection"); + } + return ClassID.value(); +} + +[[nodiscard]] inline bool isUnavailablePhysicalRegister(const MachineFunction &Function, + const TargetRegisterInfo &Registers, + PhysicalRegister Reg) noexcept { + const PhysicalRegisterDesc *Description = Registers.physicalRegister(Reg); + return Description == nullptr || Description->IsConstant || Description->IsArtificial || + Description->IsReserved || Registers.isReservedRegister(Reg) || + Function.registerInfo().isReserved(Reg); +} + +[[nodiscard]] inline bool candidateSupportsSubRegisters(const MachineFunction &Function, + const TargetRegisterInfo &Registers, + Register VirtualReg, + PhysicalRegister Candidate) noexcept { + for (const MachineOperand &Operand : + const_cast<MachineRegisterInfo &>(Function.registerInfo()).regOperands(VirtualReg)) { + if (Operand.subReg().value() != 0 && !Registers.subRegister(Candidate, Operand.subReg())) + return false; + } + return true; +} + +[[nodiscard]] inline Expected<std::vector<PhysicalRegister>> +allocationOrder(const MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, Register VirtualReg) { + auto ClassID = registerClassFor(Function, VirtualReg); + if (!ClassID) + return std::unexpected(ClassID.error()); + const RegisterClassDesc *Class = Target.registers().registerClass(*ClassID); + if (Class == nullptr || !Class->Allocatable || Class->AllocationOrder.empty()) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "virtual register class has no allocatable physical registers"); + } + + auto Info = Function.registerInfo().virtualRegisterInfo(VirtualReg); + if (!Info) + return std::unexpected(Info.error()); + std::vector<PhysicalRegister> Result; + Result.reserve(Class->AllocationOrder.size()); + for (const PhysicalRegister Hint : Info->get().AllocationHints) { + if (std::ranges::find(Class->AllocationOrder, Hint) != Class->AllocationOrder.end() && + std::ranges::find(Result, Hint) == Result.end()) + Result.push_back(Hint); + } + for (const PhysicalRegister Candidate : Class->AllocationOrder) { + if (std::ranges::find(Result, Candidate) == Result.end()) + Result.push_back(Candidate); + } + std::erase_if(Result, [&](PhysicalRegister Candidate) { + return isUnavailablePhysicalRegister(Function, Target.registers(), Candidate) || + TargetInfo.isReservedForFunction(Function, Candidate, Target) || + !candidateSupportsSubRegisters(Function, Target.registers(), VirtualReg, Candidate); + }); + return Result; +} + +[[nodiscard]] inline std::vector<PhysicalRegister> +functionReservedRegisters(const MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo) { + std::vector<PhysicalRegister> Roots; + for (const PhysicalRegisterDesc &Description : Target.registers().physicalRegisters()) { + if (TargetInfo.isReservedForFunction(Function, Description.Reg, Target)) + Roots.push_back(Description.Reg); + } + std::vector<PhysicalRegister> Result; + for (const PhysicalRegisterDesc &Description : Target.registers().physicalRegisters()) { + if (std::ranges::any_of(Roots, [&](PhysicalRegister Root) { + return Target.registers().regsOverlap(Description.Reg, Root); + })) { + Result.push_back(Description.Reg); + } + } + return Result; +} + +[[nodiscard]] inline Expected<bool> +reserveFunctionRegisters(MachineFunction &Function, std::span<const PhysicalRegister> Registers) { + bool Changed = false; + for (const PhysicalRegister Reg : Registers) { + if (Function.registerInfo().isReserved(Reg)) + continue; + auto Reserved = Function.registerInfo().reservePhysicalRegister(Reg); + if (!Reserved) + return std::unexpected(Reserved.error()); + Changed = true; + } + return Changed; +} + +[[nodiscard]] inline MachineInstr *firstTerminator(MachineBasicBlock &Block, + const TargetDescription &Target) noexcept { + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Terminator)) + return Instruction; + } + return nullptr; +} + +[[nodiscard]] inline bool hasTerminatorDefinition(const MachineFunction &Function, + const TargetDescription &Target, + Register VirtualReg) noexcept { + for (const MachineOperand &Operand : + const_cast<MachineRegisterInfo &>(Function.registerInfo()).defOperands(VirtualReg)) { + const MachineInstr *Instruction = Operand.parent(); + const InstructionDescriptor *Descriptor = + Instruction == nullptr ? nullptr : Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Terminator)) + return true; + } + return false; +} + +[[nodiscard]] inline std::uint64_t allocationFingerprint(const MachineFunction &Function) noexcept { + std::uint64_t Hash = 1469598103934665603ULL; + auto Mix = [&Hash](std::uint64_t Value) { + Hash ^= Value; + Hash *= 1099511628211ULL; + }; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + Mix(Block->stableID().value()); + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + Mix(Instruction->opcode().value()); + Mix(Instruction->stableID().value()); + for (const MachineOperand &Operand : Instruction->operands()) { + Mix(static_cast<std::uint64_t>(Operand.kind())); + if (const Register *Reg = Operand.getIf<Register>()) + Mix(Reg->opaqueValue()); + } + } + } + return Hash; +} + +[[nodiscard]] inline Expected<void> initializeMapping(MachineFunction &Function, + VirtRegMap &Mapping) { + for (std::size_t Index = 0; Index < Function.registerInfo().virtualRegisters().size(); ++Index) { + auto Reg = virtualRegisterAt(Index); + if (!Reg) + return std::unexpected(Reg.error()); + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) + continue; + auto Added = Mapping.add(*Reg); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; +} + +[[nodiscard]] inline MachinePassResult passResult(const AllocationRunResult &Run) { + MachinePassResult Result; + Result.Changed = Run.Changed; + if (Run.Changed) { + Result.Changes.add(MachinePassChange::Registers); + Result.Changes.add(MachinePassChange::Operands); + } + if (Run.ChangedInstructions) + Result.Changes.add(MachinePassChange::Instructions); + if (Run.ChangedFrame) + Result.Changes.add(MachinePassChange::Frame); + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +[[nodiscard]] bool +intervalsOverlapOnPhysicalRegisters(const LiveInterval &Left, PhysicalRegister LeftReg, + const LiveInterval &Right, PhysicalRegister RightReg, + const TargetRegisterInfo &Registers) noexcept; + +} // namespace rcc::regalloc_detail + +#endif // RCC_LIB_CODEGEN_REGALLOCINTERNAL_H diff --git a/backend/lib/CodeGen/RegAllocPBQP.cpp b/backend/lib/CodeGen/RegAllocPBQP.cpp new file mode 100644 index 0000000..59f07da --- /dev/null +++ b/backend/lib/CodeGen/RegAllocPBQP.cpp @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegAllocPBQP.h" + +#include "RegAllocBase.h" +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/LiveIntervals.h" +#include "rcc/CodeGen/LiveRegMatrix.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/PBQP/Solver.h" + +#include <algorithm> +#include <cstdint> +#include <limits> +#include <map> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct PBQPAllocationNode final { + Register VirtualReg; + pbqp::NodeID Node; + std::vector<std::optional<PhysicalRegister>> Options; +}; + +[[nodiscard]] std::map<std::pair<std::uint32_t, std::uint32_t>, MachineInstrStableID> +copyRelatedRegisters(const MachineFunction &Function) { + std::map<std::pair<std::uint32_t, std::uint32_t>, MachineInstrStableID> Result; + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (Instruction->opcode() != opcode(GenericOpcode::COPY) || Instruction->operandCount() < 2) + continue; + const Register *Destination = Instruction->operands()[0].getIf<Register>(); + const Register *Source = Instruction->operands()[1].getIf<Register>(); + if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() || + !Source->isVirtual() || *Destination == *Source) + continue; + std::uint32_t Left = Destination->opaqueValue(); + std::uint32_t Right = Source->opaqueValue(); + if (Right < Left) + std::swap(Left, Right); + Result.try_emplace({Left, Right}, Instruction->stableID()); + } + } + return Result; +} + +[[nodiscard]] std::string_view reductionName(pbqp::ReductionKind Kind) noexcept { + switch (Kind) { + case pbqp::ReductionKind::DegreeZero: + return "degree-zero"; + case pbqp::ReductionKind::DegreeOne: + return "degree-one"; + case pbqp::ReductionKind::DegreeTwo: + return "degree-two"; + case pbqp::ReductionKind::Heuristic: + return "heuristic"; + } + return "unknown"; +} + +[[nodiscard]] MachineInstrStableID actionInstructionForRegister(const MachineFunction &Function, + Register Reg) noexcept { + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg); + const MachineInstr *Instruction = Definition == nullptr ? nullptr : Definition->parent(); + return Instruction == nullptr ? MachineInstrStableID{} : Instruction->stableID(); +} + +[[nodiscard]] pbqp::Cost boundedCost(std::uint64_t Value) noexcept { + const std::uint64_t Limit = static_cast<std::uint64_t>(pbqp::InfiniteCost - 1); + return static_cast<pbqp::Cost>(std::min(Value, Limit)); +} + +class PBQPAllocatorEngine final : public RegAllocBase { +public: + PBQPAllocatorEngine(MachineFunction &Function, const TargetDescription &Target, + const TargetRegisterAllocationInfo &TargetInfo, + MachineFunctionAnalysisManager &Analyses, RegisterAllocationOptions Options) + : RegAllocBase(Function, Target, TargetInfo, Analyses, Options) {} + +private: + [[nodiscard]] Expected<AllocationRoundResult> + allocateRound(const LiveIntervalsAnalysis::Result &Intervals, VirtRegMap &Mapping) override { + LiveRegMatrix FixedMatrix(Function, Target, Intervals); + pbqp::Graph Problem; + std::vector<PBQPAllocationNode> Nodes; + for (const LiveInterval *Interval : orderedAllocationIntervals(Intervals, Function)) { + const Register VirtualReg = Interval->Reg; + const bool TerminatorDefined = + regalloc_detail::hasTerminatorDefinition(Function, Target, VirtualReg); + if (TerminatorDefined) { + auto Marked = Mapping.setSpillable(VirtualReg, false); + if (!Marked) + return std::unexpected(Marked.error()); + } + auto ClassID = regalloc_detail::registerClassFor(Function, VirtualReg); + if (!ClassID) + return std::unexpected(ClassID.error()); + auto Candidates = regalloc_detail::allocationOrder(Function, Target, TargetInfo, VirtualReg); + if (!Candidates) + return std::unexpected(Candidates.error()); + std::erase_if(*Candidates, [&](PhysicalRegister Candidate) { + return FixedMatrix.checkInterference(VirtualReg, Candidate).Kind != + RegisterInterferenceKind::Free; + }); + + std::vector<std::optional<PhysicalRegister>> NodeOptions; + NodeOptions.reserve(Candidates->size() + 1U); + NodeOptions.push_back(std::nullopt); + for (const PhysicalRegister Candidate : *Candidates) + NodeOptions.push_back(Candidate); + pbqp::CostVector Costs(NodeOptions.size(), 0); + if (!Mapping.isSpillable(VirtualReg)) { + Costs[0] = pbqp::InfiniteCost; + } else { + const std::uint64_t TargetSpillCost = TargetInfo.spillCost(Function, *Interval, *ClassID); + const std::uint64_t Weight = std::max<std::uint64_t>(1, Interval->SpillWeight); + const std::uint64_t Scaled = + TargetSpillCost > std::numeric_limits<std::uint64_t>::max() / Weight + ? std::numeric_limits<std::uint64_t>::max() + : TargetSpillCost * Weight; + const std::uint64_t Adjusted = + Scaled == std::numeric_limits<std::uint64_t>::max() ? Scaled : Scaled + 1U; + Costs[0] = boundedCost(Adjusted); + } + auto Info = Function.registerInfo().virtualRegisterInfo(VirtualReg); + if (!Info) + return std::unexpected(Info.error()); + for (std::size_t Index = 1; Index < NodeOptions.size(); ++Index) { + const auto CandidateOption = NodeOptions[Index]; + if (!CandidateOption) { + return makeUnexpected(ErrorCode::InternalError, + "PBQP allocation node has an unexpected spill option"); + } + const PhysicalRegister Candidate = CandidateOption.value(); + std::uint64_t Cost = + TargetInfo.physicalRegisterCost(Function, VirtualReg, Candidate, *ClassID, true); + const auto Hint = std::ranges::find(Info->get().AllocationHints, Candidate); + const std::uint64_t HintCost = + Hint == Info->get().AllocationHints.end() + ? 2U + : static_cast<std::uint64_t>(Hint - Info->get().AllocationHints.begin()); + Cost = Cost > std::numeric_limits<std::uint64_t>::max() - HintCost + ? std::numeric_limits<std::uint64_t>::max() + : Cost + HintCost; + Costs[Index] = boundedCost(Cost); + } + auto NodeID = Problem.addNode(std::move(Costs), VirtualReg.opaqueValue()); + if (!NodeID) + return std::unexpected(NodeID.error()); + Nodes.push_back(PBQPAllocationNode{VirtualReg, *NodeID, std::move(NodeOptions)}); + } + + const auto CopyPairs = copyRelatedRegisters(Function); + for (std::size_t LeftIndex = 0; LeftIndex < Nodes.size(); ++LeftIndex) { + const PBQPAllocationNode &Left = Nodes[LeftIndex]; + const LiveInterval *LeftInterval = Intervals.interval(Left.VirtualReg); + for (std::size_t RightIndex = LeftIndex + 1U; RightIndex < Nodes.size(); ++RightIndex) { + const PBQPAllocationNode &Right = Nodes[RightIndex]; + const LiveInterval *RightInterval = Intervals.interval(Right.VirtualReg); + if (LeftInterval == nullptr || RightInterval == nullptr) + continue; + std::uint32_t LeftKey = Left.VirtualReg.opaqueValue(); + std::uint32_t RightKey = Right.VirtualReg.opaqueValue(); + if (RightKey < LeftKey) + std::swap(LeftKey, RightKey); + const auto CopyPosition = CopyPairs.find({LeftKey, RightKey}); + bool CopyRelated = Options.EnableCopyCoalescing && CopyPosition != CopyPairs.end(); + if (CopyRelated) { + auto Accepted = Analyses.requestAction("coalesce-copy", Function, CopyPosition->second); + if (!Accepted) + return std::unexpected(Accepted.error()); + CopyRelated = *Accepted; + } + pbqp::CostMatrix Costs(Left.Options.size(), Right.Options.size(), 0); + bool HasCost = false; + for (std::size_t LeftOption = 1; LeftOption < Left.Options.size(); ++LeftOption) { + for (std::size_t RightOption = 1; RightOption < Right.Options.size(); ++RightOption) { + const auto LeftRegister = Left.Options[LeftOption]; + const auto RightRegister = Right.Options[RightOption]; + if (!LeftRegister || !RightRegister) { + return makeUnexpected(ErrorCode::InternalError, + "PBQP edge has an unexpected spill option"); + } + const PhysicalRegister LeftReg = LeftRegister.value(); + const PhysicalRegister RightReg = RightRegister.value(); + if (regalloc_detail::intervalsOverlapOnPhysicalRegisters( + *LeftInterval, LeftReg, *RightInterval, RightReg, Target.registers())) { + Costs.at(LeftOption, RightOption) = pbqp::InfiniteCost; + HasCost = true; + } else if (CopyRelated && LeftReg == RightReg) { + Costs.at(LeftOption, RightOption) = -1; + HasCost = true; + } + } + } + if (!HasCost) + continue; + const std::uint64_t StableKey = + (static_cast<std::uint64_t>(Left.VirtualReg.opaqueValue()) << 32U) | + Right.VirtualReg.opaqueValue(); + auto Edge = Problem.addEdge(Left.Node, Right.Node, std::move(Costs), StableKey); + if (!Edge) + return std::unexpected(Edge.error()); + } + } + + pbqp::DeterministicSolver Solver; + auto Solution = Solver.solve(Problem, [&](const pbqp::SolverAction &Action) -> Expected<bool> { + if (Action.Node.value() >= Nodes.size()) { + return makeUnexpected(ErrorCode::InternalError, + "PBQP action names an unknown allocation node"); + } + const Register VirtualReg = Nodes[Action.Node.value()].VirtualReg; + std::string Rule = + Action.Kind == pbqp::SolverActionKind::Reduction ? "pbqp-reduce." : "pbqp-choice."; + Rule.append(reductionName(Action.Reduction)); + if (Action.Kind == pbqp::SolverActionKind::Choice) { + Rule.append(".option-"); + Rule.append(std::to_string(Action.Value)); + } + return Analyses.requestAction(Rule, Function, + actionInstructionForRegister(Function, VirtualReg)); + }); + if (!Solution) + return std::unexpected(Solution.error()); + + LiveRegMatrix AssignedMatrix(Function, Target, Intervals); + AllocationRoundResult Result; + for (const PBQPAllocationNode &Node : Nodes) { + const auto Selection = Solution->selection(Node.Node); + if (!Selection) + return makeUnexpected(ErrorCode::InternalError, + "PBQP solver returned an invalid selection"); + const std::size_t SelectedOption = Selection.value(); + if (SelectedOption >= Node.Options.size()) { + return makeUnexpected(ErrorCode::InternalError, + "PBQP solver selected an out-of-range option"); + } + const auto AssignedOption = Node.Options[SelectedOption]; + if (!AssignedOption) { + Result.Spilled.push_back(Node.VirtualReg); + continue; + } + const PhysicalRegister Assigned = AssignedOption.value(); + auto MatrixAssignment = AssignedMatrix.assign(Node.VirtualReg, Assigned); + if (!MatrixAssignment) + return makeUnexpected(ErrorCode::InternalError, + "PBQP solution violates its encoded interference constraints"); + auto MapAssignment = Mapping.assign(Node.VirtualReg, Assigned); + if (!MapAssignment) + return std::unexpected(MapAssignment.error()); + } + return Result; + } +}; + +[[nodiscard]] const MachinePassContract &pbqpContract() { + static const MachinePassContract Contract = [] { + MachinePassContract Value{ + .Name = "PBQPAllocator", + .Requires = {MachineFunctionProperty::NoPHIs}, + .Sets = {}, + .Clears = {MachineFunctionProperty::IsSSA}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<LiveIntervalsAnalysis>(), + MachineFunctionAnalysisManager::id<VirtRegMapAnalysis>()}, + .VerificationStage = std::nullopt, + }; + Value.ThreadSafety = PassThreadSafety::ThreadSafe; + return Value; + }(); + return Contract; +} + +} // namespace + +const MachinePassContract &PBQPAllocatorPass::contract() const noexcept { + return pbqpContract(); +} + +Expected<MachinePassResult> PBQPAllocatorPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + if (Info == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "PBQPAllocator requires target behavior"); + const std::vector<PhysicalRegister> FunctionReserved = + regalloc_detail::functionReservedRegisters(Function, Target, *Info); + auto Reserved = regalloc_detail::reserveFunctionRegisters(Function, FunctionReserved); + if (!Reserved) + return std::unexpected(Reserved.error()); + PBQPAllocatorEngine Allocator(Function, Target, *Info, Analyses, Options); + auto Mapping = Analyses.getResult<VirtRegMapAnalysis>(Function, Target); + if (!Mapping) + return std::unexpected(Mapping.error()); + auto Allocated = Allocator.run(**Mapping); + if (!Allocated) + return std::unexpected(Allocated.error()); + Allocated->Changed |= *Reserved; + MachinePassResult Result = regalloc_detail::passResult(*Allocated); + Result.Preserved.preserve<VirtRegMapAnalysis>(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/Register.cpp b/backend/lib/CodeGen/Register.cpp new file mode 100644 index 0000000..0af26c7 --- /dev/null +++ b/backend/lib/CodeGen/Register.cpp @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/Register.h" + +#include <string> + +namespace rcc { +namespace { + +constexpr std::uint64_t MaximumRegisterIndex = (std::uint64_t{1} << 31) - 1; + +} // namespace + +Expected<PhysicalRegister> PhysicalRegister::create(std::uint64_t Number) { + if (Number == 0 || Number > MaximumRegisterIndex) { + return makeUnexpected(ErrorCode::InvalidArgument, + "physical register number must be in the range [1, 2147483647]"); + } + return PhysicalRegister(static_cast<std::uint32_t>(Number)); +} + +Expected<Register> Register::makePhysical(std::uint64_t Number) { + auto PhysReg = PhysicalRegister::create(Number); + if (!PhysReg) + return std::unexpected(PhysReg.error()); + return fromPhysical(*PhysReg); +} + +Expected<Register> Register::makeVirtual(std::uint64_t Index) { + if (Index > MaximumRegisterIndex) { + return makeUnexpected(ErrorCode::InvalidArgument, "virtual register index must fit in 31 bits"); + } + return Register(VirtualBit | static_cast<std::uint32_t>(Index)); +} + +std::optional<PhysicalRegister> Register::physicalRegister() const noexcept { + if (!isPhysical()) + return std::nullopt; + return PhysicalRegister(OpaqueValue); +} + +std::optional<std::uint32_t> Register::virtualIndex() const noexcept { + if (!isVirtual()) + return std::nullopt; + return OpaqueValue & IndexMask; +} + +std::string Register::toString() const { + if (isNoRegister()) + return "noreg"; + if (isVirtual()) + return "%" + std::to_string(OpaqueValue & IndexMask); + return "$physreg" + std::to_string(OpaqueValue); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegisterMask.cpp b/backend/lib/CodeGen/RegisterMask.cpp new file mode 100644 index 0000000..027ee26 --- /dev/null +++ b/backend/lib/CodeGen/RegisterMask.cpp @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegisterMask.h" + +namespace rcc { + +bool RegisterMask::preserves(PhysicalRegister PhysReg) const noexcept { + if (!PhysReg.isValid()) + return false; + + constexpr std::uint32_t BitsPerWord = 64; + const std::uint32_t RegisterNumber = PhysReg.number(); + const std::size_t WordIndex = RegisterNumber / BitsPerWord; + if (WordIndex >= PreservedWords.size()) + return false; + + const std::uint32_t BitIndex = RegisterNumber % BitsPerWord; + return (PreservedWords[WordIndex] & (std::uint64_t{1} << BitIndex)) != 0; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegisterPressure.cpp b/backend/lib/CodeGen/RegisterPressure.cpp new file mode 100644 index 0000000..201e85b --- /dev/null +++ b/backend/lib/CodeGen/RegisterPressure.cpp @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegisterPressure.h" + +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +using EntityID = std::size_t; + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits<std::uint64_t>::max() - Left < Right + ? std::numeric_limits<std::uint64_t>::max() + : Left + Right; +} + +template <typename ValueT> void sortAndUnique(std::vector<ValueT> &Values) { + std::ranges::sort(Values); + const auto [FirstDuplicate, End] = std::ranges::unique(Values); + Values.erase(FirstDuplicate, End); +} + +} // namespace + +namespace detail { + +struct RegisterPressureBuilder final { + RegisterPressureTracker Result; + const ScheduleDAG *DAG = nullptr; + const MachineFunction *Function = nullptr; + const TargetDescription *Target = nullptr; + const TargetSchedulingModel *Model = nullptr; + ScheduleDAGMode Mode = ScheduleDAGMode::PreRegisterAllocation; + std::unordered_map<Register, EntityID> CurrentVirtualEntities; + std::unordered_map<RegisterUnitID, EntityID> CurrentPhysicalEntities; + std::unordered_map<RegisterUnitID, std::vector<RegisterPressureSetID>> PhysicalSets; + + [[nodiscard]] Expected<void> initializePressureSets() { + Result.Pressure.reserve(Model->pressureSets().size()); + for (const RegisterPressureSetDesc &Set : Model->pressureSets()) { + Result.PressureIndexes.emplace(Set.ID, Result.Pressure.size()); + Result.Pressure.push_back({Set.ID, 0, 0, Set.Limit}); + for (const RegisterClassID ClassID : Set.RegisterClasses) { + const RegisterClassDesc *Class = Target->registers().registerClass(ClassID); + if (Class == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure set references an unknown target register class"); + } + for (const PhysicalRegister Physical : Class->Registers) { + const PhysicalRegisterDesc *Description = Target->registers().physicalRegister(Physical); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "register class references an unknown physical register"); + } + for (const RegisterUnitLane &Unit : Description->Units) + PhysicalSets[Unit.Unit].push_back(Set.ID); + } + } + } + for (auto &[Unit, Sets] : PhysicalSets) { + (void)Unit; + sortAndUnique(Sets); + } + return {}; + } + + [[nodiscard]] EntityID createEntity(std::vector<RegisterPressureSetID> Sets) { + sortAndUnique(Sets); + const EntityID ID = Result.Entities.size(); + Result.Entities.push_back({std::move(Sets), 0, false, false, false, false}); + return ID; + } + + [[nodiscard]] Expected<std::vector<RegisterPressureSetID>> + virtualPressureSets(Register Reg) const { + if (Mode == ScheduleDAGMode::PostRegisterAllocation) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA pressure tracking encountered a virtual register"); + } + auto Info = Function->registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + const auto &RegClass = Info->get().RegClass; + if (!RegClass) { + return makeUnexpected(ErrorCode::VerificationError, + "pre-RA pressure tracking requires constrained virtual registers"); + } + const std::span<const RegisterPressureSetID> Sets = Model->pressureSetsForClass(*RegClass); + return std::vector<RegisterPressureSetID>{Sets.begin(), Sets.end()}; + } + + [[nodiscard]] Expected<EntityID> boundaryVirtualEntity(Register Reg) { + if (const auto Position = CurrentVirtualEntities.find(Reg); + Position != CurrentVirtualEntities.end()) { + return Position->second; + } + auto Sets = virtualPressureSets(Reg); + if (!Sets) + return std::unexpected(Sets.error()); + const EntityID ID = createEntity(std::move(*Sets)); + CurrentVirtualEntities.emplace(Reg, ID); + return ID; + } + + [[nodiscard]] Expected<EntityID> defineVirtualEntity(Register Reg) { + auto Sets = virtualPressureSets(Reg); + if (!Sets) + return std::unexpected(Sets.error()); + const EntityID ID = createEntity(std::move(*Sets)); + CurrentVirtualEntities.insert_or_assign(Reg, ID); + return ID; + } + + [[nodiscard]] EntityID boundaryPhysicalEntity(RegisterUnitID Unit) { + if (const auto Position = CurrentPhysicalEntities.find(Unit); + Position != CurrentPhysicalEntities.end()) { + return Position->second; + } + const auto Sets = PhysicalSets.find(Unit); + const EntityID ID = createEntity( + Sets == PhysicalSets.end() ? std::vector<RegisterPressureSetID>{} : Sets->second); + CurrentPhysicalEntities.emplace(Unit, ID); + return ID; + } + + [[nodiscard]] EntityID definePhysicalEntity(RegisterUnitID Unit) { + const auto Sets = PhysicalSets.find(Unit); + const EntityID ID = createEntity( + Sets == PhysicalSets.end() ? std::vector<RegisterPressureSetID>{} : Sets->second); + CurrentPhysicalEntities.insert_or_assign(Unit, ID); + return ID; + } + + [[nodiscard]] Expected<std::vector<RegisterUnitID>> physicalUnits(Register Reg, + SubRegIndex SubReg) const { + std::optional<PhysicalRegister> Physical = Reg.physicalRegister(); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an invalid physical register"); + } + if (SubReg.value() != 0) { + Physical = Target->registers().subRegister(*Physical, SubReg); + if (!Physical) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an invalid physical subregister"); + } + } + const PhysicalRegisterDesc *Description = Target->registers().physicalRegister(*Physical); + if (Description == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered an unknown physical register"); + } + std::vector<RegisterUnitID> Units; + Units.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) + Units.push_back(Unit.Unit); + sortAndUnique(Units); + return Units; + } + + [[nodiscard]] Expected<void> appendRegisterUse(RegisterPressureTracker::Event &Event, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Entity = boundaryVirtualEntity(*Reg); + if (!Entity) + return std::unexpected(Entity.error()); + Event.Uses.push_back(*Entity); + return {}; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) + Event.Uses.push_back(boundaryPhysicalEntity(Unit)); + return {}; + } + + [[nodiscard]] Expected<void> appendRegisterDefinition(RegisterPressureTracker::Event &Event, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return {}; + if (Reg->isVirtual()) { + auto Entity = defineVirtualEntity(*Reg); + if (!Entity) + return std::unexpected(Entity.error()); + Event.Definitions.push_back(*Entity); + Result.Entities[*Entity].HasDefinition = true; + Result.Entities[*Entity].DefinitionPending = true; + return {}; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) { + const EntityID Entity = definePhysicalEntity(Unit); + Event.Definitions.push_back(Entity); + Result.Entities[Entity].HasDefinition = true; + Result.Entities[Entity].DefinitionPending = true; + } + return {}; + } + + [[nodiscard]] Expected<void> appendMaskDefinition(RegisterPressureTracker::Event &Event, + const RegisterMask &Mask) { + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask.clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) { + const EntityID Entity = definePhysicalEntity(Unit.Unit); + Event.Definitions.push_back(Entity); + Result.Entities[Entity].HasDefinition = true; + Result.Entities[Entity].DefinitionPending = true; + } + } + return {}; + } + + void appendLiveOutMaskUse(RegisterPressureTracker::Event &Event, const RegisterMask &Mask) { + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask.preserves(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + Event.Uses.push_back(boundaryPhysicalEntity(Unit.Unit)); + } + } + + [[nodiscard]] Expected<RegisterPressureTracker::Event> + buildEvent(const MachineInstr &Instruction) { + RegisterPressureTracker::Event Event; + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered a null live-out register mask"); + } + appendLiveOutMaskUse(Event, *Mask->Mask); + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse() || + Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + if (Operand.isUse()) { + auto Added = appendRegisterUse(Event, Operand); + if (!Added) + return std::unexpected(Added.error()); + } + } + for (const MachineOperand &Operand : Instruction.operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure tracking encountered a null register mask"); + } + auto Added = appendMaskDefinition(Event, *Mask->Mask); + if (!Added) + return std::unexpected(Added.error()); + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse()) + continue; + if (Operand.isDef()) { + auto Added = appendRegisterDefinition(Event, Operand); + if (!Added) + return std::unexpected(Added.error()); + } + } + sortAndUnique(Event.Uses); + sortAndUnique(Event.Definitions); + return Event; + } + + [[nodiscard]] Expected<void> buildEffects() { + Result.Effects.resize(DAG->units().size()); + Result.Scheduled.assign(DAG->units().size(), false); + for (const ScheduleUnit &Unit : DAG->units()) { + RegisterPressureTracker::UnitEffects &Effects = Result.Effects[Unit.ID]; + Effects.Events.reserve(Unit.Instructions.size()); + for (const MachineInstr *Instruction : Unit.Instructions) { + auto Event = buildEvent(*Instruction); + if (!Event) + return std::unexpected(Event.error()); + for (const EntityID Entity : Event->Uses) + ++Result.Entities[Entity].RemainingUses; + Effects.Events.push_back(std::move(*Event)); + } + } + return {}; + } + + void markEntityLiveOut(EntityID Entity) { + RegisterPressureTracker::Entity &Value = Result.Entities[Entity]; + if (Value.LiveOut) + return; + Value.LiveOut = true; + ++Value.RemainingUses; + } + + [[nodiscard]] Expected<void> markBlockLiveOut(const MachineBlockLiveness &Liveness) { + for (const auto &[Reg, Lanes] : Liveness.VirtualLiveOut.entries()) { + (void)Lanes; + auto Entity = boundaryVirtualEntity(Reg); + if (!Entity) + return std::unexpected(Entity.error()); + markEntityLiveOut(*Entity); + } + for (const auto &[Unit, Lanes] : Liveness.PhysicalLiveOut.entries()) { + (void)Lanes; + markEntityLiveOut(boundaryPhysicalEntity(Unit)); + } + return {}; + } + + [[nodiscard]] Expected<void> markSuffixUses() { + if (DAG->units().empty()) + return {}; + const MachineInstr *Instruction = DAG->units().back().Instructions.back()->next(); + std::unordered_set<Register> KilledVirtual; + std::unordered_set<RegisterUnitID> KilledPhysical; + for (; Instruction != nullptr; Instruction = Instruction->next()) { + for (const MachineOperand &Operand : Instruction->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>()) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "pressure suffix scan encountered a null register mask"); + } + for (const PhysicalRegisterDesc &Register : Target->registers().physicalRegisters()) { + if (!Mask->Mask->clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + KilledPhysical.insert(Unit.Unit); + } + continue; + } + if (!Operand.isRegister() || Operand.isDebugUse() || + Operand.registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + continue; + if (Reg->isVirtual()) { + if (Operand.isUse() && !KilledVirtual.contains(*Reg)) { + const auto Position = CurrentVirtualEntities.find(*Reg); + if (Position != CurrentVirtualEntities.end()) + markEntityLiveOut(Position->second); + } + if (Operand.isDef()) + KilledVirtual.insert(*Reg); + continue; + } + auto Units = physicalUnits(*Reg, Operand.subReg()); + if (!Units) + return std::unexpected(Units.error()); + for (const RegisterUnitID Unit : *Units) { + if (Operand.isUse() && !KilledPhysical.contains(Unit)) { + const auto Position = CurrentPhysicalEntities.find(Unit); + if (Position != CurrentPhysicalEntities.end()) + markEntityLiveOut(Position->second); + } + if (Operand.isDef()) + KilledPhysical.insert(Unit); + } + } + } + return {}; + } + + [[nodiscard]] Expected<void> initializeLivePressure() { + for (EntityID ID = 0; ID < Result.Entities.size(); ++ID) { + RegisterPressureTracker::Entity &Entity = Result.Entities[ID]; + const bool InitiallyLive = Result.Direction == RegisterPressureDirection::TopDown + ? !Entity.HasDefinition && Entity.RemainingUses != 0 + : Entity.LiveOut; + if (!InitiallyLive) + continue; + auto Added = Result.makeLive(ID); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; + } +}; + +} // namespace detail + +Expected<RegisterPressureTracker> +RegisterPressureTracker::create(const ScheduleDAG &DAG, const MachineFunction &Function, + const TargetDescription &Target, const TargetSchedulingModel &Model, + ScheduleDAGMode Mode, const MachineBlockLiveness &Liveness, + RegisterPressureDirection Direction) { + detail::RegisterPressureBuilder Builder; + Builder.DAG = &DAG; + Builder.Function = &Function; + Builder.Target = &Target; + Builder.Model = &Model; + Builder.Mode = Mode; + Builder.Result.Direction = Direction; + auto Sets = Builder.initializePressureSets(); + if (!Sets) + return std::unexpected(Sets.error()); + auto Effects = Builder.buildEffects(); + if (!Effects) + return std::unexpected(Effects.error()); + auto Suffix = Builder.markSuffixUses(); + if (!Suffix) + return std::unexpected(Suffix.error()); + auto LiveOut = Builder.markBlockLiveOut(Liveness); + if (!LiveOut) + return std::unexpected(LiveOut.error()); + auto Initial = Builder.initializeLivePressure(); + if (!Initial) + return std::unexpected(Initial.error()); + return std::move(Builder.Result); +} + +Expected<RegisterPressureCost> RegisterPressureTracker::evaluate(std::size_t UnitID) const { + RegisterPressureTracker Trial = *this; + const RegisterPressureCost Before = cost(); + auto ScheduledUnit = Trial.schedule(UnitID); + if (!ScheduledUnit) + return std::unexpected(ScheduledUnit.error()); + RegisterPressureCost After = Trial.cost(); + After.PeakIncrease = + After.PeakExcess > Before.PeakExcess ? After.PeakExcess - Before.PeakExcess : 0; + return After; +} + +Expected<void> RegisterPressureTracker::schedule(std::size_t UnitID) { + if (UnitID >= Effects.size()) + return makeUnexpected(ErrorCode::InvalidArgument, "pressure tracker unit ID is out of range"); + if (Scheduled[UnitID]) + return makeUnexpected(ErrorCode::InvalidArgument, "schedule unit was already issued"); + RegisterPressureTracker Trial = *this; + if (Trial.Direction == RegisterPressureDirection::TopDown) { + for (const Event &Value : Trial.Effects[UnitID].Events) { + auto Applied = Trial.applyEvent(Value); + if (!Applied) + return std::unexpected(Applied.error()); + } + } else { + for (auto Position = Trial.Effects[UnitID].Events.rbegin(); + Position != Trial.Effects[UnitID].Events.rend(); ++Position) { + auto Applied = Trial.applyEvent(*Position); + if (!Applied) + return std::unexpected(Applied.error()); + } + } + Trial.Scheduled[UnitID] = true; + *this = std::move(Trial); + return {}; +} + +RegisterPressureCost RegisterPressureTracker::cost() const noexcept { + RegisterPressureCost Result; + for (const RegisterPressureValue &Value : Pressure) { + Result.CurrentPressure = saturatingAdd(Result.CurrentPressure, Value.Current); + if (Value.Current > Value.Limit) { + Result.CurrentExcess = saturatingAdd(Result.CurrentExcess, Value.Current - Value.Limit); + } + if (Value.Maximum > Value.Limit) + Result.PeakExcess = saturatingAdd(Result.PeakExcess, Value.Maximum - Value.Limit); + } + return Result; +} + +Expected<void> RegisterPressureTracker::applyEvent(const Event &Value) { + if (Direction == RegisterPressureDirection::BottomUp) { + for (const EntityID Entity : Value.Definitions) { + if (Entity >= Entities.size() || !Entities[Entity].DefinitionPending) { + return makeUnexpected(ErrorCode::VerificationError, + "bottom-up scheduler crossed an unavailable definition"); + } + Entities[Entity].DefinitionPending = false; + if (Entities[Entity].Live) { + auto Removed = makeDead(Entity); + if (!Removed) + return std::unexpected(Removed.error()); + } + } + for (const EntityID Entity : Value.Uses) { + if (Entity >= Entities.size()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure event references an unknown use entity"); + } + if (!Entities[Entity].Live) { + auto Added = makeLive(Entity); + if (!Added) + return std::unexpected(Added.error()); + } + } + return {}; + } + for (const EntityID Entity : Value.Uses) { + if (Entity >= Entities.size() || !Entities[Entity].Live || + Entities[Entity].RemainingUses == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler selected a use before its pressure value became live"); + } + --Entities[Entity].RemainingUses; + if (Entities[Entity].RemainingUses == 0) { + auto Removed = makeDead(Entity); + if (!Removed) + return std::unexpected(Removed.error()); + } + } + for (const EntityID Entity : Value.Definitions) { + if (Entity >= Entities.size()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure event references an unknown definition entity"); + } + if (Entities[Entity].Live) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler redefined a pressure value while it was live"); + } + if (Entities[Entity].RemainingUses != 0) { + auto Added = makeLive(Entity); + if (!Added) + return std::unexpected(Added.error()); + } + } + return {}; +} + +Expected<void> RegisterPressureTracker::makeLive(EntityID EntityIDValue) { + if (EntityIDValue >= Entities.size()) + return makeUnexpected(ErrorCode::InternalError, "unknown pressure entity became live"); + Entity &Value = Entities[EntityIDValue]; + if (Value.Live) + return {}; + for (const RegisterPressureSetID Set : Value.Sets) { + const auto Position = PressureIndexes.find(Set); + if (Position == PressureIndexes.end()) { + return makeUnexpected(ErrorCode::InternalError, + "pressure entity references an unknown pressure set"); + } + RegisterPressureValue &SetPressure = Pressure[Position->second]; + if (SetPressure.Current == std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, "register pressure counter overflowed"); + } + ++SetPressure.Current; + SetPressure.Maximum = std::max(SetPressure.Maximum, SetPressure.Current); + } + Value.Live = true; + return {}; +} + +Expected<void> RegisterPressureTracker::makeDead(EntityID EntityIDValue) { + if (EntityIDValue >= Entities.size()) + return makeUnexpected(ErrorCode::InternalError, "unknown pressure entity became dead"); + Entity &Value = Entities[EntityIDValue]; + if (!Value.Live) + return makeUnexpected(ErrorCode::InternalError, "dead pressure entity was not live"); + for (const RegisterPressureSetID Set : Value.Sets) { + const auto Position = PressureIndexes.find(Set); + if (Position == PressureIndexes.end() || Pressure[Position->second].Current == 0) { + return makeUnexpected(ErrorCode::InternalError, "register pressure counter underflowed"); + } + --Pressure[Position->second].Current; + } + Value.Live = false; + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/RegisterScavenger.cpp b/backend/lib/CodeGen/RegisterScavenger.cpp new file mode 100644 index 0000000..386c327 --- /dev/null +++ b/backend/lib/CodeGen/RegisterScavenger.cpp @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/RegisterScavenger.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <optional> +#include <ranges> +#include <unordered_map> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] std::optional<PhysicalRegister> +physicalRegister(const MachineOperand &Operand) noexcept { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isPhysical()) + return std::nullopt; + return Reg->physicalRegister(); +} + +[[nodiscard]] bool maskClobbersRegisterUnits(const RegisterMask &Mask, PhysicalRegister Reg, + const TargetDescription &Target) noexcept { + return std::ranges::any_of(Target.registers().physicalRegisters(), + [&](const PhysicalRegisterDesc &Description) { + return Target.registers().regsOverlap(Description.Reg, Reg) && + Mask.clobbers(Description.Reg); + }); +} + +[[nodiscard]] bool maskPreservesRegisterUnits(const RegisterMask &Mask, PhysicalRegister Reg, + const TargetDescription &Target) noexcept { + return std::ranges::any_of(Target.registers().physicalRegisters(), + [&](const PhysicalRegisterDesc &Description) { + return Target.registers().regsOverlap(Description.Reg, Reg) && + Mask.preserves(Description.Reg); + }); +} + +[[nodiscard]] bool isUnavailable(const MachineFunction &Function, const TargetDescription &Target, + PhysicalRegister Reg) noexcept { + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(Reg); + if (Description == nullptr || Description->IsConstant || Description->IsArtificial || + Description->IsReserved || Target.registers().isReservedRegister(Reg)) { + return true; + } + return std::ranges::any_of( + Function.registerInfo().reservedRegisters(), + [&](PhysicalRegister Reserved) { return Target.registers().regsOverlap(Reg, Reserved); }); +} + +} // namespace + +bool RegisterScavenger::instructionUses(const MachineInstr &Instruction, + PhysicalRegister Reg) const noexcept { + for (const MachineOperand &Operand : Instruction.operands()) { + const auto Physical = physicalRegister(Operand); + if (Physical && Target->registers().regsOverlap(*Physical, Reg)) + return true; + } + return false; +} + +Expected<bool> RegisterScavenger::isLiveBefore(const MachineInstr &Instruction, + PhysicalRegister Reg) const { + const MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr || &Instruction.function() != Function || &Block->function() != Function) { + return makeUnexpected(ErrorCode::VerificationError, + "register scavenging anchor is detached or belongs to another function"); + } + + std::vector<const MachineBasicBlock *> Blocks; + std::unordered_map<const MachineBasicBlock *, std::size_t> BlockIndices; + for (const MachineBasicBlock *Current = Function->firstBlock(); Current != nullptr; + Current = Current->next()) { + BlockIndices.emplace(Current, Blocks.size()); + Blocks.push_back(Current); + } + + if (!BlockIndices.contains(Block)) { + return makeUnexpected(ErrorCode::VerificationError, + "register scavenging anchor block is not linked in its function"); + } + for (const MachineBasicBlock *Current : Blocks) { + for (const MachineSuccessor &Successor : Current->successors()) { + if (Successor.Block == nullptr || !BlockIndices.contains(Successor.Block)) { + return makeUnexpected(ErrorCode::VerificationError, + "register scavenging CFG has an unknown successor"); + } + } + for (const MachineInstr *CurrentInstruction = Current->firstInstruction(); + CurrentInstruction != nullptr; CurrentInstruction = CurrentInstruction->next()) { + for (const MachineOperand &Operand : CurrentInstruction->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && !Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "register scavenging found a null register mask"); + } + if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); + Mask != nullptr && !Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "register scavenging found a null live-out register mask"); + } + } + } + } + + const auto Transfer = [&](const MachineBasicBlock &Current, bool LiveOut) noexcept { + bool Live = LiveOut; + for (const MachineInstr *CurrentInstruction = Current.lastInstruction(); + CurrentInstruction != nullptr; CurrentInstruction = CurrentInstruction->previous()) { + for (const MachineOperand &Operand : CurrentInstruction->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && maskClobbersRegisterUnits(*Mask->Mask, Reg, *Target)) { + Live = false; + } + const auto Physical = physicalRegister(Operand); + if (Physical && Operand.isDef() && Target->registers().regsOverlap(*Physical, Reg)) { + Live = false; + } + } + for (const MachineOperand &Operand : CurrentInstruction->operands()) { + if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); + Mask != nullptr && maskPreservesRegisterUnits(*Mask->Mask, Reg, *Target)) { + Live = true; + } + const auto Physical = physicalRegister(Operand); + if (Physical && Operand.isUse() && !Operand.isDebugUse() && + !Operand.registerFlags().has(RegisterOperandFlag::Undef) && + Target->registers().regsOverlap(*Physical, Reg)) { + Live = true; + } + } + } + return Live; + }; + + std::vector<bool> LiveIns(Blocks.size(), false); + for (std::size_t Index = 0; Index < Blocks.size(); ++Index) { + for (const MachineBasicBlockLiveIn &LiveIn : Blocks[Index]->liveIns()) { + if (Target->registers().regsOverlap(LiveIn.PhysReg, Reg)) + LiveIns[Index] = true; + } + } + bool Changed = true; + while (Changed) { + Changed = false; + for (std::size_t Reverse = Blocks.size(); Reverse != 0; --Reverse) { + const std::size_t Index = Reverse - 1U; + bool LiveOut = false; + for (const MachineSuccessor &Successor : Blocks[Index]->successors()) { + LiveOut |= LiveIns[BlockIndices.at(Successor.Block)]; + } + const bool NewLiveIn = LiveIns[Index] || Transfer(*Blocks[Index], LiveOut); + if (NewLiveIn != LiveIns[Index]) { + LiveIns[Index] = NewLiveIn; + Changed = true; + } + } + } + + bool Live = false; + for (const MachineSuccessor &Successor : Block->successors()) { + Live |= LiveIns[BlockIndices.at(Successor.Block)]; + } + + for (const MachineInstr *Current = Block->lastInstruction(); Current != nullptr; + Current = Current->previous()) { + for (const MachineOperand &Operand : Current->operands()) { + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); + Mask != nullptr && maskClobbersRegisterUnits(*Mask->Mask, Reg, *Target)) { + Live = false; + } + const auto Physical = physicalRegister(Operand); + if (Physical && Operand.isDef() && Target->registers().regsOverlap(*Physical, Reg)) + Live = false; + } + for (const MachineOperand &Operand : Current->operands()) { + if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); + Mask != nullptr && maskPreservesRegisterUnits(*Mask->Mask, Reg, *Target)) { + Live = true; + } + const auto Physical = physicalRegister(Operand); + if (Physical && Operand.isUse() && !Operand.isDebugUse() && + !Operand.registerFlags().has(RegisterOperandFlag::Undef) && + Target->registers().regsOverlap(*Physical, Reg)) + Live = true; + } + if (Current == &Instruction) + return Live; + } + return true; +} + +Expected<ScavengedRegister> +RegisterScavenger::scavenge(const MachineInstr &Instruction, + std::span<const PhysicalRegister> Candidates) const { + std::optional<PhysicalRegister> SpillCandidate; + for (const PhysicalRegister Candidate : Candidates) { + if (!Candidate.isValid() || isUnavailable(*Function, *Target, Candidate) || + instructionUses(Instruction, Candidate)) { + continue; + } + if (!SpillCandidate) + SpillCandidate = Candidate; + auto Live = isLiveBefore(Instruction, Candidate); + if (!Live) + return std::unexpected(Live.error()); + if (!*Live) + return ScavengedRegister{Candidate, false}; + } + if (SpillCandidate) + return ScavengedRegister{*SpillCandidate, true}; + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "register scavenging found no usable physical register"); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/ScheduleDAG.cpp b/backend/lib/CodeGen/ScheduleDAG.cpp new file mode 100644 index 0000000..9fba862 --- /dev/null +++ b/backend/lib/CodeGen/ScheduleDAG.cpp @@ -0,0 +1,637 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/ScheduleDAG.h" + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineAnalyses.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/RegisterMask.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <span> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct RegisterDependencyKey final { + bool IsUnit = false; + std::uint32_t Value = 0; + + friend constexpr bool operator==(RegisterDependencyKey, RegisterDependencyKey) noexcept = default; +}; + +struct RegisterDependencyKeyHash final { + [[nodiscard]] std::size_t operator()(RegisterDependencyKey Key) const noexcept { + const std::uint64_t Value = (static_cast<std::uint64_t>(Key.IsUnit) << 32U) | Key.Value; + return std::hash<std::uint64_t>{}(Value); + } +}; + +struct RegisterDependencyState final { + struct Definition final { + std::size_t Unit = 0; + MachineInstr *Instruction = nullptr; + std::size_t OperandIndex = 0; + }; + + std::optional<Definition> LastDefinition; + std::vector<std::size_t> LastUses; +}; + +struct RegisterAccess final { + RegisterDependencyKey Key; + MachineInstr *Instruction = nullptr; + std::size_t OperandIndex = 0; +}; + +[[nodiscard]] bool keyLess(RegisterDependencyKey Left, RegisterDependencyKey Right) noexcept { + return Left.IsUnit == Right.IsUnit ? Left.Value < Right.Value : Left.IsUnit < Right.IsUnit; +} + +[[nodiscard]] std::uint32_t saturatingAdd(std::uint32_t Left, std::uint32_t Right) noexcept { + return std::numeric_limits<std::uint32_t>::max() - Left < Right + ? std::numeric_limits<std::uint32_t>::max() + : Left + Right; +} + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits<std::uint64_t>::max() - Left < Right + ? std::numeric_limits<std::uint64_t>::max() + : Left + Right; +} + +[[nodiscard]] Expected<std::vector<RegisterDependencyKey>> +operandKeys(const MachineOperand &Operand, const MachineFunction &Function, + const TargetDescription &Target, ScheduleDAGMode Mode) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return std::vector<RegisterDependencyKey>{}; + if (Reg->isVirtual()) { + if (Mode == ScheduleDAGMode::PostRegisterAllocation) { + return makeUnexpected(ErrorCode::VerificationError, + "post-RA scheduling region contains a virtual register"); + } + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown virtual register"); + } + return std::vector<RegisterDependencyKey>{{false, Reg->opaqueValue()}}; + } + + std::optional<PhysicalRegister> Physical = Reg->physicalRegister(); + if (!Physical) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an invalid physical register"); + if (Operand.subReg().value() != 0) { + const std::optional<PhysicalRegister> SubReg = + Target.registers().subRegister(*Physical, Operand.subReg()); + if (!SubReg) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an invalid physical subregister"); + } + Physical = SubReg; + } + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(*Physical); + if (Description == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown physical register"); + std::vector<RegisterDependencyKey> Result; + Result.reserve(Description->Units.size()); + for (const RegisterUnitLane &Unit : Description->Units) + Result.push_back({true, Unit.Unit.value()}); + std::ranges::sort(Result, {}, &RegisterDependencyKey::Value); + const auto [FirstDuplicate, End] = std::ranges::unique(Result); + Result.erase(FirstDuplicate, End); + return Result; +} + +[[nodiscard]] Expected<std::vector<RegisterDependencyKey>> +maskKeys(const RegisterMask &Mask, const TargetDescription &Target) { + std::vector<RegisterDependencyKey> Result; + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask.clobbers(Register.Reg)) + continue; + for (const RegisterUnitLane &Unit : Register.Units) + Result.push_back({true, Unit.Unit.value()}); + } + std::ranges::sort(Result, {}, &RegisterDependencyKey::Value); + const auto [FirstDuplicate, End] = std::ranges::unique(Result); + Result.erase(FirstDuplicate, End); + return Result; +} + +[[nodiscard]] bool mayAlias(const MachineInstr &Earlier, const MachineInstr &Later, + const MachineAliasAnalysis::Result &Aliases) noexcept { + if (Earlier.memOperands().empty() || Later.memOperands().empty()) + return true; + for (const MachineMemOperand *Left : Earlier.memOperands()) { + if (Left == nullptr) + return true; + for (const MachineMemOperand *Right : Later.memOperands()) { + if (Right == nullptr || Aliases.alias(*Left, *Right) != MachineAliasResult::NoAlias) + return true; + } + } + return false; +} + +[[nodiscard]] bool hasOrderedMemory(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isAtomic() || Memory->isVolatile(); + }); +} + +[[nodiscard]] bool hasLoad(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isLoad(); + }); +} + +[[nodiscard]] bool hasStore(const MachineInstr &Instruction) noexcept { + return std::ranges::any_of(Instruction.memOperands(), [](const MachineMemOperand *Memory) { + return Memory == nullptr || Memory->isStore(); + }); +} + +[[nodiscard]] bool isHazardDependency(ScheduleDependencyKind Kind) noexcept { + return Kind == ScheduleDependencyKind::Anti || Kind == ScheduleDependencyKind::Output; +} + +} // namespace + +Expected<ScheduleDAG> +ScheduleDAG::build(MachineBasicBlock &Block, std::span<MachineInstr *const> Instructions, + const MachineFunction &Function, const TargetDescription &Target, + const TargetSchedulingModel &Model, ScheduleDAGMode Mode, bool EmitDebugOutput) { + if (&Block.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule DAG block belongs to another function"); + ScheduleDAG Result(Block); + auto Units = Result.buildUnits(Instructions, Target, Model); + if (!Units) + return std::unexpected(Units.error()); + auto Registers = Result.buildRegisterDependencies(Function, Target, Model, Mode); + if (!Registers) + return std::unexpected(Registers.error()); + auto Memory = Result.buildMemoryDependencies(Target); + if (!Memory) + return std::unexpected(Memory.error()); + auto Order = Result.buildOrderDependencies(Target); + if (!Order) + return std::unexpected(Order.error()); + auto Mutated = Model.hooks().mutateScheduleDAG(Result, Function, Target, Mode, EmitDebugOutput); + if (!Mutated) + return std::unexpected(Mutated.error()); + auto CriticalPaths = Result.computeCriticalPaths(); + if (!CriticalPaths) + return std::unexpected(CriticalPaths.error()); + return Result; +} + +Expected<void> ScheduleDAG::buildUnits(std::span<MachineInstr *const> Instructions, + const TargetDescription &Target, + const TargetSchedulingModel &Model) { + MachineInstr *ExpectedInstruction = Instructions.empty() ? nullptr : Instructions.front(); + for (std::size_t Index = 0; Index < Instructions.size();) { + MachineInstr *Instruction = Instructions[Index]; + if (Instruction == nullptr || Instruction->parent() != Block) + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule region contains a detached or foreign instruction"); + if (Instruction != ExpectedInstruction) { + return makeUnexpected(ErrorCode::InvalidArgument, + "schedule region is not a contiguous instruction sequence"); + } + if (Instruction->flags().has(MachineInstrFlag::BundledWithPred)) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule region begins in the middle of an instruction bundle"); + } + + ScheduleUnit Unit; + Unit.ID = Units.size(); + Unit.StableID = Instruction->stableID(); + std::unordered_map<ProcessorResourceID, std::uint32_t> ResourceCycles; + for (;;) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + } + const SchedulingClassModelDesc *Class = Model.schedulingClass(Descriptor->SchedulingClass); + if (Class == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction references an unknown scheduling class"); + } + Unit.MicroOps = saturatingAdd(Unit.MicroOps, Class->MicroOps); + Unit.Latency = std::max(Unit.Latency, Class->Latency); + for (const SchedulingResourceUse &Use : Class->Resources) { + ResourceCycles[Use.Resource] = saturatingAdd(ResourceCycles[Use.Resource], Use.Cycles); + } + Unit.Instructions.push_back(Instruction); + const bool HasSuccessor = Instruction->flags().has(MachineInstrFlag::BundledWithSucc); + ++Index; + if (!HasSuccessor) + break; + if (Index >= Instructions.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule region ends in the middle of an instruction bundle"); + } + MachineInstr *Next = Instructions[Index]; + if (Next == nullptr || Next->parent() != Block || + !Next->flags().has(MachineInstrFlag::BundledWithPred) || Instruction->next() != Next) { + return makeUnexpected(ErrorCode::VerificationError, + "instruction bundle links are not reciprocal and contiguous"); + } + Instruction = Next; + } + Unit.Resources.reserve(ResourceCycles.size()); + for (const auto &[Resource, Cycles] : ResourceCycles) + Unit.Resources.push_back({Resource, Cycles}); + std::ranges::sort(Unit.Resources, {}, &SchedulingResourceUse::Resource); + ExpectedInstruction = Unit.Instructions.back()->next(); + Units.push_back(std::move(Unit)); + } + return {}; +} + +Expected<void> ScheduleDAG::buildRegisterDependencies(const MachineFunction &Function, + const TargetDescription &Target, + const TargetSchedulingModel &Model, + ScheduleDAGMode Mode) { + std::unordered_map<RegisterDependencyKey, RegisterDependencyState, RegisterDependencyKeyHash> + States; + for (ScheduleUnit &Unit : Units) { + std::vector<RegisterAccess> Uses; + std::vector<RegisterAccess> Definitions; + for (MachineInstr *Instruction : Unit.Instructions) { + for (std::size_t OperandIndex = 0; OperandIndex < Instruction->operandCount(); + ++OperandIndex) { + const MachineOperand *Operand = Instruction->operandAt(OperandIndex); + if (Operand == nullptr) + continue; + if (const auto *Mask = Operand->getIf<RegisterMaskOperand>()) { + if (!Mask->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered a null register mask"); + auto Keys = maskKeys(*Mask->Mask, Target); + if (!Keys) + return std::unexpected(Keys.error()); + for (const RegisterDependencyKey Key : *Keys) + Definitions.push_back({Key, Instruction, OperandIndex}); + continue; + } + if (const auto *Mask = Operand->getIf<LiveOutRegisterMaskOperand>()) { + if (!Mask->Mask) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered a null live-out register mask"); + for (const PhysicalRegisterDesc &Register : Target.registers().physicalRegisters()) { + if (!Mask->Mask->preserves(Register.Reg)) + continue; + for (const RegisterUnitLane &RegisterUnit : Register.Units) + Uses.push_back({{true, RegisterUnit.Unit.value()}, Instruction, OperandIndex}); + } + continue; + } + if (!Operand->isRegister() || Operand->isDebugUse() || + Operand->registerFlags().has(RegisterOperandFlag::Undef)) { + continue; + } + auto Keys = operandKeys(*Operand, Function, Target, Mode); + if (!Keys) + return std::unexpected(Keys.error()); + if (Operand->isUse()) { + for (const RegisterDependencyKey Key : *Keys) + Uses.push_back({Key, Instruction, OperandIndex}); + } + if (Operand->isDef()) { + for (const RegisterDependencyKey Key : *Keys) + Definitions.push_back({Key, Instruction, OperandIndex}); + } + } + } + std::ranges::stable_sort(Uses, [](const RegisterAccess &Left, const RegisterAccess &Right) { + return keyLess(Left.Key, Right.Key); + }); + std::ranges::stable_sort(Definitions, + [](const RegisterAccess &Left, const RegisterAccess &Right) { + return keyLess(Left.Key, Right.Key); + }); + + for (std::size_t Begin = 0; Begin < Uses.size();) { + std::size_t End = Begin + 1; + while (End < Uses.size() && Uses[End].Key == Uses[Begin].Key) + ++End; + RegisterDependencyState &State = States[Uses[Begin].Key]; + if (State.LastDefinition) { + std::uint32_t Latency = 0; + for (std::size_t Index = Begin; Index < End; ++Index) { + const InstructionDescriptor *Producer = + Target.instructionDescriptor(State.LastDefinition->Instruction->opcode()); + const InstructionDescriptor *Consumer = + Target.instructionDescriptor(Uses[Index].Instruction->opcode()); + if (Producer == nullptr || Consumer == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler latency lookup encountered an unknown instruction"); + } + auto OperandLatency = + Model.operandLatency(Producer->SchedulingClass, State.LastDefinition->OperandIndex, + Consumer->SchedulingClass, Uses[Index].OperandIndex); + if (!OperandLatency) + return std::unexpected(OperandLatency.error()); + Latency = std::max(Latency, *OperandLatency); + } + addDependency(State.LastDefinition->Unit, Unit.ID, + Uses[Begin].Key.IsUnit ? ScheduleDependencyKind::RegisterUnit + : ScheduleDependencyKind::Data, + Latency); + } + if (State.LastUses.empty() || State.LastUses.back() != Unit.ID) + State.LastUses.push_back(Unit.ID); + Begin = End; + } + + for (std::size_t Begin = 0; Begin < Definitions.size();) { + std::size_t End = Begin + 1; + while (End < Definitions.size() && Definitions[End].Key == Definitions[Begin].Key) { + ++End; + } + RegisterDependencyState &State = States[Definitions[Begin].Key]; + if (State.LastDefinition) { + addDependency(State.LastDefinition->Unit, Unit.ID, ScheduleDependencyKind::Output, 0); + } + for (const std::size_t Use : State.LastUses) + addDependency(Use, Unit.ID, ScheduleDependencyKind::Anti, 0); + State.LastUses.clear(); + const RegisterAccess &Last = Definitions[End - 1]; + State.LastDefinition = + RegisterDependencyState::Definition{Unit.ID, Last.Instruction, Last.OperandIndex}; + Begin = End; + } + } + return {}; +} + +Expected<void> ScheduleDAG::addArtificialDependency(std::size_t Source, std::size_t Destination, + std::uint32_t Latency) { + if (Source >= Units.size() || Destination >= Units.size() || Source == Destination) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target schedule mutation requires distinct known schedule units"); + } + if (hasPath(Destination, Source)) + return makeUnexpected(ErrorCode::SchedulingFailed, + "target schedule mutation would introduce a dependency cycle"); + addDependency(Source, Destination, ScheduleDependencyKind::Artificial, Latency); + return {}; +} + +bool ScheduleDAG::hasPath(std::size_t Source, std::size_t Destination) const { + if (Source >= Units.size() || Destination >= Units.size()) + return false; + std::vector<bool> Visited(Units.size()); + std::vector<std::size_t> Pending = {Source}; + Visited[Source] = true; + while (!Pending.empty()) { + const std::size_t Current = Pending.back(); + Pending.pop_back(); + for (const ScheduleDependency &Dependency : Units[Current].Successors) { + if (Dependency.Destination == Destination) + return true; + if (!Visited[Dependency.Destination]) { + Visited[Dependency.Destination] = true; + Pending.push_back(Dependency.Destination); + } + } + } + return false; +} + +bool ScheduleDAG::hasDirectDependency(std::size_t Source, std::size_t Destination) const { + if (Source >= Units.size() || Destination >= Units.size()) + return false; + return std::ranges::any_of(Units[Source].Successors, + [Destination](const ScheduleDependency &Dependency) { + return Dependency.Destination == Destination; + }); +} + +Expected<bool> ScheduleDAG::fuseInstructionPair(std::size_t First, std::size_t Second) { + if (First >= Units.size() || Second >= Units.size() || First == Second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "macro fusion requires two distinct known schedule units"); + } + ScheduleUnit &FirstUnit = Units[First]; + ScheduleUnit &SecondUnit = Units[Second]; + if (FirstUnit.ClusterPredecessor || FirstUnit.ClusterSuccessor || SecondUnit.ClusterPredecessor || + SecondUnit.ClusterSuccessor || hasPath(Second, First)) { + return false; + } + + struct Binding final { + std::size_t Source = 0; + std::size_t Destination = 0; + }; + std::vector<Binding> Bindings; + const std::vector<ScheduleDependency> FirstSuccessors = FirstUnit.Successors; + const std::vector<ScheduleDependency> SecondPredecessors = SecondUnit.Predecessors; + for (const ScheduleDependency &Dependency : FirstSuccessors) { + if (Dependency.Kind == ScheduleDependencyKind::Cluster || isHazardDependency(Dependency.Kind) || + Dependency.Destination == Second || hasDirectDependency(Dependency.Destination, Second)) { + continue; + } + if (hasPath(Dependency.Destination, Second)) + return false; + Bindings.push_back({Second, Dependency.Destination}); + } + for (const ScheduleDependency &Dependency : SecondPredecessors) { + if (Dependency.Kind == ScheduleDependencyKind::Cluster || isHazardDependency(Dependency.Kind) || + Dependency.Source == First || hasDirectDependency(First, Dependency.Source)) { + continue; + } + if (hasPath(First, Dependency.Source)) + return false; + Bindings.push_back({Dependency.Source, First}); + } + + addDependency(First, Second, ScheduleDependencyKind::Cluster, 0); + FirstUnit.ClusterSuccessor = Second; + SecondUnit.ClusterPredecessor = First; + for (ScheduleDependency &Dependency : FirstUnit.Successors) { + if (Dependency.Destination == Second) + Dependency.Latency = 0; + } + for (ScheduleDependency &Dependency : SecondUnit.Predecessors) { + if (Dependency.Source == First) + Dependency.Latency = 0; + } + for (const Binding &Binding : Bindings) + addDependency(Binding.Source, Binding.Destination, ScheduleDependencyKind::Artificial, 0); + return true; +} + +Expected<void> ScheduleDAG::buildMemoryDependencies(const TargetDescription &Target) { + MachineAliasAnalysis::Result Aliases; + for (std::size_t LaterIndex = 0; LaterIndex < Units.size(); ++LaterIndex) { + for (std::size_t EarlierIndex = 0; EarlierIndex < LaterIndex; ++EarlierIndex) { + bool NeedsEdge = false; + for (MachineInstr *Earlier : Units[EarlierIndex].Instructions) { + const InstructionDescriptor *EarlierDescriptor = + Target.instructionDescriptor(Earlier->opcode()); + if (EarlierDescriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + const bool EarlierMemory = EarlierDescriptor->Flags.has(InstructionFlag::MayLoad) || + EarlierDescriptor->Flags.has(InstructionFlag::MayStore) || + !Earlier->memOperands().empty(); + if (!EarlierMemory) + continue; + for (MachineInstr *Later : Units[LaterIndex].Instructions) { + const InstructionDescriptor *LaterDescriptor = + Target.instructionDescriptor(Later->opcode()); + if (LaterDescriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + const bool LaterMemory = LaterDescriptor->Flags.has(InstructionFlag::MayLoad) || + LaterDescriptor->Flags.has(InstructionFlag::MayStore) || + !Later->memOperands().empty(); + if (!LaterMemory) + continue; + if (hasOrderedMemory(*Earlier) || hasOrderedMemory(*Later) || + ((hasStore(*Earlier) || hasStore(*Later)) && mayAlias(*Earlier, *Later, Aliases)) || + (Earlier->memOperands().empty() || Later->memOperands().empty()) || + (!hasLoad(*Earlier) && !hasStore(*Earlier)) || + (!hasLoad(*Later) && !hasStore(*Later))) { + NeedsEdge = true; + break; + } + } + if (NeedsEdge) + break; + } + if (NeedsEdge) + addDependency(EarlierIndex, LaterIndex, ScheduleDependencyKind::Memory, 0); + } + } + return {}; +} + +Expected<void> ScheduleDAG::buildOrderDependencies(const TargetDescription &Target) { + for (std::size_t Barrier = 0; Barrier < Units.size(); ++Barrier) { + bool IsBarrier = false; + for (const MachineInstr *Instruction : Units[Barrier].Instructions) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "scheduler encountered an unknown instruction descriptor"); + } + const InstructionFlags Flags = Descriptor->Flags; + IsBarrier |= Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) || + Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) || + Flags.has(InstructionFlag::Barrier) || + Flags.has(InstructionFlag::HasSideEffects) || + Flags.has(InstructionFlag::UnmodeledSideEffects) || + Flags.has(InstructionFlag::Convergent) || Flags.has(InstructionFlag::Phi) || + Flags.has(InstructionFlag::Meta); + } + if (!IsBarrier) + continue; + for (std::size_t Earlier = 0; Earlier < Barrier; ++Earlier) + addDependency(Earlier, Barrier, ScheduleDependencyKind::Barrier, 0); + for (std::size_t Later = Barrier + 1; Later < Units.size(); ++Later) + addDependency(Barrier, Later, ScheduleDependencyKind::Barrier, 0); + } + return {}; +} + +void ScheduleDAG::addDependency(std::size_t Source, std::size_t Destination, + ScheduleDependencyKind Kind, std::uint32_t Latency) { + if (Source >= Units.size() || Destination >= Units.size() || Source == Destination) + return; + const auto Existing = std::ranges::find_if( + Units[Source].Successors, [Destination, Kind](const ScheduleDependency &Dependency) { + return Dependency.Destination == Destination && Dependency.Kind == Kind; + }); + if (Existing != Units[Source].Successors.end()) { + Existing->Latency = std::max(Existing->Latency, Latency); + const auto Predecessor = std::ranges::find_if( + Units[Destination].Predecessors, [Source, Kind](const ScheduleDependency &Dependency) { + return Dependency.Source == Source && Dependency.Kind == Kind; + }); + if (Predecessor != Units[Destination].Predecessors.end()) + Predecessor->Latency = Existing->Latency; + return; + } + const ScheduleDependency Dependency{Source, Destination, Kind, Latency}; + Units[Source].Successors.push_back(Dependency); + Units[Destination].Predecessors.push_back(Dependency); +} + +Expected<void> ScheduleDAG::computeCriticalPaths() { + std::vector<std::size_t> RemainingPredecessors; + RemainingPredecessors.reserve(Units.size()); + for (const ScheduleUnit &Unit : Units) + RemainingPredecessors.push_back(Unit.Predecessors.size()); + std::vector<std::size_t> TopologicalOrder; + TopologicalOrder.reserve(Units.size()); + std::vector<bool> Emitted(Units.size()); + for (std::size_t Count = 0; Count < Units.size(); ++Count) { + std::optional<std::size_t> Next; + for (const ScheduleUnit &Unit : Units) { + if (!Emitted[Unit.ID] && RemainingPredecessors[Unit.ID] == 0) { + Next = Unit.ID; + break; + } + } + if (!Next) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "schedule DAG contains a cyclic target mutation"); + } + Emitted[*Next] = true; + TopologicalOrder.push_back(*Next); + for (const ScheduleDependency &Dependency : Units[*Next].Successors) { + if (RemainingPredecessors[Dependency.Destination] == 0) { + return makeUnexpected(ErrorCode::InternalError, + "schedule DAG predecessor count underflowed"); + } + --RemainingPredecessors[Dependency.Destination]; + } + } + for (const std::size_t ID : TopologicalOrder) { + ScheduleUnit &Unit = Units[ID]; + Unit.Depth = 0; + for (const ScheduleDependency &Dependency : Unit.Predecessors) { + Unit.Depth = + std::max(Unit.Depth, saturatingAdd(Units[Dependency.Source].Depth, + static_cast<std::uint64_t>(Dependency.Latency))); + } + } + for (auto Position = TopologicalOrder.rbegin(); Position != TopologicalOrder.rend(); ++Position) { + ScheduleUnit &Unit = Units[*Position]; + Unit.Height = Unit.Latency; + for (const ScheduleDependency &Dependency : Unit.Successors) { + Unit.Height = + std::max(Unit.Height, saturatingAdd(Units[Dependency.Destination].Height, + static_cast<std::uint64_t>(Dependency.Latency))); + } + } + for (ScheduleUnit &Unit : Units) { + std::ranges::sort(Unit.Predecessors, {}, &ScheduleDependency::Source); + std::ranges::sort(Unit.Successors, {}, &ScheduleDependency::Destination); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp b/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp new file mode 100644 index 0000000..28c032d --- /dev/null +++ b/backend/lib/CodeGen/ScheduleHazardRecognizer.cpp @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/ScheduleHazardRecognizer.h" + +#include "rcc/CodeGen/ScheduleDAG.h" +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <utility> + +namespace rcc { +namespace { + +[[nodiscard]] std::uint64_t saturatingAdd(std::uint64_t Left, std::uint64_t Right) noexcept { + return std::numeric_limits<std::uint64_t>::max() - Left < Right + ? std::numeric_limits<std::uint64_t>::max() + : Left + Right; +} + +} // namespace + +Expected<ScheduleHazardRecognizer> +ScheduleHazardRecognizer::create(const TargetSchedulingModel &Model, ScheduleDirection Direction) { + ScheduleHazardRecognizer Result(Model, Direction); + Result.TargetHazards = Model.hooks().createHazardRecognizer(Direction); + auto Reset = Result.reset(); + if (!Reset) + return std::unexpected(Reset.error()); + return Result; +} + +ScheduleHazardRecognizer::ScheduleHazardRecognizer(const ScheduleHazardRecognizer &Other) + : Model(Other.Model), Direction(Other.Direction), + TargetHazards(Other.TargetHazards ? Other.TargetHazards->clone() : nullptr), + Resources(Other.Resources), ResourceIndexes(Other.ResourceIndexes), + CurrentCycle(Other.CurrentCycle), DispatchAvailableAt(Other.DispatchAvailableAt), + IssuedMicroOps(Other.IssuedMicroOps), DispatchedUnits(Other.DispatchedUnits) {} + +ScheduleHazardRecognizer & +ScheduleHazardRecognizer::operator=(const ScheduleHazardRecognizer &Other) { + if (this == &Other) + return *this; + Model = Other.Model; + Direction = Other.Direction; + TargetHazards = Other.TargetHazards ? Other.TargetHazards->clone() : nullptr; + Resources = Other.Resources; + ResourceIndexes = Other.ResourceIndexes; + CurrentCycle = Other.CurrentCycle; + DispatchAvailableAt = Other.DispatchAvailableAt; + IssuedMicroOps = Other.IssuedMicroOps; + DispatchedUnits = Other.DispatchedUnits; + return *this; +} + +Expected<void> ScheduleHazardRecognizer::reset() { + Resources.clear(); + ResourceIndexes.clear(); + Resources.reserve(Model->processorResources().size()); + for (const ProcessorResourceDesc &Resource : Model->processorResources()) { + ResourceIndexes.emplace(Resource.ID, Resources.size()); + Resources.push_back({Resource.ID, std::vector<std::uint64_t>(Resource.Units, 0)}); + } + CurrentCycle = 0; + DispatchAvailableAt = 0; + IssuedMicroOps = 0; + DispatchedUnits = 0; + if (TargetHazards != nullptr) { + auto Reset = TargetHazards->reset(); + if (!Reset) + return std::unexpected(Reset.error()); + } + return {}; +} + +Expected<bool> ScheduleHazardRecognizer::canIssue(const ScheduleUnit &Unit, + std::uint64_t Cycle) const { + ScheduleHazardRecognizer Trial = *this; + auto Issued = Trial.issueImplementation(Unit, Cycle); + if (!Issued) { + if (Issued.error().code() == ErrorCode::SchedulingFailed) + return false; + return std::unexpected(Issued.error()); + } + return true; +} + +Expected<std::uint64_t> ScheduleHazardRecognizer::earliestIssueCycle(const ScheduleUnit &Unit, + std::uint64_t Earliest) const { + std::uint64_t Candidate = std::max(Earliest, CurrentCycle); + for (;;) { + auto Available = canIssue(Unit, Candidate); + if (!Available) + return std::unexpected(Available.error()); + if (*Available) + return Candidate; + if (Candidate == std::numeric_limits<std::uint64_t>::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduler resource timeline is exhausted"); + } + ++Candidate; + } +} + +Expected<void> ScheduleHazardRecognizer::issue(const ScheduleUnit &Unit, std::uint64_t Cycle) { + ScheduleHazardRecognizer Trial = *this; + auto Issued = Trial.issueImplementation(Unit, Cycle); + if (!Issued) + return std::unexpected(Issued.error()); + *this = std::move(Trial); + return {}; +} + +Expected<void> ScheduleHazardRecognizer::issueImplementation(const ScheduleUnit &Unit, + std::uint64_t Cycle) { + if (Cycle < CurrentCycle) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduler cannot issue into an earlier cycle"); + } + auto Moved = moveToCycle(Cycle); + if (!Moved) + return std::unexpected(Moved.error()); + if (Cycle < DispatchAvailableAt) { + return makeUnexpected(ErrorCode::SchedulingFailed, "scheduler dispatch pipeline is occupied"); + } + if (TargetHazards != nullptr) { + auto Hazard = TargetHazards->getHazardType(Unit); + if (!Hazard) + return std::unexpected(Hazard.error()); + if (*Hazard != ScheduleHazardType::NoHazard) { + return makeUnexpected(ErrorCode::SchedulingFailed, + *Hazard == ScheduleHazardType::NoopHazard + ? "target hazard requires an unavailable scheduling noop" + : "target dynamic scheduling hazard prevents issue"); + } + } + + const std::uint32_t Width = Model->issueWidth(); + if (Unit.MicroOps <= Width) { + if (Unit.MicroOps > Width - IssuedMicroOps || DispatchedUnits == Model->dispatchLimit()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduler issue or dispatch width is exhausted"); + } + IssuedMicroOps += Unit.MicroOps; + ++DispatchedUnits; + } else { + if (IssuedMicroOps != 0 || DispatchedUnits != 0) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "multi-cycle dispatch requires an empty issue cycle"); + } + const std::uint64_t DispatchCycles = + (static_cast<std::uint64_t>(Unit.MicroOps) + Width - 1U) / Width; + DispatchAvailableAt = saturatingAdd(Cycle, DispatchCycles); + IssuedMicroOps = Width; + DispatchedUnits = 1; + } + + for (const SchedulingResourceUse &Use : Unit.Resources) { + auto Reserved = reserveResource(Use.Resource, Use.Cycles, Cycle); + if (!Reserved) + return std::unexpected(Reserved.error()); + } + if (TargetHazards != nullptr) { + auto Emitted = TargetHazards->emitInstruction(Unit); + if (!Emitted) + return std::unexpected(Emitted.error()); + } + return {}; +} + +Expected<void> ScheduleHazardRecognizer::moveToCycle(std::uint64_t Cycle) { + while (CurrentCycle < Cycle) { + if (TargetHazards != nullptr) { + Expected<void> Advanced = Direction == ScheduleDirection::TopDown + ? TargetHazards->advanceCycle() + : TargetHazards->recedeCycle(); + if (!Advanced) + return std::unexpected(Advanced.error()); + } + ++CurrentCycle; + IssuedMicroOps = 0; + DispatchedUnits = 0; + } + return {}; +} + +Expected<void> ScheduleHazardRecognizer::reserveResource(ProcessorResourceID Resource, + std::uint32_t Cycles, + std::uint64_t Cycle) { + const auto Position = ResourceIndexes.find(Resource); + const ProcessorResourceDesc *Description = Model->processorResource(Resource); + if (Position == ResourceIndexes.end() || Description == nullptr || Cycles == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "schedule unit references an invalid processor resource"); + } + ResourceState &State = Resources[Position->second]; + std::optional<std::size_t> Selected; + for (std::size_t Index = 0; Index < State.AvailableAt.size(); ++Index) { + if (State.AvailableAt[Index] <= Cycle && + (!Selected || State.AvailableAt[Index] < State.AvailableAt[*Selected])) { + Selected = Index; + } + } + if (!Selected) { + return makeUnexpected(ErrorCode::SchedulingFailed, "processor resource has no available unit"); + } + State.AvailableAt[*Selected] = saturatingAdd(Cycle, Cycles); + if (Description->Parent) { + auto Parent = reserveResource(*Description->Parent, Cycles, Cycle); + if (!Parent) + return std::unexpected(Parent.error()); + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/SlotIndexes.cpp b/backend/lib/CodeGen/SlotIndexes.cpp new file mode 100644 index 0000000..4fe753a --- /dev/null +++ b/backend/lib/CodeGen/SlotIndexes.cpp @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/SlotIndexes.h" + +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" + +#include <cstdint> +#include <limits> + +namespace rcc { + +namespace { +constexpr std::uint64_t SlotSpacing = 16; +constexpr std::uint64_t SlotAlignmentMask = 3; +} // namespace + +SlotIndex +SlotIndexesAnalysis::Result::instructionIndex(const MachineInstr &Instruction) const noexcept { + const auto Position = InstructionSlots.find(Instruction.stableID()); + return Position == InstructionSlots.end() ? SlotIndex{} : Position->second; +} + +SlotIndex SlotIndexesAnalysis::Result::blockStart(const MachineBasicBlock &Block) const noexcept { + const auto Position = BlockStarts.find(&Block); + return Position == BlockStarts.end() ? SlotIndex{} : Position->second; +} + +SlotIndex SlotIndexesAnalysis::Result::blockEnd(const MachineBasicBlock &Block) const noexcept { + const auto Position = BlockEnds.find(&Block); + return Position == BlockEnds.end() ? SlotIndex{} : Position->second; +} + +const MachineInstr * +SlotIndexesAnalysis::Result::instructionFromIndex(SlotIndex Index) const noexcept { + if (!Index.isValid()) + return nullptr; + const auto Position = SlotToInstruction.find(Index.baseIndex().value()); + return Position == SlotToInstruction.end() ? nullptr : Position->second; +} + +const MachineBasicBlock * +SlotIndexesAnalysis::Result::blockFromIndex(SlotIndex Index) const noexcept { + if (!Index.isValid()) + return nullptr; + for (const MachineBasicBlock *Block : BlocksByStart) { + const SlotIndex Start = blockStart(*Block); + const SlotIndex End = blockEnd(*Block); + if (!(Index < Start) && Index < End) + return Block; + } + return nullptr; +} + +Expected<void> SlotIndexesAnalysis::Result::insertInstruction(const MachineInstr &Instruction) { + const MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr || !BlockStarts.contains(Block) || !BlockEnds.contains(Block)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "SlotIndex insertion requires a linked indexed instruction"); + } + if (InstructionSlots.contains(Instruction.stableID())) { + return makeUnexpected(ErrorCode::InvalidArgument, "instruction already has a SlotIndex"); + } + + const MachineInstr *Previous = Instruction.previous(); + const MachineInstr *Next = Instruction.next(); + const SlotIndex LowerIndex = + Previous == nullptr ? blockStart(*Block) : instructionIndex(*Previous); + const SlotIndex UpperIndex = Next == nullptr ? blockEnd(*Block) : instructionIndex(*Next); + if (!LowerIndex.isValid() || !UpperIndex.isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "adjacent instructions are missing SlotIndexes"); + } + const std::uint64_t Lower = LowerIndex.baseIndex().value(); + const std::uint64_t Upper = UpperIndex.baseIndex().value(); + if (Upper <= Lower || Upper - Lower <= SlotAlignmentMask + 1U) { + return makeUnexpected(ErrorCode::Unsupported, + "local SlotIndex space is exhausted; renumbering is required"); + } + std::uint64_t Candidate = ((Lower + Upper) / 2U) & ~SlotAlignmentMask; + if (Candidate <= Lower) + Candidate = Lower + SlotAlignmentMask + 1U; + if (Candidate >= Upper) { + return makeUnexpected(ErrorCode::Unsupported, + "local SlotIndex space is exhausted; renumbering is required"); + } + const SlotIndex Index = SlotIndex::fromValue(Candidate).registerSlot(); + InstructionSlots.emplace(Instruction.stableID(), Index); + SlotToInstruction.emplace(Index.baseIndex().value(), &Instruction); + return {}; +} + +Expected<void> SlotIndexesAnalysis::Result::eraseInstruction(const MachineInstr &Instruction) { + const auto Position = InstructionSlots.find(Instruction.stableID()); + if (Position == InstructionSlots.end()) + return makeUnexpected(ErrorCode::InvalidArgument, "instruction has no SlotIndex"); + SlotToInstruction.erase(Position->second.baseIndex().value()); + InstructionSlots.erase(Position); + return {}; +} + +Expected<void> SlotIndexesAnalysis::Result::renumber(MachineFunction &Function) { + Result NewResult; + std::uint64_t Position = SlotSpacing; + auto Advance = [&Position]() -> Expected<void> { + if (Position > std::numeric_limits<std::uint64_t>::max() - SlotSpacing) { + return makeUnexpected(ErrorCode::Unsupported, "machine SlotIndex space is exhausted"); + } + Position += SlotSpacing; + return {}; + }; + + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + const SlotIndex Start = SlotIndex::fromValue(Position).withKind(SlotIndexKind::Block); + NewResult.BlockStarts.emplace(Block, Start); + NewResult.BlocksByStart.push_back(Block); + auto Advanced = Advance(); + if (!Advanced) + return std::unexpected(Advanced.error()); + + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const SlotIndex Index = SlotIndex::fromValue(Position).registerSlot(); + if (!NewResult.InstructionSlots.emplace(Instruction->stableID(), Index).second || + !NewResult.SlotToInstruction.emplace(Index.baseIndex().value(), Instruction).second) { + return makeUnexpected(ErrorCode::VerificationError, + "duplicate instruction identity during SlotIndex construction"); + } + Advanced = Advance(); + if (!Advanced) + return std::unexpected(Advanced.error()); + } + NewResult.BlockEnds.emplace(Block, + SlotIndex::fromValue(Position).withKind(SlotIndexKind::Block)); + Advanced = Advance(); + if (!Advanced) + return std::unexpected(Advanced.error()); + } + *this = std::move(NewResult); + return {}; +} + +bool SlotIndexesAnalysis::Result::invalidate(const MachineFunction &, + const PreservedAnalyses &Preserved) const noexcept { + return !Preserved.preserves(MachineFunctionAnalysisManager::id<SlotIndexesAnalysis>()); +} + +Expected<SlotIndexesAnalysis::Result> SlotIndexesAnalysis::run(MachineFunction &Function, + const TargetDescription &, + MachineFunctionAnalysisManager &) { + Result ResultValue; + auto Numbered = ResultValue.renumber(Function); + if (!Numbered) + return std::unexpected(Numbered.error()); + return ResultValue; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetCallLowering.cpp b/backend/lib/CodeGen/TargetCallLowering.cpp new file mode 100644 index 0000000..adbe406 --- /dev/null +++ b/backend/lib/CodeGen/TargetCallLowering.cpp @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetCallLowering.h" + +#include "rcc/CodeGen/MachineIRBuilder.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] bool isKnownExtension(ABIExtensionKind Extension) noexcept { + return Extension < ABIExtensionKind::Count; +} + +[[nodiscard]] bool isKnownLocationKind(CCLocationKind Kind) noexcept { + return Kind <= CCLocationKind::Custom; +} + +[[nodiscard]] bool isKnownTailCallKind(TailCallKind Kind) noexcept { + return Kind <= TailCallKind::MustTail; +} + +[[nodiscard]] bool addOverflows(std::uint64_t Left, std::uint64_t Right) noexcept { + return Left > std::numeric_limits<std::uint64_t>::max() - Right; +} + +[[nodiscard]] Expected<void> verifyAssignment(const CCAssignment &Assignment, + bool AllowNoRegister) { + if (Assignment.OriginalValue.isNoRegister() && !AllowNoRegister) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering assignment requires a machine register value"); + } + if (!Assignment.ValueType.isValid() || !Assignment.LocationType.isValid() || + !Assignment.ValueType.isScalarValue() || !Assignment.LocationType.isScalarValue()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering assignments require scalar value and location LLTs"); + } + if (Assignment.FragmentCount == 0 || Assignment.FragmentIndex >= Assignment.FragmentCount) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering assignment has invalid fragment coordinates"); + } + if (Assignment.LocationSizeBytes == 0 || Assignment.StackSlotSizeBytes == 0 || + Assignment.LocationSizeBytes > Assignment.StackSlotSizeBytes || + !isKnownExtension(Assignment.Extension) || !isKnownLocationKind(Assignment.LocationKind)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering assignment has invalid size or extension metadata"); + } + + const bool HasRegister = Assignment.LocationRegister.has_value(); + const bool HasStack = Assignment.StackOffsetBytes.has_value(); + if (HasRegister && !Assignment.LocationRegister->isValid()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering assignment names an invalid physical register"); + } + if (HasStack && *Assignment.StackOffsetBytes % Assignment.LocationAlignment.bytes() != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "stack assignment offset violates its location alignment"); + } + switch (Assignment.LocationKind) { + case CCLocationKind::Register: + if (!HasRegister || HasStack) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register assignment must name exactly one physical register"); + } + break; + case CCLocationKind::Stack: + if (HasRegister || !HasStack) { + return makeUnexpected(ErrorCode::InvalidArgument, + "stack assignment must name exactly one stack offset"); + } + break; + case CCLocationKind::Indirect: + if (HasRegister == HasStack || !Assignment.CustomFlags.has(CCAssignmentFlag::Indirect)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "indirect assignment must have one transport location and its flag"); + } + break; + case CCLocationKind::Custom: + if (HasRegister && HasStack) { + return makeUnexpected(ErrorCode::InvalidArgument, + "custom assignment cannot use register and stack simultaneously"); + } + break; + } + return {}; +} + +[[nodiscard]] Expected<void> verifyAssignments(std::span<const CCAssignment> Assignments, + bool AllowNoRegister, + const StackArgumentArea &StackArea) { + std::unordered_set<PhysicalRegister> Registers; + std::vector<std::pair<std::uint64_t, std::uint64_t>> StackRanges; + StackRanges.reserve(Assignments.size()); + for (const CCAssignment &Assignment : Assignments) { + auto Verification = verifyAssignment(Assignment, AllowNoRegister); + if (!Verification) + return std::unexpected(Verification.error()); + if (Assignment.LocationRegister && !Registers.insert(*Assignment.LocationRegister).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering assignments reuse a physical location register"); + } + if (!Assignment.StackOffsetBytes) + continue; + if (addOverflows(*Assignment.StackOffsetBytes, Assignment.StackSlotSizeBytes)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering stack assignment range overflows uint64"); + } + const std::uint64_t End = *Assignment.StackOffsetBytes + Assignment.StackSlotSizeBytes; + if (End > StackArea.SizeBytes) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering stack assignment exceeds its argument area"); + } + StackRanges.emplace_back(*Assignment.StackOffsetBytes, End); + } + + std::ranges::sort(StackRanges); + for (std::size_t Index = 1; Index < StackRanges.size(); ++Index) { + if (StackRanges[Index].first < StackRanges[Index - 1].second) { + return makeUnexpected(ErrorCode::InvalidArgument, "call-lowering stack assignments overlap"); + } + } + return {}; +} + +[[nodiscard]] Expected<void> verifyVarArgs(const VarArgLoweringState &State) { + if (!State.IsVariadic) { + if (State.FixedArgumentCount != 0 || State.FirstUnallocatedIntegerRegister != 0 || + State.OverflowAreaOffsetBytes != 0 || State.RegisterSaveAreaOffsetBytes != 0 || + State.RegisterSaveAreaSizeBytes != 0 || State.RegisterSavePaddingBytes != 0 || + !State.RegistersToSave.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "non-variadic plan contains variadic lowering state"); + } + return {}; + } + + if (State.RegisterSaveAreaSizeBytes == 0 && !State.RegistersToSave.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "variadic register saves require a nonempty save area"); + } + if (State.RegisterSaveAreaSizeBytes % 8 != 0 || State.RegisterSavePaddingBytes % 8 != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "variadic register-save size and padding must be XLEN aligned"); + } + std::unordered_set<PhysicalRegister> SavedRegisters; + std::unordered_set<std::uint64_t> SavedOffsets; + for (const VarArgRegisterSave Save : State.RegistersToSave) { + if (!Save.Reg.isValid() || !SavedRegisters.insert(Save.Reg).second || + Save.OffsetBytes % 8 != 0 || !SavedOffsets.insert(Save.OffsetBytes).second || + addOverflows(Save.OffsetBytes, std::uint64_t{8}) || + Save.OffsetBytes + 8 > State.RegisterSaveAreaSizeBytes) { + return makeUnexpected(ErrorCode::InvalidArgument, + "variadic register-save entry is invalid or outside its area"); + } + } + return {}; +} + +} // namespace + +ABIValueFlags::ABIValueFlags(std::initializer_list<ABIValueFlag> Flags) noexcept { + for (const ABIValueFlag Flag : Flags) + set(Flag); +} + +CCAssignmentFlags::CCAssignmentFlags(std::initializer_list<CCAssignmentFlag> Flags) noexcept { + for (const CCAssignmentFlag Flag : Flags) + set(Flag); +} + +Expected<MachineCallTarget> MachineCallTarget::globalSymbol(std::string Name, + std::uint32_t TargetFlags) { + if (Name.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "global call target name cannot be empty"); + return MachineCallTarget(MachineCallTargetKind::GlobalSymbol, std::move(Name), TargetFlags); +} + +Expected<MachineCallTarget> MachineCallTarget::externalSymbol(std::string Name, + std::uint32_t TargetFlags) { + if (Name.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "external call target name cannot be empty"); + return MachineCallTarget(MachineCallTargetKind::ExternalSymbol, std::move(Name), TargetFlags); +} + +Expected<MachineCallTarget> MachineCallTarget::indirect(Register Callee, + std::uint32_t TargetFlags) { + if (Callee.isNoRegister()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "indirect call target requires a machine register"); + } + return MachineCallTarget(MachineCallTargetKind::IndirectRegister, Callee, TargetFlags); +} + +std::string_view MachineCallTarget::symbolName() const noexcept { + const auto *Name = std::get_if<std::string>(&Value); + return Name == nullptr ? std::string_view{} : std::string_view(*Name); +} + +Register MachineCallTarget::indirectRegister() const noexcept { + const auto *Reg = std::get_if<Register>(&Value); + return Reg == nullptr ? Register{} : *Reg; +} + +Expected<CallLoweringPlan> CallLoweringPlan::create(CallLoweringPlanDesc Description) { + if (Description.Target.empty() || Description.Convention.value() == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering plan requires target and calling-convention identities"); + } + if (Description.Kind > CallLoweringPlanKind::Return || + !isKnownTailCallKind(Description.TailDecision)) { + return makeUnexpected(ErrorCode::InvalidArgument, "call-lowering plan kind is invalid"); + } + if (Description.StackArguments.ShadowSpaceBytes > Description.StackArguments.SizeBytes || + Description.StackArguments.SizeBytes % Description.StackArguments.Align.bytes() != 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering stack argument area is malformed"); + } + if (Description.CallFrame.SizeBytes % Description.CallFrame.Align.bytes() != 0 || + Description.CallFrame.EmitSetup != Description.CallFrame.EmitDestroy) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering call-frame requirements are inconsistent"); + } + + auto ArgumentVerification = + verifyAssignments(Description.ArgumentAssignments, false, Description.StackArguments); + if (!ArgumentVerification) + return std::unexpected(ArgumentVerification.error()); + const bool AllowNoReturnRegister = Description.Kind == CallLoweringPlanKind::Call && + Description.TailDecision != TailCallKind::NotTail; + auto ReturnVerification = verifyAssignments(Description.ReturnAssignments, AllowNoReturnRegister, + StackArgumentArea{0, 0, Alignment{}}); + if (!ReturnVerification) + return std::unexpected(ReturnVerification.error()); + auto VarArgVerification = verifyVarArgs(Description.VarArgs); + if (!VarArgVerification) + return std::unexpected(VarArgVerification.error()); + + switch (Description.Kind) { + case CallLoweringPlanKind::FormalArguments: + if (!Description.ReturnAssignments.empty() || Description.Callee || Description.PreservedMask || + Description.TailDecision != TailCallKind::NotTail || Description.CallFrame.EmitSetup || + Description.CallFrame.ReusesIncomingArea || !Description.ByValCopies.empty() || + !Description.TailRejectionReason.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "formal-argument plan contains call or return state"); + } + break; + case CallLoweringPlanKind::Return: + if (!Description.ArgumentAssignments.empty() || Description.Callee || + Description.PreservedMask || Description.TailDecision != TailCallKind::NotTail || + Description.CallFrame.EmitSetup || Description.CallFrame.ReusesIncomingArea || + !Description.ByValCopies.empty() || Description.VarArgs.IsVariadic || + !Description.TailRejectionReason.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "return plan contains formal-argument or call state"); + } + break; + case CallLoweringPlanKind::Call: + if (!Description.Callee || !Description.Callee->isValid() || !Description.PreservedMask) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call plan requires a callee and preserved mask"); + } + if (Description.TailDecision == TailCallKind::NotTail) { + if (!Description.CallFrame.EmitSetup || Description.CallFrame.ReusesIncomingArea) { + return makeUnexpected(ErrorCode::InvalidArgument, + "ordinary call plan requires a private call frame"); + } + } else if (Description.CallFrame.EmitSetup || !Description.CallFrame.ReusesIncomingArea || + !Description.TailRejectionReason.empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "tail-call plan must reuse the caller's incoming argument area"); + } + if (Description.CallFrame.SizeBytes < Description.StackArguments.SizeBytes) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call frame does not contain the stack-argument area"); + } + break; + } + + std::unordered_set<std::size_t> ByValAssignments; + std::vector<std::pair<std::uint64_t, std::uint64_t>> ByValRanges; + for (const ByValCopyAssignment &Copy : Description.ByValCopies) { + if (Copy.ArgumentAssignmentIndex >= Description.ArgumentAssignments.size() || + !ByValAssignments.insert(Copy.ArgumentAssignmentIndex).second || + Copy.Source.isNoRegister() || !Copy.PointerType.isPointer() || Copy.SizeBytes == 0 || + Copy.StackOffsetBytes % Copy.Align.bytes() != 0 || + Copy.StackOffsetBytes < Description.StackArguments.SizeBytes || + !Description.ArgumentAssignments[Copy.ArgumentAssignmentIndex].CustomFlags.has( + CCAssignmentFlag::ByVal) || + addOverflows(Copy.StackOffsetBytes, Copy.SizeBytes) || + Copy.StackOffsetBytes + Copy.SizeBytes > Description.CallFrame.SizeBytes) { + return makeUnexpected(ErrorCode::InvalidArgument, + "call-lowering byval copy is malformed or outside the call frame"); + } + ByValRanges.emplace_back(Copy.StackOffsetBytes, Copy.StackOffsetBytes + Copy.SizeBytes); + } + std::ranges::sort(ByValRanges); + for (std::size_t Index = 1; Index < ByValRanges.size(); ++Index) { + if (ByValRanges[Index].first < ByValRanges[Index - 1].second) { + return makeUnexpected(ErrorCode::InvalidArgument, "call-lowering byval copy ranges overlap"); + } + } + + return CallLoweringPlan(std::move(Description)); +} + +Expected<void> +TargetCallLowering::lowerFormalArguments(MachineIRBuilder &Builder, + const FormalArgumentRequest &Request) const { + auto Plan = planFormalArguments(Request); + if (!Plan) + return std::unexpected(Plan.error()); + if (Plan->kind() != CallLoweringPlanKind::FormalArguments) { + return makeUnexpected(ErrorCode::InternalError, + "target returned the wrong call-lowering plan kind for formal arguments"); + } + return emitPlan(Builder, *Plan); +} + +Expected<void> TargetCallLowering::lowerCall(MachineIRBuilder &Builder, + const CallLoweringRequest &Request) const { + auto Plan = planCall(Request); + if (!Plan) + return std::unexpected(Plan.error()); + if (Plan->kind() != CallLoweringPlanKind::Call) { + return makeUnexpected(ErrorCode::InternalError, + "target returned the wrong call-lowering plan kind for a call"); + } + return emitPlan(Builder, *Plan); +} + +Expected<void> TargetCallLowering::lowerReturn(MachineIRBuilder &Builder, + const ReturnLoweringRequest &Request) const { + auto Plan = planReturn(Request); + if (!Plan) + return std::unexpected(Plan.error()); + if (Plan->kind() != CallLoweringPlanKind::Return) { + return makeUnexpected(ErrorCode::InternalError, + "target returned the wrong call-lowering plan kind for a return"); + } + return emitPlan(Builder, *Plan); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetConstant.cpp b/backend/lib/CodeGen/TargetConstant.cpp new file mode 100644 index 0000000..8859325 --- /dev/null +++ b/backend/lib/CodeGen/TargetConstant.cpp @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetConstant.h" + +#include <cstddef> +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <unordered_set> +#include <utility> + +namespace rcc { + +Expected<void> TargetConstantEmission::verify() const { + if (Bytes.empty()) + return makeUnexpected(ErrorCode::VerificationError, + "target constant emission must contain at least one byte"); + + std::unordered_set<std::uint64_t> FixupOffsets; + for (const TargetConstantFixup &Fixup : Fixups) { + if (Fixup.Offset >= Bytes.size()) { + return makeUnexpected(ErrorCode::VerificationError, + "target constant fixup offset is outside emitted bytes"); + } + if (Fixup.Kind.value() == 0) + return makeUnexpected(ErrorCode::VerificationError, + "target constant fixup relocation kind must be nonzero"); + if (Fixup.Symbol.empty()) + return makeUnexpected(ErrorCode::VerificationError, + "target constant fixup symbol cannot be empty"); + if (!FixupOffsets.insert(Fixup.Offset).second) { + return makeUnexpected(ErrorCode::VerificationError, + "target constant emission has duplicate fixup offsets"); + } + } + return {}; +} + +TargetConstant::TargetConstant(TargetConstant &&) noexcept = default; +TargetConstant &TargetConstant::operator=(TargetConstant &&) noexcept = default; +TargetConstant::~TargetConstant() = default; + +std::optional<TargetConstantTypeID> TargetConstant::typeID() const noexcept { + if (Storage == nullptr) + return std::nullopt; + return Storage->typeID(); +} + +std::optional<std::uint32_t> TargetConstant::schemaVersion() const noexcept { + if (Storage == nullptr) + return std::nullopt; + return Storage->schemaVersion(); +} + +Expected<void> TargetConstant::verify() const { + if (Storage == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "target constant is empty"); + if (Storage->typeID().value() == 0 || Storage->schemaVersion() == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "target constant identity or schema version is invalid"); + } + return Storage->verify(); +} + +Expected<TargetConstant> TargetConstant::clone() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + auto ClonedStorage = Storage->cloneStorage(); + if (!ClonedStorage) + return std::unexpected(ClonedStorage.error()); + TargetConstant Result(std::move(*ClonedStorage)); + auto CloneVerification = Result.verify(); + if (!CloneVerification) + return std::unexpected(CloneVerification.error()); + return Result; +} + +Expected<std::uint64_t> TargetConstant::stableHash() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return Storage->stableHash(); +} + +Expected<std::string> TargetConstant::print() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return Storage->print(); +} + +Expected<EncodedTargetConstant> TargetConstant::encode() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + auto Payload = Storage->encodePayload(); + if (!Payload) + return std::unexpected(Payload.error()); + return EncodedTargetConstant{Storage->typeID(), Storage->schemaVersion(), std::move(*Payload)}; +} + +Expected<TargetConstantEmission> TargetConstant::emit() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + auto Emission = Storage->emit(); + if (!Emission) + return std::unexpected(Emission.error()); + auto EmissionVerification = Emission->verify(); + if (!EmissionVerification) + return std::unexpected(EmissionVerification.error()); + return Emission; +} + +Expected<Alignment> TargetConstant::requiredAlignment() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return Storage->requiredAlignment(); +} + +Expected<std::optional<TargetRelocationKindID>> TargetConstant::relocationKind() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + const std::optional<TargetRelocationKindID> Kind = Storage->relocationKind(); + if (Kind && Kind->value() == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "target constant relocation kind must be nonzero"); + } + return Kind; +} + +bool TargetConstant::equals(const TargetConstant &Other) const noexcept { + if (Storage == nullptr || Other.Storage == nullptr) + return Storage == nullptr && Other.Storage == nullptr; + if (Storage->typeID() != Other.Storage->typeID() || + Storage->localTypeToken() != Other.Storage->localTypeToken()) { + return false; + } + return Storage->equals(*Other.Storage); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetDescription.cpp b/backend/lib/CodeGen/TargetDescription.cpp new file mode 100644 index 0000000..3701add --- /dev/null +++ b/backend/lib/CodeGen/TargetDescription.cpp @@ -0,0 +1,776 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetDescription.h" + +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineCFIInfo.h" +#include "rcc/CodeGen/MachineFrameInfo.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <span> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] const OperandDescriptor &descriptorForOperand(const InstructionDescriptor &Descriptor, + std::size_t Index) { + if (Index < Descriptor.Operands.size()) + return Descriptor.Operands[Index]; + return Descriptor.Operands.back(); +} + +[[nodiscard]] std::uint64_t minimumTypeSizeInBits(LowLevelType Type) noexcept { + if (Type.isToken()) + return 0; + const std::uint64_t ScalarSize = Type.scalarBitWidth(); + if (!Type.isVector()) + return ScalarSize; + return ScalarSize * Type.minimumElementCount(); +} + +[[nodiscard]] bool isPureDef(const MachineOperand &Operand) noexcept { + return Operand.isRegister() && Operand.isDef() && !Operand.isUse(); +} + +[[nodiscard]] bool isPureUse(const MachineOperand &Operand) noexcept { + return Operand.isRegister() && Operand.isUse() && !Operand.isDef(); +} + +} // namespace + +Expected<TargetDescription> TargetDescription::create(InstructionSet Instructions, + TargetRegisterInfo Registers, + TargetMIRTextInfoDesc MIRTextDescription, + TargetBooleanContentInfo BooleanInfo, + TargetStackInfo StackInfo) { + auto MIRText = TargetMIRTextInfo::create(std::move(MIRTextDescription), Registers); + if (!MIRText) + return std::unexpected(MIRText.error()); + + if (StackInfo.StackPointerRegisterToSaveRestore.isValid() && + Registers.physicalRegister(StackInfo.StackPointerRegisterToSaveRestore) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target stack pointer is not a known physical register"); + } + + for (const InstructionDescriptor &Instruction : Instructions.descriptors()) { + for (const OperandDescriptor &Operand : Instruction.Operands) { + if (Operand.AllowAnyTargetFlags && Operand.AllowedTargetFlags != 0) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "instruction operand cannot combine unrestricted and explicit target flags"); + } + if (!Operand.AllowAnyTargetFlags && + (Operand.AllowedTargetFlags & ~MIRText->serializableOperandFlagMask()) != 0) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "instruction operand allows a target flag outside the MIR text schema"); + } + if (Operand.RegClass && Registers.registerClass(*Operand.RegClass) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction operand references an unknown target register class"); + } + if (Operand.RegBank && Registers.registerBank(*Operand.RegBank) == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction operand references an unknown target register bank"); + } + if (Operand.RegClass && Operand.RegBank && + !Registers.bankCoversClass(*Operand.RegBank, *Operand.RegClass)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "instruction operand register class is not covered by its bank"); + } + } + + auto ValidateImplicit = + [&Registers](const ImplicitRegisterDescriptor &Implicit) -> Expected<void> { + const PhysicalRegisterDesc *Register = Registers.physicalRegister(Implicit.Reg); + if (Register == nullptr) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "instruction implicit operand references an unknown physical register"); + } + if (Implicit.Renamable && (Register->IsConstant || Register->IsArtificial)) { + return makeUnexpected( + ErrorCode::InvalidArgument, + "renamable implicit operand references a constant or artificial register"); + } + return {}; + }; + for (const ImplicitRegisterDescriptor &Implicit : Instruction.ImplicitDefs) { + auto Validation = ValidateImplicit(Implicit); + if (!Validation) + return std::unexpected(Validation.error()); + } + for (const ImplicitRegisterDescriptor &Implicit : Instruction.ImplicitUses) { + auto Validation = ValidateImplicit(Implicit); + if (!Validation) + return std::unexpected(Validation.error()); + } + for (const ImplicitRegisterDescriptor &Def : Instruction.ImplicitDefs) { + if (!Def.EarlyClobber) + continue; + if (std::ranges::any_of(Instruction.ImplicitUses, + [&Registers, Def](const ImplicitRegisterDescriptor &Use) { + return Registers.regsOverlap(Def.Reg, Use.Reg); + })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "early-clobber implicit definition overlaps an implicit use"); + } + } + } + + return TargetDescription(std::move(Instructions), std::move(Registers), std::move(*MIRText), + BooleanInfo, StackInfo); +} + +Expected<void> TargetDescription::validateVirtualRegister(Register Reg, + const VirtualRegisterInfo &Info) const { + if (!Reg.isVirtual()) { + return makeUnexpected(ErrorCode::InternalError, + "target virtual-register validation received a non-virtual register"); + } + + auto TargetFlagValidation = MIRText.validateVirtualRegisterFlags(Info.TargetFlags); + if (!TargetFlagValidation) + return std::unexpected(TargetFlagValidation.error()); + + const RegisterBankDesc *Bank = nullptr; + if (Info.Bank) { + Bank = Registers.registerBank(*Info.Bank); + if (Bank == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register references an unknown target register bank"); + } + } + + const RegisterClassDesc *Class = nullptr; + if (Info.RegClass) { + Class = Registers.registerClass(*Info.RegClass); + if (Class == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register references an unknown target register class"); + } + } + + if (Info.Bank && Info.RegClass && !Registers.bankCoversClass(*Info.Bank, *Info.RegClass)) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register class is not covered by its selected register bank"); + } + + if (Info.Type) { + const std::uint64_t MinimumSize = minimumTypeSizeInBits(*Info.Type); + if (MinimumSize == 0 && (Bank != nullptr || Class != nullptr)) { + return makeUnexpected(ErrorCode::VerificationError, + "token virtual register cannot have a bank or register class"); + } + if (Bank != nullptr && MinimumSize > Bank->MaximumSizeInBits) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register type exceeds its register bank maximum size"); + } + if (Class != nullptr && MinimumSize > Class->SizeInBits) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register type exceeds its register class size"); + } + } + + for (const PhysicalRegister Hint : Info.AllocationHints) { + const PhysicalRegisterDesc *HintDesc = Registers.physicalRegister(Hint); + if (HintDesc == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register allocation hint names an unknown physical register"); + } + if (HintDesc->IsConstant || HintDesc->IsArtificial || HintDesc->IsReserved) { + return makeUnexpected( + ErrorCode::VerificationError, + "virtual register allocation hint names a constant, artificial, or reserved register"); + } + if (Info.RegClass && !Registers.registerClassContains(*Info.RegClass, Hint)) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register allocation hint is outside its register class"); + } + if (!Info.RegClass && Info.Bank && !Registers.bankContainsRegister(*Info.Bank, Hint)) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register allocation hint is outside its register bank"); + } + } + return {}; +} + +std::optional<RegisterClassID> +TargetDescription::effectiveRegisterClass(const MachineOperand &Operand, + const MachineRegisterInfo &RegInfo) const { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return std::nullopt; + + auto Info = RegInfo.virtualRegisterInfo(*Reg); + if (!Info) + return std::nullopt; + const std::optional<RegisterClassID> RegClass = Info->get().RegClass; + if (!RegClass) + return std::nullopt; + if (Operand.subReg().value() == 0) + return RegClass; + return Registers.subRegClass(*RegClass, Operand.subReg()); +} + +Expected<void> TargetDescription::validateTargetOperand(const OperandDescriptor *Descriptor, + const MachineOperand &Operand, + const MachineRegisterInfo &RegInfo) const { + auto TargetFlagValidation = MIRText.validateOperandFlags(Operand.targetFlags()); + if (!TargetFlagValidation) + return std::unexpected(TargetFlagValidation.error()); + + if (const auto *Index = Operand.getIf<TargetIndexOperand>(); + Index != nullptr && !MIRText.targetIndexName(Index->Index)) { + return makeUnexpected(ErrorCode::VerificationError, + "target-index operand is not registered for MIR serialization"); + } + if (const auto *Intrinsic = Operand.getIf<IntrinsicOperand>(); + Intrinsic != nullptr && !MIRText.intrinsicName(Intrinsic->ID)) { + return makeUnexpected(ErrorCode::VerificationError, + "intrinsic operand is not registered for MIR serialization"); + } + + if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); Mask != nullptr) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "register-mask operand contains a null mask"); + } + return Registers.validateRegisterMask(*Mask->Mask); + } + if (const auto *Mask = Operand.getIf<LiveOutRegisterMaskOperand>(); Mask != nullptr) { + if (!Mask->Mask) { + return makeUnexpected(ErrorCode::VerificationError, + "live-out register-mask operand contains a null mask"); + } + return Registers.validateRegisterMask(*Mask->Mask); + } + + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr) + return {}; + if (Reg->isNoRegister()) { + if (Operand.subReg().value() != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "noreg operand cannot carry a subregister index"); + } + if (Operand.registerFlags().has(RegisterOperandFlag::Renamable)) { + return makeUnexpected(ErrorCode::VerificationError, "noreg operand cannot be renamable"); + } + return {}; + } + + if (const auto PhysReg = Reg->physicalRegister()) { + const PhysicalRegister PhysRegValue = *PhysReg; + const PhysicalRegisterDesc *PhysRegDesc = Registers.physicalRegister(PhysRegValue); + if (PhysRegDesc == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "operand references an unknown physical register"); + } + if (Operand.subReg().value() != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "physical register operand cannot carry a subregister index"); + } + if (Descriptor != nullptr && Descriptor->RegClass && + !Registers.registerClassContains(*Descriptor->RegClass, PhysRegValue)) { + return makeUnexpected(ErrorCode::VerificationError, + "physical register operand is outside its required register class"); + } + if (Descriptor != nullptr && Descriptor->RegBank && + !Registers.bankContainsRegister(*Descriptor->RegBank, PhysRegValue)) { + return makeUnexpected(ErrorCode::VerificationError, + "physical register operand is outside its required register bank"); + } + const bool IsReservedOrAlias = std::ranges::any_of( + RegInfo.reservedRegisters(), [this, PhysRegValue](PhysicalRegister Reserved) { + return Registers.regsOverlap(Reserved, PhysRegValue); + }); + if (Operand.registerFlags().has(RegisterOperandFlag::Renamable) && + (IsReservedOrAlias || PhysRegDesc->IsConstant || PhysRegDesc->IsArtificial || + PhysRegDesc->IsReserved)) { + return makeUnexpected(ErrorCode::VerificationError, + "fixed physical register operand cannot be renamable"); + } + return {}; + } + + if (Operand.registerFlags().has(RegisterOperandFlag::Renamable)) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register operand cannot be renamable"); + } + + auto Info = RegInfo.virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + auto RegisterValidation = validateVirtualRegister(*Reg, Info->get()); + if (!RegisterValidation) + return std::unexpected(RegisterValidation.error()); + + if (Operand.subReg().value() != 0) { + if (Registers.subRegIndex(Operand.subReg()) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "operand references an unknown subregister index"); + } + const std::optional<RegisterClassID> RegClass = Info->get().RegClass; + if (!RegClass) { + return makeUnexpected(ErrorCode::VerificationError, + "generic virtual register cannot carry a subregister index"); + } + if (!Registers.subRegClass(*RegClass, Operand.subReg())) { + return makeUnexpected( + ErrorCode::VerificationError, + "virtual register class does not support the operand subregister index"); + } + } + + if (Descriptor != nullptr && Descriptor->RegBank && Info->get().Bank != Descriptor->RegBank) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register bank does not match the operand descriptor"); + } + if (Descriptor != nullptr && Descriptor->RegClass) { + const auto EffectiveClass = effectiveRegisterClass(Operand, RegInfo); + if (!EffectiveClass || !Registers.isSubclass(*EffectiveClass, *Descriptor->RegClass)) { + const std::string ActualClass = + EffectiveClass ? std::to_string(EffectiveClass->value()) : "<none>"; + return makeUnexpected(ErrorCode::VerificationError, + "virtual register class " + ActualClass + + " does not satisfy required class " + + std::to_string(Descriptor->RegClass->value())); + } + } + return {}; +} + +Expected<void> TargetDescription::validateMemOperands( + Opcode InstructionOpcode, std::span<const MachineMemOperand *const> MemOperands) const { + auto DescriptorValidation = Instructions.validateMemOperands(InstructionOpcode, MemOperands); + if (!DescriptorValidation) + return std::unexpected(DescriptorValidation.error()); + for (const MachineMemOperand *MemOperand : MemOperands) { + if (MemOperand == nullptr) + continue; + auto TargetFlagValidation = + MIRText.validateMachineMemOperandFlags(MemOperand->description().TargetFlags); + if (!TargetFlagValidation) + return std::unexpected(TargetFlagValidation.error()); + } + return {}; +} + +Expected<void> TargetDescription::validateCFIInfo(const MachineCFIInfo &Info) const { + auto ValidateRegister = [this](DwarfRegisterNumber Register) -> Expected<void> { + if (!MIRText.physicalRegister(Register.value())) { + return makeUnexpected(ErrorCode::VerificationError, + "CFI instruction references an unmapped DWARF register"); + } + return {}; + }; + + for (const MachineCFIInstruction &Instruction : Info.instructions()) { + if (const auto *Rule = Instruction.getIf<MachineCFIRegisterRule>()) { + auto Result = ValidateRegister(Rule->Register); + if (!Result) + return Result; + } + if (const auto *Rule = Instruction.getIf<MachineCFIRegisterOffsetRule>()) { + auto Result = ValidateRegister(Rule->Register); + if (!Result) + return Result; + } + if (const auto *Rule = Instruction.getIf<MachineCFIRegisterRegisterRule>()) { + auto RegisterResult = ValidateRegister(Rule->Register); + if (!RegisterResult) + return RegisterResult; + auto SourceResult = ValidateRegister(Rule->SourceRegister); + if (!SourceResult) + return SourceResult; + } + if (const auto *Rule = Instruction.getIf<MachineCFIAddressSpaceCFARule>()) { + auto Result = ValidateRegister(Rule->Register); + if (!Result) + return Result; + } + if (const auto *Rule = Instruction.getIf<MachineCFIRegisterPairRule>()) { + for (const DwarfRegisterNumber Register : + {Rule->Register, Rule->FirstRegister, Rule->SecondRegister}) { + auto Result = ValidateRegister(Register); + if (!Result) + return Result; + } + } + if (const auto *Rule = Instruction.getIf<MachineCFIVectorRegistersRule>()) { + auto Result = ValidateRegister(Rule->Register); + if (!Result) + return Result; + for (const MachineCFIVectorRegisterLane &Lane : Rule->VectorRegisters) { + Result = ValidateRegister(Lane.Register); + if (!Result) + return Result; + } + } + if (const auto *Rule = Instruction.getIf<MachineCFIVectorOffsetRule>()) { + auto RegisterResult = ValidateRegister(Rule->Register); + if (!RegisterResult) + return RegisterResult; + auto MaskResult = ValidateRegister(Rule->MaskRegister); + if (!MaskResult) + return MaskResult; + } + if (const auto *Rule = Instruction.getIf<MachineCFIVectorRegisterMaskRule>()) { + for (const DwarfRegisterNumber Register : + {Rule->Register, Rule->SpillRegister, Rule->MaskRegister}) { + auto Result = ValidateRegister(Register); + if (!Result) + return Result; + } + } + } + return {}; +} + +Expected<void> TargetDescription::validateFrameInfo(const MachineFrameInfo &Info) const { + for (const StackObject &Object : Info.objects()) { + const StackObjectDesc *Description = Object.description(); + if (Description != nullptr && !MIRText.supportsStackID(Description->Stack)) { + return makeUnexpected(ErrorCode::VerificationError, + "stack object uses a stack ID unsupported by the target"); + } + } + return {}; +} + +Expected<void> +TargetDescription::validateExplicitOperands(Opcode InstructionOpcode, + std::span<const MachineOperand> Operands, + const MachineRegisterInfo &RegInfo) const { + auto StructuralValidation = + Instructions.validateExplicitOperands(InstructionOpcode, Operands, RegInfo); + if (!StructuralValidation) + return std::unexpected(StructuralValidation.error()); + + const InstructionDescriptor *Descriptor = Instructions.descriptor(InstructionOpcode); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, "instruction opcode is not registered"); + } + for (std::size_t Index = 0; Index < Operands.size(); ++Index) { + auto TargetValidation = + validateTargetOperand(&descriptorForOperand(*Descriptor, Index), Operands[Index], RegInfo); + if (!TargetValidation) + return std::unexpected( + withErrorContext(TargetValidation.error(), "operand " + std::to_string(Index) + " of '" + + Descriptor->Name + "': ")); + } + return {}; +} + +Expected<void> +TargetDescription::validateTieCompatibility(const MachineOperand &Def, const MachineOperand &Use, + const MachineRegisterInfo &RegInfo) const { + const Register *DefReg = Def.getIf<Register>(); + const Register *UseReg = Use.getIf<Register>(); + if (DefReg == nullptr || UseReg == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "target tie validation received a non-register operand"); + } + if (DefReg->isNoRegister() || UseReg->isNoRegister()) + return {}; + + const auto DefPhys = DefReg->physicalRegister(); + const auto UsePhys = UseReg->physicalRegister(); + if (DefPhys && UsePhys) { + if (*DefPhys != *UsePhys) { + return makeUnexpected(ErrorCode::VerificationError, + "tied physical register operands must name the same register"); + } + return {}; + } + + const auto DefClass = effectiveRegisterClass(Def, RegInfo); + const auto UseClass = effectiveRegisterClass(Use, RegInfo); + if (DefClass && UseClass && !Registers.registerClassesOverlap(*DefClass, *UseClass)) { + return makeUnexpected(ErrorCode::VerificationError, + "tied virtual register classes have no compatible physical register"); + } + + auto BankFor = [&RegInfo](Register Reg) -> std::optional<RegisterBankID> { + if (!Reg.isVirtual()) + return std::nullopt; + auto Info = RegInfo.virtualRegisterInfo(Reg); + return Info ? Info->get().Bank : std::nullopt; + }; + const auto DefBank = BankFor(*DefReg); + const auto UseBank = BankFor(*UseReg); + + auto ClassIntersectsBank = [this](RegisterClassID ClassID, RegisterBankID BankID) { + const RegisterClassDesc *Class = Registers.registerClass(ClassID); + return Class != nullptr && + std::ranges::any_of(Class->Registers, [this, BankID](PhysicalRegister Reg) { + return Registers.bankContainsRegister(BankID, Reg); + }); + }; + + if (DefPhys && UseClass && !Registers.registerClassContains(*UseClass, *DefPhys)) { + return makeUnexpected(ErrorCode::VerificationError, + "tied physical register is outside the virtual use register class"); + } + if (UsePhys && DefClass && !Registers.registerClassContains(*DefClass, *UsePhys)) { + return makeUnexpected(ErrorCode::VerificationError, + "tied physical register is outside the virtual def register class"); + } + if (DefPhys && UseBank && !Registers.bankContainsRegister(*UseBank, *DefPhys)) { + return makeUnexpected(ErrorCode::VerificationError, + "tied physical register is outside the virtual use register bank"); + } + if (UsePhys && DefBank && !Registers.bankContainsRegister(*DefBank, *UsePhys)) { + return makeUnexpected(ErrorCode::VerificationError, + "tied physical register is outside the virtual def register bank"); + } + if (DefClass && UseBank && !ClassIntersectsBank(*DefClass, *UseBank)) { + return makeUnexpected(ErrorCode::VerificationError, + "tied def register class and use register bank do not intersect"); + } + if (UseClass && DefBank && !ClassIntersectsBank(*UseClass, *DefBank)) { + return makeUnexpected(ErrorCode::VerificationError, + "tied use register class and def register bank do not intersect"); + } + if (!DefClass && !UseClass && DefBank && UseBank && + !Registers.registerBanksOverlap(*DefBank, *UseBank)) { + return makeUnexpected(ErrorCode::VerificationError, + "tied virtual register banks have no compatible physical register"); + } + return {}; +} + +Expected<void> TargetDescription::validateOperandTies( + Opcode InstructionOpcode, std::span<const MachineOperand> ExplicitOperands, + std::span<const MachineOperandTie> Ties, const MachineRegisterInfo &RegInfo) const { + auto StructuralValidation = + Instructions.validateOperandTies(InstructionOpcode, ExplicitOperands, Ties); + if (!StructuralValidation) + return std::unexpected(StructuralValidation.error()); + + for (const MachineOperandTie Tie : Ties) { + auto Compatibility = validateTieCompatibility(ExplicitOperands[Tie.DefOperand], + ExplicitOperands[Tie.UseOperand], RegInfo); + if (!Compatibility) + return std::unexpected(Compatibility.error()); + } + return {}; +} + +Expected<void> +TargetDescription::validateImplicitOperands(Opcode InstructionOpcode, + std::span<const MachineOperand *const> ImplicitOperands, + const MachineRegisterInfo &RegInfo) const { + auto StructuralValidation = + Instructions.validateImplicitOperands(InstructionOpcode, ImplicitOperands); + if (!StructuralValidation) + return std::unexpected(StructuralValidation.error()); + + for (const MachineOperand *Operand : ImplicitOperands) { + if (Operand == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "implicit operand sequence contains a null operand"); + } + auto TargetValidation = validateTargetOperand(nullptr, *Operand, RegInfo); + if (!TargetValidation) + return std::unexpected(TargetValidation.error()); + } + return {}; +} + +Expected<void> +TargetDescription::validateEarlyClobber(std::span<const MachineOperand> ExplicitOperands, + std::span<const MachineOperand *const> ImplicitOperands, + std::span<const MachineOperandTie> Ties) const { + struct IndexedOperand final { + const MachineOperand *Operand = nullptr; + std::optional<std::size_t> ExplicitIndex; + }; + + std::vector<IndexedOperand> Operands; + Operands.reserve(ExplicitOperands.size() + ImplicitOperands.size()); + for (std::size_t Index = 0; Index < ExplicitOperands.size(); ++Index) + Operands.push_back(IndexedOperand{&ExplicitOperands[Index], Index}); + for (const MachineOperand *Operand : ImplicitOperands) + Operands.push_back(IndexedOperand{Operand, std::nullopt}); + + auto AreTied = [Ties](const IndexedOperand &Def, const IndexedOperand &Use) { + if (!Def.ExplicitIndex || !Use.ExplicitIndex) + return false; + return std::ranges::any_of(Ties, [&Def, &Use](MachineOperandTie Tie) { + return Tie.DefOperand == *Def.ExplicitIndex && Tie.UseOperand == *Use.ExplicitIndex; + }); + }; + + for (const IndexedOperand &DefEntry : Operands) { + if (DefEntry.Operand == nullptr) + continue; + const MachineOperand &Def = *DefEntry.Operand; + if (!Def.isDef() || !Def.registerFlags().has(RegisterOperandFlag::EarlyClobber)) + continue; + + const Register *DefReg = Def.getIf<Register>(); + if (DefReg == nullptr || DefReg->isNoRegister()) + continue; + for (const IndexedOperand &UseEntry : Operands) { + if (UseEntry.Operand == nullptr || UseEntry.Operand == DefEntry.Operand) + continue; + const MachineOperand &Use = *UseEntry.Operand; + if (!Use.isUse() || AreTied(DefEntry, UseEntry)) + continue; + + const Register *UseReg = Use.getIf<Register>(); + if (UseReg == nullptr || UseReg->isNoRegister()) + continue; + if (DefReg->isVirtual() && UseReg->isVirtual() && *DefReg == *UseReg) { + return makeUnexpected(ErrorCode::VerificationError, + "early-clobber virtual definition overlaps an untied use"); + } + + const auto DefPhys = DefReg->physicalRegister(); + const auto UsePhys = UseReg->physicalRegister(); + if (DefPhys && UsePhys && Registers.regsOverlap(*DefPhys, *UsePhys)) { + return makeUnexpected(ErrorCode::VerificationError, + "early-clobber physical definition overlaps an untied use"); + } + } + } + return {}; +} + +Expected<void> TargetDescription::validateInstruction(const MachineInstr &Instruction, + const MachineRegisterInfo &RegInfo) const { + auto StructuralValidation = Instructions.validateInstruction(Instruction, RegInfo); + if (!StructuralValidation) + return std::unexpected(StructuralValidation.error()); + + const auto Operands = Instruction.operands(); + std::size_t ExplicitCount = 0; + while (ExplicitCount < Operands.size() && !Operands[ExplicitCount].isImplicit()) { + ++ExplicitCount; + } + + auto ExplicitValidation = + validateExplicitOperands(Instruction.opcode(), Operands.first(ExplicitCount), RegInfo); + if (!ExplicitValidation) + return std::unexpected(ExplicitValidation.error()); + + std::vector<const MachineOperand *> ImplicitOperands; + for (std::size_t Index = ExplicitCount; Index < Operands.size(); ++Index) + ImplicitOperands.push_back(&Operands[Index]); + auto ImplicitValidation = + validateImplicitOperands(Instruction.opcode(), ImplicitOperands, RegInfo); + if (!ImplicitValidation) + return std::unexpected(ImplicitValidation.error()); + + std::vector<MachineOperandTie> Ties; + for (std::size_t Index = 0; Index < ExplicitCount; ++Index) { + const MachineOperand &Operand = Operands[Index]; + if (!Operand.tiedOperandIndex() || Index > *Operand.tiedOperandIndex()) { + continue; + } + const std::size_t OtherIndex = *Operand.tiedOperandIndex(); + if (isPureDef(Operand) && isPureUse(Operands[OtherIndex])) { + Ties.push_back(MachineOperandTie{Index, OtherIndex}); + } else { + Ties.push_back(MachineOperandTie{OtherIndex, Index}); + } + } + auto TieValidation = + validateOperandTies(Instruction.opcode(), Operands.first(ExplicitCount), Ties, RegInfo); + if (!TieValidation) + return std::unexpected(TieValidation.error()); + + return validateEarlyClobber(Operands.first(ExplicitCount), ImplicitOperands, Ties); +} + +Expected<void> TargetDescription::validateRegisterInfo(const MachineRegisterInfo &RegInfo) const { + for (const PhysicalRegisterLiveIn &LiveIn : RegInfo.liveIns()) { + if (!Registers.isKnownPhysicalRegister(LiveIn.PhysReg)) { + return makeUnexpected(ErrorCode::VerificationError, + "function live-in names an unknown physical register"); + } + if (!LiveIn.InitialVirtualRegister) + continue; + auto Info = RegInfo.virtualRegisterInfo(*LiveIn.InitialVirtualRegister); + if (!Info) + return std::unexpected(Info.error()); + const std::optional<RegisterClassID> RegClass = Info->get().RegClass; + if (RegClass && !Registers.registerClassContains(*RegClass, LiveIn.PhysReg)) { + return makeUnexpected( + ErrorCode::VerificationError, + "function live-in physical register is outside its initial virtual register class"); + } + const std::optional<RegisterBankID> Bank = Info->get().Bank; + if (Bank && !Registers.bankContainsRegister(*Bank, LiveIn.PhysReg)) { + return makeUnexpected( + ErrorCode::VerificationError, + "function live-in physical register is outside its initial virtual register bank"); + } + } + for (const PhysicalRegister Reg : RegInfo.reservedRegisters()) { + if (!Registers.isKnownPhysicalRegister(Reg)) { + return makeUnexpected(ErrorCode::VerificationError, + "reserved-register list names an unknown physical register"); + } + } + + const auto VirtualRegisters = RegInfo.virtualRegisters(); + for (std::size_t Index = 0; Index < VirtualRegisters.size(); ++Index) { + const VirtualRegisterInfo &Info = VirtualRegisters[Index]; + if (Info.IsTombstone) + continue; + auto Reg = Register::makeVirtual(Index); + if (!Reg) + return std::unexpected(Reg.error()); + auto Validation = validateVirtualRegister(*Reg, Info); + if (!Validation) + return std::unexpected(Validation.error()); + for (const PhysicalRegister Hint : Info.AllocationHints) { + const bool IsStaticReservedOrAlias = std::ranges::any_of( + Registers.physicalRegisters(), [this, Hint](const PhysicalRegisterDesc &Register) { + return Register.IsReserved && Registers.regsOverlap(Register.Reg, Hint); + }); + const bool IsFunctionReservedOrAlias = + std::ranges::any_of(RegInfo.reservedRegisters(), [this, Hint](PhysicalRegister Reserved) { + return Registers.regsOverlap(Reserved, Hint); + }); + if (IsStaticReservedOrAlias || IsFunctionReservedOrAlias) { + return makeUnexpected( + ErrorCode::VerificationError, + "virtual register allocation hint overlaps a reserved physical register"); + } + } + } + return {}; +} + +Expected<void> TargetDescription::validateBlockLiveIns(const MachineBasicBlock &Block) const { + for (const MachineBasicBlockLiveIn &LiveIn : Block.liveIns()) { + if (!Registers.isKnownPhysicalRegister(LiveIn.PhysReg)) { + return makeUnexpected(ErrorCode::VerificationError, + "block live-in names an unknown physical register"); + } + if (!Registers.isValidLaneMask(LiveIn.PhysReg, LiveIn.Lanes)) { + return makeUnexpected(ErrorCode::VerificationError, + "block live-in lane mask is not valid for its physical register"); + } + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetFunctionState.cpp b/backend/lib/CodeGen/TargetFunctionState.cpp new file mode 100644 index 0000000..56ff363 --- /dev/null +++ b/backend/lib/CodeGen/TargetFunctionState.cpp @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetFunctionState.h" + +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <utility> + +namespace rcc { + +TargetFunctionState::TargetFunctionState(TargetFunctionState &&) noexcept = default; +TargetFunctionState &TargetFunctionState::operator=(TargetFunctionState &&) noexcept = default; +TargetFunctionState::~TargetFunctionState() = default; + +std::optional<TargetFunctionStateTypeID> TargetFunctionState::typeID() const noexcept { + if (Storage == nullptr) + return std::nullopt; + return Storage->typeID(); +} + +std::optional<std::uint32_t> TargetFunctionState::schemaVersion() const noexcept { + if (Storage == nullptr) + return std::nullopt; + return Storage->schemaVersion(); +} + +Expected<void> TargetFunctionState::verify() const { + if (Storage == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "target function state is empty"); + if (Storage->typeID().value() == 0 || Storage->schemaVersion() == 0) { + return makeUnexpected(ErrorCode::VerificationError, + "target function state identity or schema version is invalid"); + } + return Storage->verify(); +} + +Expected<TargetFunctionState> +TargetFunctionState::clone(const MachineFunctionCloneContext &CloneContext) const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + auto ClonedStorage = Storage->cloneStorage(CloneContext); + if (!ClonedStorage) + return std::unexpected(ClonedStorage.error()); + TargetFunctionState Result(std::move(*ClonedStorage)); + auto CloneVerification = Result.verify(); + if (!CloneVerification) + return std::unexpected(CloneVerification.error()); + return Result; +} + +Expected<std::string> TargetFunctionState::print() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return Storage->print(); +} + +Expected<EncodedTargetFunctionState> TargetFunctionState::encode() const { + auto Verification = verify(); + if (!Verification) + return std::unexpected(Verification.error()); + auto Payload = Storage->encodePayload(); + if (!Payload) + return std::unexpected(Payload.error()); + return EncodedTargetFunctionState{Storage->typeID(), Storage->schemaVersion(), + std::move(*Payload)}; +} + +bool TargetFunctionState::equals(const TargetFunctionState &Other) const noexcept { + if (Storage == nullptr || Other.Storage == nullptr) + return Storage == nullptr && Other.Storage == nullptr; + if (Storage->typeID() != Other.Storage->typeID() || + Storage->localTypeToken() != Other.Storage->localTypeToken()) { + return false; + } + return Storage->equals(*Other.Storage); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetInstrInfo.cpp b/backend/lib/CodeGen/TargetInstrInfo.cpp new file mode 100644 index 0000000..14e56ad --- /dev/null +++ b/backend/lib/CodeGen/TargetInstrInfo.cpp @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetInstrInfo.h" + +namespace rcc { + +TargetInstrInfo::~TargetInstrInfo() = default; + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetMIRTextInfo.cpp b/backend/lib/CodeGen/TargetMIRTextInfo.cpp new file mode 100644 index 0000000..938f513 --- /dev/null +++ b/backend/lib/CodeGen/TargetMIRTextInfo.cpp @@ -0,0 +1,545 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetMIRTextInfo.h" + +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <bit> +#include <cstddef> +#include <cstdint> +#include <string> +#include <string_view> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] std::unexpected<Error> invalidDescription(std::string Message) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target MIR text description: " + std::move(Message)); +} + +[[nodiscard]] bool isASCIIAlpha(char Character) noexcept { + return (Character >= 'a' && Character <= 'z') || (Character >= 'A' && Character <= 'Z'); +} + +[[nodiscard]] bool isASCIIDigit(char Character) noexcept { + return Character >= '0' && Character <= '9'; +} + +[[nodiscard]] bool isMIRIdentifier(std::string_view Name) noexcept { + if (Name.empty() || (!isASCIIAlpha(Name.front()) && Name.front() != '_')) + return false; + for (const char Character : Name) { + if (!isASCIIAlpha(Character) && !isASCIIDigit(Character) && Character != '_' && + Character != '-' && Character != '.' && Character != '$') { + return false; + } + } + return true; +} + +[[nodiscard]] std::string lowerASCII(std::string_view Name) { + std::string Result(Name); + for (char &Character : Result) { + if (Character >= 'A' && Character <= 'Z') + Character = static_cast<char>(Character - 'A' + 'a'); + } + return Result; +} + +} // namespace + +MIRTargetNode::MIRTargetNode() = default; +MIRTargetNode::MIRTargetNode(const MIRTargetNode &) = default; +MIRTargetNode &MIRTargetNode::operator=(const MIRTargetNode &) = default; +MIRTargetNode::MIRTargetNode(MIRTargetNode &&) noexcept = default; +MIRTargetNode &MIRTargetNode::operator=(MIRTargetNode &&) noexcept = default; +MIRTargetNode::~MIRTargetNode() = default; + +MIRTargetNode MIRTargetNode::null() { + return {}; +} + +MIRTargetNode MIRTargetNode::scalar(std::string Value) { + MIRTargetNode Result; + Result.Kind = MIRTargetNodeKind::Scalar; + Result.Scalar = std::move(Value); + return Result; +} + +MIRTargetNode MIRTargetNode::sequence(std::vector<MIRTargetNode> Values) { + MIRTargetNode Result; + Result.Kind = MIRTargetNodeKind::Sequence; + Result.Sequence = std::move(Values); + return Result; +} + +MIRTargetNode MIRTargetNode::mapping(std::vector<MIRTargetMappingEntry> Entries) { + MIRTargetNode Result; + Result.Kind = MIRTargetNodeKind::Mapping; + Result.Mapping = std::move(Entries); + return Result; +} + +bool operator==(const MIRTargetNode &Left, const MIRTargetNode &Right) { + return Left.Kind == Right.Kind && Left.Scalar == Right.Scalar && + Left.Sequence == Right.Sequence && Left.Mapping == Right.Mapping; +} + +const MIRTargetNode *MIRTargetNode::mappingValue(std::string_view Key) const noexcept { + if (Kind != MIRTargetNodeKind::Mapping) + return nullptr; + for (const MIRTargetMappingEntry &Entry : Mapping) { + if (Entry.Key == Key) + return &Entry.Value; + } + return nullptr; +} + +Expected<void> MIRTargetNode::verify() const { + if (Kind >= MIRTargetNodeKind::Count) + return makeUnexpected(ErrorCode::InvalidArgument, "target MIR node kind is invalid"); + + const bool HasScalar = !Scalar.empty(); + const bool HasSequence = !Sequence.empty(); + const bool HasMapping = !Mapping.empty(); + if ((Kind != MIRTargetNodeKind::Scalar && HasScalar) || + (Kind != MIRTargetNodeKind::Sequence && HasSequence) || + (Kind != MIRTargetNodeKind::Mapping && HasMapping)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target MIR node contains storage for a different node kind"); + } + + if (Kind == MIRTargetNodeKind::Sequence) { + for (const MIRTargetNode &Element : Sequence) { + auto ElementResult = Element.verify(); + if (!ElementResult) + return ElementResult; + } + } + if (Kind == MIRTargetNodeKind::Mapping) { + std::unordered_set<std::string_view> Keys; + for (const MIRTargetMappingEntry &Entry : Mapping) { + if (Entry.Key.empty() || !Keys.insert(Entry.Key).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target MIR mapping keys must be nonempty and unique"); + } + auto ValueResult = Entry.Value.verify(); + if (!ValueResult) + return ValueResult; + } + } + return {}; +} + +Expected<TargetMIRTextInfo> TargetMIRTextInfo::create(TargetMIRTextInfoDesc Description, + const TargetRegisterInfo &Registers) { + Description.Intrinsics.push_back({"llvm.va_copy", intrinsicID(GenericIntrinsic::VACopy)}); + TargetMIRTextInfo Result(std::move(Description)); + + for (std::size_t Index = 0; Index < Result.Description.RegisterMasks.size(); ++Index) { + MIRRegisterMaskDesc &Mask = Result.Description.RegisterMasks[Index]; + Mask.Name = lowerASCII(Mask.Name); + if (!isMIRIdentifier(Mask.Name) || Mask.Mask == nullptr) { + return invalidDescription("named register masks require an identifier and nonnull mask"); + } + auto Validation = Registers.validateRegisterMask(*Mask.Mask); + if (!Validation) + return invalidDescription("named register mask contains an unknown physical register"); + if (!Result.RegisterMaskNameToIndex.emplace(Mask.Name, Index).second) { + return invalidDescription("register mask names must be unique case-insensitively"); + } + for (std::size_t Previous = 0; Previous < Index; ++Previous) { + if (*Result.Description.RegisterMasks[Previous].Mask == *Mask.Mask) { + return invalidDescription("named register mask values must have one canonical name"); + } + } + } + + for (std::size_t Index = 0; Index < Result.Description.TargetIndices.size(); ++Index) { + const MIRTargetIndexDesc &TargetIndex = Result.Description.TargetIndices[Index]; + if (!isMIRIdentifier(TargetIndex.Name)) { + return invalidDescription("target index names must be MIR identifiers"); + } + if (!Result.TargetIndexNameToIndex.emplace(TargetIndex.Name, Index).second || + !Result.TargetIndexToIndex.emplace(TargetIndex.Index, Index).second) { + return invalidDescription("target index names and values must be unique"); + } + } + + if (Result.Description.DirectOperandFlags.empty() != + (Result.Description.DirectOperandFlagMask == 0)) { + return invalidDescription( + "direct operand flags and the direct operand flag mask must be present together"); + } + + std::unordered_set<std::string> OperandFlagNames; + for (std::size_t Index = 0; Index < Result.Description.DirectOperandFlags.size(); ++Index) { + const MIROperandFlagDesc &Flag = Result.Description.DirectOperandFlags[Index]; + if (!isMIRIdentifier(Flag.Name) || Flag.Value == 0 || + (Flag.Value & ~Result.Description.DirectOperandFlagMask) != 0) { + return invalidDescription( + "direct operand flags require an identifier and a nonzero value inside the direct mask"); + } + if (!OperandFlagNames.insert(Flag.Name).second || + !Result.DirectOperandFlagNameToIndex.emplace(Flag.Name, Index).second || + !Result.DirectOperandFlagToIndex.emplace(Flag.Value, Index).second) { + return invalidDescription("direct operand flag names and values must be unique"); + } + } + + std::uint32_t BitmaskOperandFlags = 0; + for (std::size_t Index = 0; Index < Result.Description.BitmaskOperandFlags.size(); ++Index) { + const MIROperandFlagDesc &Flag = Result.Description.BitmaskOperandFlags[Index]; + if (!isMIRIdentifier(Flag.Name) || !std::has_single_bit(Flag.Value) || + (Flag.Value & Result.Description.DirectOperandFlagMask) != 0) { + return invalidDescription( + "bitmask operand flags require an identifier and one bit outside the direct mask"); + } + if (!OperandFlagNames.insert(Flag.Name).second || + !Result.BitmaskOperandFlagNameToIndex.emplace(Flag.Name, Index).second || + (BitmaskOperandFlags & Flag.Value) != 0) { + return invalidDescription("operand flag names and bitmask values must be unique"); + } + BitmaskOperandFlags |= Flag.Value; + } + Result.SerializableOperandFlagMask = + Result.Description.DirectOperandFlagMask | BitmaskOperandFlags; + + for (std::size_t Index = 0; Index < Result.Description.MachineMemOperandFlags.size(); ++Index) { + const MIRMachineMemOperandFlagDesc &Flag = Result.Description.MachineMemOperandFlags[Index]; + if (!isMIRIdentifier(Flag.Name) || !std::has_single_bit(Flag.Value)) { + return invalidDescription( + "machine memory operand flags require an identifier and a single-bit value"); + } + if (!Result.MachineMemOperandFlagNameToIndex.emplace(Flag.Name, Index).second || + (Result.SerializableMachineMemOperandFlagMask & Flag.Value) != 0) { + return invalidDescription("machine memory operand flag names and values must be unique"); + } + Result.SerializableMachineMemOperandFlagMask |= Flag.Value; + } + + for (std::size_t Index = 0; Index < Result.Description.VirtualRegisterFlags.size(); ++Index) { + const MIRVirtualRegisterFlagDesc &Flag = Result.Description.VirtualRegisterFlags[Index]; + if (!isMIRIdentifier(Flag.Name) || !std::has_single_bit(Flag.Value)) { + return invalidDescription( + "virtual register flags require an identifier and a single-bit value"); + } + if (!Result.VirtualRegisterFlagNameToIndex.emplace(Flag.Name, Index).second || + (Result.SerializableVirtualRegisterFlagMask & Flag.Value) != 0) { + return invalidDescription("virtual register flag names and values must be unique"); + } + Result.SerializableVirtualRegisterFlagMask |= Flag.Value; + } + + for (std::size_t Index = 0; Index < Result.Description.Intrinsics.size(); ++Index) { + const MIRIntrinsicDesc &Intrinsic = Result.Description.Intrinsics[Index]; + if (!isMIRIdentifier(Intrinsic.Name) || !Intrinsic.Name.starts_with("llvm.") || + Intrinsic.ID.value() == 0) { + return invalidDescription("intrinsics require an llvm-prefixed MIR name and a nonzero ID"); + } + if (!Result.IntrinsicNameToIndex.emplace(Intrinsic.Name, Index).second || + !Result.IntrinsicToIndex.emplace(Intrinsic.ID, Index).second) { + return invalidDescription("intrinsic names and IDs must be unique"); + } + } + + std::unordered_set<std::uint32_t> ReferencedDwarfRegisters; + for (const MIRCFIRegisterMappingDesc &Mapping : Result.Description.CFIRegisters) { + if (!Registers.isKnownPhysicalRegister(Mapping.Register) || + !Result.RegisterToDwarf.emplace(Mapping.Register, Mapping.DwarfRegister).second) { + return invalidDescription("CFI register mappings require unique, known physical registers"); + } + ReferencedDwarfRegisters.insert(Mapping.DwarfRegister); + if (Mapping.IsCanonical && + !Result.DwarfToRegister.emplace(Mapping.DwarfRegister, Mapping.Register).second) { + return invalidDescription( + "each DWARF register must have exactly one canonical physical register"); + } + } + if (ReferencedDwarfRegisters.size() != Result.DwarfToRegister.size()) { + return invalidDescription( + "every referenced DWARF register requires a canonical physical register"); + } + + for (const StackID ID : Result.Description.SupportedStackIDs) { + if (!ID.isValid() || !Result.SupportedStackIDs.insert(ID.value()).second) + return invalidDescription("supported stack IDs must be valid and unique"); + } + if (!Result.SupportedStackIDs.contains(StackID::defaultStack().value()) || + !Result.SupportedStackIDs.contains(StackID::noAllocStack().value())) { + return invalidDescription("targets must support the default and noalloc stack IDs"); + } + + if (Result.Description.FunctionStateType && Result.Description.FunctionStateType->value() == 0) { + return invalidDescription("target function state TypeID must be nonzero"); + } + const bool HasStateType = Result.Description.FunctionStateType.has_value(); + const bool HasParser = Result.Description.ParseFunctionState != nullptr; + const bool HasPrinter = Result.Description.PrintFunctionState != nullptr; + if (HasStateType != HasParser || HasStateType != HasPrinter) { + return invalidDescription( + "target function state TypeID, parser, and printer must be provided together"); + } + const bool HasConstantParser = Result.Description.ParseTargetConstant != nullptr; + const bool HasConstantPrinter = Result.Description.PrintTargetConstant != nullptr; + if (HasConstantParser != HasConstantPrinter) { + return invalidDescription("target constant parser and printer must be provided together"); + } + + return Result; +} + +std::shared_ptr<const RegisterMask> +TargetMIRTextInfo::registerMask(std::string_view Name) const noexcept { + const auto Position = RegisterMaskNameToIndex.find(std::string(Name)); + if (Position == RegisterMaskNameToIndex.end()) + return {}; + return Description.RegisterMasks[Position->second].Mask; +} + +std::optional<std::string_view> +TargetMIRTextInfo::registerMaskName(const RegisterMask &Mask) const noexcept { + for (const MIRRegisterMaskDesc &NamedMask : Description.RegisterMasks) { + if (*NamedMask.Mask == Mask) + return NamedMask.Name; + } + return std::nullopt; +} + +std::optional<TargetIndex> TargetMIRTextInfo::targetIndex(std::string_view Name) const noexcept { + const auto Position = TargetIndexNameToIndex.find(std::string(Name)); + if (Position == TargetIndexNameToIndex.end()) + return std::nullopt; + return Description.TargetIndices[Position->second].Index; +} + +std::optional<std::string_view> +TargetMIRTextInfo::targetIndexName(TargetIndex Index) const noexcept { + const auto Position = TargetIndexToIndex.find(Index); + if (Position == TargetIndexToIndex.end()) + return std::nullopt; + return Description.TargetIndices[Position->second].Name; +} + +std::optional<std::uint32_t> +TargetMIRTextInfo::directOperandFlag(std::string_view Name) const noexcept { + const auto Position = DirectOperandFlagNameToIndex.find(std::string(Name)); + if (Position == DirectOperandFlagNameToIndex.end()) + return std::nullopt; + return Description.DirectOperandFlags[Position->second].Value; +} + +std::optional<std::uint32_t> +TargetMIRTextInfo::bitmaskOperandFlag(std::string_view Name) const noexcept { + const auto Position = BitmaskOperandFlagNameToIndex.find(std::string(Name)); + if (Position == BitmaskOperandFlagNameToIndex.end()) + return std::nullopt; + return Description.BitmaskOperandFlags[Position->second].Value; +} + +Expected<void> TargetMIRTextInfo::validateOperandFlags(std::uint32_t Flags) const { + if ((Flags & ~SerializableOperandFlagMask) != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "machine operand contains an unknown target flag bit"); + } + const std::uint32_t DirectFlag = Flags & Description.DirectOperandFlagMask; + if (DirectFlag != 0 && !DirectOperandFlagToIndex.contains(DirectFlag)) { + return makeUnexpected(ErrorCode::VerificationError, + "machine operand contains an unregistered direct target flag"); + } + return {}; +} + +Expected<std::vector<std::string_view>> +TargetMIRTextInfo::operandFlagNames(std::uint32_t Flags) const { + auto Validation = validateOperandFlags(Flags); + if (!Validation) + return std::unexpected(Validation.error()); + + std::vector<std::string_view> Names; + const std::uint32_t DirectFlag = Flags & Description.DirectOperandFlagMask; + if (DirectFlag != 0) { + Names.push_back(Description.DirectOperandFlags[DirectOperandFlagToIndex.at(DirectFlag)].Name); + } + for (const MIROperandFlagDesc &Flag : Description.BitmaskOperandFlags) { + if ((Flags & Flag.Value) != 0) + Names.push_back(Flag.Name); + } + return Names; +} + +std::optional<std::uint16_t> +TargetMIRTextInfo::machineMemOperandFlag(std::string_view Name) const noexcept { + const auto Position = MachineMemOperandFlagNameToIndex.find(std::string(Name)); + if (Position == MachineMemOperandFlagNameToIndex.end()) + return std::nullopt; + return Description.MachineMemOperandFlags[Position->second].Value; +} + +Expected<void> TargetMIRTextInfo::validateMachineMemOperandFlags(std::uint16_t Flags) const { + if ((Flags & static_cast<std::uint16_t>(~SerializableMachineMemOperandFlagMask)) != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "machine memory operand contains an unknown target flag"); + } + return {}; +} + +Expected<std::vector<std::string_view>> +TargetMIRTextInfo::machineMemOperandFlagNames(std::uint16_t Flags) const { + auto Validation = validateMachineMemOperandFlags(Flags); + if (!Validation) + return std::unexpected(Validation.error()); + + std::vector<std::string_view> Names; + for (const MIRMachineMemOperandFlagDesc &Flag : Description.MachineMemOperandFlags) { + if ((Flags & Flag.Value) != 0) + Names.push_back(Flag.Name); + } + return Names; +} + +std::optional<std::uint8_t> +TargetMIRTextInfo::virtualRegisterFlag(std::string_view Name) const noexcept { + const auto Position = VirtualRegisterFlagNameToIndex.find(std::string(Name)); + if (Position == VirtualRegisterFlagNameToIndex.end()) + return std::nullopt; + return Description.VirtualRegisterFlags[Position->second].Value; +} + +Expected<void> TargetMIRTextInfo::validateVirtualRegisterFlags(std::uint8_t Flags) const { + if ((Flags & static_cast<std::uint8_t>(~SerializableVirtualRegisterFlagMask)) != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "virtual register contains an unknown target flag"); + } + return {}; +} + +Expected<std::vector<std::string_view>> +TargetMIRTextInfo::virtualRegisterFlagNames(std::uint8_t Flags) const { + auto Validation = validateVirtualRegisterFlags(Flags); + if (!Validation) + return std::unexpected(Validation.error()); + + std::vector<std::string_view> Names; + for (const MIRVirtualRegisterFlagDesc &Flag : Description.VirtualRegisterFlags) { + if ((Flags & Flag.Value) != 0) + Names.push_back(Flag.Name); + } + return Names; +} + +std::optional<IntrinsicID> TargetMIRTextInfo::intrinsic(std::string_view Name) const noexcept { + const auto Position = IntrinsicNameToIndex.find(std::string(Name)); + if (Position == IntrinsicNameToIndex.end()) + return std::nullopt; + return Description.Intrinsics[Position->second].ID; +} + +std::optional<std::string_view> TargetMIRTextInfo::intrinsicName(IntrinsicID ID) const noexcept { + const auto Position = IntrinsicToIndex.find(ID); + if (Position == IntrinsicToIndex.end()) + return std::nullopt; + return Description.Intrinsics[Position->second].Name; +} + +std::optional<std::uint32_t> +TargetMIRTextInfo::dwarfRegister(PhysicalRegister Register) const noexcept { + const auto Position = RegisterToDwarf.find(Register); + if (Position == RegisterToDwarf.end()) + return std::nullopt; + return Position->second; +} + +std::optional<PhysicalRegister> +TargetMIRTextInfo::physicalRegister(std::uint32_t DwarfRegister) const noexcept { + const auto Position = DwarfToRegister.find(DwarfRegister); + if (Position == DwarfToRegister.end()) + return std::nullopt; + return Position->second; +} + +Expected<TargetFunctionState> +TargetMIRTextInfo::parseFunctionState(const MIRTargetNode &Node) const { + if (Description.ParseFunctionState == nullptr || !Description.FunctionStateType) { + return makeUnexpected(ErrorCode::Unsupported, + "target does not define a machineFunctionInfo MIR schema"); + } + auto NodeResult = Node.verify(); + if (!NodeResult) + return std::unexpected(NodeResult.error()); + auto State = Description.ParseFunctionState(Node); + if (!State) + return std::unexpected(State.error()); + auto StateResult = State->verify(); + if (!StateResult) + return std::unexpected(StateResult.error()); + if (State->typeID() != Description.FunctionStateType) { + return makeUnexpected(ErrorCode::ParseError, + "parsed target function state has the wrong schema TypeID"); + } + return State; +} + +Expected<MIRTargetNode> +TargetMIRTextInfo::printFunctionState(const TargetFunctionState &State) const { + if (Description.PrintFunctionState == nullptr || !Description.FunctionStateType) { + return makeUnexpected(ErrorCode::Unsupported, + "target does not define a machineFunctionInfo MIR schema"); + } + auto StateResult = State.verify(); + if (!StateResult) + return std::unexpected(StateResult.error()); + if (State.typeID() != Description.FunctionStateType) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target function state has the wrong MIR schema TypeID"); + } + auto Node = Description.PrintFunctionState(State); + if (!Node) + return std::unexpected(Node.error()); + auto NodeResult = Node->verify(); + if (!NodeResult) + return std::unexpected(NodeResult.error()); + if (Node->Kind != MIRTargetNodeKind::Mapping) { + return makeUnexpected(ErrorCode::InvalidArgument, + "machineFunctionInfo printer must produce a mapping"); + } + return Node; +} + +Expected<TargetConstant> TargetMIRTextInfo::parseTargetConstant(std::string_view Text) const { + if (Description.ParseTargetConstant == nullptr) { + return makeUnexpected(ErrorCode::Unsupported, + "target does not define a target constant MIR schema"); + } + auto Constant = Description.ParseTargetConstant(Text); + if (!Constant) + return std::unexpected(Constant.error()); + auto Verification = Constant->verify(); + if (!Verification) + return std::unexpected(Verification.error()); + return Constant; +} + +Expected<std::string> TargetMIRTextInfo::printTargetConstant(const TargetConstant &Constant) const { + if (Description.PrintTargetConstant == nullptr) { + return makeUnexpected(ErrorCode::Unsupported, + "target does not define a target constant MIR schema"); + } + auto Verification = Constant.verify(); + if (!Verification) + return std::unexpected(Verification.error()); + auto Text = Description.PrintTargetConstant(Constant); + if (!Text) + return std::unexpected(Text.error()); + if (Text->empty()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target constant printer produced an empty MIR value"); + } + return Text; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetRegisterInfo.cpp b/backend/lib/CodeGen/TargetRegisterInfo.cpp new file mode 100644 index 0000000..b6631eb --- /dev/null +++ b/backend/lib/CodeGen/TargetRegisterInfo.cpp @@ -0,0 +1,689 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetRegisterInfo.h" + +#include <algorithm> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <ranges> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] std::unexpected<Error> invalidDescription(std::string Message) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target register description: " + std::move(Message)); +} + +[[nodiscard]] std::string lowerASCII(std::string_view Name) { + std::string Result(Name); + for (char &Character : Result) { + if (Character >= 'A' && Character <= 'Z') + Character = static_cast<char>(Character - 'A' + 'a'); + } + return Result; +} + +template <typename RangeT, typename ValueT> +[[nodiscard]] bool contains(const RangeT &Range, const ValueT &Value) { + return std::ranges::find(Range, Value) != std::ranges::end(Range); +} + +[[nodiscard]] const RegisterUnitLane *findUnit(const PhysicalRegisterDesc &Register, + RegisterUnitID Unit) noexcept { + const auto Position = std::ranges::find(Register.Units, Unit, &RegisterUnitLane::Unit); + return Position == Register.Units.end() ? nullptr : &*Position; +} + +} // namespace + +Expected<TargetRegisterInfo> TargetRegisterInfo::create(TargetRegisterInfoDesc Description) { + if (Description.RegisterUnits.empty() || Description.PhysicalRegisters.empty() || + Description.RegisterClasses.empty() || Description.RegisterBanks.empty()) { + return invalidDescription( + "register units, physical registers, classes, and banks must be nonempty"); + } + + TargetRegisterInfo Result(std::move(Description)); + + std::unordered_set<std::string> UnitNames; + for (std::size_t Index = 0; Index < Result.Description.RegisterUnits.size(); ++Index) { + const RegisterUnitDesc &Unit = Result.Description.RegisterUnits[Index]; + if (Unit.ID.value() == 0 || Unit.Name.empty()) { + return invalidDescription("register unit requires a nonzero ID and name"); + } + if (!Result.UnitToIndex.emplace(Unit.ID, Index).second || !UnitNames.insert(Unit.Name).second) { + return invalidDescription("register unit IDs and names must be unique"); + } + } + + for (std::size_t Index = 0; Index < Result.Description.PhysicalRegisters.size(); ++Index) { + const PhysicalRegisterDesc &Register = Result.Description.PhysicalRegisters[Index]; + if (!Register.Reg.isValid() || Register.Name.empty()) { + return invalidDescription("physical register requires a valid number and name"); + } + std::string MIRName = lowerASCII(Register.Name); + if (!Result.RegisterToIndex.emplace(Register.Reg, Index).second || + Result.AlternativeRegisterNameToIndex.contains(MIRName) || + !Result.RegisterNameToIndex.emplace(MIRName, Index).second) { + return invalidDescription( + "physical register numbers and names must be unique case-insensitively"); + } + for (const std::string &AlternativeName : Register.AlternativeNames) { + const std::string NormalizedName = lowerASCII(AlternativeName); + if (AlternativeName.empty() || Result.RegisterNameToIndex.contains(NormalizedName) || + !Result.AlternativeRegisterNameToIndex.emplace(NormalizedName, Index).second) { + return invalidDescription( + "physical register alternative names must be nonempty and unique case-insensitively"); + } + } + Result.RegisterMIRNames.push_back(std::move(MIRName)); + } + + std::unordered_set<std::string> SubRegNames; + for (std::size_t Index = 0; Index < Result.Description.SubRegIndices.size(); ++Index) { + const SubRegIndexDesc &SubReg = Result.Description.SubRegIndices[Index]; + if (SubReg.ID.value() == 0 || SubReg.Name.empty() || SubReg.Lanes.empty() || + SubReg.SizeInBits == 0 || + static_cast<std::uint64_t>(SubReg.OffsetInBits) + SubReg.SizeInBits > + std::numeric_limits<std::uint32_t>::max()) { + return invalidDescription( + "subregister index requires a nonzero ID, name, lanes, and valid bit range"); + } + if (!Result.SubRegIndexToIndex.emplace(SubReg.ID, Index).second || + !SubRegNames.insert(SubReg.Name).second || + !Result.SubRegNameToIndex.emplace(SubReg.Name, Index).second) { + return invalidDescription("subregister index IDs and names must be unique"); + } + } + + for (std::size_t Index = 0; Index < Result.Description.RegisterClasses.size(); ++Index) { + const RegisterClassDesc &Class = Result.Description.RegisterClasses[Index]; + if (Class.ID.value() == 0 || Class.Name.empty()) { + return invalidDescription("register class requires a nonzero ID and name"); + } + std::string MIRName = lowerASCII(Class.Name); + if (!Result.ClassToIndex.emplace(Class.ID, Index).second || + !Result.ClassNameToIndex.emplace(MIRName, Index).second) { + return invalidDescription("register class IDs and names must be unique case-insensitively"); + } + Result.ClassMIRNames.push_back(std::move(MIRName)); + } + + for (std::size_t Index = 0; Index < Result.Description.RegisterBanks.size(); ++Index) { + const RegisterBankDesc &Bank = Result.Description.RegisterBanks[Index]; + if (Bank.ID.value() == 0 || Bank.Name.empty() || Bank.MaximumSizeInBits == 0) { + return invalidDescription("register bank requires a nonzero ID, name, and maximum size"); + } + std::string MIRName = lowerASCII(Bank.Name); + if (!Result.BankToIndex.emplace(Bank.ID, Index).second || + !Result.BankNameToIndex.emplace(MIRName, Index).second) { + return invalidDescription("register bank IDs and names must be unique case-insensitively"); + } + Result.BankMIRNames.push_back(std::move(MIRName)); + } + + std::unordered_set<RegisterUnitID> ReferencedUnits; + std::unordered_map<RegisterUnitID, int> UnitLaneCardinalities; + for (const PhysicalRegisterDesc &Register : Result.Description.PhysicalRegisters) { + if (Register.Units.empty()) + return invalidDescription("physical register must reference at least one register unit"); + + std::unordered_set<RegisterUnitID> SeenUnits; + LaneMask RegisterLanes; + for (const RegisterUnitLane &Unit : Register.Units) { + if (Result.registerUnit(Unit.Unit) == nullptr || Unit.Lanes.empty() || + !SeenUnits.insert(Unit.Unit).second) { + return invalidDescription( + "physical register has an unknown, empty-lane, or duplicate register unit"); + } + const int Cardinality = std::popcount(Unit.Lanes.rawValue()); + const auto [Position, Inserted] = UnitLaneCardinalities.emplace(Unit.Unit, Cardinality); + if (!Inserted && Position->second != Cardinality) { + return invalidDescription( + "every physical-register view of a register unit must have the same lane count"); + } + ReferencedUnits.insert(Unit.Unit); + RegisterLanes = RegisterLanes | Unit.Lanes; + } + if (RegisterLanes.empty()) + return invalidDescription("physical register has no addressable lanes"); + } + if (ReferencedUnits.size() != Result.Description.RegisterUnits.size()) + return invalidDescription("every declared register unit must be used by a physical register"); + + const std::size_t RegisterCount = Result.Description.PhysicalRegisters.size(); + std::vector<std::vector<bool>> IsSubRegister(RegisterCount, + std::vector<bool>(RegisterCount, false)); + for (const PhysicalSubRegisterDesc &Mapping : Result.Description.PhysicalSubRegisters) { + const PhysicalRegisterDesc *Super = Result.physicalRegister(Mapping.SuperReg); + const PhysicalRegisterDesc *Sub = Result.physicalRegister(Mapping.SubReg); + const SubRegIndexDesc *Index = Result.subRegIndex(Mapping.Index); + if (Super == nullptr || Sub == nullptr || Index == nullptr || + Mapping.SuperReg == Mapping.SubReg) { + return invalidDescription( + "physical subregister mapping references an unknown or identical register"); + } + + const std::uint64_t Key = pairKey(Mapping.SuperReg.number(), Mapping.Index.value()); + if (!Result.SubRegisterMap.emplace(Key, Mapping.SubReg).second) + return invalidDescription("physical subregister mapping key must be unique"); + + LaneMask ProjectedLanes; + for (const RegisterUnitLane &SubUnit : Sub->Units) { + const RegisterUnitLane *SuperUnit = findUnit(*Super, SubUnit.Unit); + if (SuperUnit == nullptr) { + return invalidDescription( + "physical subregister units must be a subset of its superregister units"); + } + ProjectedLanes = ProjectedLanes | SuperUnit->Lanes; + } + if (ProjectedLanes != Index->Lanes) { + return invalidDescription( + "physical subregister units do not match the subregister index lane mask"); + } + + const std::size_t SuperIndex = Result.RegisterToIndex.at(Mapping.SuperReg); + const std::size_t SubIndex = Result.RegisterToIndex.at(Mapping.SubReg); + IsSubRegister[SuperIndex][SubIndex] = true; + } + + for (std::size_t Through = 0; Through < RegisterCount; ++Through) { + for (std::size_t Super = 0; Super < RegisterCount; ++Super) { + if (!IsSubRegister[Super][Through]) + continue; + for (std::size_t Sub = 0; Sub < RegisterCount; ++Sub) + IsSubRegister[Super][Sub] = IsSubRegister[Super][Sub] || IsSubRegister[Through][Sub]; + } + } + for (std::size_t Index = 0; Index < RegisterCount; ++Index) { + if (IsSubRegister[Index][Index]) + return invalidDescription("physical subregister graph must be acyclic"); + } + + Result.SuperRegisters.resize(RegisterCount); + for (std::size_t Sub = 0; Sub < RegisterCount; ++Sub) { + for (std::size_t Super = 0; Super < RegisterCount; ++Super) { + if (IsSubRegister[Super][Sub]) { + Result.SuperRegisters[Sub].push_back(Result.Description.PhysicalRegisters[Super].Reg); + } + } + } + + for (const SubRegIndexComposition &Composition : Result.Description.SubRegCompositions) { + const SubRegIndexDesc *First = Result.subRegIndex(Composition.First); + const SubRegIndexDesc *Second = Result.subRegIndex(Composition.Second); + const SubRegIndexDesc *Composed = Result.subRegIndex(Composition.Result); + if (First == nullptr || Second == nullptr || Composed == nullptr) { + return invalidDescription("subregister composition references an unknown index"); + } + const std::uint64_t ComposedOffset = + static_cast<std::uint64_t>(First->OffsetInBits) + Second->OffsetInBits; + if (static_cast<std::uint64_t>(Second->OffsetInBits) + Second->SizeInBits > First->SizeInBits || + Composed->SizeInBits != Second->SizeInBits || Composed->OffsetInBits != ComposedOffset) { + return invalidDescription("subregister composition has inconsistent bit ranges"); + } + const std::uint64_t Key = pairKey(Composition.First.value(), Composition.Second.value()); + if (!Result.CompositionMap.emplace(Key, Composition.Result).second) + return invalidDescription("subregister composition key must be unique"); + } + + for (const SubRegIndexComposition &Composition : Result.Description.SubRegCompositions) { + for (const PhysicalRegisterDesc &Super : Result.Description.PhysicalRegisters) { + const auto Intermediate = Result.subRegister(Super.Reg, Composition.First); + if (!Intermediate) + continue; + const auto Final = Result.subRegister(*Intermediate, Composition.Second); + if (!Final) + continue; + const auto Composed = Result.subRegister(Super.Reg, Composition.Result); + if (!Composed || *Composed != *Final) { + return invalidDescription( + "subregister composition conflicts with physical subregister mappings"); + } + } + } + + for (const RegisterBankDesc &Bank : Result.Description.RegisterBanks) { + if (Bank.RegisterClasses.empty()) + return invalidDescription("register bank must cover at least one register class"); + + std::unordered_set<RegisterClassID> SeenClasses; + for (const RegisterClassID ClassID : Bank.RegisterClasses) { + const RegisterClassDesc *Class = Result.registerClass(ClassID); + if (Class == nullptr || !SeenClasses.insert(ClassID).second) { + return invalidDescription("register bank has an unknown or duplicate register class"); + } + if (Class->SizeInBits > Bank.MaximumSizeInBits) { + return invalidDescription( + "register bank maximum size is smaller than a covered register class"); + } + } + } + + for (const RegisterClassDesc &Class : Result.Description.RegisterClasses) { + if (Class.Registers.empty() || Class.SizeInBits == 0 || + Class.SpillSizeInBits < Class.SizeInBits || + !std::has_single_bit(Class.SpillAlignmentInBits) || Class.Lanes.empty()) { + return invalidDescription( + "register class requires members, sizes, power-of-two spill alignment, and lanes"); + } + + std::unordered_set<PhysicalRegister> Members; + for (const PhysicalRegister Reg : Class.Registers) { + if (!Result.isKnownPhysicalRegister(Reg) || !Members.insert(Reg).second || + !Result.isValidLaneMask(Reg, Class.Lanes)) { + return invalidDescription( + "register class has an unknown, duplicate, or lane-incompatible member"); + } + } + + std::unordered_set<PhysicalRegister> AllocationRegisters; + for (const PhysicalRegister Reg : Class.AllocationOrder) { + const PhysicalRegisterDesc *Register = Result.physicalRegister(Reg); + const bool IsUnavailable = + Register != nullptr && + (Register->IsConstant || Register->IsArtificial || Register->IsReserved); + if (!Members.contains(Reg) || IsUnavailable || !AllocationRegisters.insert(Reg).second) { + return invalidDescription( + "register class allocation order must contain unique allocatable class members"); + } + } + if ((Class.Allocatable && Class.AllocationOrder.empty()) || + (!Class.Allocatable && !Class.AllocationOrder.empty())) { + return invalidDescription( + "allocatable classes require an allocation order and nonallocatable classes forbid one"); + } + + std::unordered_set<SubRegIndex> SupportedSubRegs; + LaneMask CoveredSubRegLanes; + for (const SubRegIndex SubReg : Class.SupportedSubRegs) { + const SubRegIndexDesc *SubRegDesc = Result.subRegIndex(SubReg); + if (SubRegDesc == nullptr || !SupportedSubRegs.insert(SubReg).second || + !Class.Lanes.contains(SubRegDesc->Lanes)) { + return invalidDescription( + "register class has an unknown, duplicate, or lane-incompatible subregister index"); + } + CoveredSubRegLanes = CoveredSubRegLanes | SubRegDesc->Lanes; + } + if (Class.CoveredBySubRegs && + (Class.SupportedSubRegs.empty() || !CoveredSubRegLanes.contains(Class.Lanes))) { + return invalidDescription( + "subregister-covered class must cover every class lane with supported indices"); + } + + std::unordered_set<RegisterClassID> SuperClasses; + for (const RegisterClassID SuperClass : Class.SuperClasses) { + if (Result.registerClass(SuperClass) == nullptr || SuperClass == Class.ID || + !SuperClasses.insert(SuperClass).second) { + return invalidDescription("register class has an unknown, self, or duplicate superclass"); + } + Result.SubclassRelations.insert(pairKey(Class.ID.value(), SuperClass.value())); + } + Result.SubclassRelations.insert(pairKey(Class.ID.value(), Class.ID.value())); + + if (Result.registerBank(Class.DefaultBank) == nullptr || + !Result.bankCoversClass(Class.DefaultBank, Class.ID)) { + return invalidDescription( + "register class default bank must exist and explicitly cover the class"); + } + } + + bool RelationsChanged = true; + while (RelationsChanged) { + RelationsChanged = false; + for (const RegisterClassDesc &Candidate : Result.Description.RegisterClasses) { + for (const RegisterClassDesc &Intermediate : Result.Description.RegisterClasses) { + if (!Result.isSubclass(Candidate.ID, Intermediate.ID)) + continue; + for (const RegisterClassDesc &Required : Result.Description.RegisterClasses) { + if (!Result.isSubclass(Intermediate.ID, Required.ID)) + continue; + RelationsChanged |= + Result.SubclassRelations.insert(pairKey(Candidate.ID.value(), Required.ID.value())) + .second; + } + } + } + } + + for (const RegisterClassDesc &Class : Result.Description.RegisterClasses) { + for (const RegisterClassDesc &Other : Result.Description.RegisterClasses) { + if (Class.ID != Other.ID && Result.isSubclass(Class.ID, Other.ID) && + Result.isSubclass(Other.ID, Class.ID)) { + return invalidDescription("register class hierarchy must be acyclic"); + } + } + for (const RegisterClassID SuperID : Class.SuperClasses) { + const RegisterClassDesc *Super = Result.registerClass(SuperID); + if (Super == nullptr || Class.SizeInBits != Super->SizeInBits || + Class.SpillSizeInBits != Super->SpillSizeInBits || + Class.SpillAlignmentInBits != Super->SpillAlignmentInBits || + !std::ranges::all_of(Class.Registers, [Super](PhysicalRegister Reg) { + return contains(Super->Registers, Reg); + })) { + return invalidDescription( + "register class must be storage-compatible with each declared superclass"); + } + } + } + + for (const SubRegClassMapping &Mapping : Result.Description.SubRegClassMappings) { + const RegisterClassDesc *Source = Result.registerClass(Mapping.SourceClass); + const SubRegIndexDesc *Index = Result.subRegIndex(Mapping.Index); + const RegisterClassDesc *Mapped = Result.registerClass(Mapping.ResultClass); + if (Source == nullptr || Index == nullptr || Mapped == nullptr || + !contains(Source->SupportedSubRegs, Mapping.Index)) { + return invalidDescription( + "subregister class mapping references an unknown or unsupported entity"); + } + if (Mapped->SizeInBits != Index->SizeInBits) { + return invalidDescription("subregister class mapping result size must match its index size"); + } + if (static_cast<std::uint64_t>(Index->OffsetInBits) + Index->SizeInBits > Source->SizeInBits) { + return invalidDescription("subregister class mapping index exceeds its source class size"); + } + + const std::uint64_t Key = pairKey(Mapping.SourceClass.value(), Mapping.Index.value()); + if (!Result.SubRegClassMap.emplace(Key, Mapping.ResultClass).second) { + return invalidDescription("subregister class mapping key must be unique"); + } + for (const PhysicalRegister SuperReg : Source->Registers) { + const auto SubReg = Result.subRegister(SuperReg, Mapping.Index); + if (!SubReg || !Result.registerClassContains(Mapping.ResultClass, *SubReg)) { + return invalidDescription("subregister class mapping does not cover every source register"); + } + } + } + + for (const RegisterClassDesc &Class : Result.Description.RegisterClasses) { + for (const SubRegIndex SubReg : Class.SupportedSubRegs) { + if (!Result.subRegClass(Class.ID, SubReg)) { + return invalidDescription( + "every supported class subregister index requires a class mapping"); + } + } + } + + for (const SubRegIndexComposition &Composition : Result.Description.SubRegCompositions) { + for (const RegisterClassDesc &Source : Result.Description.RegisterClasses) { + const auto Intermediate = Result.subRegClass(Source.ID, Composition.First); + if (!Intermediate) + continue; + const auto Final = Result.subRegClass(*Intermediate, Composition.Second); + if (!Final) + continue; + const auto Composed = Result.subRegClass(Source.ID, Composition.Result); + if (!Composed || !Result.isSubclass(*Final, *Composed)) { + return invalidDescription("subregister composition conflicts with register class mappings"); + } + } + } + + for (const PhysicalRegisterDesc &Register : Result.Description.PhysicalRegisters) { + if (std::ranges::none_of(Result.Description.RegisterClasses, + [&Register](const RegisterClassDesc &Class) { + return contains(Class.Registers, Register.Reg); + })) { + return invalidDescription( + "every physical register must belong to at least one register class"); + } + } + + Result.Aliases.resize(RegisterCount); + for (std::size_t Left = 0; Left < RegisterCount; ++Left) { + for (std::size_t Right = 0; Right < RegisterCount; ++Right) { + if (Left != Right && Result.regsOverlap(Result.Description.PhysicalRegisters[Left].Reg, + Result.Description.PhysicalRegisters[Right].Reg)) { + Result.Aliases[Left].push_back(Result.Description.PhysicalRegisters[Right].Reg); + } + } + } + + return Result; +} + +const RegisterUnitDesc *TargetRegisterInfo::registerUnit(RegisterUnitID ID) const noexcept { + const auto Position = UnitToIndex.find(ID); + return Position == UnitToIndex.end() ? nullptr : &Description.RegisterUnits[Position->second]; +} + +const PhysicalRegisterDesc * +TargetRegisterInfo::physicalRegister(PhysicalRegister Reg) const noexcept { + const auto Position = RegisterToIndex.find(Reg); + return Position == RegisterToIndex.end() ? nullptr + : &Description.PhysicalRegisters[Position->second]; +} + +const PhysicalRegisterDesc * +TargetRegisterInfo::physicalRegister(std::string_view Name) const noexcept { + const auto Position = RegisterNameToIndex.find(std::string(Name)); + return Position == RegisterNameToIndex.end() ? nullptr + : &Description.PhysicalRegisters[Position->second]; +} + +const PhysicalRegisterDesc * +TargetRegisterInfo::physicalRegisterByAlternativeName(std::string_view Name) const noexcept { + const auto Position = AlternativeRegisterNameToIndex.find(lowerASCII(Name)); + return Position == AlternativeRegisterNameToIndex.end() + ? nullptr + : &Description.PhysicalRegisters[Position->second]; +} + +bool TargetRegisterInfo::isReservedRegister(PhysicalRegister Reg) const noexcept { + const PhysicalRegisterDesc *Register = physicalRegister(Reg); + return Register != nullptr && Register->IsReserved; +} + +const SubRegIndexDesc *TargetRegisterInfo::subRegIndex(SubRegIndex Index) const noexcept { + const auto Position = SubRegIndexToIndex.find(Index); + return Position == SubRegIndexToIndex.end() ? nullptr + : &Description.SubRegIndices[Position->second]; +} + +const SubRegIndexDesc *TargetRegisterInfo::subRegIndex(std::string_view Name) const noexcept { + const auto Position = SubRegNameToIndex.find(std::string(Name)); + return Position == SubRegNameToIndex.end() ? nullptr + : &Description.SubRegIndices[Position->second]; +} + +const RegisterClassDesc *TargetRegisterInfo::registerClass(RegisterClassID ID) const noexcept { + const auto Position = ClassToIndex.find(ID); + return Position == ClassToIndex.end() ? nullptr : &Description.RegisterClasses[Position->second]; +} + +const RegisterClassDesc *TargetRegisterInfo::registerClass(std::string_view Name) const noexcept { + const auto Position = ClassNameToIndex.find(std::string(Name)); + return Position == ClassNameToIndex.end() ? nullptr + : &Description.RegisterClasses[Position->second]; +} + +const RegisterBankDesc *TargetRegisterInfo::registerBank(RegisterBankID ID) const noexcept { + const auto Position = BankToIndex.find(ID); + return Position == BankToIndex.end() ? nullptr : &Description.RegisterBanks[Position->second]; +} + +const RegisterBankDesc *TargetRegisterInfo::registerBank(std::string_view Name) const noexcept { + const auto Position = BankNameToIndex.find(std::string(Name)); + return Position == BankNameToIndex.end() ? nullptr : &Description.RegisterBanks[Position->second]; +} + +std::optional<std::string_view> +TargetRegisterInfo::physicalRegisterMIRName(PhysicalRegister Reg) const noexcept { + const auto Position = RegisterToIndex.find(Reg); + if (Position == RegisterToIndex.end()) + return std::nullopt; + return RegisterMIRNames[Position->second]; +} + +std::optional<std::string_view> +TargetRegisterInfo::registerClassMIRName(RegisterClassID ID) const noexcept { + const auto Position = ClassToIndex.find(ID); + if (Position == ClassToIndex.end()) + return std::nullopt; + return ClassMIRNames[Position->second]; +} + +std::optional<std::string_view> +TargetRegisterInfo::registerBankMIRName(RegisterBankID ID) const noexcept { + const auto Position = BankToIndex.find(ID); + if (Position == BankToIndex.end()) + return std::nullopt; + return BankMIRNames[Position->second]; +} + +LaneMask TargetRegisterInfo::registerLaneMask(PhysicalRegister Reg) const noexcept { + const PhysicalRegisterDesc *Register = physicalRegister(Reg); + if (Register == nullptr) + return {}; + + LaneMask Result; + for (const RegisterUnitLane &Unit : Register->Units) + Result = Result | Unit.Lanes; + return Result; +} + +bool TargetRegisterInfo::isValidLaneMask(PhysicalRegister Reg, LaneMask Lanes) const noexcept { + if (physicalRegister(Reg) == nullptr || Lanes.empty()) + return false; + if (Lanes == LaneMask::all()) + return true; + return registerLaneMask(Reg).contains(Lanes); +} + +bool TargetRegisterInfo::regsOverlap(PhysicalRegister Left, PhysicalRegister Right) const noexcept { + const PhysicalRegisterDesc *LeftDesc = physicalRegister(Left); + const PhysicalRegisterDesc *RightDesc = physicalRegister(Right); + if (LeftDesc == nullptr || RightDesc == nullptr) + return false; + + return std::ranges::any_of(LeftDesc->Units, [RightDesc](const RegisterUnitLane &Unit) { + return findUnit(*RightDesc, Unit.Unit) != nullptr; + }); +} + +std::span<const PhysicalRegister> TargetRegisterInfo::aliases(PhysicalRegister Reg) const noexcept { + const auto Position = RegisterToIndex.find(Reg); + if (Position == RegisterToIndex.end()) + return {}; + return Aliases[Position->second]; +} + +std::optional<PhysicalRegister> TargetRegisterInfo::subRegister(PhysicalRegister SuperReg, + SubRegIndex Index) const noexcept { + const auto Position = SubRegisterMap.find(pairKey(SuperReg.number(), Index.value())); + if (Position == SubRegisterMap.end()) + return std::nullopt; + return Position->second; +} + +bool TargetRegisterInfo::isSubRegister(PhysicalRegister SuperReg, + PhysicalRegister SubReg) const noexcept { + const auto Position = RegisterToIndex.find(SubReg); + if (Position == RegisterToIndex.end()) + return false; + return contains(SuperRegisters[Position->second], SuperReg); +} + +std::span<const PhysicalRegister> +TargetRegisterInfo::superRegisters(PhysicalRegister Reg) const noexcept { + const auto Position = RegisterToIndex.find(Reg); + if (Position == RegisterToIndex.end()) + return {}; + return SuperRegisters[Position->second]; +} + +std::optional<SubRegIndex> +TargetRegisterInfo::composeSubRegIndices(SubRegIndex First, SubRegIndex Second) const noexcept { + const auto Position = CompositionMap.find(pairKey(First.value(), Second.value())); + if (Position == CompositionMap.end()) + return std::nullopt; + return Position->second; +} + +bool TargetRegisterInfo::registerClassContains(RegisterClassID Class, + PhysicalRegister Reg) const noexcept { + const RegisterClassDesc *ClassDesc = registerClass(Class); + return ClassDesc != nullptr && contains(ClassDesc->Registers, Reg); +} + +bool TargetRegisterInfo::isSubclass(RegisterClassID Candidate, + RegisterClassID Required) const noexcept { + return SubclassRelations.contains(pairKey(Candidate.value(), Required.value())); +} + +bool TargetRegisterInfo::registerClassesOverlap(RegisterClassID Left, + RegisterClassID Right) const noexcept { + const RegisterClassDesc *LeftClass = registerClass(Left); + const RegisterClassDesc *RightClass = registerClass(Right); + if (LeftClass == nullptr || RightClass == nullptr) + return false; + return std::ranges::any_of(LeftClass->Registers, [RightClass](PhysicalRegister Reg) { + return contains(RightClass->Registers, Reg); + }); +} + +std::optional<RegisterClassID> TargetRegisterInfo::subRegClass(RegisterClassID Source, + SubRegIndex Index) const noexcept { + const auto Position = SubRegClassMap.find(pairKey(Source.value(), Index.value())); + if (Position == SubRegClassMap.end()) + return std::nullopt; + return Position->second; +} + +bool TargetRegisterInfo::bankCoversClass(RegisterBankID Bank, + RegisterClassID Class) const noexcept { + const RegisterBankDesc *BankDesc = registerBank(Bank); + return BankDesc != nullptr && contains(BankDesc->RegisterClasses, Class); +} + +bool TargetRegisterInfo::bankContainsRegister(RegisterBankID Bank, + PhysicalRegister Reg) const noexcept { + const RegisterBankDesc *BankDesc = registerBank(Bank); + if (BankDesc == nullptr) + return false; + return std::ranges::any_of(BankDesc->RegisterClasses, [this, Reg](RegisterClassID Class) { + return registerClassContains(Class, Reg); + }); +} + +bool TargetRegisterInfo::registerBanksOverlap(RegisterBankID Left, + RegisterBankID Right) const noexcept { + const RegisterBankDesc *LeftBank = registerBank(Left); + const RegisterBankDesc *RightBank = registerBank(Right); + if (LeftBank == nullptr || RightBank == nullptr) + return false; + return std::ranges::any_of(Description.PhysicalRegisters, + [this, Left, Right](const PhysicalRegisterDesc &Register) { + return bankContainsRegister(Left, Register.Reg) && + bankContainsRegister(Right, Register.Reg); + }); +} + +Expected<void> TargetRegisterInfo::validateRegisterMask(const RegisterMask &Mask) const { + constexpr std::uint64_t BitsPerWord = 64; + for (std::size_t WordIndex = 0; WordIndex < Mask.words().size(); ++WordIndex) { + std::uint64_t Word = Mask.words()[WordIndex]; + while (Word != 0) { + const std::uint64_t BitIndex = static_cast<std::uint64_t>(std::countr_zero(Word)); + const std::uint64_t RegisterNumber = + static_cast<std::uint64_t>(WordIndex) * BitsPerWord + BitIndex; + const auto Reg = PhysicalRegister::create(RegisterNumber); + if (!Reg || !isKnownPhysicalRegister(*Reg)) { + return makeUnexpected(ErrorCode::VerificationError, + "register mask contains an unknown physical register"); + } + Word &= Word - 1; + } + } + return {}; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TargetSchedulingModel.cpp b/backend/lib/CodeGen/TargetSchedulingModel.cpp new file mode 100644 index 0000000..f462c21 --- /dev/null +++ b/backend/lib/CodeGen/TargetSchedulingModel.cpp @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TargetSchedulingModel.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <limits> +#include <memory> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <utility> + +namespace rcc { +namespace { + +template <typename IDT> [[nodiscard]] bool isValidID(IDT ID) noexcept { + return ID.value() != 0; +} + +} // namespace + +Expected<ScheduleHazardType> +TargetScheduleHazardRecognizer::getHazardType(const ScheduleUnit &) const { + return ScheduleHazardType::NoHazard; +} + +Expected<void> TargetScheduleHazardRecognizer::emitInstruction(const ScheduleUnit &) { + return {}; +} + +Expected<void> TargetScheduleHazardRecognizer::advanceCycle() { + return {}; +} + +Expected<void> TargetScheduleHazardRecognizer::recedeCycle() { + return {}; +} + +Expected<void> TargetScheduleHazardRecognizer::reset() { + return {}; +} + +bool TargetSchedulingHooks::isSchedulingBoundary(const MachineInstr &, + const TargetDescription &) const noexcept { + return false; +} + +Expected<void> TargetSchedulingHooks::mutateScheduleDAG(ScheduleDAG &, const MachineFunction &, + const TargetDescription &, ScheduleDAGMode, + bool) const { + return {}; +} + +std::int64_t TargetSchedulingHooks::candidatePreference(const ScheduleUnit &, ScheduleDirection, + const MachineBasicBlock &, + std::uint64_t) const noexcept { + return 0; +} + +std::unique_ptr<TargetScheduleHazardRecognizer> +TargetSchedulingHooks::createHazardRecognizer(ScheduleDirection) const { + return nullptr; +} + +Expected<TargetSchedulingModel> +TargetSchedulingModel::create(TargetSchedulingModelDesc Description) { + if (Description.IssueWidth == 0 || Description.DispatchLimit == 0) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model requires nonzero issue and dispatch widths"); + } + + TargetSchedulingModel Result(std::move(Description)); + std::unordered_set<std::string> Names; + for (std::size_t Index = 0; Index < Result.Description.ProcessorResources.size(); ++Index) { + const ProcessorResourceDesc &Resource = Result.Description.ProcessorResources[Index]; + if (!isValidID(Resource.ID) || Resource.Name.empty() || Resource.Units == 0 || + !Result.ResourceToIndex.emplace(Resource.ID, Index).second || + !Names.insert(Resource.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid processor resource"); + } + } + for (const ProcessorResourceDesc &Resource : Result.Description.ProcessorResources) { + if (Resource.Parent && + (Resource.Parent == Resource.ID || !Result.ResourceToIndex.contains(*Resource.Parent))) { + return makeUnexpected(ErrorCode::InvalidArgument, + "processor resource has an invalid parent resource"); + } + std::unordered_set<ProcessorResourceID> Path; + const ProcessorResourceDesc *Current = &Resource; + while (Current->Parent) { + if (!Path.insert(Current->ID).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "processor resource hierarchy contains a cycle"); + } + Current = &Result.Description.ProcessorResources[Result.ResourceToIndex.at(*Current->Parent)]; + } + } + + Names.clear(); + for (std::size_t Index = 0; Index < Result.Description.SchedulingClasses.size(); ++Index) { + SchedulingClassModelDesc &Class = Result.Description.SchedulingClasses[Index]; + if (!isValidID(Class.ID) || Class.Name.empty() || Class.MicroOps == 0 || Class.Latency == 0 || + !Result.ClassToIndex.emplace(Class.ID, Index).second || !Names.insert(Class.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid scheduling class"); + } + } + for (SchedulingClassModelDesc &Class : Result.Description.SchedulingClasses) { + std::unordered_set<ProcessorResourceID> UsedResources; + for (const SchedulingResourceUse &Use : Class.Resources) { + if (!isValidID(Use.Resource) || Use.Cycles == 0 || + !Result.ResourceToIndex.contains(Use.Resource) || + !UsedResources.insert(Use.Resource).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has an invalid resource use"); + } + } + std::ranges::sort(Class.Resources, {}, + [](const SchedulingResourceUse &Use) { return Use.Resource.value(); }); + + std::unordered_set<std::uint32_t> WriteOperands; + for (const SchedulingWriteLatency &Latency : Class.WriteLatencies) { + if (!WriteOperands.insert(Latency.OperandIndex).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has duplicate write latencies"); + } + } + std::ranges::sort(Class.WriteLatencies, {}, &SchedulingWriteLatency::OperandIndex); + + std::unordered_set<std::uint64_t> ReadAdvanceKeys; + for (const SchedulingReadAdvance &Advance : Class.ReadAdvances) { + if (Advance.ProducerClass && !Result.ClassToIndex.contains(*Advance.ProducerClass)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "read advance references an unknown producer class"); + } + const std::uint64_t Producer = Advance.ProducerClass ? Advance.ProducerClass->value() : 0; + const std::uint64_t Key = (Producer << 32U) | Advance.OperandIndex; + if (!ReadAdvanceKeys.insert(Key).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "scheduling class has duplicate read advances"); + } + } + std::ranges::sort(Class.ReadAdvances, + [](const SchedulingReadAdvance &Left, const SchedulingReadAdvance &Right) { + if (Left.OperandIndex != Right.OperandIndex) + return Left.OperandIndex < Right.OperandIndex; + return Left.ProducerClass.value_or(SchedulingClassID{}).value() < + Right.ProducerClass.value_or(SchedulingClassID{}).value(); + }); + } + + Names.clear(); + for (std::size_t Index = 0; Index < Result.Description.PressureSets.size(); ++Index) { + RegisterPressureSetDesc &Set = Result.Description.PressureSets[Index]; + if (!isValidID(Set.ID) || Set.Name.empty() || Set.Limit == 0 || Set.RegisterClasses.empty() || + !Result.PressureSetToIndex.emplace(Set.ID, Index).second || + !Names.insert(Set.Name).second) { + return makeUnexpected(ErrorCode::InvalidArgument, + "target scheduling model has an invalid register pressure set"); + } + std::ranges::sort(Set.RegisterClasses, [](RegisterClassID Left, RegisterClassID Right) { + return Left.value() < Right.value(); + }); + if (std::ranges::adjacent_find(Set.RegisterClasses) != Set.RegisterClasses.end() || + std::ranges::any_of(Set.RegisterClasses, + [](RegisterClassID Class) { return !isValidID(Class); })) { + return makeUnexpected(ErrorCode::InvalidArgument, + "register pressure set has duplicate or invalid register classes"); + } + for (const RegisterClassID Class : Set.RegisterClasses) + Result.ClassToPressureSets[Class].push_back(Set.ID); + } + for (auto &[Class, Sets] : Result.ClassToPressureSets) { + (void)Class; + std::ranges::sort(Sets, [](RegisterPressureSetID Left, RegisterPressureSetID Right) { + return Left.value() < Right.value(); + }); + } + return Result; +} + +const ProcessorResourceDesc * +TargetSchedulingModel::processorResource(ProcessorResourceID ID) const noexcept { + const auto Position = ResourceToIndex.find(ID); + return Position == ResourceToIndex.end() ? nullptr + : &Description.ProcessorResources[Position->second]; +} + +const SchedulingClassModelDesc * +TargetSchedulingModel::schedulingClass(SchedulingClassID ID) const noexcept { + const auto Position = ClassToIndex.find(ID); + return Position == ClassToIndex.end() ? nullptr + : &Description.SchedulingClasses[Position->second]; +} + +const RegisterPressureSetDesc * +TargetSchedulingModel::pressureSet(RegisterPressureSetID ID) const noexcept { + const auto Position = PressureSetToIndex.find(ID); + return Position == PressureSetToIndex.end() ? nullptr + : &Description.PressureSets[Position->second]; +} + +std::span<const RegisterPressureSetID> +TargetSchedulingModel::pressureSetsForClass(RegisterClassID Class) const noexcept { + const auto Position = ClassToPressureSets.find(Class); + return Position == ClassToPressureSets.end() + ? std::span<const RegisterPressureSetID>{} + : std::span<const RegisterPressureSetID>(Position->second); +} + +Expected<std::uint32_t> TargetSchedulingModel::operandLatency(SchedulingClassID ProducerClass, + std::size_t DefinitionOperand, + SchedulingClassID ConsumerClass, + std::size_t UseOperand) const { + if (DefinitionOperand > std::numeric_limits<std::uint32_t>::max() || + UseOperand > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::SchedulingFailed, + "scheduling operand index exceeds the target model range"); + } + const SchedulingClassModelDesc *Producer = schedulingClass(ProducerClass); + const SchedulingClassModelDesc *Consumer = schedulingClass(ConsumerClass); + if (Producer == nullptr || Consumer == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "operand latency references an unknown scheduling class"); + } + + const std::uint32_t DefinitionIndex = static_cast<std::uint32_t>(DefinitionOperand); + const std::uint32_t UseIndex = static_cast<std::uint32_t>(UseOperand); + std::uint32_t Latency = Producer->Latency; + if (const auto Position = std::ranges::find(Producer->WriteLatencies, DefinitionIndex, + &SchedulingWriteLatency::OperandIndex); + Position != Producer->WriteLatencies.end()) { + Latency = Position->Cycles; + } + + const SchedulingReadAdvance *Wildcard = nullptr; + const SchedulingReadAdvance *Exact = nullptr; + for (const SchedulingReadAdvance &Advance : Consumer->ReadAdvances) { + if (Advance.OperandIndex != UseIndex) + continue; + if (!Advance.ProducerClass) + Wildcard = &Advance; + else if (*Advance.ProducerClass == ProducerClass) + Exact = &Advance; + } + const SchedulingReadAdvance *Advance = Exact != nullptr ? Exact : Wildcard; + return Advance == nullptr || Advance->Cycles >= Latency ? (Advance == nullptr ? Latency : 0U) + : Latency - Advance->Cycles; +} + +const TargetSchedulingHooks &TargetSchedulingModel::hooks() const noexcept { + static const TargetSchedulingHooks DefaultHooks; + return Description.Hooks == nullptr ? DefaultHooks : *Description.Hooks; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/TwoAddressInstructionPass.cpp b/backend/lib/CodeGen/TwoAddressInstructionPass.cpp new file mode 100644 index 0000000..d7a5ef1 --- /dev/null +++ b/backend/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -0,0 +1,621 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/TwoAddressInstructionPass.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/CodeGen/TargetInstrInfo.h" + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +struct RegisterValue final { + Register Reg; + SubRegIndex SubReg; + RegisterOperandFlags Flags; + std::uint32_t TargetFlags = 0; +}; + +[[nodiscard]] RegisterOperandFlags generatedDefFlags(RegisterOperandFlags Original) noexcept { + RegisterOperandFlags Result; + if (Original.has(RegisterOperandFlag::Undef)) + Result.set(RegisterOperandFlag::Undef); + return Result; +} + +[[nodiscard]] Expected<void> copyDebugLocation(MachineInstrBuilder &Builder, + const MachineInstr *Source) { + if (Source == nullptr || !Source->debugLocation()) + return {}; + return Builder.setDebugLocation(*Source->debugLocation()); +} + +[[nodiscard]] Expected<MachineInstr *> +buildCopy(MachineFunction &Function, const TargetDescription &Target, + const TargetInstrInfo &TargetInfo, MachineBasicBlock &Block, MachineInstr *Before, + RegisterValue Destination, RegisterValue Source, const MachineInstr *DebugSource) { + auto Legality = TargetInfo.validateCopy(Destination.Reg, Destination.SubReg, Source.Reg, + Source.SubReg, Function, Target); + if (!Legality) + return std::unexpected(withErrorContext(Legality.error(), "illegal COPY: ")); + + MachineIRBuilder Builder(Function, Target); + auto Point = + Before == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Before); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::COPY)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Destination.Reg, generatedDefFlags(Destination.Flags), + Destination.SubReg, Destination.TargetFlags); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Pending->addUse(Source.Reg, {}, Source.SubReg, Source.TargetFlags); + if (!Use) + return std::unexpected(Use.error()); + auto Debug = copyDebugLocation(*Pending, DebugSource); + if (!Debug) + return std::unexpected(Debug.error()); + return Pending->build(); +} + +[[nodiscard]] Expected<MachineInstr *> +buildImplicitDef(MachineFunction &Function, const TargetDescription &Target, + MachineBasicBlock &Block, MachineInstr *Before, RegisterValue Destination, + const MachineInstr *DebugSource) { + MachineIRBuilder Builder(Function, Target); + auto Point = + Before == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Before); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::IMPLICIT_DEF)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Destination.Reg, generatedDefFlags(Destination.Flags), + Destination.SubReg, Destination.TargetFlags); + if (!Def) + return std::unexpected(Def.error()); + auto Debug = copyDebugLocation(*Pending, DebugSource); + if (!Debug) + return std::unexpected(Debug.error()); + return Pending->build(); +} + +[[nodiscard]] bool isImplicitlyUndef(const MachineFunction &Function, + const MachineOperand &Operand) { + if (Operand.registerFlags().has(RegisterOperandFlag::Undef)) + return true; + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || Reg->isNoRegister()) + return true; + if (!Reg->isVirtual()) + return false; + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg); + if (Definition == nullptr || Definition->parent() == nullptr) + return false; + if (Definition->subReg().value() != 0 && Definition->subReg() != Operand.subReg()) + return false; + const Opcode DefinitionOpcode = Definition->parent()->opcode(); + return DefinitionOpcode == opcode(GenericOpcode::IMPLICIT_DEF) || + DefinitionOpcode == opcode(GenericOpcode::G_IMPLICIT_DEF) || + Definition->registerFlags().has(RegisterOperandFlag::Undef); +} + +[[nodiscard]] Expected<void> repairTrackedLiveness(MachineFunction &Function, + const TargetDescription &Target, + MachinePassResult &Result) { + if (!Result.Changed || !Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + return {}; + } + auto Rebuilt = rebuildMachineLiveness(Function, Target); + if (!Rebuilt) + return std::unexpected(Rebuilt.error()); + if (*Rebuilt) + Result.Changes.add(MachinePassChange::Operands); + return {}; +} + +[[nodiscard]] Expected<SubRegIndex> subRegIndex(const MachineOperand &Operand, + const TargetDescription &Target) { + std::uint64_t Value = 0; + if (const auto *Immediate = Operand.getIf<std::int64_t>(); Immediate != nullptr) { + if (*Immediate <= 0) + return makeUnexpected(ErrorCode::VerificationError, + "subregister pseudo requires a positive subregister index"); + Value = static_cast<std::uint64_t>(*Immediate); + } else if (const auto *WideImmediate = Operand.getIf<IntegerImmediateOperand>(); + WideImmediate != nullptr) { + if (WideImmediate->Value.isNegative() || WideImmediate->Value.bitWidth() > 64) + return makeUnexpected(ErrorCode::VerificationError, + "subregister pseudo index does not fit an unsigned integer"); + Value = WideImmediate->Value.words().front(); + } else { + return makeUnexpected(ErrorCode::VerificationError, + "subregister pseudo is missing its immediate index"); + } + if (Value == 0 || Value > std::numeric_limits<std::uint32_t>::max()) { + return makeUnexpected(ErrorCode::VerificationError, + "subregister pseudo index is outside the supported range"); + } + const SubRegIndex Result = SubRegIndex::fromValue(static_cast<std::uint32_t>(Value)); + if (Target.registers().subRegIndex(Result) == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "subregister pseudo references an unknown target index"); + } + return Result; +} + +[[nodiscard]] Expected<SubRegIndex> composeSubReg(SubRegIndex Outer, SubRegIndex Inner, + const TargetDescription &Target) { + if (Outer.value() == 0) + return Inner; + if (Inner.value() == 0) + return Outer; + const auto Composed = Target.registers().composeSubRegIndices(Outer, Inner); + if (!Composed) { + return makeUnexpected(ErrorCode::Unsupported, + "target cannot compose the requested subregister indices"); + } + return *Composed; +} + +[[nodiscard]] Expected<void> expandRegSequence(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &TargetInfo) { + if (Instruction.operandCount() < 3 || (Instruction.operandCount() - 1U) % 2U != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "REG_SEQUENCE requires a destination and source/subregister pairs"); + } + MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr || Destination->isNoRegister() || !DestinationOperand->isDef()) { + return makeUnexpected(ErrorCode::VerificationError, + "REG_SEQUENCE requires a register destination definition"); + } + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "REG_SEQUENCE is not linked to a block"); + + RegisterValue DestinationValue{*Destination, DestinationOperand->subReg(), + DestinationOperand->registerFlags(), + DestinationOperand->targetFlags()}; + auto Initial = + buildImplicitDef(Function, Target, *Block, &Instruction, DestinationValue, &Instruction); + if (!Initial) + return std::unexpected(Initial.error()); + + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2U) { + MachineOperand *SourceOperand = Instruction.operandAt(Index); + MachineOperand *IndexOperand = Instruction.operandAt(Index + 1U); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Source == nullptr || SourceOperand == nullptr || !SourceOperand->isUse() || + IndexOperand == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "REG_SEQUENCE contains a malformed source/subregister pair"); + } + auto InnerSubReg = subRegIndex(*IndexOperand, Target); + if (!InnerSubReg) + return std::unexpected(InnerSubReg.error()); + auto DestinationSubReg = composeSubReg(DestinationOperand->subReg(), *InnerSubReg, Target); + if (!DestinationSubReg) + return std::unexpected(DestinationSubReg.error()); + + RegisterValue LaneDestination{ + *Destination, *DestinationSubReg, {}, DestinationOperand->targetFlags()}; + if (isImplicitlyUndef(Function, *SourceOperand)) { + auto Undef = + buildImplicitDef(Function, Target, *Block, &Instruction, LaneDestination, &Instruction); + if (!Undef) + return std::unexpected(Undef.error()); + continue; + } + RegisterValue SourceValue{*Source, SourceOperand->subReg(), SourceOperand->registerFlags(), + SourceOperand->targetFlags()}; + auto Copy = buildCopy(Function, Target, TargetInfo, *Block, &Instruction, LaneDestination, + SourceValue, &Instruction); + if (!Copy) + return std::unexpected(Copy.error()); + } + return Instruction.eraseFromParent(); +} + +[[nodiscard]] Expected<void> expandInsertSubreg(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + const TargetInstrInfo &TargetInfo) { + if (Instruction.operandCount() != 4) { + return makeUnexpected(ErrorCode::VerificationError, + "INSERT_SUBREG requires destination, base, inserted value, and index"); + } + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *BaseOperand = Instruction.operandAt(1); + MachineOperand *InsertedOperand = Instruction.operandAt(2); + MachineOperand *IndexOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Base = BaseOperand == nullptr ? nullptr : BaseOperand->getIf<Register>(); + const Register *Inserted = + InsertedOperand == nullptr ? nullptr : InsertedOperand->getIf<Register>(); + if (Destination == nullptr || Base == nullptr || Inserted == nullptr || + Destination->isNoRegister() || DestinationOperand == nullptr || + !DestinationOperand->isDef() || BaseOperand == nullptr || !BaseOperand->isUse() || + InsertedOperand == nullptr || !InsertedOperand->isUse() || IndexOperand == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "INSERT_SUBREG contains malformed register operands"); + } + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "INSERT_SUBREG is not linked to a block"); + + RegisterValue DestinationValue{*Destination, DestinationOperand->subReg(), + DestinationOperand->registerFlags(), + DestinationOperand->targetFlags()}; + if (isImplicitlyUndef(Function, *BaseOperand)) { + auto Undef = + buildImplicitDef(Function, Target, *Block, &Instruction, DestinationValue, &Instruction); + if (!Undef) + return std::unexpected(Undef.error()); + } else if (*Destination != *Base || DestinationOperand->subReg() != BaseOperand->subReg()) { + RegisterValue BaseValue{*Base, BaseOperand->subReg(), BaseOperand->registerFlags(), + BaseOperand->targetFlags()}; + auto Copy = buildCopy(Function, Target, TargetInfo, *Block, &Instruction, DestinationValue, + BaseValue, &Instruction); + if (!Copy) + return std::unexpected(Copy.error()); + } + + auto InnerSubReg = subRegIndex(*IndexOperand, Target); + if (!InnerSubReg) + return std::unexpected(InnerSubReg.error()); + auto DestinationSubReg = composeSubReg(DestinationOperand->subReg(), *InnerSubReg, Target); + if (!DestinationSubReg) + return std::unexpected(DestinationSubReg.error()); + RegisterValue LaneDestination{ + *Destination, *DestinationSubReg, {}, DestinationOperand->targetFlags()}; + if (isImplicitlyUndef(Function, *InsertedOperand)) { + auto Undef = + buildImplicitDef(Function, Target, *Block, &Instruction, LaneDestination, &Instruction); + if (!Undef) + return std::unexpected(Undef.error()); + } else { + RegisterValue InsertedValue{*Inserted, InsertedOperand->subReg(), + InsertedOperand->registerFlags(), InsertedOperand->targetFlags()}; + auto Copy = buildCopy(Function, Target, TargetInfo, *Block, &Instruction, LaneDestination, + InsertedValue, &Instruction); + if (!Copy) + return std::unexpected(Copy.error()); + } + return Instruction.eraseFromParent(); +} + +[[nodiscard]] Expected<void> swapRegisterUses(MachineOperand &Left, MachineOperand &Right) { + const std::optional<Register> LeftReg = Left.registerValue(); + const std::optional<Register> RightReg = Right.registerValue(); + if (!LeftReg || !RightReg || !Left.isUse() || Left.isDef() || !Right.isUse() || Right.isDef()) { + return makeUnexpected(ErrorCode::InvalidArgument, + "commutation requires two pure register uses"); + } + const SubRegIndex LeftSubReg = Left.subReg(); + const SubRegIndex RightSubReg = Right.subReg(); + const RegisterOperandFlags LeftFlags = Left.registerFlags(); + const RegisterOperandFlags RightFlags = Right.registerFlags(); + const std::uint32_t LeftTargetFlags = Left.targetFlags(); + const std::uint32_t RightTargetFlags = Right.targetFlags(); + + auto SetLeftReg = Left.setRegister(*RightReg); + if (!SetLeftReg) + return std::unexpected(SetLeftReg.error()); + auto SetRightReg = Right.setRegister(*LeftReg); + if (!SetRightReg) + return std::unexpected(SetRightReg.error()); + auto SetLeftSubReg = Left.setSubReg(RightSubReg); + if (!SetLeftSubReg) + return std::unexpected(SetLeftSubReg.error()); + auto SetRightSubReg = Right.setSubReg(LeftSubReg); + if (!SetRightSubReg) + return std::unexpected(SetRightSubReg.error()); + auto SetLeftFlags = Left.setRegisterFlags(RightFlags); + if (!SetLeftFlags) + return std::unexpected(SetLeftFlags.error()); + auto SetRightFlags = Right.setRegisterFlags(LeftFlags); + if (!SetRightFlags) + return std::unexpected(SetRightFlags.error()); + Left.setTargetFlags(RightTargetFlags); + Right.setTargetFlags(LeftTargetFlags); + return {}; +} + +[[nodiscard]] Expected<bool> tryCommute(MachineInstr &Instruction, std::size_t TiedUseIndex, + Register Destination, const TargetDescription &Target, + MachineFunction &Function) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr || !Descriptor->Flags.has(InstructionFlag::Commutable)) + return false; + MachineOperand *TiedUse = Instruction.operandAt(TiedUseIndex); + if (TiedUse == nullptr) + return makeUnexpected(ErrorCode::InternalError, "tied use disappeared during commutation"); + + for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) { + if (Index == TiedUseIndex) + continue; + MachineOperand *Candidate = Instruction.operandAt(Index); + if (Candidate == nullptr || Candidate->isImplicit() || !Candidate->isUse() || + Candidate->isDef() || Candidate->tiedOperandIndex() || + Candidate->registerValue() != Destination || isImplicitlyUndef(Function, *Candidate)) { + continue; + } + auto Swapped = swapRegisterUses(*TiedUse, *Candidate); + if (!Swapped) + return std::unexpected(Swapped.error()); + auto Validation = Target.validateInstruction(Instruction, Function.registerInfo()); + if (Validation) + return true; + auto Restored = swapRegisterUses(*TiedUse, *Candidate); + if (!Restored) + return std::unexpected(Restored.error()); + } + return false; +} + +[[nodiscard]] Expected<void> rewriteUndefTie(MachineOperand &Use, const MachineOperand &Def) { + const std::optional<Register> Destination = Def.registerValue(); + if (!Destination) + return makeUnexpected(ErrorCode::VerificationError, "tied definition is not a register"); + auto SetRegister = Use.setRegister(*Destination); + if (!SetRegister) + return std::unexpected(SetRegister.error()); + auto SetSubReg = Use.setSubReg(Def.subReg()); + if (!SetSubReg) + return std::unexpected(SetSubReg.error()); + RegisterOperandFlags Flags = Use.registerFlags(); + Flags.set(RegisterOperandFlag::Undef); + Flags.clear(RegisterOperandFlag::Kill); + return Use.setRegisterFlags(Flags); +} + +[[nodiscard]] Expected<void> verifyNoStageTenPseudos(const MachineFunction &Function, + const TargetDescription &Target) { + for (const MachineInstr *Instruction : Function.allInstructions()) { + if (Instruction->parent() == nullptr) + continue; + if (Instruction->opcode() == opcode(GenericOpcode::G_PHI) || + Instruction->opcode() == opcode(GenericOpcode::PHI) || + Instruction->opcode() == opcode(GenericOpcode::REG_SEQUENCE) || + Instruction->opcode() == opcode(GenericOpcode::INSERT_SUBREG)) { + return makeUnexpected(ErrorCode::VerificationError, + "SSA exit left a Stage 10 pseudo instruction in MachineIR"); + } + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "SSA exit produced an instruction with no descriptor"); + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + const MachineOperand *Operand = Instruction->operandAt(Index); + if (Operand == nullptr || !Operand->isUse()) + continue; + const std::optional<std::size_t> Tie = Operand->tiedOperandIndex(); + if (!Tie) + continue; + const MachineOperand *Def = Instruction->operandAt(Tie.value()); + if (Def == nullptr || Def->registerValue() != Operand->registerValue()) { + return makeUnexpected(ErrorCode::VerificationError, + "two-address rewrite left an unequal tied register"); + } + } + } + return {}; +} + +} // namespace + +const MachinePassContract &TwoAddressInstructionPass::contract() const noexcept { + static const MachinePassContract Contract{ + .Name = "twoaddressinstruction", + .Requires = {MachineFunctionProperty::Selected, MachineFunctionProperty::NoPHIs}, + .Sets = {MachineFunctionProperty::TiedOpsRewritten}, + .Clears = {MachineFunctionProperty::IsSSA}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::TiedOpsRewritten, + }; + return Contract; +} + +Expected<MachinePassResult> +TwoAddressInstructionPass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + return runImplementation(Function, Target, Analyses); +} + +Expected<MachinePassResult> +TwoAddressInstructionPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + MachinePassResult Result; + const std::vector<MachineInstr *> Instructions(Function.allInstructions().begin(), + Function.allInstructions().end()); + for (MachineInstr *Instruction : Instructions) { + if (Instruction->parent() == nullptr) + continue; + if (Instruction->opcode() == opcode(GenericOpcode::REG_SEQUENCE)) { + auto Accepted = + Analyses.requestAction("expand-reg-sequence", Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + return makeUnexpected(ErrorCode::VerificationError, + "required REG_SEQUENCE expansion action was rejected"); + auto Expanded = expandRegSequence(*Instruction, Function, Target, Info); + if (!Expanded) + return std::unexpected(Expanded.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + continue; + } + if (Instruction->opcode() == opcode(GenericOpcode::INSERT_SUBREG)) { + auto Accepted = + Analyses.requestAction("expand-insert-subreg", Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + return makeUnexpected(ErrorCode::VerificationError, + "required INSERT_SUBREG expansion action was rejected"); + auto Expanded = expandInsertSubreg(*Instruction, Function, Target, Info); + if (!Expanded) + return std::unexpected(Expanded.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + continue; + } + + std::size_t RewriteAttempts = 0; + bool Restart = true; + while (Restart && Instruction->parent() != nullptr) { + Restart = false; + if (++RewriteAttempts > Instruction->operandCount() * 2U + 8U) { + return makeUnexpected(ErrorCode::InternalError, + "two-address target conversion did not make progress"); + } + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "two-address rewrite encountered an unknown opcode"); + } + + for (std::size_t UseIndex = 0; UseIndex < Instruction->operandCount(); ++UseIndex) { + MachineOperand *Use = Instruction->operandAt(UseIndex); + if (Use == nullptr || !Use->isUse()) + continue; + const std::optional<std::size_t> TiedDefIndex = Use->tiedOperandIndex(); + if (!TiedDefIndex) + continue; + const std::size_t DefIndex = TiedDefIndex.value(); + MachineOperand *Def = Instruction->operandAt(DefIndex); + if (Def == nullptr || !Def->isDef()) { + return makeUnexpected(ErrorCode::VerificationError, + "tied use does not reference a definition"); + } + const std::optional<Register> DefReg = Def->registerValue(); + const std::optional<Register> UseReg = Use->registerValue(); + if (!DefReg || !UseReg || DefReg->isNoRegister()) { + return makeUnexpected(ErrorCode::Unsupported, + "two-address rewrite requires register ties"); + } + if (*DefReg == *UseReg) + continue; + + if (isImplicitlyUndef(Function, *Use)) { + auto Accepted = + Analyses.requestAction("rewrite-undef-tie", Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) + return makeUnexpected(ErrorCode::VerificationError, + "required undef tie rewrite action was rejected"); + auto Rewritten = rewriteUndefTie(*Use, *Def); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Operands); + continue; + } + + auto AllowCommute = + Analyses.requestAction("commute-tied-operands", Function, Instruction->stableID()); + if (!AllowCommute) + return std::unexpected(AllowCommute.error()); + if (*AllowCommute) { + auto Commuted = tryCommute(*Instruction, UseIndex, *DefReg, Target, Function); + if (!Commuted) + return std::unexpected(Commuted.error()); + if (*Commuted) { + Result.Changed = true; + Result.Changes.add(MachinePassChange::Operands); + continue; + } + } + + auto AllowConversion = + Analyses.requestAction("convert-three-address", Function, Instruction->stableID()); + if (!AllowConversion) + return std::unexpected(AllowConversion.error()); + if (*AllowConversion) { + auto Converted = + Info.tryConvertToThreeAddress(*Instruction, DefIndex, UseIndex, Function, Target); + if (!Converted) + return std::unexpected(Converted.error()); + if (*Converted) { + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + Result.Changes.add(MachinePassChange::Operands); + Result.Changes.add(MachinePassChange::Registers); + Restart = true; + break; + } + } + + auto AllowCopy = + Analyses.requestAction("insert-tied-copy", Function, Instruction->stableID()); + if (!AllowCopy) + return std::unexpected(AllowCopy.error()); + if (!*AllowCopy) + return makeUnexpected(ErrorCode::VerificationError, + "required tied COPY insertion action was rejected"); + MachineBasicBlock *Block = Instruction->parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "two-address instruction disappeared before COPY insertion"); + RegisterValue Destination{*DefReg, Def->subReg(), Def->registerFlags(), Def->targetFlags()}; + RegisterValue Source{*UseReg, Use->subReg(), Use->registerFlags(), Use->targetFlags()}; + auto Copy = buildCopy(Function, Target, Info, *Block, Instruction, Destination, Source, + Instruction); + if (!Copy) + return std::unexpected(Copy.error()); + auto SetRegister = Use->setRegister(*DefReg); + if (!SetRegister) + return std::unexpected(SetRegister.error()); + auto SetSubReg = Use->setSubReg(Def->subReg()); + if (!SetSubReg) + return std::unexpected(SetSubReg.error()); + RegisterOperandFlags Flags = Use->registerFlags(); + Flags.clear(RegisterOperandFlag::Kill); + auto SetFlags = Use->setRegisterFlags(Flags); + if (!SetFlags) + return std::unexpected(SetFlags.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + Result.Changes.add(MachinePassChange::Operands); + } + } + } + + auto FinalState = verifyNoStageTenPseudos(Function, Target); + if (!FinalState) + return std::unexpected(FinalState.error()); + if (Result.Changed && Function.properties().has(MachineFunctionProperty::TracksLiveness)) { + auto Repaired = repairTrackedLiveness(Function, Target, Result); + if (!Repaired) + return std::unexpected( + withErrorContext(Repaired.error(), "two-address liveness repair failed: ")); + } + Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/VirtRegMap.cpp b/backend/lib/CodeGen/VirtRegMap.cpp new file mode 100644 index 0000000..b764667 --- /dev/null +++ b/backend/lib/CodeGen/VirtRegMap.cpp @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/VirtRegMap.h" + +#include "RegAllocInternal.h" +#include "rcc/CodeGen/LiveIntervals.h" +#include "rcc/CodeGen/LiveRegMatrix.h" +#include "rcc/CodeGen/LiveVariables.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <cstddef> +#include <optional> +#include <ranges> +#include <utility> +#include <vector> + +namespace rcc { + +Expected<void> VirtRegMap::add(Register VirtualReg, bool Spillable, + std::optional<Register> SplitParent, + std::optional<Register> Origin) { + if (!VirtualReg.isVirtual() || !Function->registerInfo().isKnownVirtualRegister(VirtualReg)) + return makeUnexpected(ErrorCode::InvalidArgument, + "VirtRegMap requires a known virtual register"); + if (SplitParent && !SplitParent->isVirtual()) + return makeUnexpected(ErrorCode::InvalidArgument, "split parent must be a virtual register"); + if (Origin && !Origin->isVirtual()) + return makeUnexpected(ErrorCode::InvalidArgument, "split origin must be a virtual register"); + auto [Position, Inserted] = + Entries.try_emplace(VirtualReg, VirtRegMapEntry{VirtualReg, + VirtRegAllocationState::Unassigned, + std::nullopt, + SplitParent, + Origin, + Spillable, + {}}); + if (!Inserted) { + Position->second.Spillable = Spillable; + if (SplitParent) + Position->second.SplitParent = SplitParent; + if (Origin) + Position->second.Origin = Origin; + } + return {}; +} + +Expected<void> VirtRegMap::assign(Register VirtualReg, PhysicalRegister PhysicalReg) { + VirtRegMapEntry *Value = entry(VirtualReg); + if (Value == nullptr || !PhysicalReg.isValid()) + return makeUnexpected(ErrorCode::InvalidArgument, "VirtRegMap assignment is invalid"); + Value->PhysicalReg = PhysicalReg; + Value->State = VirtRegAllocationState::Assigned; + return {}; +} + +Expected<void> VirtRegMap::assignStackSlot(Register VirtualReg, FrameIndex StackSlot, + LaneMask Lanes) { + VirtRegMapEntry *Value = entry(VirtualReg); + if (Value == nullptr || Function->frameInfo().object(StackSlot) == nullptr || Lanes.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, "VirtRegMap spill assignment is invalid"); + Value->PhysicalReg.reset(); + Value->State = VirtRegAllocationState::Spilled; + Value->Spill.StackSlot = StackSlot; + Value->Spill.Lanes = Lanes; + return {}; +} + +Expected<void> VirtRegMap::markSplit(Register VirtualReg, Register Parent, Register Origin) { + VirtRegMapEntry *Value = entry(VirtualReg); + if (Value == nullptr || !Parent.isVirtual() || !Origin.isVirtual() || Parent == VirtualReg || + Origin == VirtualReg || !Function->registerInfo().isKnownVirtualRegister(Parent) || + !Function->registerInfo().isKnownVirtualRegister(Origin)) + return makeUnexpected(ErrorCode::InvalidArgument, "VirtRegMap split relationship is invalid"); + Value->State = VirtRegAllocationState::Split; + Value->SplitParent = Parent; + Value->Origin = Origin; + return {}; +} + +Expected<void> VirtRegMap::setSpillable(Register VirtualReg, bool Spillable) { + VirtRegMapEntry *Value = entry(VirtualReg); + if (Value == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "unknown virtual register in VirtRegMap"); + Value->Spillable = Spillable; + return {}; +} + +VirtRegMapEntry *VirtRegMap::entry(Register VirtualReg) noexcept { + const auto Position = Entries.find(VirtualReg); + return Position == Entries.end() ? nullptr : &Position->second; +} + +const VirtRegMapEntry *VirtRegMap::entry(Register VirtualReg) const noexcept { + const auto Position = Entries.find(VirtualReg); + return Position == Entries.end() ? nullptr : &Position->second; +} + +std::optional<PhysicalRegister> VirtRegMap::physicalRegister(Register VirtualReg) const noexcept { + const VirtRegMapEntry *Value = entry(VirtualReg); + return Value == nullptr ? std::nullopt : Value->PhysicalReg; +} + +std::optional<FrameIndex> VirtRegMap::stackSlot(Register VirtualReg) const noexcept { + const VirtRegMapEntry *Value = entry(VirtualReg); + return Value == nullptr ? std::nullopt : Value->Spill.StackSlot; +} + +bool VirtRegMap::isSpillable(Register VirtualReg) const noexcept { + const VirtRegMapEntry *Value = entry(VirtualReg); + return Value != nullptr && Value->Spillable; +} + +std::vector<const VirtRegMapEntry *> VirtRegMap::orderedEntries() const { + std::vector<const VirtRegMapEntry *> Result; + Result.reserve(Entries.size()); + for (const auto &[Reg, Value] : Entries) { + (void)Reg; + Result.push_back(&Value); + } + std::ranges::sort(Result, [](const VirtRegMapEntry *Left, const VirtRegMapEntry *Right) { + return Left->VirtualReg.opaqueValue() < Right->VirtualReg.opaqueValue(); + }); + return Result; +} + +Expected<void> VirtRegMap::verify() const { + if (Function == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "VirtRegMap has no MachineFunction"); + for (const VirtRegMapEntry *Value : orderedEntries()) { + if (Value == nullptr || !Value->VirtualReg.isVirtual() || + !Function->registerInfo().isKnownVirtualRegister(Value->VirtualReg)) + return makeUnexpected(ErrorCode::VerificationError, "VirtRegMap contains an invalid entry"); + if (Value->State == VirtRegAllocationState::Assigned && + (!Value->PhysicalReg || !Value->PhysicalReg->isValid())) + return makeUnexpected(ErrorCode::VerificationError, + "assigned VirtRegMap entry has no register"); + if (Value->State == VirtRegAllocationState::Spilled && !Value->Spill.StackSlot) + return makeUnexpected(ErrorCode::VerificationError, + "spilled VirtRegMap entry has no stack slot"); + if (Value->Spill.StackSlot && Value->Spill.Lanes.empty()) + return makeUnexpected(ErrorCode::VerificationError, + "VirtRegMap spill metadata has an empty lane mask"); + if (Value->SplitParent && + (!Value->SplitParent->isVirtual() || *Value->SplitParent == Value->VirtualReg || + !Function->registerInfo().isKnownVirtualRegister(*Value->SplitParent))) + return makeUnexpected(ErrorCode::VerificationError, "VirtRegMap has an invalid split parent"); + if (Value->Origin && (!Value->Origin->isVirtual() || *Value->Origin == Value->VirtualReg || + !Function->registerInfo().isKnownVirtualRegister(*Value->Origin))) + return makeUnexpected(ErrorCode::VerificationError, "VirtRegMap has an invalid split origin"); + if (Value->State == VirtRegAllocationState::Split && (!Value->SplitParent || !Value->Origin)) + return makeUnexpected(ErrorCode::VerificationError, + "split VirtRegMap entry lacks parent or origin metadata"); + } + return {}; +} + +Expected<bool> rewriteVirtualRegisters(MachineFunction &Function, const TargetDescription &Target, + const VirtRegMap &Mapping) { + if (&Mapping.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "VirtRegMap belongs to a different MachineFunction"); + auto MappingVerification = Mapping.verify(); + if (!MappingVerification) + return std::unexpected(MappingVerification.error()); + + MachineFunctionAnalysisManager ValidationAnalyses; + auto Registered = registerStage11MachineAnalyses(ValidationAnalyses); + if (!Registered) + return std::unexpected(Registered.error()); + auto Intervals = ValidationAnalyses.getResult<LiveIntervalsAnalysis>(Function, Target); + if (!Intervals) + return std::unexpected(Intervals.error()); + LiveRegMatrix ValidationMatrix(Function, Target, **Intervals); + for (const VirtRegMapEntry *Entry : Mapping.orderedEntries()) { + if (Entry == nullptr || !Entry->PhysicalReg) + continue; + auto ClassID = regalloc_detail::registerClassFor(Function, Entry->VirtualReg); + if (!ClassID) + return std::unexpected(ClassID.error()); + if (!Target.registers().registerClassContains(*ClassID, *Entry->PhysicalReg)) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "VirtRegMap physical register is outside the virtual register class"); + } + const LiveInterval *Interval = (*Intervals)->interval(Entry->VirtualReg); + if (Interval == nullptr || Interval->MainRange.empty()) + continue; + auto Assigned = ValidationMatrix.assign(Entry->VirtualReg, *Entry->PhysicalReg); + if (!Assigned) + return std::unexpected(Assigned.error()); + } + + struct Rewrite final { + MachineOperand *Operand = nullptr; + Register NewRegister; + }; + std::vector<Rewrite> Rewrites; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { + MachineOperand *Operand = Instruction->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + const auto Assigned = Mapping.physicalRegister(*Reg); + if (!Assigned) { + if (Operand->isDebugUse()) { + Rewrites.push_back({Operand, Register{}}); + continue; + } + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "physical rewrite found an unassigned virtual register " + + Reg->toString()); + } + PhysicalRegister Effective = *Assigned; + if (Operand->subReg().value() != 0) { + const auto Sub = Target.registers().subRegister(*Assigned, Operand->subReg()); + if (!Sub) { + return makeUnexpected(ErrorCode::RegisterAllocationFailed, + "physical assignment does not support a required subregister"); + } + Effective = *Sub; + } + Rewrites.push_back({Operand, Register::fromPhysical(Effective)}); + } + } + } + + for (const Rewrite &Plan : Rewrites) { + auto SetRegister = Plan.Operand->setRegister(Plan.NewRegister); + if (!SetRegister) + return std::unexpected(SetRegister.error()); + if (const auto Physical = Plan.NewRegister.physicalRegister()) { + const PhysicalRegisterDesc *Description = Target.registers().physicalRegister(*Physical); + if (Description != nullptr && !Description->IsConstant && !Description->IsArtificial && + !Description->IsReserved && !Target.registers().isReservedRegister(*Physical) && + !Function.registerInfo().isReserved(*Physical)) { + RegisterOperandFlags Flags = Plan.Operand->registerFlags(); + Flags.set(RegisterOperandFlag::Renamable); + auto Updated = Plan.Operand->setRegisterFlags(Flags); + if (!Updated) + return std::unexpected(Updated.error()); + } + } + if (Plan.Operand->subReg().value() != 0) { + auto Cleared = Plan.Operand->setSubReg(SubRegIndex{}); + if (!Cleared) + return std::unexpected(Cleared.error()); + } + if (Plan.NewRegister.isNoRegister()) { + RegisterOperandFlags Flags = Plan.Operand->registerFlags(); + Flags.set(RegisterOperandFlag::Undef); + auto Updated = Plan.Operand->setRegisterFlags(Flags); + if (!Updated) + return std::unexpected(Updated.error()); + } + } + + Function.registerInfo().clearLiveInVirtualRegisters(); + for (std::size_t Index = 0; Index < Function.registerInfo().virtualRegisters().size(); ++Index) { + auto Reg = regalloc_detail::virtualRegisterAt(Index); + if (!Reg) + return std::unexpected(Reg.error()); + if (!Function.registerInfo().isKnownVirtualRegister(*Reg)) + continue; + auto Erased = Function.registerInfo().eraseVirtualRegister(*Reg); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Liveness = rebuildMachineLiveness(Function, Target); + if (!Liveness) + return std::unexpected(Liveness.error()); + return !Rewrites.empty(); +} + +const MachinePassContract &VirtualRegisterRewritePass::contract() const noexcept { + static const MachinePassContract Contract{ + .Name = "virtregrewriter", + .Requires = {MachineFunctionProperty::NoPHIs}, + .Sets = {MachineFunctionProperty::NoVRegs}, + .Clears = {MachineFunctionProperty::IsSSA}, + .RequiredAnalyses = {MachineFunctionAnalysisManager::id<VirtRegMapAnalysis>()}, + .VerificationStage = PipelineStage::RegisterAllocated, + }; + return Contract; +} + +Expected<VirtRegMapAnalysis::Result> VirtRegMapAnalysis::run(MachineFunction &Function, + const TargetDescription &, + MachineFunctionAnalysisManager &) { + return VirtRegMap(Function); +} + +Expected<MachinePassResult> +VirtualRegisterRewritePass::run(MachineFunction &Function, const TargetDescription &Target, + MachineFunctionAnalysisManager &Analyses) { + auto Mapping = Analyses.getResult<VirtRegMapAnalysis>(Function, Target); + if (!Mapping) + return std::unexpected(Mapping.error()); + auto Rewritten = rewriteVirtualRegisters(Function, Target, **Mapping); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + MachinePassResult Result; + Result.Changed = *Rewritten; + if (Result.Changed) { + Result.Changes.add(MachinePassChange::Registers); + Result.Changes.add(MachinePassChange::Operands); + } + Result.Preserved = PreservedAnalyses::none(); + return Result; +} + +} // namespace rcc |