From 495c675771f43cfa12ab7a426a538be72fec1805 Mon Sep 17 00:00:00 2001 From: Richard-Qin-X Date: Sun, 9 Aug 2026 17:03:19 +0800 Subject: [CodeGen] Implement machine optimization and scheduling --- backend/lib/Target/RISCV/CMakeLists.txt | 18 + backend/lib/Target/RISCV/RISCV.rccdesc | 57 +- backend/lib/Target/RISCV/RISCVBranchInfo.cpp | 731 +++++++++++++++++++++ backend/lib/Target/RISCV/RISCVFrameLowering.cpp | 113 +++- backend/lib/Target/RISCV/RISCVMachineCombiner.cpp | 92 +++ .../lib/Target/RISCV/RISCVMachineMemoryFolding.cpp | 48 ++ .../lib/Target/RISCV/RISCVOptimizationPipeline.cpp | 19 + backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp | 512 +++++++++++++++ backend/lib/Target/RISCV/RISCVScheduling.cpp | 136 ++++ 9 files changed, 1693 insertions(+), 33 deletions(-) create mode 100644 backend/lib/Target/RISCV/RISCVBranchInfo.cpp create mode 100644 backend/lib/Target/RISCV/RISCVMachineCombiner.cpp create mode 100644 backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp create mode 100644 backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp create mode 100644 backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp create mode 100644 backend/lib/Target/RISCV/RISCVScheduling.cpp (limited to 'backend/lib/Target/RISCV') diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt index 442e75e..a29315a 100644 --- a/backend/lib/Target/RISCV/CMakeLists.txt +++ b/backend/lib/Target/RISCV/CMakeLists.txt @@ -47,10 +47,16 @@ set_source_files_properties( add_library( RCCBackendRISCV + RISCVBranchInfo.cpp RISCVCodeGen.cpp RISCVCallLowering.cpp RISCVFrameLowering.cpp + RISCVMachineCombiner.cpp + RISCVMachineMemoryFolding.cpp + RISCVOptimizationPipeline.cpp + RISCVPseudoExpansion.cpp RISCVRegisterAllocation.cpp + RISCVScheduling.cpp RISCVTarget.cpp "${RCCBackendRISCVGeneratedRegisterHeader}" "${RCCBackendRISCVGeneratedInstructionHeader}" @@ -93,19 +99,31 @@ install( ) rcc_backend_register_format_files( + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVBranchInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineMemoryFolding.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVOptimizationPipeline.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVPseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVScheduling.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCallLowering.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVTarget.h" ) rcc_backend_register_tidy_files( + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVBranchInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineCombiner.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVMachineMemoryFolding.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVOptimizationPipeline.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVPseudoExpansion.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVScheduling.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" ) diff --git a/backend/lib/Target/RISCV/RISCV.rccdesc b/backend/lib/Target/RISCV/RISCV.rccdesc index e248bc7..dce984c 100644 --- a/backend/lib/Target/RISCV/RISCV.rccdesc +++ b/backend/lib/Target/RISCV/RISCV.rccdesc @@ -245,6 +245,10 @@ mir_operand_flags: direct_mask: 31 direct: - { name: riscv-call, value: 1 } + - { name: riscv-pcrel-hi, value: 2 } + - { name: riscv-pcrel-lo, value: 4 } + - { name: riscv-copy-f32, value: 8 } + - { name: riscv-copy-f64, value: 16 } bitmask: [] calling_conventions: @@ -386,7 +390,7 @@ instructions: assembly_mnemonic: auipc operands: - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } - - { kinds: [immediate], immediate_predicate: SIMM20, name: IMM } + - { kinds: [immediate, basic_block, global_symbol, external_symbol, mc_symbol], immediate_predicate: SIMM20, allowed_target_flags: 3, name: IMM } num_explicit_defs: 1 flags: [produces_code] scheduling_class: ALU @@ -871,7 +875,7 @@ instructions: operands: - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } - - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } + - { kinds: [immediate, basic_block, global_symbol, external_symbol, mc_symbol], immediate_predicate: SIMM12, allowed_target_flags: 4, name: IMM } num_explicit_defs: 1 flags: [terminator, branch, indirect_branch, produces_code] scheduling_class: BRANCH @@ -3195,3 +3199,52 @@ instructions: flags: [pseudo, produces_code, allowed_pre_legalizer, terminator, return, barrier] owner: post_ra_expansion deadline: post_ra_finalized + - + id: 1225 + symbol: JALR_CALL + name: JALR_CALL + assembly_mnemonic: jalr + operands: + - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } + - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } + - { kinds: [immediate, mc_symbol], immediate_predicate: SIMM12, allowed_target_flags: 4, name: IMM } + - { kinds: [immediate, integer_immediate] } + - { kinds: [register_mask] } + num_explicit_defs: 1 + allow_extra_implicit_operands: true + flags: [call, may_load, may_store, has_side_effects, produces_code, not_duplicable, unmodeled_side_effects] + scheduling_class: BRANCH + encoding_category: I + assembly_format: RD_RS1_IMM + - + id: 1226 + symbol: JALR_TAIL + name: JALR_TAIL + assembly_mnemonic: jalr + operands: + - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } + - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } + - { kinds: [immediate, mc_symbol], immediate_predicate: SIMM12, allowed_target_flags: 4, name: IMM } + - { kinds: [immediate, integer_immediate] } + - { kinds: [register_mask] } + num_explicit_defs: 1 + allow_extra_implicit_operands: true + flags: [terminator, branch, indirect_branch, call, barrier, may_load, may_store, has_side_effects, produces_code, not_duplicable, unmodeled_side_effects] + scheduling_class: BRANCH + encoding_category: I + assembly_format: RD_RS1_IMM + - + id: 1227 + symbol: JALR_RET + name: JALR_RET + assembly_mnemonic: jalr + operands: + - { kinds: [register], role: def, register_class: GPR, register_bank: GPR, name: RD } + - { kinds: [register], role: use, register_class: GPR, register_bank: GPR, name: RS1 } + - { kinds: [immediate], immediate_predicate: SIMM12, name: IMM } + num_explicit_defs: 1 + allow_extra_implicit_operands: true + flags: [terminator, return, barrier, produces_code] + scheduling_class: BRANCH + encoding_category: I + assembly_format: RD_RS1_IMM diff --git a/backend/lib/Target/RISCV/RISCVBranchInfo.cpp b/backend/lib/Target/RISCV/RISCVBranchInfo.cpp new file mode 100644 index 0000000..18812d8 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVBranchInfo.cpp @@ -0,0 +1,731 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +constexpr std::uint32_t PCRelativeHighFlag = 2; +constexpr std::uint32_t PCRelativeLowFlag = 4; + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] bool isConditionalBranch(Opcode Value) noexcept { + return Value == targetOpcode(InstructionOpcode::BEQ) || + Value == targetOpcode(InstructionOpcode::BNE) || + Value == targetOpcode(InstructionOpcode::BLT) || + Value == targetOpcode(InstructionOpcode::BGE) || + Value == targetOpcode(InstructionOpcode::BLTU) || + Value == targetOpcode(InstructionOpcode::BGEU) || + Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ); +} + +[[nodiscard]] std::optional reversedOpcode(Opcode Value) noexcept { + constexpr std::array, 4> Pairs = {{ + {InstructionOpcode::BEQ, InstructionOpcode::BNE}, + {InstructionOpcode::BLT, InstructionOpcode::BGE}, + {InstructionOpcode::BLTU, InstructionOpcode::BGEU}, + {InstructionOpcode::C_BEQZ, InstructionOpcode::C_BNEZ}, + }}; + for (const auto &[Left, Right] : Pairs) { + if (Value == targetOpcode(Left)) + return targetOpcode(Right); + if (Value == targetOpcode(Right)) + return targetOpcode(Left); + } + return std::nullopt; +} + +[[nodiscard]] bool isZeroRegister(const MachineOperand *Operand) noexcept { + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); + const std::optional Physical = + Reg == nullptr ? std::nullopt : Reg->physicalRegister(); + return Physical && Physical->number() == static_cast(PhysicalRegisterID::X0); +} + +[[nodiscard]] bool sameRegister(const MachineOperand *Left, const MachineOperand *Right) noexcept { + const Register *LeftReg = Left == nullptr ? nullptr : Left->getIf(); + const Register *RightReg = Right == nullptr ? nullptr : Right->getIf(); + return LeftReg != nullptr && RightReg != nullptr && *LeftReg == *RightReg && + Left->subReg() == Right->subReg(); +} + +[[nodiscard]] bool isUnconditionalBranch(const MachineInstr &Instruction) noexcept { + if (Instruction.opcode() == targetOpcode(InstructionOpcode::C_J)) + return true; + return Instruction.opcode() == targetOpcode(InstructionOpcode::JAL) && + isZeroRegister(Instruction.operandAt(0)); +} + +[[nodiscard]] std::size_t branchTargetIndex(Opcode Value) noexcept { + if (Value == targetOpcode(InstructionOpcode::C_J)) + return 0; + if (Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ)) { + return 1; + } + if (Value == targetOpcode(InstructionOpcode::JAL)) + return 1; + return 2; +} + +[[nodiscard]] Expected directTarget(MachineInstr &Instruction) { + if (!isConditionalBranch(Instruction.opcode()) && + Instruction.opcode() != targetOpcode(InstructionOpcode::JAL) && + Instruction.opcode() != targetOpcode(InstructionOpcode::C_J)) { + return static_cast(nullptr); + } + MachineOperand *Operand = Instruction.operandAt(branchTargetIndex(Instruction.opcode())); + const BasicBlockOperand *Block = + Operand == nullptr ? nullptr : Operand->getIf(); + if (Block == nullptr || Block->Block == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V direct branch has no basic-block target"); + } + return Block->Block; +} + +[[nodiscard]] Expected setDirectTarget(MachineInstr &Instruction, + MachineBasicBlock &TargetBlock) { + MachineOperand *Operand = Instruction.operandAt(branchTargetIndex(Instruction.opcode())); + if (Operand == nullptr || Operand->kind() != MachineOperand::Kind::BasicBlock) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V direct branch has no mutable basic-block target"); + } + return Operand->setBasicBlock(TargetBlock); +} + +[[nodiscard]] Expected removeInstruction(MachineInstr &Instruction) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch instruction is not linked to a block"); + } + return Block->removeInstruction(Instruction); +} + +[[nodiscard]] Expected physical(PhysicalRegisterID ID) { + auto Reg = physicalRegister(ID); + if (!Reg) + return std::unexpected(Reg.error()); + return *Reg; +} + +[[nodiscard]] Expected +buildJAL(MachineFunction &Function, const TargetDescription &Target, MachineBasicBlock &Block, + MachineInstr *Before, PhysicalRegister Destination, MachineBasicBlock &TargetBlock) { + MachineIRBuilder Builder(Function, Target); + Expected Point = + Before == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Before); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(InstructionOpcode::JAL)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Register::fromPhysical(Destination)); + if (!Def) + return std::unexpected(Def.error()); + auto TargetOperand = Pending->addBasicBlock(TargetBlock); + if (!TargetOperand) + return std::unexpected(TargetOperand.error()); + return Pending->build(); +} + +[[nodiscard]] Expected copySourceIdentity(const MachineInstr &Source, + MachineInstr &Destination) { + if (Source.sourceLocation()) + Destination.setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Destination.setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Destination.setOrigin(*Source.origin()); + return {}; +} + +[[nodiscard]] Expected copyTrailingAnnotations(const MachineInstr &Source, + MachineInstr &Destination) { + if (Source.debugInstrNumber() != 0) { + auto DebugNumber = Destination.setDebugInstrNumber(Source.debugInstrNumber()); + if (!DebugNumber) + return std::unexpected(DebugNumber.error()); + } + if (Source.postInstrSymbol()) { + auto Symbol = Destination.setPostInstrSymbol(std::string(*Source.postInstrSymbol())); + if (!Symbol) + return std::unexpected(Symbol.error()); + } + if (Source.heapAllocMarker()) { + auto Marker = Destination.setHeapAllocMarker(*Source.heapAllocMarker()); + if (!Marker) + return std::unexpected(Marker.error()); + } + if (Source.pcSections()) { + auto Sections = Destination.setPCSections(*Source.pcSections()); + if (!Sections) + return std::unexpected(Sections.error()); + } + if (Source.mmra()) { + auto Metadata = Destination.setMMRA(*Source.mmra()); + if (!Metadata) + return std::unexpected(Metadata.error()); + } + if (Source.cfiType()) { + auto Type = Destination.setCFIType(*Source.cfiType()); + if (!Type) + return std::unexpected(Type.error()); + } + if (Source.deactivationSymbol()) { + auto Symbol = Destination.setDeactivationSymbol(std::string(*Source.deactivationSymbol())); + if (!Symbol) + return std::unexpected(Symbol.error()); + } + return {}; +} + +[[nodiscard]] Expected copySingleInstructionAnnotations(const MachineInstr &Source, + MachineInstr &Destination) { + Destination.setFlags(Source.flags()); + auto Identity = copySourceIdentity(Source, Destination); + if (!Identity) + return std::unexpected(Identity.error()); + if (Source.preInstrSymbol()) { + auto Symbol = Destination.setPreInstrSymbol(std::string(*Source.preInstrSymbol())); + if (!Symbol) + return std::unexpected(Symbol.error()); + } + return copyTrailingAnnotations(Source, Destination); +} + +[[nodiscard]] bool isZeroPhysicalRegister(PhysicalRegister Reg) noexcept { + return Reg.number() == static_cast(PhysicalRegisterID::X0); +} + +[[nodiscard]] Expected +selectLongBranchScratch(const MachineInstr &Instruction, const MachineBasicBlock &TargetBlock, + const MachineFunction &Function, const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const std::optional DestinationPhysical = + Destination == nullptr ? std::nullopt : Destination->physicalRegister(); + if (!DestinationPhysical) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V long JAL has no physical destination"); + } + if (!isZeroPhysicalRegister(*DestinationPhysical)) + return *DestinationPhysical; + + const auto GPRClass = + RegisterClassID::fromValue(static_cast(RegisterClass::GPR_NX0)); + const RegisterClassDesc *Candidates = Target.registers().registerClass(GPRClass); + if (Candidates == nullptr) { + return makeUnexpected(ErrorCode::InternalError, "RISC-V target has no nonzero GPR class"); + } + for (const PhysicalRegister Candidate : Candidates->AllocationOrder) { + if (Target.registers().isReservedRegister(Candidate) || + Function.registerInfo().isReserved(Candidate)) { + continue; + } + const bool LiveAtTarget = + std::ranges::any_of(TargetBlock.liveIns(), [&](const MachineBasicBlockLiveIn &LiveIn) { + return Target.registers().regsOverlap(Candidate, LiveIn.PhysReg); + }); + if (!LiveAtTarget) + return Candidate; + } + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "RISC-V long branch has no dead physical scratch register"); +} + +[[nodiscard]] Expected expandCompressedBranch(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "compressed branch is not linked"); + auto TargetBlock = directTarget(Instruction); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + + if (Instruction.opcode() == targetOpcode(InstructionOpcode::C_J)) { + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Replacement = buildJAL(Function, Target, *Block, &Instruction, *Zero, **TargetBlock); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Annotations = copySingleInstructionAnnotations(Instruction, **Replacement); + if (!Annotations) + return std::unexpected(Annotations.error()); + auto Removed = removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + + const MachineOperand *SourceOperand = Instruction.operandAt(0); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "compressed conditional branch has no source register"); + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + const InstructionOpcode Expanded = Instruction.opcode() == targetOpcode(InstructionOpcode::C_BEQZ) + ? InstructionOpcode::BEQ + : InstructionOpcode::BNE; + auto Pending = Builder.buildInstr(targetOpcode(Expanded)); + if (!Pending) + return std::unexpected(Pending.error()); + auto First = Pending->addUse(*Source); + if (!First) + return std::unexpected(First.error()); + auto Second = Pending->addUse(Register::fromPhysical(*Zero)); + if (!Second) + return std::unexpected(Second.error()); + auto Destination = Pending->addBasicBlock(**TargetBlock); + if (!Destination) + return std::unexpected(Destination.error()); + auto Replacement = Pending->build(); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Annotations = copySingleInstructionAnnotations(Instruction, **Replacement); + if (!Annotations) + return std::unexpected(Annotations.error()); + auto Removed = removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +[[nodiscard]] Expected relaxConditionalBranch(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "conditional branch is not linked"); + auto TrueTarget = directTarget(Instruction); + if (!TrueTarget) + return std::unexpected(TrueTarget.error()); + MachineBasicBlock *FalseTarget = Block->next(); + if (FalseTarget == nullptr || FalseTarget == *TrueTarget || + std::ranges::none_of(Block->successors(), [FalseTarget](const MachineSuccessor &Successor) { + return Successor.Block == FalseTarget; + })) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "RISC-V conditional relaxation requires its false edge as fallthrough"); + } + + auto Reversed = reversedOpcode(Instruction.opcode()); + if (!Reversed) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V conditional branch cannot be inverted"); + auto SetOpcode = Instruction.setOpcode(*Reversed); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto SetTarget = setDirectTarget(Instruction, *FalseTarget); + if (!SetTarget) + return std::unexpected(SetTarget.error()); + + MachineInstr *Following = Instruction.next(); + if (Following != nullptr && isUnconditionalBranch(*Following)) { + auto Retargeted = setDirectTarget(*Following, **TrueTarget); + if (!Retargeted) + return std::unexpected(Retargeted.error()); + return true; + } + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointAfter(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(InstructionOpcode::JAL)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Register::fromPhysical(*Zero)); + if (!Def) + return std::unexpected(Def.error()); + auto Destination = Pending->addBasicBlock(**TrueTarget); + if (!Destination) + return std::unexpected(Destination.error()); + auto LongJump = Pending->build(); + if (!LongJump) + return std::unexpected(LongJump.error()); + auto Identity = copySourceIdentity(Instruction, **LongJump); + if (!Identity) + return std::unexpected(Identity.error()); + MachineInstrFlags OriginalFlags = Instruction.flags(); + const bool HadBundledSuccessor = OriginalFlags.has(MachineInstrFlag::BundledWithSucc); + OriginalFlags.set(MachineInstrFlag::BundledWithSucc); + Instruction.setFlags(OriginalFlags); + MachineInstrFlags LongFlags; + LongFlags.set(MachineInstrFlag::BundledWithPred); + if (HadBundledSuccessor) + LongFlags.set(MachineInstrFlag::BundledWithSucc); + if (OriginalFlags.has(MachineInstrFlag::NoMerge)) + LongFlags.set(MachineInstrFlag::NoMerge); + (*LongJump)->setFlags(LongFlags); + return true; +} + +[[nodiscard]] Expected relaxJAL(MachineInstr &Instruction, std::int64_t Offset, + MachineFunction &Function, const TargetDescription &Target) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V JAL is not linked"); + if (Offset < std::numeric_limits::min() || + Offset > std::numeric_limits::max()) { + return makeUnexpected(ErrorCode::BranchRelaxationFailed, + "RISC-V PC-relative long branch exceeds the signed 32-bit range"); + } + auto DestinationBlock = directTarget(Instruction); + if (!DestinationBlock) + return std::unexpected(DestinationBlock.error()); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr || !Destination->isPhysical()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V long JAL requires a physical destination register"); + } + auto Scratch = selectLongBranchScratch(Instruction, **DestinationBlock, Function, Target); + if (!Scratch) + return std::unexpected(Scratch.error()); + const std::string Anchor = Instruction.preInstrSymbol() + ? std::string(*Instruction.preInstrSymbol()) + : ".Lrcc_long_" + std::to_string(Function.stableID().value()) + + "_" + std::to_string(Instruction.stableID().value()); + const MachineInstrFlags OriginalFlags = Instruction.flags(); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto HighPending = Builder.buildInstr(targetOpcode(InstructionOpcode::AUIPC)); + if (!HighPending) + return std::unexpected(HighPending.error()); + auto HighDef = HighPending->addDef(Register::fromPhysical(*Scratch)); + if (!HighDef) + return std::unexpected(HighDef.error()); + auto HighTarget = HighPending->addBasicBlock(**DestinationBlock, PCRelativeHighFlag); + if (!HighTarget) + return std::unexpected(HighTarget.error()); + auto High = HighPending->build(); + if (!High) + return std::unexpected(High.error()); + auto HighSymbol = (*High)->setPreInstrSymbol(Anchor); + if (!HighSymbol) + return std::unexpected(HighSymbol.error()); + auto HighIdentity = copySourceIdentity(Instruction, **High); + if (!HighIdentity) + return std::unexpected(HighIdentity.error()); + + auto LowPending = Builder.buildInstr(targetOpcode(InstructionOpcode::JALR)); + if (!LowPending) + return std::unexpected(LowPending.error()); + auto LowDef = LowPending->addDef(*Destination); + if (!LowDef) + return std::unexpected(LowDef.error()); + auto LowBase = LowPending->addUse(Register::fromPhysical(*Scratch)); + if (!LowBase) + return std::unexpected(LowBase.error()); + auto LowSymbol = LowPending->addMCSymbol(Anchor, 0, PCRelativeLowFlag); + if (!LowSymbol) + return std::unexpected(LowSymbol.error()); + auto Low = LowPending->build(); + if (!Low) + return std::unexpected(Low.error()); + auto LowIdentity = copySourceIdentity(Instruction, **Low); + if (!LowIdentity) + return std::unexpected(LowIdentity.error()); + auto LowAnnotations = copyTrailingAnnotations(Instruction, **Low); + if (!LowAnnotations) + return std::unexpected(LowAnnotations.error()); + + MachineInstrFlags HighFlags; + if (OriginalFlags.has(MachineInstrFlag::BundledWithPred)) + HighFlags.set(MachineInstrFlag::BundledWithPred); + HighFlags.set(MachineInstrFlag::BundledWithSucc); + if (OriginalFlags.has(MachineInstrFlag::NoMerge)) + HighFlags.set(MachineInstrFlag::NoMerge); + (*High)->setFlags(HighFlags); + MachineInstrFlags LowFlags = OriginalFlags; + LowFlags.clear(MachineInstrFlag::BundledWithPred); + LowFlags.set(MachineInstrFlag::BundledWithPred); + (*Low)->setFlags(LowFlags); + auto Removed = removeInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +class RISCVBranchInfo final : public TargetBranchInfo { +public: + [[nodiscard]] Expected + analyzeBranch(MachineBasicBlock &Block, MachineFunction &Function, + const TargetDescription &Target) const override { + if (&Block.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch analysis references another function"); + std::vector Terminators; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V branch analysis encountered an unknown opcode"); + if (Descriptor->Flags.has(InstructionFlag::Terminator)) + Terminators.push_back(Instruction); + } + + MachineBranchAnalysis Result; + if (Terminators.empty()) + return Result; + if (Terminators.size() > 2) { + Result.HasUnanalyzableTerminator = true; + return Result; + } + MachineInstr *First = Terminators.front(); + if (isConditionalBranch(First->opcode())) { + auto TrueTarget = directTarget(*First); + if (!TrueTarget) + return std::unexpected(TrueTarget.error()); + Result.Conditional = First; + Result.TrueTarget = *TrueTarget; + if (Terminators.size() == 2) { + MachineInstr *Second = Terminators.back(); + if (!isUnconditionalBranch(*Second)) { + Result.HasUnanalyzableTerminator = true; + return Result; + } + auto FalseTarget = directTarget(*Second); + if (!FalseTarget) + return std::unexpected(FalseTarget.error()); + Result.Unconditional = Second; + Result.FalseTarget = *FalseTarget; + } else { + for (const MachineSuccessor &Successor : Block.successors()) { + if (Successor.Block == nullptr || Successor.Block == Result.TrueTarget) + continue; + if (Result.FalseTarget != nullptr) { + Result.HasUnanalyzableTerminator = true; + return Result; + } + Result.FalseTarget = Successor.Block; + } + if (Result.FalseTarget == nullptr) + Result.FalseTarget = Block.next(); + } + return Result; + } + if (Terminators.size() == 1 && isUnconditionalBranch(*First)) { + auto TargetBlock = directTarget(*First); + if (!TargetBlock) + return std::unexpected(TargetBlock.error()); + Result.Unconditional = First; + Result.TrueTarget = *TargetBlock; + return Result; + } + Result.HasUnanalyzableTerminator = true; + return Result; + } + + [[nodiscard]] Expected reverseBranchCondition(MachineInstr &Branch, + MachineBasicBlock &NewTarget, + MachineFunction &Function, + const TargetDescription &) const override { + if (&Branch.function() != &Function || &NewTarget.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch reversal references another function"); + const std::optional Reversed = reversedOpcode(Branch.opcode()); + if (!Reversed) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch opcode has no inverse condition"); + auto OpcodeResult = Branch.setOpcode(*Reversed); + if (!OpcodeResult) + return std::unexpected(OpcodeResult.error()); + return setDirectTarget(Branch, NewTarget); + } + + [[nodiscard]] Expected removeBranch(MachineInstr &Branch, MachineFunction &Function, + const TargetDescription &) const override { + if (&Branch.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch removal references another function"); + return removeInstruction(Branch); + } + + [[nodiscard]] Expected rewriteBranchTarget(MachineBasicBlock &Block, + MachineBasicBlock &OldTarget, + MachineBasicBlock &NewTarget, + MachineFunction &Function, + const TargetDescription &) const override { + if (&Block.function() != &Function || &OldTarget.function() != &Function || + &NewTarget.function() != &Function) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch rewrite references another function"); + } + bool Changed = false; + for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (!isConditionalBranch(Instruction->opcode()) && !isUnconditionalBranch(*Instruction)) { + continue; + } + auto CurrentTarget = directTarget(*Instruction); + if (!CurrentTarget) + return std::unexpected(CurrentTarget.error()); + if (*CurrentTarget != &OldTarget) + continue; + auto Rewritten = setDirectTarget(*Instruction, NewTarget); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + Changed = true; + } + return Changed; + } + + [[nodiscard]] Expected + insertUnconditionalBranch(MachineBasicBlock &Block, MachineBasicBlock &TargetBlock, + MachineFunction &Function, + const TargetDescription &Target) const override { + if (&Block.function() != &Function || &TargetBlock.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch insertion references another function"); + auto Zero = physical(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + return buildJAL(Function, Target, Block, nullptr, *Zero, TargetBlock); + } + + [[nodiscard]] Expected + instructionSize(const MachineInstr &Instruction, const MachineFunction &, + const TargetDescription &Target) const override { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V instruction-size query encountered an unknown opcode"); + if (!Descriptor->Flags.has(InstructionFlag::ProducesCode)) + return 0; + const auto Category = + std::ranges::find_if(encodingCategories(), [Descriptor](const EncodingCategoryInfo &Info) { + return Info.ID == Descriptor->EncodingCategory; + }); + if (Category == encodingCategories().end() || Category->SizeInBytes == 0) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V producing instruction has no exact encoding size"); + return Category->SizeInBytes; + } + + [[nodiscard]] Expected + branchTarget(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &) const override { + if (&Instruction.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch target query references another function"); + return directTarget(Instruction); + } + + [[nodiscard]] Expected> + evaluateBranchCondition(const MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &) const override { + if (&Instruction.function() != &Function) { + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch evaluation references another function"); + } + const Opcode Value = Instruction.opcode(); + if (!isConditionalBranch(Value)) + return std::optional{}; + if (Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ)) { + if (!isZeroRegister(Instruction.operandAt(0))) + return std::optional{}; + return std::optional{Value == targetOpcode(InstructionOpcode::C_BEQZ)}; + } + if (!sameRegister(Instruction.operandAt(0), Instruction.operandAt(1))) + return std::optional{}; + if (Value == targetOpcode(InstructionOpcode::BEQ) || + Value == targetOpcode(InstructionOpcode::BGE) || + Value == targetOpcode(InstructionOpcode::BGEU)) { + return std::optional{true}; + } + return std::optional{false}; + } + + [[nodiscard]] bool isBranchOffsetInRange(const MachineInstr &Instruction, + std::int64_t Offset) const noexcept override { + if ((Offset & 1) != 0) + return false; + const Opcode Value = Instruction.opcode(); + if (Value == targetOpcode(InstructionOpcode::C_BEQZ) || + Value == targetOpcode(InstructionOpcode::C_BNEZ)) { + return Offset >= -256 && Offset <= 254; + } + if (Value == targetOpcode(InstructionOpcode::C_J)) + return Offset >= -2048 && Offset <= 2046; + if (isConditionalBranch(Value)) + return Offset >= -4096 && Offset <= 4094; + if (Value == targetOpcode(InstructionOpcode::JAL)) + return Offset >= -1048576 && Offset <= 1048574; + return true; + } + + [[nodiscard]] Expected relaxBranch(MachineInstr &Instruction, std::int64_t Offset, + MachineFunction &Function, + const TargetDescription &Target) const override { + if (&Instruction.function() != &Function) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V branch relaxation references another function"); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::C_J) || + Instruction.opcode() == targetOpcode(InstructionOpcode::C_BEQZ) || + Instruction.opcode() == targetOpcode(InstructionOpcode::C_BNEZ)) { + return expandCompressedBranch(Instruction, Function, Target); + } + if (isConditionalBranch(Instruction.opcode())) + return relaxConditionalBranch(Instruction, Function, Target); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::JAL)) + return relaxJAL(Instruction, Offset, Function, Target); + return false; + } +}; + +} // namespace + +const TargetBranchInfo &branchInfo() noexcept { + static const RISCVBranchInfo Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp index 0e6fec9..e47a9c8 100644 --- a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp +++ b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp @@ -1124,8 +1124,43 @@ calleeSaveOperations(MachineBasicBlock &Block, const MachineFunction &Function, return Result; } +[[nodiscard]] MachineBasicBlock *blockWithID(MachineFunction &Function, + MachineBlockStableID ID) noexcept { + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (Block->stableID() == ID) + return Block; + } + return nullptr; +} + +[[nodiscard]] MachineBasicBlock *frameSaveBlock(MachineFunction &Function) noexcept { + const auto &SavePoints = Function.frameInfo().state().SavePoints; + return SavePoints.empty() ? Function.firstBlock() + : blockWithID(Function, SavePoints.front().Block); +} + +[[nodiscard]] bool isFrameRestoreBlock(const MachineFunction &Function, + const MachineBasicBlock &Block) noexcept { + const MachineFrameState &State = Function.frameInfo().state(); + if (State.RestorePoints.empty()) + return State.SavePoints.empty(); + return std::ranges::any_of(State.RestorePoints, + [&Block](const MachineFrameSaveRestorePoint &Point) { + return Point.Block == Block.stableID(); + }); +} + class RISCVFrameLowering final : public TargetFrameLowering { public: + [[nodiscard]] bool + isCalleeSavedRegister(PhysicalRegister Reg, const MachineFunction &, + const TargetDescription &Target) const noexcept override { + return std::ranges::any_of(saveOrder(), [&](PhysicalRegisterID ID) { + const auto Candidate = physical(ID); + return Candidate && Target.registers().regsOverlap(Reg, *Candidate); + }); + } + [[nodiscard]] Expected prepareFrame(MachineFunction &Function, const TargetDescription &Target) const override { MachineFrameInfo &Frame = Function.frameInfo(); @@ -1133,6 +1168,10 @@ public: return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V frame layout is already frozen"); MachineFrameState State = Frame.state(); + const std::vector RequestedSavePoints = State.SavePoints; + const std::vector RequestedRestorePoints = State.RestorePoints; + const bool HasRequestedPlacement = + !RequestedSavePoints.empty() || !RequestedRestorePoints.empty(); for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; @@ -1249,25 +1288,32 @@ public: State.IsCalleeSavedInfoValid = true; State.SavePoints.clear(); State.RestorePoints.clear(); - if (!SavedRegisters->empty() && Function.firstBlock() != nullptr) { - State.SavePoints.push_back( - MachineFrameSaveRestorePoint{Function.firstBlock()->stableID(), *SavedRegisters}); - } - for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; - Block = Block->next()) { - bool HasEpilogue = false; - for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; - Instruction = Instruction->next()) { - HasEpilogue |= isEpilogueTerminator(*Instruction, Target); + std::vector RestoredRegisters; + std::ranges::copy_if( + *SavedRegisters, std::back_inserter(RestoredRegisters), + [&Function](PhysicalRegister Reg) { return calleeSaveIsRestored(Function, Reg); }); + if (HasRequestedPlacement) { + if (RequestedSavePoints.size() != 1 || RequestedRestorePoints.empty()) { + return makeUnexpected( + ErrorCode::VerificationError, + "RISC-V shrink-wrapped frame requires one save point and at least one restore point"); } - if (HasEpilogue && !SavedRegisters->empty()) { - std::vector Restored; - std::ranges::copy_if( - *SavedRegisters, std::back_inserter(Restored), - [&Function](PhysicalRegister Reg) { return calleeSaveIsRestored(Function, Reg); }); - if (!Restored.empty()) { - State.RestorePoints.push_back( - MachineFrameSaveRestorePoint{Block->stableID(), std::move(Restored)}); + State.SavePoints.push_back({RequestedSavePoints.front().Block, *SavedRegisters}); + for (const MachineFrameSaveRestorePoint &Point : RequestedRestorePoints) + State.RestorePoints.push_back({Point.Block, RestoredRegisters}); + } else { + if (!SavedRegisters->empty() && Function.firstBlock() != nullptr) { + State.SavePoints.push_back({Function.firstBlock()->stableID(), *SavedRegisters}); + } + for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; + Block = Block->next()) { + bool HasEpilogue = false; + for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + HasEpilogue |= isEpilogueTerminator(*Instruction, Target); + } + if (HasEpilogue && !RestoredRegisters.empty()) { + State.RestorePoints.push_back({Block->stableID(), RestoredRegisters}); } } } @@ -1290,10 +1336,10 @@ public: const std::span Saved = Function.records().calleeSavedRegisters(); if (Saved.empty()) return {}; - MachineBasicBlock *Entry = Function.firstBlock(); - if (Entry == nullptr) { + MachineBasicBlock *SaveBlock = frameSaveBlock(Function); + if (SaveBlock == nullptr) { return makeUnexpected(ErrorCode::VerificationError, - "RISC-V callee-save frame has no entry block"); + "RISC-V callee-save frame has no valid save block"); } auto FP = physical(PhysicalRegisterID::X8); auto BP = physical(PhysicalRegisterID::X9); @@ -1303,9 +1349,9 @@ public: const bool Realign = requiresStackRealignment(Function); MachineIRBuilder SaveBuilder(Function, Target); - auto SavePoint = Entry->firstInstruction() != nullptr - ? SaveBuilder.setInsertPointBefore(*Entry->firstInstruction()) - : SaveBuilder.setInsertPoint(*Entry); + auto SavePoint = SaveBlock->firstInstruction() != nullptr + ? SaveBuilder.setInsertPointBefore(*SaveBlock->firstInstruction()) + : SaveBuilder.setInsertPoint(*SaveBlock); if (!SavePoint) return std::unexpected(SavePoint.error()); for (const PhysicalRegister Reg : Saved) { @@ -1322,6 +1368,8 @@ public: for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) { + if (!isFrameRestoreBlock(Function, *Block)) + continue; std::vector Terminators; for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; Instruction = Instruction->next()) { @@ -1433,21 +1481,22 @@ public: [[nodiscard]] Expected emitPrologue(MachineFunction &Function, const TargetDescription &Target, RegisterScavenger &Scavenger) const override { - MachineBasicBlock *Entry = Function.firstBlock(); - if (Entry == nullptr) + MachineBasicBlock *SaveBlock = frameSaveBlock(Function); + if (SaveBlock == nullptr) return Function.frameInfo().state().StackSize == 0 ? Expected{} : makeUnexpected(ErrorCode::VerificationError, - "RISC-V nonempty frame has no entry block"); - const std::vector Saves = calleeSaveOperations(*Entry, Function, false); + "RISC-V nonempty frame has no save block"); + const std::vector Saves = + calleeSaveOperations(*SaveBlock, Function, false); if (Saves.size() != Function.records().calleeSavedRegisters().size()) { return makeUnexpected(ErrorCode::InternalError, "RISC-V prologue has an incomplete callee-save sequence"); } - MachineInstr *Anchor = Entry->firstInstruction(); + MachineInstr *Anchor = SaveBlock->firstInstruction(); MachineIRBuilder Builder(Function, Target); - auto Point = - Anchor != nullptr ? Builder.setInsertPointBefore(*Anchor) : Builder.setInsertPoint(*Entry); + auto Point = Anchor != nullptr ? Builder.setInsertPointBefore(*Anchor) + : Builder.setInsertPoint(*SaveBlock); if (!Point) return std::unexpected(Point.error()); @@ -1637,6 +1686,8 @@ public: MachineFunction &Function, const TargetDescription &Target, RegisterScavenger &Scavenger) const override { + if (!isFrameRestoreBlock(Function, Block)) + return {}; auto FP = physical(PhysicalRegisterID::X8); auto SP = physical(PhysicalRegisterID::X2); auto BP = physical(PhysicalRegisterID::X9); diff --git a/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp b/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp new file mode 100644 index 0000000..5c22dde --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVMachineCombiner.cpp @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] constexpr MachineCombinerStageMask allOptimizationStages() noexcept { + return machineCombinerStageMask(MachineCombinerStage::SelectedSSA) | + machineCombinerStageMask(MachineCombinerStage::PreRegisterAllocation) | + machineCombinerStageMask(MachineCombinerStage::PostRegisterAllocation) | + machineCombinerStageMask(MachineCombinerStage::PostFrame); +} + +[[nodiscard]] MachineCombinerPattern immediateIdentity(InstructionOpcode OpcodeValue, + std::string Name, std::int64_t Immediate) { + return { + .Name = std::move(Name), + .RootOpcode = targetOpcode(OpcodeValue), + .Stages = allOptimizationStages(), + .Predicates = {{MachineCombinerPredicateKind::ImmediateEquals, 2, 0, Immediate}}, + .Action = {MachineCombinerActionKind::ReplaceDefWithOperandAndErase, 0, 1}, + }; +} + +[[nodiscard]] MachineCombinerPattern zeroRegisterIdentity(InstructionOpcode OpcodeValue, + std::string Name, std::size_t ZeroOperand, + std::size_t SourceOperand) { + return { + .Name = std::move(Name), + .RootOpcode = targetOpcode(OpcodeValue), + .Stages = allOptimizationStages(), + .Predicates = {{MachineCombinerPredicateKind::PhysicalRegisterEquals, ZeroOperand, 0, + static_cast(PhysicalRegisterID::X0)}}, + .Action = {MachineCombinerActionKind::ReplaceDefWithOperandAndErase, 0, SourceOperand}, + }; +} + +[[nodiscard]] MachineCombinerPattern equalSourceIdentity(InstructionOpcode OpcodeValue, + std::string Name) { + return { + .Name = std::move(Name), + .RootOpcode = targetOpcode(OpcodeValue), + .Stages = allOptimizationStages(), + .Predicates = {{MachineCombinerPredicateKind::RegisterEquals, 1, 2, 0}}, + .Action = {MachineCombinerActionKind::ReplaceDefWithOperandAndErase, 0, 1}, + }; +} + +[[nodiscard]] std::vector createPatterns() { + std::vector Patterns; + Patterns.push_back(immediateIdentity(InstructionOpcode::ADDI, "addi-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::XORI, "xori-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::ORI, "ori-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::ANDI, "andi-all-ones", -1)); + Patterns.push_back(immediateIdentity(InstructionOpcode::SLLI, "slli-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::SRLI, "srli-zero", 0)); + Patterns.push_back(immediateIdentity(InstructionOpcode::SRAI, "srai-zero", 0)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::ADD, "add-zero-right", 2, 1)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::ADD, "add-zero-left", 1, 2)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::OR, "or-zero-right", 2, 1)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::OR, "or-zero-left", 1, 2)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::XOR, "xor-zero-right", 2, 1)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::XOR, "xor-zero-left", 1, 2)); + Patterns.push_back(zeroRegisterIdentity(InstructionOpcode::SUB, "sub-zero-right", 2, 1)); + Patterns.push_back(equalSourceIdentity(InstructionOpcode::FSGNJ_S, "fsgnj-s-copy")); + Patterns.push_back(equalSourceIdentity(InstructionOpcode::FSGNJ_D, "fsgnj-d-copy")); + return Patterns; +} + +} // namespace + +const TargetMachineCombinerInfo &machineCombinerInfo() noexcept { + static const Expected Info = + TargetMachineCombinerInfo::create(createPatterns()); + if (!Info) + std::abort(); + return *Info; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp b/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp new file mode 100644 index 0000000..2cc5f24 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVMachineMemoryFolding.cpp @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" + +#include +#include + +namespace rcc::riscv { +namespace { + +class RISCVMachineMemoryFoldingInfo final : public TargetMemoryFoldingInfo { +public: + [[nodiscard]] Expected canFoldMemoryOperand(const MachineInstr &, std::size_t, + const MachineInstr &, const MachineFunction &, + const TargetDescription &) const override { + return false; + } + + [[nodiscard]] Expected + foldMemoryOperand(MachineInstr &, std::size_t, MachineInstr &, MachineFunction &, + const TargetDescription &) const override { + return makeUnexpected(ErrorCode::Unsupported, + "RV64GC has no register instruction form with a folded memory operand"); + } + + [[nodiscard]] Expected unfoldMemoryOperand(MachineInstr &, MachineFunction &, + const TargetDescription &) const override { + return false; + } + + [[nodiscard]] Expected> + getFoldedMemorySize(const MachineInstr &, const MachineFunction &, + const TargetDescription &) const override { + return std::optional{}; + } +}; + +} // namespace + +const TargetMemoryFoldingInfo &memoryFoldingInfo() noexcept { + static const RISCVMachineMemoryFoldingInfo Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp b/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp new file mode 100644 index 0000000..2f8c5d1 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVOptimizationPipeline.cpp @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" + +namespace rcc::riscv { + +MachineOptimizationPipelineTargetInfo optimizationPipelineInfo() noexcept { + return { + .SSA = ssaInfo(), + .Combiner = machineCombinerInfo(), + .MemoryFolding = memoryFoldingInfo(), + .FrameLowering = frameLowering(), + .PseudoExpansion = pseudoExpansionInfo(), + .SchedulingModel = schedulingModel(), + .BranchInfo = branchInfo(), + }; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp new file mode 100644 index 0000000..ed73794 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVPseudoExpansion.cpp @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionRecords.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVInstructions.h" +#include "rcc/Target/RISCV/RISCVRegisters.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +constexpr std::uint32_t PCRelativeHighFlag = 2; +constexpr std::uint32_t PCRelativeLowFlag = 4; +constexpr std::uint32_t CopyF32Flag = 8; +constexpr std::uint32_t CopyF64Flag = 16; + +[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] Expected physical(PhysicalRegisterID ID) { + auto Reg = physicalRegister(ID); + if (!Reg) + return std::unexpected(Reg.error()); + return *Reg; +} + +[[nodiscard]] std::optional +registerBank(Register Reg, const MachineFunction &Function, const TargetDescription &Target) { + if (Reg.isVirtual()) { + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + return Info ? Info->get().Bank : std::nullopt; + } + const std::optional Physical = Reg.physicalRegister(); + if (!Physical) + return std::nullopt; + for (const RegisterBankDesc &Bank : Target.registers().registerBanks()) { + if (Target.registers().bankContainsRegister(Bank.ID, *Physical)) + return Bank.ID; + } + return std::nullopt; +} + +[[nodiscard]] std::optional registerType(Register Reg, const MachineFunction &Function) { + if (!Reg.isVirtual()) + return std::nullopt; + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + return Info ? Info->get().Type : std::nullopt; +} + +[[nodiscard]] Expected copyLocation(const MachineInstr &Source, MachineInstr &Destination) { + if (Source.sourceLocation()) + Destination.setSourceLocation(*Source.sourceLocation()); + if (Source.debugLocation()) { + auto Debug = Destination.setDebugLocation(*Source.debugLocation()); + if (!Debug) + return std::unexpected(Debug.error()); + } + if (Source.origin()) + Destination.setOrigin(*Source.origin()); + return {}; +} + +[[nodiscard]] Expected copyCommonDecoration(const MachineInstr &Source, + MachineInstr &Destination, bool CopyPreSymbol, + bool CopyPostSymbol, bool CopyDebugNumber) { + auto Location = copyLocation(Source, Destination); + if (!Location) + return std::unexpected(Location.error()); + Destination.setFlags(Source.flags()); + if (CopyDebugNumber && Source.debugInstrNumber() != 0) { + auto Number = Destination.setDebugInstrNumber(Source.debugInstrNumber()); + if (!Number) + return std::unexpected(Number.error()); + } + if (CopyPreSymbol) { + if (const auto Symbol = Source.preInstrSymbol()) { + auto Set = Destination.setPreInstrSymbol(std::string(*Symbol)); + if (!Set) + return std::unexpected(Set.error()); + } + } + if (CopyPostSymbol) { + if (const auto Symbol = Source.postInstrSymbol()) { + auto Set = Destination.setPostInstrSymbol(std::string(*Symbol)); + if (!Set) + return std::unexpected(Set.error()); + } + } + if (Source.heapAllocMarker()) { + auto Set = Destination.setHeapAllocMarker(*Source.heapAllocMarker()); + if (!Set) + return std::unexpected(Set.error()); + } + if (Source.pcSections()) { + auto Set = Destination.setPCSections(*Source.pcSections()); + if (!Set) + return std::unexpected(Set.error()); + } + if (Source.mmra()) { + auto Set = Destination.setMMRA(*Source.mmra()); + if (!Set) + return std::unexpected(Set.error()); + } + if (Source.cfiType()) { + auto Set = Destination.setCFIType(*Source.cfiType()); + if (!Set) + return std::unexpected(Set.error()); + } + if (const auto Symbol = Source.deactivationSymbol()) { + auto Set = Destination.setDeactivationSymbol(std::string(*Symbol)); + if (!Set) + return std::unexpected(Set.error()); + } + return {}; +} + +[[nodiscard]] Expected removeOldInstruction(MachineInstr &Instruction) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InvalidArgument, "pseudo instruction is not linked"); + return Block->removeInstruction(Instruction); +} + +[[nodiscard]] Expected> +copyRegisters(const MachineInstr &Instruction) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "post-RA copy pseudo requires two register operands"); + return std::pair{*Destination, *Source}; +} + +[[nodiscard]] Expected prepareCopy(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + if (Instruction.opcode() != opcode(GenericOpcode::COPY)) + return false; + auto Registers = copyRegisters(Instruction); + if (!Registers) + return std::unexpected(Registers.error()); + const auto DestinationBank = registerBank(Registers->first, Function, Target); + const auto SourceBank = registerBank(Registers->second, Function, Target); + if (!DestinationBank || !SourceBank) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V copy preparation requires constrained register banks"); + if (*DestinationBank == *SourceBank) + return false; + + const auto DestinationType = registerType(Registers->first, Function); + const auto SourceType = registerType(Registers->second, Function); + const std::optional Type = DestinationType ? DestinationType : SourceType; + if (!Type || Type->isVector() || (Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64)) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V cross-bank COPY requires a scalar 32-bit or 64-bit type"); + } + if (DestinationType && SourceType && + DestinationType->scalarBitWidth() != SourceType->scalarBitWidth()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V cross-bank COPY has mismatched value widths"); + } + const std::uint32_t Flag = Type->scalarBitWidth() == 32 ? CopyF32Flag : CopyF64Flag; + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *SourceOperand = Instruction.operandAt(1); + const bool Changed = + DestinationOperand->targetFlags() != Flag || SourceOperand->targetFlags() != Flag; + DestinationOperand->setTargetFlags(Flag); + SourceOperand->setTargetFlags(Flag); + return Changed; +} + +[[nodiscard]] Expected expandCopy(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + auto Registers = copyRegisters(Instruction); + if (!Registers) + return std::unexpected(Registers.error()); + const Register Destination = Registers->first; + const Register Source = Registers->second; + if (!Destination.isPhysical() || !Source.isPhysical()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V post-RA COPY retains a virtual register"); + } + if (Destination == Source) { + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; + } + const auto DestinationBank = registerBank(Destination, Function, Target); + const auto SourceBank = registerBank(Source, Function, Target); + if (!DestinationBank || !SourceBank) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V post-RA COPY references an unknown register bank"); + const RegisterBankID GPR = + RegisterBankID::fromValue(static_cast(RegisterBank::GPR)); + const RegisterBankID FPR = + RegisterBankID::fromValue(static_cast(RegisterBank::FPR)); + + InstructionOpcode Selected = InstructionOpcode::Invalid; + bool DuplicateSource = false; + bool AddZeroImmediate = false; + if (*DestinationBank == GPR && *SourceBank == GPR) { + Selected = InstructionOpcode::ADDI; + AddZeroImmediate = true; + } else if (*DestinationBank == FPR && *SourceBank == FPR) { + Selected = InstructionOpcode::FSGNJ_D; + DuplicateSource = true; + } else { + const std::uint32_t DestinationFlags = Instruction.operandAt(0)->targetFlags(); + const std::uint32_t SourceFlags = Instruction.operandAt(1)->targetFlags(); + if (DestinationFlags != SourceFlags || + (DestinationFlags != CopyF32Flag && DestinationFlags != CopyF64Flag)) { + return makeUnexpected(ErrorCode::PseudoExpansionFailed, + "RISC-V cross-bank COPY has no prepared transfer width"); + } + if (*DestinationBank == GPR && *SourceBank == FPR) { + Selected = + DestinationFlags == CopyF32Flag ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D; + } else if (*DestinationBank == FPR && *SourceBank == GPR) { + Selected = + DestinationFlags == CopyF32Flag ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; + } + } + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V COPY crosses unsupported banks"); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Selected)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Destination); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Pending->addUse(Source); + if (!Use) + return std::unexpected(Use.error()); + if (DuplicateSource) { + auto SecondUse = Pending->addUse(Source); + if (!SecondUse) + return std::unexpected(SecondUse.error()); + } + if (AddZeroImmediate) { + auto Immediate = Pending->addImmediate(0); + if (!Immediate) + return std::unexpected(Immediate.error()); + } + auto Replacement = Pending->build(); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Decoration = copyCommonDecoration(Instruction, **Replacement, true, true, true); + if (!Decoration) + return std::unexpected(Decoration.error()); + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +[[nodiscard]] Expected expandSubregisterCopy(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const Opcode Value = Instruction.opcode(); + if (Value == opcode(GenericOpcode::EXTRACT_SUBREG) || + Value == opcode(GenericOpcode::SUBREG_TO_REG)) { + const MachineOperand *Index = Instruction.operandAt(2); + const auto *Immediate = Index == nullptr ? nullptr : Index->getIf(); + if (Immediate == nullptr || *Immediate != 0) + return makeUnexpected(ErrorCode::Unsupported, + "RV64GC does not support nonzero subregister indices"); + } else if (Value == opcode(GenericOpcode::COPY_LANEMASK)) { + const MachineOperand *MaskOperand = Instruction.operandAt(2); + const auto *Mask = MaskOperand == nullptr ? nullptr : MaskOperand->getIf(); + if (Mask == nullptr || Mask->Mask.empty()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V lane copy requires a nonempty lane mask"); + } else { + return false; + } + return expandCopy(Instruction, Function, Target); +} + +[[nodiscard]] Expected addDirectTarget(MachineInstrBuilder &Builder, + const MachineOperand &Callee) { + if (const auto *Global = Callee.getIf()) + return Builder.addGlobalSymbol(Global->Name, Global->Offset, PCRelativeHighFlag); + if (const auto *External = Callee.getIf()) + return Builder.addExternalSymbol(External->Name, External->Offset, PCRelativeHighFlag); + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V direct call pseudo has no symbol target"); +} + +[[nodiscard]] bool isRegister(const MachineOperand &Operand, PhysicalRegister Reg) noexcept { + const Register *Value = Operand.getIf(); + return Value != nullptr && Value->physicalRegister() == Reg; +} + +[[nodiscard]] Expected copyExtraImplicitOperands( + const MachineInstr &Source, std::size_t FirstImplicit, MachineInstrBuilder &Destination, + std::optional ExplicitDef, std::optional ExplicitUse) { + for (std::size_t Index = FirstImplicit; Index < Source.operandCount(); ++Index) { + const MachineOperand *Operand = Source.operandAt(Index); + if (Operand == nullptr || !Operand->isImplicit()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V call pseudo has malformed implicit operands"); + if ((ExplicitDef && Operand->isDef() && isRegister(*Operand, *ExplicitDef)) || + (ExplicitUse && Operand->isUse() && isRegister(*Operand, *ExplicitUse))) { + continue; + } + auto Added = Destination.addImplicitOperand(Operand->cloneDetached()); + if (!Added) + return std::unexpected(Added.error()); + } + return {}; +} + +[[nodiscard]] Expected expandCall(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, bool IsTail) { + MachineBasicBlock *Block = Instruction.parent(); + const MachineOperand *Callee = Instruction.operandAt(0); + const MachineOperand *CallSite = Instruction.operandAt(1); + const MachineOperand *Mask = Instruction.operandAt(2); + if (Block == nullptr || Callee == nullptr || CallSite == nullptr || Mask == nullptr || + Mask->kind() != MachineOperand::Kind::RegisterMask) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V call pseudo has malformed explicit operands"); + } + auto Zero = physical(PhysicalRegisterID::X0); + auto Link = physical(PhysicalRegisterID::X1); + auto Scratch = physical(PhysicalRegisterID::X6); + if (!Zero || !Link || !Scratch) + return makeUnexpected(ErrorCode::InternalError, "RISC-V call registers are unavailable"); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + Register Base; + std::optional Anchor; + const bool Direct = Callee->kind() == MachineOperand::Kind::GlobalSymbol || + Callee->kind() == MachineOperand::Kind::ExternalSymbol; + if (Direct) { + const PhysicalRegister HighRegister = IsTail ? *Scratch : *Link; + Base = Register::fromPhysical(HighRegister); + Anchor = Instruction.preInstrSymbol() + ? std::optional{std::string(*Instruction.preInstrSymbol())} + : std::optional{".Lrcc_call_" + + std::to_string(Function.stableID().value()) + "_" + + std::to_string(Instruction.stableID().value())}; + auto HighPending = Builder.buildInstr(targetOpcode(InstructionOpcode::AUIPC)); + if (!HighPending) + return std::unexpected(HighPending.error()); + auto HighDef = HighPending->addDef(Base); + if (!HighDef) + return std::unexpected(HighDef.error()); + auto HighTarget = addDirectTarget(*HighPending, *Callee); + if (!HighTarget) + return std::unexpected(HighTarget.error()); + auto High = HighPending->build(); + if (!High) + return std::unexpected(High.error()); + auto Symbol = (*High)->setPreInstrSymbol(*Anchor); + if (!Symbol) + return std::unexpected(Symbol.error()); + auto Decoration = copyLocation(Instruction, **High); + if (!Decoration) + return std::unexpected(Decoration.error()); + } else { + const Register *CalleeRegister = Callee->getIf(); + if (CalleeRegister == nullptr || !CalleeRegister->isPhysical()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V indirect call retains a virtual callee register"); + Base = *CalleeRegister; + } + + const InstructionOpcode LowOpcode = + IsTail ? InstructionOpcode::JALR_TAIL : InstructionOpcode::JALR_CALL; + auto LowPending = Builder.buildInstr(targetOpcode(LowOpcode)); + if (!LowPending) + return std::unexpected(LowPending.error()); + const PhysicalRegister DestinationRegister = IsTail ? *Zero : *Link; + auto LowDef = LowPending->addDef(Register::fromPhysical(DestinationRegister)); + if (!LowDef) + return std::unexpected(LowDef.error()); + auto LowBase = LowPending->addUse(Base); + if (!LowBase) + return std::unexpected(LowBase.error()); + Expected LowOffset = + Anchor ? LowPending->addMCSymbol(*Anchor, 0, PCRelativeLowFlag) : LowPending->addImmediate(0); + if (!LowOffset) + return std::unexpected(LowOffset.error()); + auto LowCallSite = LowPending->addOperand(CallSite->cloneDetached()); + if (!LowCallSite) + return std::unexpected(LowCallSite.error()); + auto LowMask = LowPending->addOperand(Mask->cloneDetached()); + if (!LowMask) + return std::unexpected(LowMask.error()); + auto Implicit = copyExtraImplicitOperands( + Instruction, 3, *LowPending, IsTail ? std::nullopt : std::optional{*Link}, + std::nullopt); + if (!Implicit) + return std::unexpected(Implicit.error()); + auto Low = LowPending->build(); + if (!Low) + return std::unexpected(Low.error()); + auto Decoration = copyCommonDecoration(Instruction, **Low, !Direct, true, true); + if (!Decoration) + return std::unexpected(Decoration.error()); + auto Records = + Function.records().replaceInstructionReference(Instruction.stableID(), (*Low)->stableID()); + if (!Records) + return std::unexpected(Records.error()); + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +[[nodiscard]] Expected expandReturn(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + auto Zero = physical(PhysicalRegisterID::X0); + auto Link = physical(PhysicalRegisterID::X1); + if (!Zero || !Link) + return makeUnexpected(ErrorCode::InternalError, "RISC-V return registers are unavailable"); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(InstructionOpcode::JALR_RET)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Register::fromPhysical(*Zero)); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Pending->addUse(Register::fromPhysical(*Link)); + if (!Use) + return std::unexpected(Use.error()); + auto Immediate = Pending->addImmediate(0); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto Implicit = copyExtraImplicitOperands(Instruction, 0, *Pending, std::nullopt, *Link); + if (!Implicit) + return std::unexpected(Implicit.error()); + auto Replacement = Pending->build(); + if (!Replacement) + return std::unexpected(Replacement.error()); + auto Decoration = copyCommonDecoration(Instruction, **Replacement, true, true, true); + if (!Decoration) + return std::unexpected(Decoration.error()); + auto Removed = removeOldInstruction(Instruction); + if (!Removed) + return std::unexpected(Removed.error()); + return true; +} + +class RISCVPseudoExpansionInfo final : public TargetPseudoExpansionInfo { +public: + [[nodiscard]] Expected preparePostRAPseudo(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) const override { + return prepareCopy(Instruction, Function, Target); + } + + [[nodiscard]] Expected expandPseudo(MachineInstr &Instruction, + MachinePseudoExpansionPhase Phase, + MachineFunction &Function, + const TargetDescription &Target) const override { + if (Phase != MachinePseudoExpansionPhase::PostRA) + return false; + if (Instruction.opcode() == opcode(GenericOpcode::COPY)) + return expandCopy(Instruction, Function, Target); + auto Subregister = expandSubregisterCopy(Instruction, Function, Target); + if (!Subregister || *Subregister) + return Subregister; + if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoCALL)) + return expandCall(Instruction, Function, Target, false); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoTAIL)) + return expandCall(Instruction, Function, Target, true); + if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoRET)) + return expandReturn(Instruction, Function, Target); + return false; + } +}; + +} // namespace + +const TargetPseudoExpansionInfo &pseudoExpansionInfo() noexcept { + static const RISCVPseudoExpansionInfo Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/lib/Target/RISCV/RISCVScheduling.cpp b/backend/lib/Target/RISCV/RISCVScheduling.cpp new file mode 100644 index 0000000..8ef8d91 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVScheduling.cpp @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +enum class Resource : std::uint32_t { + Core = 1, + IntegerALU = 2, + Memory = 3, + Branch = 4, + MultiplyDivide = 5, + FloatingPoint = 6, + System = 7, +}; + +enum class PressureSet : std::uint32_t { + GPR = 1, + FPR = 2, +}; + +[[nodiscard]] constexpr ProcessorResourceID resource(Resource Value) noexcept { + return ProcessorResourceID::fromValue(static_cast(Value)); +} + +[[nodiscard]] constexpr RegisterPressureSetID pressureSet(PressureSet Value) noexcept { + return RegisterPressureSetID::fromValue(static_cast(Value)); +} + +[[nodiscard]] constexpr RegisterClassID registerClass(RegisterClass Value) noexcept { + return RegisterClassID::fromValue(static_cast(Value)); +} + +[[nodiscard]] std::vector resourceUses(SchedulingClass Class) { + switch (Class) { + case SchedulingClass::ALU: + case SchedulingClass::COMPRESSED: + return {{resource(Resource::IntegerALU), 1}}; + case SchedulingClass::LOAD: + case SchedulingClass::STORE: + return {{resource(Resource::Memory), 1}}; + case SchedulingClass::BRANCH: + return {{resource(Resource::Branch), 1}}; + case SchedulingClass::SYSTEM: + case SchedulingClass::FENCE: + return {{resource(Resource::System), 1}}; + case SchedulingClass::MUL: + return {{resource(Resource::MultiplyDivide), 3}}; + case SchedulingClass::DIV: + return {{resource(Resource::MultiplyDivide), 20}}; + case SchedulingClass::ATOMIC: + return {{resource(Resource::Memory), 4}, {resource(Resource::System), 1}}; + case SchedulingClass::FP_ADD: + case SchedulingClass::FP_MUL: + case SchedulingClass::FP_CONVERT: + return {{resource(Resource::FloatingPoint), 4}}; + case SchedulingClass::FP_DIV: + case SchedulingClass::FP_SQRT: + return {{resource(Resource::FloatingPoint), 20}}; + case SchedulingClass::None: + return {}; + } + return {}; +} + +[[nodiscard]] TargetSchedulingModelDesc createDescription() { + TargetSchedulingModelDesc Description; + Description.IssueWidth = 2; + Description.DispatchLimit = 2; + Description.ProcessorResources = { + {resource(Resource::Core), "core", 2, std::nullopt}, + {resource(Resource::IntegerALU), "integer-alu", 2, resource(Resource::Core)}, + {resource(Resource::Memory), "memory", 1, resource(Resource::Core)}, + {resource(Resource::Branch), "branch", 1, resource(Resource::Core)}, + {resource(Resource::MultiplyDivide), "multiply-divide", 1, resource(Resource::Core)}, + {resource(Resource::FloatingPoint), "floating-point", 1, resource(Resource::Core)}, + {resource(Resource::System), "system", 1, resource(Resource::Core)}, + }; + for (const SchedulingClassInfo &Class : schedulingClasses()) { + SchedulingClassModelDesc Model{ + .ID = Class.ID, + .Name = std::string(Class.Name), + .MicroOps = Class.MicroOps, + .Latency = Class.Latency, + .Resources = resourceUses(static_cast(Class.ID.value())), + .WriteLatencies = {}, + .ReadAdvances = {}, + }; + Model.WriteLatencies.push_back({0, Class.Latency}); + switch (static_cast(Class.ID.value())) { + case SchedulingClass::ALU: + Model.ReadAdvances = { + {1, SchedulingClassID::fromValue(static_cast(SchedulingClass::LOAD)), 1}, + {2, SchedulingClassID::fromValue(static_cast(SchedulingClass::LOAD)), 1}, + }; + break; + default: + break; + } + Description.SchedulingClasses.push_back(std::move(Model)); + } + Description.PressureSets = { + {pressureSet(PressureSet::GPR), + "gpr", + {registerClass(RegisterClass::GPR), registerClass(RegisterClass::GPRC), + registerClass(RegisterClass::GPR_NX0), registerClass(RegisterClass::GPR_NX0_NX2)}, + 27}, + {pressureSet(PressureSet::FPR), + "fpr", + {registerClass(RegisterClass::FPR64), registerClass(RegisterClass::FPR64C), + registerClass(RegisterClass::FPR32), registerClass(RegisterClass::FPR32C)}, + 32}, + }; + return Description; +} + +} // namespace + +const TargetSchedulingModel &schedulingModel() noexcept { + static const Expected Model = + TargetSchedulingModel::create(createDescription()); + if (!Model) + std::abort(); + return *Model; +} + +} // namespace rcc::riscv -- cgit v1.2.3