aboutsummaryrefslogtreecommitdiff
path: root/backend/unittests
diff options
context:
space:
mode:
Diffstat (limited to 'backend/unittests')
-rw-r--r--backend/unittests/CodeGen/CMakeLists.txt22
-rw-r--r--backend/unittests/CodeGen/InstructionDescriptorTest.cpp10
-rw-r--r--backend/unittests/CodeGen/MachineOperandTest.cpp43
-rw-r--r--backend/unittests/CodeGen/MachineOptimizationTest.cpp772
-rw-r--r--backend/unittests/CodeGen/MachineSchedulingTest.cpp518
-rw-r--r--backend/unittests/Support/ErrorTest.cpp10
-rw-r--r--backend/unittests/Target/CMakeLists.txt13
-rw-r--r--backend/unittests/Target/DescriptionGeneratorTest.cpp2
-rw-r--r--backend/unittests/Target/RISCVLateCodeGenTest.cpp1432
-rw-r--r--backend/unittests/Target/RISCVTargetTest.cpp4
10 files changed, 2823 insertions, 3 deletions
diff --git a/backend/unittests/CodeGen/CMakeLists.txt b/backend/unittests/CodeGen/CMakeLists.txt
index 65d9168..1b66194 100644
--- a/backend/unittests/CodeGen/CMakeLists.txt
+++ b/backend/unittests/CodeGen/CMakeLists.txt
@@ -20,8 +20,10 @@ set(
MachineLivenessTest.cpp
MachineMetadataTest.cpp
MachineOperandTest.cpp
+ MachineOptimizationTest.cpp
MachinePassTest.cpp
MachinePipelinePassesTest.cpp
+ MachineSchedulingTest.cpp
PBQPTest.cpp
RegisterAllocationTest.cpp
MachineSSAExitTest.cpp
@@ -355,6 +357,22 @@ target_link_libraries(
add_test(NAME RCCBackend.CodeGen.MachineAnalyses COMMAND RCCBackendCodeGenMachineAnalysesTests)
+add_executable(RCCBackendCodeGenMachineOptimizationTests MachineOptimizationTest.cpp)
+rcc_backend_configure_target(RCCBackendCodeGenMachineOptimizationTests)
+target_link_libraries(
+ RCCBackendCodeGenMachineOptimizationTests
+ PRIVATE RCCBackend::CodeGen RCCBackend::RISCV
+)
+add_test(NAME RCCBackend.CodeGen.MachineOptimization COMMAND RCCBackendCodeGenMachineOptimizationTests)
+
+add_executable(RCCBackendCodeGenMachineSchedulingTests MachineSchedulingTest.cpp)
+rcc_backend_configure_target(RCCBackendCodeGenMachineSchedulingTests)
+target_link_libraries(
+ RCCBackendCodeGenMachineSchedulingTests
+ PRIVATE RCCBackend::CodeGen RCCBackend::RISCV
+)
+add_test(NAME RCCBackend.CodeGen.MachineScheduling COMMAND RCCBackendCodeGenMachineSchedulingTests)
+
add_executable(RCCBackendCodeGenMachineSSAExitTests MachineSSAExitTest.cpp)
rcc_backend_configure_target(RCCBackendCodeGenMachineSSAExitTests)
target_link_libraries(
@@ -439,8 +457,10 @@ rcc_backend_register_format_files(
"${CMAKE_CURRENT_SOURCE_DIR}/MachineLivenessTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadataTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachinePassTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePassesTest.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/MachineSchedulingTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/PBQPTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RegisterAllocationTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachineSSAExitTest.cpp"
@@ -479,8 +499,10 @@ rcc_backend_register_tidy_files(
"${CMAKE_CURRENT_SOURCE_DIR}/MachineLivenessTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadataTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/MachineOptimizationTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachinePassTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePassesTest.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/MachineSchedulingTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/PBQPTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RegisterAllocationTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/MachineSSAExitTest.cpp"
diff --git a/backend/unittests/CodeGen/InstructionDescriptorTest.cpp b/backend/unittests/CodeGen/InstructionDescriptorTest.cpp
index 424d2e4..687e723 100644
--- a/backend/unittests/CodeGen/InstructionDescriptorTest.cpp
+++ b/backend/unittests/CodeGen/InstructionDescriptorTest.cpp
@@ -86,6 +86,7 @@ private:
RegisterOrImmediate.Kinds = {
rcc::MachineOperand::Kind::Register,
rcc::MachineOperand::Kind::Immediate,
+ rcc::MachineOperand::Kind::GlobalSymbol,
};
RegisterOrImmediate.RegisterRole = rcc::RegisterOperandRole::Use;
RegisterOrImmediate.ImmediatePredicate = rcc::ImmediatePredicateID::fromValue(1);
@@ -218,6 +219,15 @@ void testOperandValidation(TestContext &Context) {
.has_value(),
"matching roles, constraints, flags, and predicate must validate");
+ std::vector<rcc::MachineOperand> SymbolOperands;
+ SymbolOperands.push_back(*registerOperand(*Destination, {rcc::RegisterOperandFlag::Define}));
+ SymbolOperands.push_back(*registerOperand(*Source, {rcc::RegisterOperandFlag::Use}));
+ SymbolOperands.push_back(*rcc::MachineOperand::globalSymbol("object", 0, 0x1));
+ Context.expect(
+ InstructionSet->validateExplicitOperands(rcc::Opcode::fromValue(1), SymbolOperands, RegInfo)
+ .has_value(),
+ "an immediate predicate must not reject another allowed symbolic operand kind");
+
std::vector<rcc::MachineOperand> WrongRole;
WrongRole.push_back(*registerOperand(*Destination, {rcc::RegisterOperandFlag::Use}));
WrongRole.push_back(*registerOperand(*Source, {rcc::RegisterOperandFlag::Use}));
diff --git a/backend/unittests/CodeGen/MachineOperandTest.cpp b/backend/unittests/CodeGen/MachineOperandTest.cpp
index 1b246ca..3cc7060 100644
--- a/backend/unittests/CodeGen/MachineOperandTest.cpp
+++ b/backend/unittests/CodeGen/MachineOperandTest.cpp
@@ -264,6 +264,48 @@ void testOperandFactories(TestContext &Context) {
"detached operand cloning must preserve the new payload kinds");
}
+void testFrameIndexRewrite(TestContext &Context) {
+ auto Function = createFunction(Context, 5, "frame-index-rewrite");
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+ auto Align = rcc::Alignment::create(8);
+ if (Block == nullptr || !Align)
+ return;
+
+ rcc::StackObjectDesc Description;
+ Description.Kind = rcc::StackObjectKind::SpillSlot;
+ Description.Size = 8;
+ Description.Align = *Align;
+ auto First = Function->frameInfo().createObject(Description);
+ auto Second = Function->frameInfo().createObject(Description);
+ Context.expect(First.has_value() && Second.has_value(),
+ "frame-index rewrite objects must be created");
+ if (!First || !Second)
+ return;
+
+ auto Instruction = Function->createInstruction(rcc::Opcode::fromValue(OperandOpcode));
+ Context.expect(Instruction.has_value() &&
+ (*Instruction)->addOperand(rcc::MachineOperand::frameIndex(*First, 4)) &&
+ Block->appendInstruction(**Instruction),
+ "frame-index rewrite operand must be linked to its function");
+ if (!Instruction || (*Instruction)->operandCount() == 0)
+ return;
+ rcc::MachineOperand *Operand = (*Instruction)->operandAt(0);
+ Context.expect(Operand != nullptr && Operand->setFrameIndex(*Second).has_value() &&
+ Operand->getIf<rcc::FrameIndexOperand>()->Index == *Second &&
+ Operand->getIf<rcc::FrameIndexOperand>()->Offset == 4,
+ "setFrameIndex must rewrite only the object identity");
+
+ const rcc::FrameIndex Unknown = rcc::FrameIndex::fromValue(99);
+ Context.expect(Operand != nullptr && !Operand->setFrameIndex(Unknown) &&
+ Operand->getIf<rcc::FrameIndexOperand>()->Index == *Second,
+ "setFrameIndex must reject an unknown attached object atomically");
+ auto Immediate = rcc::MachineOperand::immediate(0);
+ Context.expect(!Immediate.setFrameIndex(*First),
+ "setFrameIndex must reject non-frame-index operands");
+}
+
void testCFIInfo(TestContext &Context) {
const auto R1 = rcc::DwarfRegisterNumber::fromValue(1);
const auto R2 = rcc::DwarfRegisterNumber::fromValue(2);
@@ -476,6 +518,7 @@ int main() {
TestContext Context;
testIntegerImmediate(Context);
testOperandFactories(Context);
+ testFrameIndexRewrite(Context);
testCFIInfo(Context);
testBuilderVerifierAndClone(Context);
return Context.result();
diff --git a/backend/unittests/CodeGen/MachineOptimizationTest.cpp b/backend/unittests/CodeGen/MachineOptimizationTest.cpp
new file mode 100644
index 0000000..8d01a65
--- /dev/null
+++ b/backend/unittests/CodeGen/MachineOptimizationTest.cpp
@@ -0,0 +1,772 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineCFGUpdater.h"
+#include "rcc/CodeGen/MachineCSE.h"
+#include "rcc/CodeGen/MachineCombiner.h"
+#include "rcc/CodeGen/MachineDCE.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineLICM.h"
+#include "rcc/CodeGen/MachineLiveness.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineMemoryFolding.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/MachineSink.h"
+#include "rcc/CodeGen/TargetMemoryFolding.h"
+#include "rcc/Target/RISCV/RISCVCodeGen.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <cstdint>
+#include <iostream>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+
+namespace {
+
+class TestContext final {
+public:
+ void expect(bool Condition, std::string_view Message) {
+ if (Condition)
+ return;
+ std::cerr << Message << '\n';
+ ++Failures;
+ }
+
+ [[nodiscard]] int result() const noexcept {
+ return Failures == 0 ? 0 : 1;
+ }
+
+private:
+ int Failures = 0;
+};
+
+struct Fixture final {
+ std::unique_ptr<rcc::MachineFunction> Function;
+ rcc::MachineBasicBlock *Entry = nullptr;
+};
+
+[[nodiscard]] rcc::Opcode opcode(rcc::riscv::InstructionOpcode Value) noexcept;
+
+class TestMemoryFoldingInfo final : public rcc::TargetMemoryFoldingInfo {
+public:
+ explicit TestMemoryFoldingInfo(bool DropMemory = false) noexcept : DropMemory(DropMemory) {}
+
+ [[nodiscard]] rcc::Expected<bool>
+ canFoldMemoryOperand(const rcc::MachineInstr &Consumer, std::size_t OperandIndex,
+ const rcc::MachineInstr &MemoryInstruction, const rcc::MachineFunction &,
+ const rcc::TargetDescription &) const override {
+ return Consumer.opcode() == opcode(rcc::riscv::InstructionOpcode::ADDI) && OperandIndex == 1 &&
+ MemoryInstruction.opcode() == opcode(rcc::riscv::InstructionOpcode::LD);
+ }
+
+ [[nodiscard]] rcc::Expected<rcc::MachineInstr *>
+ foldMemoryOperand(rcc::MachineInstr &Consumer, std::size_t, rcc::MachineInstr &MemoryInstruction,
+ rcc::MachineFunction &Function,
+ const rcc::TargetDescription &Target) const override {
+ const rcc::MachineOperand *DestinationOperand = Consumer.operandAt(0);
+ const rcc::MachineOperand *BaseOperand = MemoryInstruction.operandAt(1);
+ const rcc::MachineOperand *OffsetOperand = MemoryInstruction.operandAt(2);
+ const rcc::Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<rcc::Register>();
+ const rcc::Register *Base =
+ BaseOperand == nullptr ? nullptr : BaseOperand->getIf<rcc::Register>();
+ const std::int64_t *Offset =
+ OffsetOperand == nullptr ? nullptr : OffsetOperand->getIf<std::int64_t>();
+ rcc::MachineBasicBlock *Block = Consumer.parent();
+ if (Destination == nullptr || Base == nullptr || Offset == nullptr || Block == nullptr ||
+ MemoryInstruction.parent() != Block) {
+ return rcc::makeUnexpected(rcc::ErrorCode::VerificationError,
+ "test memory fold received an invalid candidate");
+ }
+
+ rcc::MachineIRBuilder Builder(Function, Target);
+ if (auto Point = Builder.setInsertPointBefore(Consumer); !Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::LD));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ if (auto Added = Pending->addDef(*Destination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Pending->addUse(*Base); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Pending->addImmediate(*Offset); !Added)
+ return std::unexpected(Added.error());
+ for (const rcc::MachineMemOperand *Memory : MemoryInstruction.memOperands()) {
+ if (Memory == nullptr)
+ return rcc::makeUnexpected(rcc::ErrorCode::VerificationError,
+ "test memory fold received a null MMO");
+ if (auto Added = Pending->addMemOperand(*Memory); !Added)
+ return std::unexpected(Added.error());
+ }
+ auto Folded = Pending->build();
+ if (!Folded)
+ return std::unexpected(Folded.error());
+ if (DropMemory) {
+ while (!(*Folded)->memOperands().empty()) {
+ if (auto Erased = (*Folded)->eraseMemOperand(0); !Erased)
+ return std::unexpected(Erased.error());
+ }
+ }
+ if (auto Removed = Block->removeInstruction(MemoryInstruction); !Removed)
+ return std::unexpected(Removed.error());
+ if (auto Removed = Block->removeInstruction(Consumer); !Removed)
+ return std::unexpected(Removed.error());
+ return *Folded;
+ }
+
+ [[nodiscard]] rcc::Expected<bool>
+ unfoldMemoryOperand(rcc::MachineInstr &, rcc::MachineFunction &,
+ const rcc::TargetDescription &) const override {
+ return false;
+ }
+
+ [[nodiscard]] rcc::Expected<std::optional<rcc::MemorySize>>
+ getFoldedMemorySize(const rcc::MachineInstr &, const rcc::MachineFunction &,
+ const rcc::TargetDescription &) const override {
+ auto Size = rcc::MemorySize::fixed(8);
+ if (!Size)
+ return std::unexpected(Size.error());
+ return std::optional<rcc::MemorySize>(*Size);
+ }
+
+private:
+ bool DropMemory;
+};
+
+[[nodiscard]] rcc::Opcode opcode(rcc::riscv::InstructionOpcode Value) noexcept {
+ return rcc::Opcode::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+[[nodiscard]] Fixture makeFixture(TestContext &Context, std::string Name, std::uint64_t ID) {
+ auto Function =
+ rcc::MachineFunction::create(std::move(Name), rcc::MachineFunctionStableID::fromValue(ID));
+ Context.expect(Function.has_value(), "optimization fixture function creation must succeed");
+ if (!Function)
+ return {};
+ auto Entry = (*Function)->createBasicBlock();
+ Context.expect(Entry.has_value(), "optimization fixture block creation must succeed");
+ if (!Entry || !(*Function)->appendBlock(**Entry))
+ return {};
+ (*Function)->properties().set(rcc::MachineFunctionProperty::IsSSA);
+ (*Function)->properties().set(rcc::MachineFunctionProperty::Legalized);
+ (*Function)->properties().set(rcc::MachineFunctionProperty::RegBankSelected);
+ (*Function)->properties().set(rcc::MachineFunctionProperty::Selected);
+ return {std::move(*Function), *Entry};
+}
+
+void setPostRAProperties(rcc::MachineFunction &Function) {
+ Function.properties().clear(rcc::MachineFunctionProperty::IsSSA);
+ Function.properties().set(rcc::MachineFunctionProperty::NoPHIs);
+ Function.properties().set(rcc::MachineFunctionProperty::TiedOpsRewritten);
+ Function.properties().set(rcc::MachineFunctionProperty::NoVRegs);
+ Function.properties().set(rcc::MachineFunctionProperty::TracksLiveness);
+}
+
+[[nodiscard]] rcc::Register makeGPR(TestContext &Context, rcc::MachineFunction &Function) {
+ auto Type = rcc::LLT::makeInteger(64);
+ auto Reg = Type ? Function.registerInfo().createVirtualRegister(*Type)
+ : rcc::Expected<rcc::Register>(std::unexpected(
+ rcc::Error(rcc::ErrorCode::InternalError, "test LLT creation failed")));
+ Context.expect(Reg.has_value(), "optimization virtual register creation must succeed");
+ if (!Reg)
+ return {};
+ const auto Bank =
+ rcc::RegisterBankID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterBank::GPR));
+ const auto Class =
+ rcc::RegisterClassID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterClass::GPR));
+ Context.expect(Function.registerInfo().setRegisterBank(*Reg, Bank).has_value() &&
+ Function.registerInfo().setRegisterClass(*Reg, Class).has_value(),
+ "optimization register constraints must be assigned");
+ return *Reg;
+}
+
+[[nodiscard]] rcc::PhysicalRegister physical(TestContext &Context,
+ rcc::riscv::PhysicalRegisterID ID) {
+ auto Reg = rcc::riscv::physicalRegister(ID);
+ Context.expect(Reg.has_value(), "optimization physical register lookup must succeed");
+ return Reg.value_or(rcc::PhysicalRegister{});
+}
+
+[[nodiscard]] rcc::MachineInstr *buildADDI(TestContext &Context, rcc::MachineIRBuilder &Builder,
+ rcc::Register Destination, rcc::Register Source,
+ std::int64_t Immediate) {
+ auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::ADDI));
+ if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Source) ||
+ !Pending->addImmediate(Immediate)) {
+ Context.expect(false, "ADDI construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "ADDI insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+[[nodiscard]] rcc::MachineInstr *buildADD(TestContext &Context, rcc::MachineIRBuilder &Builder,
+ rcc::Register Destination, rcc::Register Left,
+ rcc::Register Right) {
+ auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::ADD));
+ if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Left) ||
+ !Pending->addUse(Right)) {
+ Context.expect(false, "ADD construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "ADD insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+[[nodiscard]] const rcc::MachineMemOperand *
+makeMemory(TestContext &Context, rcc::MachineFunction &Function, bool Store, bool Invariant) {
+ auto Size = rcc::MemorySize::fixed(8);
+ auto Align = rcc::Alignment::create(8);
+ Context.expect(Size.has_value() && Align.has_value(), "memory shape creation must succeed");
+ if (!Size || !Align)
+ return nullptr;
+ rcc::MemoryAccessFlags Flags{Store ? rcc::MemoryAccessFlag::Store : rcc::MemoryAccessFlag::Load};
+ if (Invariant) {
+ Flags.set(rcc::MemoryAccessFlag::Invariant);
+ Flags.set(rcc::MemoryAccessFlag::Dereferenceable);
+ }
+ rcc::MachineMemOperandDesc Description;
+ Description.Accesses = Flags;
+ Description.Size = *Size;
+ Description.Align = *Align;
+ Description.Base = Invariant ? rcc::MemoryBase::constantPoolPseudo() : rcc::MemoryBase::stack();
+ auto Memory = Function.createMemOperand(std::move(Description));
+ Context.expect(Memory.has_value(), "memory operand creation must succeed");
+ return Memory.value_or(nullptr);
+}
+
+[[nodiscard]] rcc::MachineInstr *buildStore(TestContext &Context, rcc::MachineIRBuilder &Builder,
+ rcc::Register Value, rcc::PhysicalRegister StackPointer,
+ const rcc::MachineMemOperand &Memory) {
+ auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::SD));
+ if (!Pending || !Pending->addUse(Value) ||
+ !Pending->addUse(rcc::Register::fromPhysical(StackPointer)) || !Pending->addImmediate(0) ||
+ !Pending->addMemOperand(Memory)) {
+ Context.expect(false, "store construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "store insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+[[nodiscard]] rcc::MachineInstr *buildLoad(TestContext &Context, rcc::MachineIRBuilder &Builder,
+ rcc::Register Destination, rcc::Register Address,
+ const rcc::MachineMemOperand &Memory) {
+ auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::LD));
+ if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Address) ||
+ !Pending->addImmediate(0) || !Pending->addMemOperand(Memory)) {
+ Context.expect(false, "load construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "load insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+void testDCEFixedPointAndDebugUses(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "dce", 1);
+ if (!Fixture.Function || Fixture.Entry == nullptr)
+ return;
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(),
+ "DCE insert point must be established");
+ const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+ const rcc::PhysicalRegister X2 = physical(Context, rcc::riscv::PhysicalRegisterID::X2);
+
+ const rcc::Register LiveRoot = makeGPR(Context, *Fixture.Function);
+ const rcc::Register LiveValue = makeGPR(Context, *Fixture.Function);
+ const rcc::Register DeadRoot = makeGPR(Context, *Fixture.Function);
+ const rcc::Register DeadValue = makeGPR(Context, *Fixture.Function);
+ (void)buildADDI(Context, Builder, LiveRoot, rcc::Register::fromPhysical(X0), 1);
+ (void)buildADDI(Context, Builder, LiveValue, LiveRoot, 2);
+ (void)buildADDI(Context, Builder, DeadRoot, rcc::Register::fromPhysical(X0), 3);
+ (void)buildADDI(Context, Builder, DeadValue, DeadRoot, 4);
+
+ auto Debug = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::DBG_VALUE));
+ Context.expect(Debug.has_value() &&
+ Debug->addUse(DeadValue, {rcc::RegisterOperandFlag::Debug}).has_value() &&
+ Debug->build().has_value(),
+ "debug-only use construction must succeed");
+ const rcc::MachineMemOperand *Memory = makeMemory(Context, *Fixture.Function, true, false);
+ if (Memory == nullptr)
+ return;
+ rcc::MachineInstr *Store = buildStore(Context, Builder, LiveValue, X2, *Memory);
+ const std::size_t Before = Fixture.Entry->instructionCount();
+
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineDCEPass Pass;
+ auto Result = Pass.run(*Fixture.Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "machine DCE failure: " << Result.error().message() << '\n';
+ Context.expect(Result.has_value() && Result->Changed,
+ "machine DCE must remove the debug-only dead chain");
+ Context.expect(Fixture.Entry->instructionCount() + 2 == Before,
+ "machine DCE must reach a fixed point across the dead chain");
+ Context.expect(Store != nullptr && Store->isLinked(), "machine DCE must preserve a store");
+ Context.expect(!Fixture.Function->registerInfo().isKnownVirtualRegister(DeadRoot) &&
+ !Fixture.Function->registerInfo().isKnownVirtualRegister(DeadValue),
+ "machine DCE must retire removed virtual registers");
+ rcc::MachineInstr *DebugInstruction = nullptr;
+ for (rcc::MachineInstr *Instruction = Fixture.Entry->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ if (Instruction->opcode() == rcc::opcode(rcc::GenericOpcode::DBG_VALUE))
+ DebugInstruction = Instruction;
+ }
+ const rcc::MachineOperand *DebugOperand =
+ DebugInstruction == nullptr ? nullptr : DebugInstruction->operandAt(0);
+ Context.expect(DebugOperand != nullptr && DebugOperand->registerValue().has_value() &&
+ DebugOperand->registerValue()->isNoRegister(),
+ "machine DCE must clear a dangling debug register use");
+}
+
+void testMachineCombinerIdentityRules(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "combiner", 6);
+ if (!Fixture.Function || Fixture.Entry == nullptr)
+ return;
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(),
+ "combiner insert point must be established");
+ const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+ const rcc::PhysicalRegister X2 = physical(Context, rcc::riscv::PhysicalRegisterID::X2);
+ const rcc::Register Seed = makeGPR(Context, *Fixture.Function);
+ const rcc::Register Identity = makeGPR(Context, *Fixture.Function);
+ const rcc::Register Consumer = makeGPR(Context, *Fixture.Function);
+ (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 7);
+ rcc::MachineInstr *IdentityInstruction = buildADDI(Context, Builder, Identity, Seed, 0);
+ rcc::MachineInstr *ConsumerInstruction = buildADD(Context, Builder, Consumer, Identity, Seed);
+ const rcc::MachineMemOperand *Memory = makeMemory(Context, *Fixture.Function, true, false);
+ if (Memory == nullptr)
+ return;
+ (void)buildStore(Context, Builder, Consumer, X2, *Memory);
+
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineCombinerPass Pass(rcc::riscv::machineCombinerInfo(),
+ rcc::MachineCombinerStage::SelectedSSA);
+ auto Result = Pass.run(*Fixture.Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "machine combiner failure: " << Result.error().message() << '\n';
+ Context.expect(Result.has_value() && Result->Changed,
+ "machine combiner must apply a registered RV64GC identity rule");
+ Context.expect(IdentityInstruction != nullptr && !IdentityInstruction->isLinked() &&
+ !Fixture.Function->registerInfo().isKnownVirtualRegister(Identity),
+ "machine combiner must erase the identity and retire its result");
+ const rcc::MachineOperand *ConsumerUse =
+ ConsumerInstruction == nullptr ? nullptr : ConsumerInstruction->operandAt(1);
+ Context.expect(ConsumerUse != nullptr && ConsumerUse->registerValue() == Seed,
+ "machine combiner executor must rewrite all result uses");
+}
+
+void testPostRAMachineCombinerIdentity(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "post_ra_combiner", 7);
+ if (!Fixture.Function || Fixture.Entry == nullptr)
+ return;
+ Fixture.Function->properties().clear(rcc::MachineFunctionProperty::IsSSA);
+ Fixture.Function->properties().set(rcc::MachineFunctionProperty::NoPHIs);
+ Fixture.Function->properties().set(rcc::MachineFunctionProperty::TiedOpsRewritten);
+ Fixture.Function->properties().set(rcc::MachineFunctionProperty::NoVRegs);
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(),
+ "post-RA combiner insert point must be established");
+ const rcc::PhysicalRegister X10 = physical(Context, rcc::riscv::PhysicalRegisterID::X10);
+ rcc::MachineInstr *Identity = buildADDI(Context, Builder, rcc::Register::fromPhysical(X10),
+ rcc::Register::fromPhysical(X10), 0);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineCombinerPass Pass(rcc::riscv::machineCombinerInfo(),
+ rcc::MachineCombinerStage::PostRegisterAllocation);
+ auto Result = Pass.run(*Fixture.Function, Target, Analyses);
+ Context.expect(
+ Result.has_value() && Result->Changed && Identity != nullptr && !Identity->isLinked(),
+ "post-RA combiner must erase a physical-register no-op only when identities match");
+}
+
+void testCSERegisterAndMemoryRules(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "cse", 2);
+ if (!Fixture.Function || Fixture.Entry == nullptr)
+ return;
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(),
+ "CSE insert point must be established");
+ const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+
+ const rcc::Register Seed = makeGPR(Context, *Fixture.Function);
+ const rcc::Register First = makeGPR(Context, *Fixture.Function);
+ const rcc::Register Redundant = makeGPR(Context, *Fixture.Function);
+ const rcc::Register Consumer = makeGPR(Context, *Fixture.Function);
+ (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 9);
+ rcc::MachineInstr *FirstInstruction = buildADDI(Context, Builder, First, Seed, 7);
+ rcc::MachineInstr *RedundantInstruction = buildADDI(Context, Builder, Redundant, Seed, 7);
+ rcc::MachineInstr *ConsumerInstruction = buildADD(Context, Builder, Consumer, Redundant, Seed);
+
+ const rcc::Register MutableLoadOne = makeGPR(Context, *Fixture.Function);
+ const rcc::Register MutableLoadTwo = makeGPR(Context, *Fixture.Function);
+ const rcc::Register InvariantLoadOne = makeGPR(Context, *Fixture.Function);
+ const rcc::Register InvariantLoadTwo = makeGPR(Context, *Fixture.Function);
+ const rcc::MachineMemOperand *MutableMemory =
+ makeMemory(Context, *Fixture.Function, false, false);
+ const rcc::MachineMemOperand *InvariantMemory =
+ makeMemory(Context, *Fixture.Function, false, true);
+ if (MutableMemory == nullptr || InvariantMemory == nullptr)
+ return;
+ rcc::MachineInstr *MutableOne = buildLoad(Context, Builder, MutableLoadOne, Seed, *MutableMemory);
+ rcc::MachineInstr *MutableTwo = buildLoad(Context, Builder, MutableLoadTwo, Seed, *MutableMemory);
+ rcc::MachineInstr *InvariantOne =
+ buildLoad(Context, Builder, InvariantLoadOne, Seed, *InvariantMemory);
+ rcc::MachineInstr *InvariantTwo =
+ buildLoad(Context, Builder, InvariantLoadTwo, Seed, *InvariantMemory);
+
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineCSEPass Pass;
+ auto Result = Pass.run(*Fixture.Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "machine CSE failure: " << Result.error().message() << '\n';
+ Context.expect(Result.has_value() && Result->Changed,
+ "machine CSE must eliminate safe redundant expressions");
+ Context.expect(FirstInstruction != nullptr && FirstInstruction->isLinked() &&
+ RedundantInstruction != nullptr && !RedundantInstruction->isLinked(),
+ "machine CSE must keep the dominating expression only");
+ const rcc::MachineOperand *ConsumerUse =
+ ConsumerInstruction == nullptr ? nullptr : ConsumerInstruction->operandAt(1);
+ Context.expect(ConsumerUse != nullptr && ConsumerUse->registerValue() == First,
+ "machine CSE must rewrite uses to the dominating result");
+ Context.expect(MutableOne != nullptr && MutableOne->isLinked() && MutableTwo != nullptr &&
+ MutableTwo->isLinked(),
+ "machine CSE must not merge mutable loads");
+ Context.expect(InvariantOne != nullptr && InvariantOne->isLinked() && InvariantTwo != nullptr &&
+ !InvariantTwo->isLinked(),
+ "machine CSE must merge invariant dereferenceable loads");
+ Context.expect(!Fixture.Function->registerInfo().isKnownVirtualRegister(Redundant) &&
+ !Fixture.Function->registerInfo().isKnownVirtualRegister(InvariantLoadTwo),
+ "machine CSE must retire redundant result registers");
+}
+
+void testCSEBudgetPreservesLegalMIR(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "cse_budget", 3);
+ if (!Fixture.Function || Fixture.Entry == nullptr)
+ return;
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(),
+ "CSE budget insert point must be established");
+ const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+ const rcc::Register Seed = makeGPR(Context, *Fixture.Function);
+ const rcc::Register First = makeGPR(Context, *Fixture.Function);
+ const rcc::Register Second = makeGPR(Context, *Fixture.Function);
+ (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 1);
+ (void)buildADDI(Context, Builder, First, Seed, 2);
+ rcc::MachineInstr *SecondInstruction = buildADDI(Context, Builder, Second, Seed, 2);
+
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineCSEPass Pass({.MaximumComparisons = 0});
+ auto Result = Pass.run(*Fixture.Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "machine CSE budget failure: " << Result.error().message() << '\n';
+ Context.expect(Result.has_value() && !Result->Changed && SecondInstruction != nullptr &&
+ SecondInstruction->isLinked(),
+ "machine CSE budget exhaustion must stop without rejecting or mutating MIR");
+}
+
+void testLICMHoistsOnlyInvariantInstructions(TestContext &Context,
+ const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "licm", 4);
+ if (!Fixture.Function || Fixture.Entry == nullptr)
+ return;
+ auto Header = Fixture.Function->createBasicBlock();
+ auto Body = Fixture.Function->createBasicBlock();
+ auto Exit = Fixture.Function->createBasicBlock();
+ Context.expect(Header.has_value() && Body.has_value() && Exit.has_value() &&
+ Fixture.Function->appendBlock(**Header).has_value() &&
+ Fixture.Function->appendBlock(**Body).has_value() &&
+ Fixture.Function->appendBlock(**Exit).has_value(),
+ "LICM loop blocks must be created");
+ if (!Header || !Body || !Exit)
+ return;
+ auto Half = rcc::BranchProbability::fromFraction(1, 2);
+ Context.expect(Half.has_value(), "LICM branch probability must be created");
+ if (!Half)
+ return;
+ auto Updater = Fixture.Function->cfgUpdater();
+ Context.expect(
+ Updater.addSuccessor(*Fixture.Entry, **Header, rcc::BranchProbability::one()).has_value() &&
+ Updater.addSuccessor(**Header, **Body, *Half).has_value() &&
+ Updater.addSuccessor(**Header, **Exit, *Half).has_value() &&
+ Updater.addSuccessor(**Body, **Header, rcc::BranchProbability::one()).has_value(),
+ "LICM loop CFG must be connected");
+
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(),
+ "LICM preheader insertion point must be established");
+ const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+ const rcc::Register Seed = makeGPR(Context, *Fixture.Function);
+ (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 1);
+
+ Context.expect(Builder.setInsertPoint(**Body).has_value(),
+ "LICM body insertion point must be established");
+ const rcc::Register HoistedValue = makeGPR(Context, *Fixture.Function);
+ const rcc::Register DependentValue = makeGPR(Context, *Fixture.Function);
+ rcc::MachineInstr *Hoisted = buildADDI(Context, Builder, HoistedValue, Seed, 8);
+ rcc::MachineInstr *Dependent = buildADDI(Context, Builder, DependentValue, HoistedValue, 9);
+ const rcc::Register MutableLoad = makeGPR(Context, *Fixture.Function);
+ const rcc::MachineMemOperand *Memory = makeMemory(Context, *Fixture.Function, false, false);
+ rcc::MachineInstr *Load =
+ Memory == nullptr ? nullptr : buildLoad(Context, Builder, MutableLoad, Seed, *Memory);
+
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineLICMPass Pass;
+ auto Result = Pass.run(*Fixture.Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "machine LICM failure: " << Result.error().message() << '\n';
+ Context.expect(Result.has_value() && Result->Changed,
+ "machine LICM must hoist loop-invariant instructions");
+ Context.expect(Hoisted != nullptr && Hoisted->parent() == Fixture.Entry && Dependent != nullptr &&
+ Dependent->parent() == Fixture.Entry,
+ "machine LICM must reach a fixed point for dependent invariants");
+ Context.expect(Load != nullptr && Load->parent() == *Body,
+ "machine LICM must keep mutable loads in the loop");
+}
+
+void testMachineSinkUsesColdUniqueSuccessor(TestContext &Context,
+ const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "sink", 5);
+ if (!Fixture.Function || Fixture.Entry == nullptr)
+ return;
+ auto Cold = Fixture.Function->createBasicBlock();
+ auto Hot = Fixture.Function->createBasicBlock();
+ Context.expect(Cold.has_value() && Hot.has_value() &&
+ Fixture.Function->appendBlock(**Cold).has_value() &&
+ Fixture.Function->appendBlock(**Hot).has_value(),
+ "machine sink successor blocks must be created");
+ if (!Cold || !Hot)
+ return;
+ auto ColdProbability = rcc::BranchProbability::fromFraction(1, 10);
+ auto HotProbability = rcc::BranchProbability::fromFraction(9, 10);
+ Context.expect(ColdProbability.has_value() && HotProbability.has_value(),
+ "machine sink probabilities must be created");
+ if (!ColdProbability || !HotProbability)
+ return;
+ auto Updater = Fixture.Function->cfgUpdater();
+ Context.expect(Updater.addSuccessor(*Fixture.Entry, **Cold, *ColdProbability).has_value() &&
+ Updater.addSuccessor(*Fixture.Entry, **Hot, *HotProbability).has_value(),
+ "machine sink CFG must be connected");
+
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(),
+ "machine sink source insertion point must be established");
+ const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+ const rcc::Register Seed = makeGPR(Context, *Fixture.Function);
+ const rcc::Register SunkValue = makeGPR(Context, *Fixture.Function);
+ (void)buildADDI(Context, Builder, Seed, rcc::Register::fromPhysical(X0), 1);
+ rcc::MachineInstr *Sunk = buildADDI(Context, Builder, SunkValue, Seed, 4);
+
+ Context.expect(Builder.setInsertPoint(**Cold).has_value(),
+ "machine sink destination insertion point must be established");
+ const rcc::Register Consumer = makeGPR(Context, *Fixture.Function);
+ rcc::MachineInstr *Use = buildADD(Context, Builder, Consumer, SunkValue, Seed);
+
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineSinkPass Pass;
+ auto Result = Pass.run(*Fixture.Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "machine sink failure: " << Result.error().message() << '\n';
+ Context.expect(Result.has_value() && Result->Changed,
+ "machine sink must move a cold-path-only expression");
+ Context.expect(Sunk != nullptr && Sunk->parent() == *Cold && Use != nullptr &&
+ Sunk->next() == Use,
+ "machine sink must place the definition immediately before its first use");
+}
+
+void testPostRACodeMotion(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto LICMFixture = makeFixture(Context, "post-ra-licm", 10);
+ if (!LICMFixture.Function || LICMFixture.Entry == nullptr)
+ return;
+ auto Header = LICMFixture.Function->createBasicBlock();
+ auto Body = LICMFixture.Function->createBasicBlock();
+ auto Exit = LICMFixture.Function->createBasicBlock();
+ if (!Header || !Body || !Exit || !LICMFixture.Function->appendBlock(**Header) ||
+ !LICMFixture.Function->appendBlock(**Body) || !LICMFixture.Function->appendBlock(**Exit)) {
+ Context.expect(false, "post-RA LICM blocks must be created");
+ return;
+ }
+ auto Half = rcc::BranchProbability::fromFraction(1, 2);
+ if (!Half)
+ return;
+ auto LICMUpdater = LICMFixture.Function->cfgUpdater();
+ if (!LICMUpdater.addSuccessor(*LICMFixture.Entry, **Header, rcc::BranchProbability::one()) ||
+ !LICMUpdater.addSuccessor(**Header, **Body, *Half) ||
+ !LICMUpdater.addSuccessor(**Header, **Exit, *Half) ||
+ !LICMUpdater.addSuccessor(**Body, **Header, rcc::BranchProbability::one())) {
+ Context.expect(false, "post-RA LICM CFG must be connected");
+ return;
+ }
+ rcc::MachineIRBuilder LICMBuilder(*LICMFixture.Function, Target);
+ if (!LICMBuilder.setInsertPoint(*LICMFixture.Entry))
+ return;
+ const rcc::Register X0 =
+ rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X0));
+ const rcc::Register X2 =
+ rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X2));
+ const rcc::Register X10 =
+ rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X10));
+ const rcc::Register X12 =
+ rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X12));
+ const rcc::Register X13 =
+ rcc::Register::fromPhysical(physical(Context, rcc::riscv::PhysicalRegisterID::X13));
+ (void)buildADDI(Context, LICMBuilder, X10, X0, 1);
+ if (!LICMBuilder.setInsertPoint(**Header))
+ return;
+ rcc::MachineInstr *Hoisted = buildADDI(Context, LICMBuilder, X12, X10, 4);
+ if (Hoisted == nullptr)
+ return;
+ Hoisted->setFlags({rcc::MachineInstrFlag::NoFPExcept});
+ if (!LICMBuilder.setInsertPoint(**Body))
+ return;
+ (void)buildADDI(Context, LICMBuilder, X13, X12, 1);
+ if (!LICMBuilder.setInsertPoint(**Exit))
+ return;
+ const rcc::MachineMemOperand *LICMMemory =
+ makeMemory(Context, *LICMFixture.Function, true, false);
+ if (LICMMemory == nullptr)
+ return;
+ (void)buildStore(Context, LICMBuilder, X12, *X2.physicalRegister(), *LICMMemory);
+ setPostRAProperties(*LICMFixture.Function);
+ Context.expect(rcc::rebuildMachineLiveness(*LICMFixture.Function, Target).has_value(),
+ "post-RA LICM fixture liveness must be built");
+ rcc::MachineAnalysisManager LICMAnalyses;
+ rcc::MachineLICMPass PostRALICM(rcc::MachineCodeMotionStage::PostRegisterAllocation);
+ auto LICMResult = PostRALICM.run(*LICMFixture.Function, Target, LICMAnalyses);
+ Context.expect(LICMResult && LICMResult->Changed && Hoisted->parent() == LICMFixture.Entry &&
+ rcc::verifyTrackedMachineLiveness(*LICMFixture.Function, Target).has_value(),
+ "post-RA LICM must hoist a safe physical-register invariant and repair liveness");
+
+ auto SinkFixture = makeFixture(Context, "post-ra-sink", 11);
+ if (!SinkFixture.Function || SinkFixture.Entry == nullptr)
+ return;
+ auto Cold = SinkFixture.Function->createBasicBlock();
+ auto Hot = SinkFixture.Function->createBasicBlock();
+ if (!Cold || !Hot || !SinkFixture.Function->appendBlock(**Cold) ||
+ !SinkFixture.Function->appendBlock(**Hot)) {
+ return;
+ }
+ auto ColdProbability = rcc::BranchProbability::fromFraction(1, 10);
+ auto HotProbability = rcc::BranchProbability::fromFraction(9, 10);
+ if (!ColdProbability || !HotProbability)
+ return;
+ auto SinkUpdater = SinkFixture.Function->cfgUpdater();
+ if (!SinkUpdater.addSuccessor(*SinkFixture.Entry, **Cold, *ColdProbability) ||
+ !SinkUpdater.addSuccessor(*SinkFixture.Entry, **Hot, *HotProbability)) {
+ return;
+ }
+ rcc::MachineIRBuilder SinkBuilder(*SinkFixture.Function, Target);
+ if (!SinkBuilder.setInsertPoint(*SinkFixture.Entry))
+ return;
+ (void)buildADDI(Context, SinkBuilder, X10, X0, 2);
+ rcc::MachineInstr *Sunk = buildADDI(Context, SinkBuilder, X12, X10, 8);
+ if (Sunk == nullptr)
+ return;
+ Sunk->setFlags({rcc::MachineInstrFlag::NoFPExcept});
+ if (!SinkBuilder.setInsertPoint(**Cold))
+ return;
+ rcc::MachineInstr *ColdUse = buildADDI(Context, SinkBuilder, X13, X12, 1);
+ setPostRAProperties(*SinkFixture.Function);
+ Context.expect(rcc::rebuildMachineLiveness(*SinkFixture.Function, Target).has_value(),
+ "post-RA sink fixture liveness must be built");
+ rcc::MachineAnalysisManager SinkAnalyses;
+ rcc::MachineSinkPass PostRASink(rcc::MachineCodeMotionStage::PostRegisterAllocation);
+ auto SinkResult = PostRASink.run(*SinkFixture.Function, Target, SinkAnalyses);
+ Context.expect(SinkResult && SinkResult->Changed && Sunk->parent() == *Cold &&
+ Sunk->next() == ColdUse &&
+ rcc::verifyTrackedMachineLiveness(*SinkFixture.Function, Target).has_value(),
+ "post-RA sinking must move a physical definition to its sole cold use");
+}
+
+void testMemoryFoldingContract(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto BuildCandidate = [&](std::string Name, std::uint64_t ID) {
+ Fixture Candidate = makeFixture(Context, std::move(Name), ID);
+ if (!Candidate.Function || Candidate.Entry == nullptr)
+ return Candidate;
+ rcc::MachineIRBuilder Builder(*Candidate.Function, Target);
+ if (!Builder.setInsertPoint(*Candidate.Entry))
+ return Candidate;
+ const rcc::PhysicalRegister X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+ const rcc::PhysicalRegister X2 = physical(Context, rcc::riscv::PhysicalRegisterID::X2);
+ const rcc::Register Address = makeGPR(Context, *Candidate.Function);
+ const rcc::Register Loaded = makeGPR(Context, *Candidate.Function);
+ const rcc::Register FoldedResult = makeGPR(Context, *Candidate.Function);
+ (void)buildADDI(Context, Builder, Address, rcc::Register::fromPhysical(X0), 0);
+ const rcc::MachineMemOperand *LoadMemory =
+ makeMemory(Context, *Candidate.Function, false, false);
+ const rcc::MachineMemOperand *StoreMemory =
+ makeMemory(Context, *Candidate.Function, true, false);
+ if (LoadMemory == nullptr || StoreMemory == nullptr)
+ return Candidate;
+ (void)buildLoad(Context, Builder, Loaded, Address, *LoadMemory);
+ (void)buildADDI(Context, Builder, FoldedResult, Loaded, 0);
+ (void)buildStore(Context, Builder, FoldedResult, X2, *StoreMemory);
+ return Candidate;
+ };
+
+ Fixture Folded = BuildCandidate("memory-fold", 8);
+ if (!Folded.Function || Folded.Entry == nullptr)
+ return;
+ TestMemoryFoldingInfo GoodInfo;
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineMemoryFoldingPass Pass(GoodInfo);
+ auto Result = Pass.run(*Folded.Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "machine memory folding failure: " << Result.error().message() << '\n';
+ std::size_t LoadCount = 0;
+ rcc::MachineInstr *FoldedLoad = nullptr;
+ for (rcc::MachineInstr *Instruction = Folded.Entry->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ if (Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::LD)) {
+ ++LoadCount;
+ FoldedLoad = Instruction;
+ }
+ }
+ Context.expect(Result && Result->Changed && LoadCount == 1 && FoldedLoad != nullptr &&
+ FoldedLoad->memOperands().size() == 1,
+ "memory folding must replace an adjacent load-use pair and migrate its MMO");
+
+ Fixture Rejected = BuildCandidate("memory-fold-missing-mmo", 9);
+ if (!Rejected.Function || Rejected.Entry == nullptr)
+ return;
+ rcc::MachineInstr *OriginalLoad = Rejected.Entry->firstInstruction();
+ OriginalLoad = OriginalLoad == nullptr ? nullptr : OriginalLoad->next();
+ rcc::MachineInstr *OriginalConsumer = OriginalLoad == nullptr ? nullptr : OriginalLoad->next();
+ const std::size_t OriginalCount = Rejected.Entry->instructionCount();
+ TestMemoryFoldingInfo BadInfo(true);
+ rcc::MachineMemoryFoldingPass BadPass(BadInfo);
+ auto Failure = BadPass.run(*Rejected.Function, Target, Analyses);
+ Context.expect(!Failure && Failure.error().code() == rcc::ErrorCode::VerificationError &&
+ Rejected.Entry->instructionCount() == OriginalCount &&
+ OriginalLoad != nullptr && OriginalLoad->isLinked() &&
+ OriginalConsumer != nullptr && OriginalConsumer->isLinked(),
+ "memory folding must reject missing MMO migration without mutating the function");
+}
+
+} // namespace
+
+int main() {
+ TestContext Context;
+ auto Target = rcc::riscv::createTargetDescription();
+ Context.expect(Target.has_value(), "optimization target construction must succeed");
+ if (!Target)
+ return Context.result();
+ testDCEFixedPointAndDebugUses(Context, *Target);
+ testMachineCombinerIdentityRules(Context, *Target);
+ testPostRAMachineCombinerIdentity(Context, *Target);
+ testCSERegisterAndMemoryRules(Context, *Target);
+ testCSEBudgetPreservesLegalMIR(Context, *Target);
+ testLICMHoistsOnlyInvariantInstructions(Context, *Target);
+ testMachineSinkUsesColdUniqueSuccessor(Context, *Target);
+ testPostRACodeMotion(Context, *Target);
+ testMemoryFoldingContract(Context, *Target);
+ return Context.result();
+}
diff --git a/backend/unittests/CodeGen/MachineSchedulingTest.cpp b/backend/unittests/CodeGen/MachineSchedulingTest.cpp
new file mode 100644
index 0000000..a485146
--- /dev/null
+++ b/backend/unittests/CodeGen/MachineSchedulingTest.cpp
@@ -0,0 +1,518 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineLiveness.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/MachineScheduler.h"
+#include "rcc/CodeGen/PostRAScheduler.h"
+#include "rcc/CodeGen/ScheduleDAG.h"
+#include "rcc/CodeGen/ScheduleHazardRecognizer.h"
+#include "rcc/CodeGen/TargetSchedulingModel.h"
+#include "rcc/Target/RISCV/RISCVCodeGen.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <array>
+#include <cstdint>
+#include <iostream>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace {
+
+class TestContext final {
+public:
+ void expect(bool Condition, std::string_view Message) {
+ if (Condition)
+ return;
+ std::cerr << Message << '\n';
+ ++Failures;
+ }
+
+ [[nodiscard]] int result() const noexcept {
+ return Failures == 0 ? 0 : 1;
+ }
+
+private:
+ int Failures = 0;
+};
+
+struct Fixture final {
+ std::unique_ptr<rcc::MachineFunction> Function;
+ rcc::MachineBasicBlock *Block = nullptr;
+};
+
+class OnePerCycleHazard final : public rcc::TargetScheduleHazardRecognizer {
+public:
+ [[nodiscard]] std::unique_ptr<rcc::TargetScheduleHazardRecognizer> clone() const override {
+ return std::make_unique<OnePerCycleHazard>(*this);
+ }
+
+ [[nodiscard]] rcc::Expected<rcc::ScheduleHazardType>
+ getHazardType(const rcc::ScheduleUnit &) const override {
+ return Emitted ? rcc::ScheduleHazardType::Hazard : rcc::ScheduleHazardType::NoHazard;
+ }
+
+ [[nodiscard]] rcc::Expected<void> emitInstruction(const rcc::ScheduleUnit &) override {
+ Emitted = true;
+ return {};
+ }
+
+ [[nodiscard]] rcc::Expected<void> advanceCycle() override {
+ Emitted = false;
+ return {};
+ }
+
+ [[nodiscard]] rcc::Expected<void> recedeCycle() override {
+ Emitted = false;
+ return {};
+ }
+
+ [[nodiscard]] rcc::Expected<void> reset() override {
+ Emitted = false;
+ return {};
+ }
+
+private:
+ bool Emitted = false;
+};
+
+class TestSchedulingHooks final : public rcc::TargetSchedulingHooks {
+public:
+ bool MakeADDIBoundary = false;
+ bool AddMutation = false;
+ bool PreferSecondUnit = false;
+ bool EnableDynamicHazards = false;
+
+ [[nodiscard]] bool isSchedulingBoundary(const rcc::MachineInstr &Instruction,
+ const rcc::TargetDescription &) const noexcept override {
+ return MakeADDIBoundary &&
+ Instruction.opcode() == rcc::Opcode::fromValue(static_cast<std::uint32_t>(
+ rcc::riscv::InstructionOpcode::ADDI));
+ }
+
+ [[nodiscard]] rcc::Expected<void> mutateScheduleDAG(rcc::ScheduleDAG &DAG,
+ const rcc::MachineFunction &,
+ const rcc::TargetDescription &,
+ rcc::ScheduleDAGMode) const override {
+ if (!AddMutation || DAG.units().size() < 2)
+ return {};
+ return DAG.addArtificialDependency(0, 1, 2);
+ }
+
+ [[nodiscard]] std::int64_t candidatePreference(const rcc::ScheduleUnit &Unit,
+ rcc::ScheduleDirection,
+ const rcc::MachineBasicBlock &,
+ std::uint64_t) const noexcept override {
+ return PreferSecondUnit && Unit.ID == 1 ? 100 : 0;
+ }
+
+ [[nodiscard]] std::unique_ptr<rcc::TargetScheduleHazardRecognizer>
+ createHazardRecognizer(rcc::ScheduleDirection) const override {
+ return EnableDynamicHazards ? std::make_unique<OnePerCycleHazard>() : nullptr;
+ }
+};
+
+[[nodiscard]] rcc::Opcode opcode(rcc::riscv::InstructionOpcode Value) noexcept {
+ return rcc::Opcode::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+[[nodiscard]] rcc::Expected<rcc::TargetSchedulingModel>
+makeSchedulingModel(const rcc::TargetSchedulingHooks *Hooks = nullptr) {
+ rcc::TargetSchedulingModelDesc Description;
+ Description.IssueWidth = 2;
+ Description.DispatchLimit = 2;
+ for (const rcc::riscv::SchedulingClassInfo &Class : rcc::riscv::schedulingClasses()) {
+ Description.SchedulingClasses.push_back({
+ .ID = Class.ID,
+ .Name = std::string(Class.Name),
+ .MicroOps = Class.MicroOps,
+ .Latency = Class.Latency,
+ .Resources = {},
+ .WriteLatencies = {{0, Class.Latency}},
+ .ReadAdvances = {},
+ });
+ }
+ Description.PressureSets.push_back({
+ .ID = rcc::RegisterPressureSetID::fromValue(1),
+ .Name = "gpr",
+ .RegisterClasses =
+ {
+ rcc::RegisterClassID::fromValue(
+ static_cast<std::uint32_t>(rcc::riscv::RegisterClass::GPR)),
+ },
+ .Limit = 27,
+ });
+ Description.Hooks = Hooks;
+ return rcc::TargetSchedulingModel::create(std::move(Description));
+}
+
+[[nodiscard]] rcc::Register physical(TestContext &Context, rcc::riscv::PhysicalRegisterID ID) {
+ auto Value = rcc::riscv::physicalRegister(ID);
+ Context.expect(Value.has_value(), "physical register creation must succeed");
+ return Value ? rcc::Register::fromPhysical(*Value) : rcc::Register{};
+}
+
+[[nodiscard]] Fixture makeFixture(TestContext &Context, std::string Name, std::uint64_t ID) {
+ auto Function =
+ rcc::MachineFunction::create(std::move(Name), rcc::MachineFunctionStableID::fromValue(ID));
+ Context.expect(Function.has_value(), "scheduler function creation must succeed");
+ if (!Function)
+ return {};
+ auto Block = (*Function)->createBasicBlock();
+ Context.expect(Block.has_value() && (*Function)->appendBlock(**Block).has_value(),
+ "scheduler block creation must succeed");
+ if (!Block)
+ return {};
+ (*Function)->properties().set(rcc::MachineFunctionProperty::Legalized);
+ (*Function)->properties().set(rcc::MachineFunctionProperty::RegBankSelected);
+ (*Function)->properties().set(rcc::MachineFunctionProperty::Selected);
+ (*Function)->properties().set(rcc::MachineFunctionProperty::NoPHIs);
+ (*Function)->properties().set(rcc::MachineFunctionProperty::TiedOpsRewritten);
+ return {std::move(*Function), *Block};
+}
+
+[[nodiscard]] rcc::Register makeGPR(TestContext &Context, rcc::MachineFunction &Function) {
+ auto Type = rcc::LLT::makeInteger(64);
+ auto Reg = Type ? Function.registerInfo().createVirtualRegister(*Type)
+ : rcc::Expected<rcc::Register>(std::unexpected(
+ rcc::Error(rcc::ErrorCode::InternalError, "test LLT creation failed")));
+ Context.expect(Reg.has_value(), "scheduler virtual register creation must succeed");
+ if (!Reg)
+ return {};
+ const auto Bank =
+ rcc::RegisterBankID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterBank::GPR));
+ const auto Class =
+ rcc::RegisterClassID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterClass::GPR));
+ Context.expect(Function.registerInfo().setRegisterBank(*Reg, Bank).has_value() &&
+ Function.registerInfo().setRegisterClass(*Reg, Class).has_value(),
+ "scheduler virtual register constraints must be assigned");
+ return *Reg;
+}
+
+[[nodiscard]] rcc::MachineInstr *buildADDI(TestContext &Context, rcc::MachineIRBuilder &Builder,
+ rcc::Register Destination, rcc::Register Source,
+ std::int64_t Immediate) {
+ auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::ADDI));
+ if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Source) ||
+ !Pending->addImmediate(Immediate)) {
+ Context.expect(false, "scheduler ADDI construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "scheduler ADDI insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+[[nodiscard]] rcc::MachineInstr *buildBinary(TestContext &Context, rcc::MachineIRBuilder &Builder,
+ rcc::riscv::InstructionOpcode Opcode,
+ rcc::Register Destination, rcc::Register Left,
+ rcc::Register Right) {
+ auto Pending = Builder.buildInstr(opcode(Opcode));
+ if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Left) ||
+ !Pending->addUse(Right)) {
+ Context.expect(false, "scheduler binary instruction construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "scheduler binary instruction insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+[[nodiscard]] bool hasDependency(const rcc::ScheduleUnit &Unit, std::size_t Destination,
+ rcc::ScheduleDependencyKind Kind) {
+ for (const rcc::ScheduleDependency &Dependency : Unit.Successors) {
+ if (Dependency.Destination == Destination && Dependency.Kind == Kind)
+ return true;
+ }
+ return false;
+}
+
+void testDAGRegisterDependencies(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "schedule_dag", 1);
+ if (!Fixture.Function || Fixture.Block == nullptr)
+ return;
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Block).has_value(),
+ "schedule DAG insertion point must be established");
+ const rcc::Register X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+ const rcc::Register X10 = physical(Context, rcc::riscv::PhysicalRegisterID::X10);
+ const rcc::Register Value = makeGPR(Context, *Fixture.Function);
+ const rcc::Register Consumer = makeGPR(Context, *Fixture.Function);
+ rcc::MachineInstr *Definition = buildADDI(Context, Builder, Value, X0, 1);
+ rcc::MachineInstr *Use =
+ buildBinary(Context, Builder, rcc::riscv::InstructionOpcode::ADD, Consumer, Value, X10);
+ rcc::MachineInstr *Redefinition = buildADDI(Context, Builder, Value, X0, 2);
+ std::array<rcc::MachineInstr *, 3> Instructions{Definition, Use, Redefinition};
+ auto DAG = rcc::ScheduleDAG::build(*Fixture.Block, Instructions, *Fixture.Function, Target,
+ rcc::riscv::schedulingModel(),
+ rcc::ScheduleDAGMode::PreRegisterAllocation);
+ if (!DAG)
+ std::cerr << "schedule DAG failure: " << DAG.error().message() << '\n';
+ Context.expect(DAG.has_value() && DAG->units().size() == 3,
+ "schedule DAG must create one unit per instruction");
+ if (!DAG)
+ return;
+ Context.expect(hasDependency(DAG->units()[0], 1, rcc::ScheduleDependencyKind::Data),
+ "schedule DAG must model virtual-register data dependencies");
+ Context.expect(hasDependency(DAG->units()[0], 2, rcc::ScheduleDependencyKind::Output),
+ "schedule DAG must model virtual-register output dependencies");
+ Context.expect(hasDependency(DAG->units()[1], 2, rcc::ScheduleDependencyKind::Anti),
+ "schedule DAG must model virtual-register anti dependencies");
+ Context.expect(DAG->units()[0].Latency != 0 && DAG->units()[0].MicroOps != 0,
+ "schedule DAG must attach target latency and resource facts");
+}
+
+void testHazardScoreboard(TestContext &Context) {
+ auto Hazards = rcc::ScheduleHazardRecognizer::create(rcc::riscv::schedulingModel());
+ Context.expect(Hazards.has_value(), "hazard recognizer construction must succeed");
+ if (!Hazards)
+ return;
+ const auto Memory = rcc::ProcessorResourceID::fromValue(3);
+ rcc::ScheduleUnit Load;
+ Load.MicroOps = 1;
+ Load.Resources = {{Memory, 1}};
+ Context.expect(Hazards->issue(Load, 0).has_value(),
+ "first memory operation must issue in cycle zero");
+ auto SecondCycle = Hazards->earliestIssueCycle(Load, 0);
+ Context.expect(SecondCycle.has_value() && *SecondCycle == 1,
+ "single-unit memory resource must serialize operations");
+
+ rcc::ScheduleUnit Wide;
+ Wide.MicroOps = 5;
+ auto Fresh = rcc::ScheduleHazardRecognizer::create(rcc::riscv::schedulingModel());
+ Context.expect(Fresh.has_value() && Fresh->issue(Wide, 0).has_value(),
+ "multi-cycle dispatch must accept instructions wider than issue width");
+ auto AfterWide = Fresh ? Fresh->earliestIssueCycle(Load, 0)
+ : rcc::Expected<std::uint64_t>(std::unexpected(
+ rcc::Error(rcc::ErrorCode::InternalError, "no recognizer")));
+ Context.expect(AfterWide.has_value() && *AfterWide == 3,
+ "multi-cycle dispatch must block the complete dispatch interval");
+}
+
+void testOperandLatencyAndModelValidation(TestContext &Context) {
+ const rcc::SchedulingClassID ALU = rcc::SchedulingClassID::fromValue(
+ static_cast<std::uint32_t>(rcc::riscv::SchedulingClass::ALU));
+ const rcc::SchedulingClassID Load = rcc::SchedulingClassID::fromValue(
+ static_cast<std::uint32_t>(rcc::riscv::SchedulingClass::LOAD));
+ auto Forwarded = rcc::riscv::schedulingModel().operandLatency(Load, 0, ALU, 1);
+ auto Unforwarded = rcc::riscv::schedulingModel().operandLatency(Load, 0, ALU, 0);
+ auto ALUChain = rcc::riscv::schedulingModel().operandLatency(ALU, 0, ALU, 1);
+ Context.expect(Forwarded.has_value() && *Forwarded == 2 && Unforwarded.has_value() &&
+ *Unforwarded == 3 && ALUChain.has_value() && *ALUChain == 1,
+ "operand latency must combine write latency with exact read advance");
+
+ rcc::TargetSchedulingModelDesc Invalid;
+ Invalid.SchedulingClasses.push_back({
+ .ID = ALU,
+ .Name = "invalid",
+ .MicroOps = 1,
+ .Latency = 1,
+ .Resources = {},
+ .WriteLatencies = {{0, 1}, {0, 2}},
+ .ReadAdvances = {},
+ });
+ auto Rejected = rcc::TargetSchedulingModel::create(std::move(Invalid));
+ Context.expect(!Rejected && Rejected.error().code() == rcc::ErrorCode::InvalidArgument,
+ "scheduling model must reject duplicate operand latency facts");
+}
+
+void testDynamicTargetHazards(TestContext &Context) {
+ TestSchedulingHooks Hooks;
+ Hooks.EnableDynamicHazards = true;
+ auto Model = makeSchedulingModel(&Hooks);
+ Context.expect(Model.has_value(), "dynamic-hazard scheduling model construction must succeed");
+ if (!Model)
+ return;
+
+ rcc::ScheduleUnit First;
+ First.ID = 1;
+ First.MicroOps = 1;
+ rcc::ScheduleUnit Second;
+ Second.ID = 2;
+ Second.MicroOps = 1;
+ auto Top = rcc::ScheduleHazardRecognizer::create(*Model, rcc::ScheduleDirection::TopDown);
+ Context.expect(Top.has_value() && Top->issue(First, 0).has_value(),
+ "target dynamic hazard must accept the first top-down issue");
+ auto TopCycle = Top ? Top->earliestIssueCycle(Second, 0)
+ : rcc::Expected<std::uint64_t>(std::unexpected(
+ rcc::Error(rcc::ErrorCode::InternalError, "no top-down recognizer")));
+ Context.expect(TopCycle.has_value() && *TopCycle == 1,
+ "target dynamic hazard must advance the top-down cycle");
+
+ auto Bottom = rcc::ScheduleHazardRecognizer::create(*Model, rcc::ScheduleDirection::BottomUp);
+ Context.expect(Bottom.has_value() && Bottom->issue(First, 0).has_value(),
+ "target dynamic hazard must accept the first bottom-up issue");
+ auto BottomCycle = Bottom ? Bottom->earliestIssueCycle(Second, 0)
+ : rcc::Expected<std::uint64_t>(std::unexpected(rcc::Error(
+ rcc::ErrorCode::InternalError, "no bottom-up recognizer")));
+ Context.expect(BottomCycle.has_value() && *BottomCycle == 1,
+ "target dynamic hazard must recede the bottom-up cycle");
+}
+
+void testTargetSchedulingHooks(TestContext &Context, const rcc::TargetDescription &Target) {
+ TestSchedulingHooks MutationHooks;
+ MutationHooks.AddMutation = true;
+ auto MutationModel = makeSchedulingModel(&MutationHooks);
+ auto Fixture = makeFixture(Context, "schedule_mutation", 4);
+ if (!MutationModel || !Fixture.Function || Fixture.Block == nullptr) {
+ Context.expect(false, "target schedule mutation fixture must be created");
+ return;
+ }
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Block).has_value(),
+ "target schedule mutation insertion point must be established");
+ const rcc::Register X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+ rcc::MachineInstr *First =
+ buildADDI(Context, Builder, makeGPR(Context, *Fixture.Function), X0, 1);
+ rcc::MachineInstr *Second =
+ buildADDI(Context, Builder, makeGPR(Context, *Fixture.Function), X0, 2);
+ std::array<rcc::MachineInstr *, 2> Instructions{First, Second};
+ auto DAG = rcc::ScheduleDAG::build(*Fixture.Block, Instructions, *Fixture.Function, Target,
+ *MutationModel, rcc::ScheduleDAGMode::PreRegisterAllocation);
+ Context.expect(DAG.has_value() &&
+ hasDependency(DAG->units()[0], 1, rcc::ScheduleDependencyKind::Artificial),
+ "target schedule mutation must add a checked artificial dependency");
+
+ TestSchedulingHooks PreferenceHooks;
+ PreferenceHooks.PreferSecondUnit = true;
+ auto PreferenceModel = makeSchedulingModel(&PreferenceHooks);
+ if (!PreferenceModel) {
+ Context.expect(false, "target scheduling preference model must be created");
+ return;
+ }
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineSchedulerPass PreferencePass(
+ *PreferenceModel, {.MaximumRegionSize = 8, .EnableBidirectionalScheduling = false});
+ auto Preferred = PreferencePass.run(*Fixture.Function, Target, Analyses);
+ Context.expect(Preferred.has_value() && Preferred->Changed &&
+ Fixture.Block->firstInstruction() == Second,
+ "target candidate preference must affect the production scheduler");
+
+ auto BoundaryFixture = makeFixture(Context, "schedule_boundary", 5);
+ TestSchedulingHooks BoundaryHooks;
+ BoundaryHooks.MakeADDIBoundary = true;
+ auto BoundaryModel = makeSchedulingModel(&BoundaryHooks);
+ if (!BoundaryFixture.Function || BoundaryFixture.Block == nullptr || !BoundaryModel) {
+ Context.expect(false, "target schedule boundary fixture must be created");
+ return;
+ }
+ rcc::MachineIRBuilder BoundaryBuilder(*BoundaryFixture.Function, Target);
+ Context.expect(BoundaryBuilder.setInsertPoint(*BoundaryFixture.Block).has_value(),
+ "target schedule boundary insertion point must be established");
+ rcc::MachineInstr *BoundaryFirst =
+ buildADDI(Context, BoundaryBuilder, makeGPR(Context, *BoundaryFixture.Function), X0, 1);
+ (void)buildADDI(Context, BoundaryBuilder, makeGPR(Context, *BoundaryFixture.Function), X0, 2);
+ rcc::MachineAnalysisManager BoundaryAnalyses;
+ rcc::MachineSchedulerPass BoundaryPass(
+ *BoundaryModel, {.MaximumRegionSize = 8, .EnableBidirectionalScheduling = false});
+ auto BoundaryResult = BoundaryPass.run(*BoundaryFixture.Function, Target, BoundaryAnalyses);
+ Context.expect(BoundaryResult.has_value() && !BoundaryResult->Changed &&
+ BoundaryFixture.Block->firstInstruction() == BoundaryFirst,
+ "target scheduling boundary must split production schedule regions");
+}
+
+void buildSchedulingChain(TestContext &Context, Fixture &Fixture,
+ const rcc::TargetDescription &Target, bool PhysicalRegisters,
+ rcc::MachineInstr *&Short, rcc::MachineInstr *&Long) {
+ rcc::MachineIRBuilder Builder(*Fixture.Function, Target);
+ Context.expect(Builder.setInsertPoint(*Fixture.Block).has_value(),
+ "scheduler chain insertion point must be established");
+ const rcc::Register X0 = physical(Context, rcc::riscv::PhysicalRegisterID::X0);
+ const rcc::Register X10 = physical(Context, rcc::riscv::PhysicalRegisterID::X10);
+ const rcc::Register X11 = physical(Context, rcc::riscv::PhysicalRegisterID::X11);
+ const rcc::Register ShortValue = PhysicalRegisters
+ ? physical(Context, rcc::riscv::PhysicalRegisterID::X12)
+ : makeGPR(Context, *Fixture.Function);
+ const rcc::Register LongValue = PhysicalRegisters
+ ? physical(Context, rcc::riscv::PhysicalRegisterID::X13)
+ : makeGPR(Context, *Fixture.Function);
+ const rcc::Register LongResult = PhysicalRegisters
+ ? physical(Context, rcc::riscv::PhysicalRegisterID::X14)
+ : makeGPR(Context, *Fixture.Function);
+ const rcc::Register ShortResult = PhysicalRegisters
+ ? physical(Context, rcc::riscv::PhysicalRegisterID::X15)
+ : makeGPR(Context, *Fixture.Function);
+ Short = buildADDI(Context, Builder, ShortValue, X0, 1);
+ Long = buildBinary(Context, Builder, rcc::riscv::InstructionOpcode::DIV, LongValue, X10, X11);
+ (void)buildBinary(Context, Builder, rcc::riscv::InstructionOpcode::ADD, LongResult, LongValue,
+ X10);
+ (void)buildBinary(Context, Builder, rcc::riscv::InstructionOpcode::ADD, ShortResult, ShortValue,
+ X11);
+}
+
+void testPreRAScheduler(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "pre_ra_schedule", 2);
+ if (!Fixture.Function || Fixture.Block == nullptr)
+ return;
+ rcc::MachineInstr *Short = nullptr;
+ rcc::MachineInstr *Long = nullptr;
+ buildSchedulingChain(Context, Fixture, Target, false, Short, Long);
+ Context.expect(Fixture.Block->firstInstruction() == Short,
+ "pre-RA fixture must begin in intentionally suboptimal order");
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineSchedulerPass Pass(rcc::riscv::schedulingModel(),
+ {.MaximumRegionSize = 64, .EnableBidirectionalScheduling = false});
+ auto Result = Pass.run(*Fixture.Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "pre-RA scheduler failure: " << Result.error().message() << '\n';
+ Context.expect(Result.has_value() && Result->Changed,
+ "pre-RA scheduler must reorder the latency-critical chain");
+ Context.expect(Fixture.Block->firstInstruction() == Long,
+ "pre-RA scheduler must prioritize the long critical path");
+}
+
+void testPostRASchedulerAndLivenessRepair(TestContext &Context,
+ const rcc::TargetDescription &Target) {
+ auto Fixture = makeFixture(Context, "post_ra_schedule", 3);
+ if (!Fixture.Function || Fixture.Block == nullptr)
+ return;
+ Fixture.Function->properties().set(rcc::MachineFunctionProperty::NoVRegs);
+ Fixture.Function->properties().set(rcc::MachineFunctionProperty::TracksLiveness);
+ rcc::MachineInstr *Short = nullptr;
+ rcc::MachineInstr *Long = nullptr;
+ buildSchedulingChain(Context, Fixture, Target, true, Short, Long);
+ auto Rebuilt = rcc::rebuildMachineLiveness(*Fixture.Function, Target);
+ Context.expect(Rebuilt.has_value(), "post-RA fixture liveness rebuild must succeed");
+ if (!Rebuilt)
+ return;
+
+ rcc::MachineAnalysisManager Analyses;
+ rcc::PostRASchedulerPass Pass(rcc::riscv::schedulingModel(),
+ {.MaximumRegionSize = 64, .EnableBidirectionalScheduling = false});
+ auto Result = Pass.run(*Fixture.Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "post-RA scheduler failure: " << Result.error().message() << '\n';
+ Context.expect(Result.has_value() && Result->Changed,
+ "post-RA scheduler must reorder the latency-critical chain");
+ Context.expect(Fixture.Block->firstInstruction() == Long,
+ "post-RA scheduler must prioritize the long critical path");
+ auto Verified = rcc::verifyTrackedMachineLiveness(*Fixture.Function, Target);
+ if (!Verified)
+ std::cerr << "post-RA liveness verification failure: " << Verified.error().message() << '\n';
+ Context.expect(Verified.has_value(),
+ "post-RA scheduler must locally repair exact kill/dead state");
+}
+
+} // namespace
+
+int main() {
+ TestContext Context;
+ auto Target = rcc::riscv::createTargetDescription();
+ Context.expect(Target.has_value(), "scheduler target creation must succeed");
+ if (!Target)
+ return Context.result();
+ testDAGRegisterDependencies(Context, *Target);
+ testHazardScoreboard(Context);
+ testOperandLatencyAndModelValidation(Context);
+ testDynamicTargetHazards(Context);
+ testTargetSchedulingHooks(Context, *Target);
+ testPreRAScheduler(Context, *Target);
+ testPostRASchedulerAndLivenessRepair(Context, *Target);
+ return Context.result();
+}
diff --git a/backend/unittests/Support/ErrorTest.cpp b/backend/unittests/Support/ErrorTest.cpp
index af2ece9..de75aa3 100644
--- a/backend/unittests/Support/ErrorTest.cpp
+++ b/backend/unittests/Support/ErrorTest.cpp
@@ -30,6 +30,16 @@ int main() {
return fail("Error code must have a stable name");
if (rcc::getErrorCodeName(rcc::ErrorCode::FrameLayoutFailed) != "frame layout failed")
return fail("frame-layout errors must have a stable name");
+ if (rcc::getErrorCodeName(rcc::ErrorCode::SchedulingFailed) != "scheduling failed")
+ return fail("scheduling errors must have a stable name");
+ if (rcc::getErrorCodeName(rcc::ErrorCode::BranchRelaxationFailed) != "branch relaxation failed")
+ return fail("branch-relaxation errors must have a stable name");
+ if (rcc::getErrorCodeName(rcc::ErrorCode::PseudoExpansionFailed) != "pseudo expansion failed")
+ return fail("pseudo-expansion errors must have a stable name");
+ if (rcc::getErrorCodeName(rcc::ErrorCode::FinalEncodabilityFailed) != "final encodability failed")
+ return fail("final-encodability errors must have a stable name");
+ if (rcc::getErrorCodeName(rcc::ErrorCode::CFGOptimizationFailed) != "CFG optimization failed")
+ return fail("CFG-optimization errors must have a stable name");
return 0;
}
diff --git a/backend/unittests/Target/CMakeLists.txt b/backend/unittests/Target/CMakeLists.txt
index a114c7f..c1bef1d 100644
--- a/backend/unittests/Target/CMakeLists.txt
+++ b/backend/unittests/Target/CMakeLists.txt
@@ -60,6 +60,17 @@ add_test(
COMMAND RCCBackendRISCVFrameLoweringTests
)
+add_executable(RCCBackendRISCVLateCodeGenTests RISCVLateCodeGenTest.cpp)
+rcc_backend_configure_target(RCCBackendRISCVLateCodeGenTests)
+target_link_libraries(
+ RCCBackendRISCVLateCodeGenTests
+ PRIVATE RCCBackend::RISCV RCCBackend::CodeGen
+)
+add_test(
+ NAME RCCBackendRISCVLateCodeGenTests
+ COMMAND RCCBackendRISCVLateCodeGenTests
+)
+
rcc_backend_register_format_files(
"${CMAKE_CURRENT_SOURCE_DIR}/DescriptionGeneratorTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLoweringTest.cpp"
@@ -68,6 +79,7 @@ rcc_backend_register_format_files(
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVSSAExitTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocationTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLoweringTest.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/RISCVLateCodeGenTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.h"
)
@@ -79,5 +91,6 @@ rcc_backend_register_tidy_files(
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVSSAExitTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocationTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLoweringTest.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/RISCVLateCodeGenTest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.cpp"
)
diff --git a/backend/unittests/Target/DescriptionGeneratorTest.cpp b/backend/unittests/Target/DescriptionGeneratorTest.cpp
index 0f8d1e9..ea073bf 100644
--- a/backend/unittests/Target/DescriptionGeneratorTest.cpp
+++ b/backend/unittests/Target/DescriptionGeneratorTest.cpp
@@ -65,7 +65,7 @@ encoding_categories:
assembly_formats:
- { id: 1, symbol: RD_IMM, name: rd-imm }
instructions:
- - { id: 1024, symbol: LOAD_IMM, name: LOAD_IMM, assembly_mnemonic: load.imm, operands: [{ kinds: [register], role: def, register_class: GPR64, register_bank: GPR, name: RD }, { kinds: [immediate], immediate_predicate: SIMM8, name: IMM, allowed_target_flags: 33 }], num_explicit_defs: 1, flags: [produces_code], scheduling_class: ALU, encoding_category: I, assembly_format: RD_IMM }
+ - { id: 1024, symbol: LOAD_IMM, name: LOAD_IMM, assembly_mnemonic: load.imm, operands: [{ kinds: [register], role: def, register_class: GPR64, register_bank: GPR, name: RD }, { kinds: [immediate, global_symbol], immediate_predicate: SIMM8, name: IMM, allowed_target_flags: 33 }], num_explicit_defs: 1, flags: [produces_code], scheduling_class: ALU, encoding_category: I, assembly_format: RD_IMM }
)";
}
diff --git a/backend/unittests/Target/RISCVLateCodeGenTest.cpp b/backend/unittests/Target/RISCVLateCodeGenTest.cpp
new file mode 100644
index 0000000..7ab3a59
--- /dev/null
+++ b/backend/unittests/Target/RISCVLateCodeGenTest.cpp
@@ -0,0 +1,1432 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineBlockPlacement.h"
+#include "rcc/CodeGen/MachineBranchOptimization.h"
+#include "rcc/CodeGen/MachineBranchRelaxation.h"
+#include "rcc/CodeGen/MachineCFGUpdater.h"
+#include "rcc/CodeGen/MachineCopyPropagation.h"
+#include "rcc/CodeGen/MachineFallthrough.h"
+#include "rcc/CodeGen/MachineFinalization.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineLiveness.h"
+#include "rcc/CodeGen/MachineMemoryFolding.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineOptimizationPipeline.h"
+#include "rcc/CodeGen/MachinePass.h"
+#include "rcc/CodeGen/MachinePseudoExpansion.h"
+#include "rcc/CodeGen/MachineShrinkWrapping.h"
+#include "rcc/CodeGen/MachineStackSlotColoring.h"
+#include "rcc/CodeGen/MachineTailDuplication.h"
+#include "rcc/CodeGen/MachineTailMerge.h"
+#include "rcc/CodeGen/PrologueEpilogueInsertion.h"
+#include "rcc/CodeGen/RegisterMask.h"
+#include "rcc/Target/RISCV/RISCVCodeGen.h"
+#include "rcc/Target/RISCV/RISCVInstructions.h"
+#include "rcc/Target/RISCV/RISCVRegisters.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <iostream>
+#include <limits>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace {
+
+class TestContext final {
+public:
+ void expect(bool Condition, std::string_view Message) {
+ if (Condition)
+ return;
+ std::cerr << Message << '\n';
+ ++Failures;
+ }
+
+ [[nodiscard]] int result() const noexcept {
+ return Failures == 0 ? 0 : 1;
+ }
+
+private:
+ int Failures = 0;
+};
+
+[[nodiscard]] constexpr rcc::Opcode opcode(rcc::riscv::InstructionOpcode Value) noexcept {
+ return rcc::Opcode::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+constexpr rcc::Opcode GrowingLatePseudoOpcode = rcc::Opcode::fromValue(10'000);
+constexpr rcc::Opcode LateExpansionRealOpcode = rcc::Opcode::fromValue(10'001);
+constexpr rcc::Opcode ShrinkingLatePseudoOpcode = rcc::Opcode::fromValue(10'002);
+
+class GrowingLatePseudoInfo final : public rcc::TargetPseudoExpansionInfo {
+public:
+ [[nodiscard]] rcc::Expected<bool>
+ expandPseudo(rcc::MachineInstr &Instruction, rcc::MachinePseudoExpansionPhase Phase,
+ rcc::MachineFunction &Function,
+ const rcc::TargetDescription &Target) const override {
+ if (Phase != rcc::MachinePseudoExpansionPhase::TargetLate ||
+ (Instruction.opcode() != GrowingLatePseudoOpcode &&
+ Instruction.opcode() != ShrinkingLatePseudoOpcode)) {
+ return false;
+ }
+ rcc::MachineBasicBlock *Block = Instruction.parent();
+ if (Block == nullptr) {
+ return rcc::makeUnexpected(rcc::ErrorCode::VerificationError,
+ "late expansion test pseudo is detached");
+ }
+ if (Instruction.opcode() == GrowingLatePseudoOpcode) {
+ rcc::MachineIRBuilder Builder(Function, Target);
+ if (auto Point = Builder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ for (std::size_t Index = 0; Index < 2; ++Index) {
+ auto Pending = Builder.buildInstr(LateExpansionRealOpcode);
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ }
+ }
+ if (auto Removed = Block->removeInstruction(Instruction); !Removed)
+ return std::unexpected(Removed.error());
+ return true;
+ }
+};
+
+class FixedSizeBranchInfo final : public rcc::TargetBranchInfo {
+public:
+ [[nodiscard]] rcc::Expected<rcc::MachineBranchAnalysis>
+ analyzeBranch(rcc::MachineBasicBlock &, rcc::MachineFunction &,
+ const rcc::TargetDescription &) const override {
+ return rcc::MachineBranchAnalysis{};
+ }
+
+ [[nodiscard]] rcc::Expected<void>
+ reverseBranchCondition(rcc::MachineInstr &, rcc::MachineBasicBlock &, rcc::MachineFunction &,
+ const rcc::TargetDescription &) const override {
+ return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, "test target has no branches");
+ }
+
+ [[nodiscard]] rcc::Expected<void> removeBranch(rcc::MachineInstr &, rcc::MachineFunction &,
+ const rcc::TargetDescription &) const override {
+ return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, "test target has no branches");
+ }
+
+ [[nodiscard]] rcc::Expected<bool>
+ rewriteBranchTarget(rcc::MachineBasicBlock &, rcc::MachineBasicBlock &, rcc::MachineBasicBlock &,
+ rcc::MachineFunction &, const rcc::TargetDescription &) const override {
+ return false;
+ }
+
+ [[nodiscard]] rcc::Expected<rcc::MachineInstr *>
+ insertUnconditionalBranch(rcc::MachineBasicBlock &, rcc::MachineBasicBlock &,
+ rcc::MachineFunction &, const rcc::TargetDescription &) const override {
+ return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, "test target has no branches");
+ }
+
+ [[nodiscard]] rcc::Expected<std::uint32_t>
+ instructionSize(const rcc::MachineInstr &, const rcc::MachineFunction &,
+ const rcc::TargetDescription &) const override {
+ return 4;
+ }
+
+ [[nodiscard]] rcc::Expected<rcc::MachineBasicBlock *>
+ branchTarget(rcc::MachineInstr &, rcc::MachineFunction &,
+ const rcc::TargetDescription &) const override {
+ return static_cast<rcc::MachineBasicBlock *>(nullptr);
+ }
+
+ [[nodiscard]] bool isBranchOffsetInRange(const rcc::MachineInstr &,
+ std::int64_t) const noexcept override {
+ return true;
+ }
+
+ [[nodiscard]] rcc::Expected<bool> relaxBranch(rcc::MachineInstr &, std::int64_t,
+ rcc::MachineFunction &,
+ const rcc::TargetDescription &) const override {
+ return false;
+ }
+};
+
+[[nodiscard]] rcc::Expected<rcc::TargetDescription> createLateExpansionSizeTarget() {
+ rcc::InstructionDescriptor Pseudo;
+ Pseudo.InstructionOpcode = GrowingLatePseudoOpcode;
+ Pseudo.Name = "TEST_GROWING_LATE_PSEUDO";
+ Pseudo.Flags = {rcc::InstructionFlag::Pseudo, rcc::InstructionFlag::ProducesCode};
+ Pseudo.Owner = rcc::PseudoOwner::TargetLateExpansion;
+ Pseudo.Deadline = rcc::PipelineStage::Emission;
+
+ rcc::InstructionDescriptor ShrinkingPseudo;
+ ShrinkingPseudo.InstructionOpcode = ShrinkingLatePseudoOpcode;
+ ShrinkingPseudo.Name = "TEST_SHRINKING_LATE_PSEUDO";
+ ShrinkingPseudo.Flags = {rcc::InstructionFlag::Pseudo, rcc::InstructionFlag::ProducesCode};
+ ShrinkingPseudo.Owner = rcc::PseudoOwner::TargetLateExpansion;
+ ShrinkingPseudo.Deadline = rcc::PipelineStage::Emission;
+
+ rcc::InstructionDescriptor Real;
+ Real.InstructionOpcode = LateExpansionRealOpcode;
+ Real.Name = "TEST_LATE_REAL";
+ Real.Flags = {rcc::InstructionFlag::ProducesCode};
+
+ std::vector<rcc::InstructionDescriptor> Descriptors;
+ Descriptors.push_back(std::move(Pseudo));
+ Descriptors.push_back(std::move(Real));
+ Descriptors.push_back(std::move(ShrinkingPseudo));
+ auto Instructions = rcc::InstructionSet::create(std::move(Descriptors));
+ if (!Instructions)
+ return std::unexpected(Instructions.error());
+ auto Registers = rcc::riscv::createRegisterInfo();
+ if (!Registers)
+ return std::unexpected(Registers.error());
+ return rcc::TargetDescription::create(std::move(*Instructions), std::move(*Registers));
+}
+
+[[nodiscard]] rcc::Register physical(rcc::riscv::PhysicalRegisterID ID) {
+ return rcc::Register::fromPhysical(*rcc::riscv::physicalRegister(ID));
+}
+
+[[nodiscard]] std::unique_ptr<rcc::MachineFunction>
+makeFunction(TestContext &Context, std::string Name, std::uint64_t StableID) {
+ auto Function = rcc::MachineFunction::create(std::move(Name),
+ rcc::MachineFunctionStableID::fromValue(StableID));
+ Context.expect(Function.has_value(), "late-codegen function creation must succeed");
+ return Function ? std::move(*Function) : nullptr;
+}
+
+[[nodiscard]] rcc::MachineBasicBlock *appendBlock(TestContext &Context,
+ rcc::MachineFunction &Function) {
+ auto Block = Function.createBasicBlock();
+ Context.expect(Block.has_value() && Function.appendBlock(**Block).has_value(),
+ "late-codegen block creation must succeed");
+ return Block.value_or(nullptr);
+}
+
+void setSelectedProperties(rcc::MachineFunction &Function) {
+ for (const rcc::MachineFunctionProperty Property : {
+ rcc::MachineFunctionProperty::Legalized,
+ rcc::MachineFunctionProperty::RegBankSelected,
+ rcc::MachineFunctionProperty::Selected,
+ rcc::MachineFunctionProperty::NoPHIs,
+ rcc::MachineFunctionProperty::TiedOpsRewritten,
+ }) {
+ Function.properties().set(Property);
+ }
+}
+
+void setPostRAProperties(rcc::MachineFunction &Function) {
+ setSelectedProperties(Function);
+ Function.properties().set(rcc::MachineFunctionProperty::TracksLiveness);
+ Function.properties().set(rcc::MachineFunctionProperty::NoVRegs);
+}
+
+void setRelaxationProperties(TestContext &Context, rcc::MachineFunction &Function,
+ const rcc::TargetDescription &Target) {
+ setPostRAProperties(Function);
+ auto StackAlignment = rcc::Alignment::create(16);
+ Context.expect(StackAlignment && Function.frameInfo().freeze(0, *StackAlignment).has_value(),
+ "relaxation fixture frame must be frozen");
+ Function.properties().set(rcc::MachineFunctionProperty::FrameFinalized);
+ Function.properties().set(rcc::MachineFunctionProperty::NoFrameIndices);
+ Function.properties().set(rcc::MachineFunctionProperty::NoExecutablePseudos);
+ auto Liveness = rcc::rebuildMachineLiveness(Function, Target);
+ Context.expect(Liveness.has_value(), "relaxation fixture liveness must be rebuilt");
+}
+
+[[nodiscard]] rcc::MachineInstr *buildADDI(TestContext &Context, rcc::MachineIRBuilder &Builder,
+ rcc::Register Destination, rcc::Register Source,
+ std::int64_t Immediate) {
+ auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::ADDI));
+ if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Source) ||
+ !Pending->addImmediate(Immediate)) {
+ Context.expect(false, "late-codegen ADDI construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "late-codegen ADDI insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+[[nodiscard]] bool constrainGPR64(TestContext &Context, rcc::MachineFunction &Function,
+ rcc::Register Reg) {
+ const auto Bank =
+ rcc::RegisterBankID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterBank::GPR));
+ const auto Class =
+ rcc::RegisterClassID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterClass::GPR));
+ const bool Valid = Function.registerInfo().setRegisterBank(Reg, Bank).has_value() &&
+ Function.registerInfo().setRegisterClass(Reg, Class).has_value();
+ Context.expect(Valid, "late-codegen GPR64 constraint must succeed");
+ return Valid;
+}
+
+[[nodiscard]] rcc::Expected<const rcc::MachineMemOperand *>
+createStackMemory(rcc::MachineFunction &Function, rcc::FrameIndex Slot,
+ rcc::MemoryAccessFlag Access) {
+ auto Size = rcc::MemorySize::fixed(8);
+ auto Align = rcc::Alignment::create(8);
+ if (!Size)
+ return std::unexpected(Size.error());
+ if (!Align)
+ return std::unexpected(Align.error());
+ rcc::MachineMemOperandDesc Description;
+ Description.Accesses = {Access};
+ Description.Size = *Size;
+ Description.Align = *Align;
+ Description.Base = rcc::MemoryBase::frameIndex(Slot);
+ return Function.createMemOperand(std::move(Description));
+}
+
+[[nodiscard]] rcc::MachineInstr *buildStackAccess(TestContext &Context,
+ rcc::MachineIRBuilder &Builder,
+ rcc::riscv::InstructionOpcode Opcode,
+ rcc::Register Reg, rcc::FrameIndex Slot,
+ rcc::MemoryAccessFlag Access) {
+ auto Memory = createStackMemory(Builder.function(), Slot, Access);
+ auto Pending = Builder.buildInstr(opcode(Opcode));
+ const bool IsLoad = Access == rcc::MemoryAccessFlag::Load;
+ if (!Memory || !Pending ||
+ !(IsLoad ? Pending->addDef(Reg).has_value() : Pending->addUse(Reg).has_value()) ||
+ !Pending->addFrameIndex(Slot) || !Pending->addImmediate(0) ||
+ !Pending->addMemOperand(**Memory)) {
+ Context.expect(false, "late-codegen stack access construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "late-codegen stack access insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+[[nodiscard]] rcc::MachineInstr *buildConditional(TestContext &Context,
+ rcc::MachineIRBuilder &Builder,
+ rcc::riscv::InstructionOpcode Opcode,
+ rcc::MachineBasicBlock &Target) {
+ auto Pending = Builder.buildInstr(opcode(Opcode));
+ if (!Pending || !Pending->addUse(physical(rcc::riscv::PhysicalRegisterID::X10)) ||
+ !Pending->addUse(physical(rcc::riscv::PhysicalRegisterID::X0)) ||
+ !Pending->addBasicBlock(Target)) {
+ Context.expect(false, "late-codegen conditional branch construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "late-codegen conditional branch insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+[[nodiscard]] rcc::MachineInstr *
+buildJAL(TestContext &Context, rcc::MachineIRBuilder &Builder, rcc::MachineBasicBlock &Target,
+ rcc::riscv::PhysicalRegisterID Destination = rcc::riscv::PhysicalRegisterID::X0) {
+ auto Pending = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::JAL));
+ if (!Pending || !Pending->addDef(physical(Destination)) || !Pending->addBasicBlock(Target)) {
+ Context.expect(false, "late-codegen JAL construction must succeed");
+ return nullptr;
+ }
+ auto Built = Pending->build();
+ Context.expect(Built.has_value(), "late-codegen JAL insertion must succeed");
+ return Built.value_or(nullptr);
+}
+
+[[nodiscard]] rcc::MachineBasicBlock *branchTarget(rcc::MachineInstr &Instruction) {
+ for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) {
+ rcc::MachineOperand *Operand = Instruction.operandAt(Index);
+ const auto *Block = Operand == nullptr ? nullptr : Operand->getIf<rcc::BasicBlockOperand>();
+ if (Block != nullptr)
+ return Block->Block;
+ }
+ return nullptr;
+}
+
+void testBlockPlacement(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "block_placement", 1);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Cold = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Hot = appendBlock(Context, *Function);
+ if (Entry == nullptr || Cold == nullptr || Hot == nullptr)
+ return;
+ auto ColdProbability = rcc::BranchProbability::fromFraction(1, 10);
+ auto HotProbability = rcc::BranchProbability::fromFraction(9, 10);
+ Context.expect(ColdProbability && HotProbability,
+ "placement probabilities must be representable");
+ if (!ColdProbability || !HotProbability)
+ return;
+ auto CFG = Function->cfgUpdater();
+ Context.expect(CFG.addSuccessor(*Entry, *Cold, *ColdProbability).has_value() &&
+ CFG.addSuccessor(*Entry, *Hot, *HotProbability).has_value(),
+ "placement CFG construction must succeed");
+ setSelectedProperties(*Function);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineBlockPlacementPass Pass;
+ auto Result = Pass.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "block placement failure: " << Result.error().message() << '\n';
+ Context.expect(Result && Result->Changed && Entry->next() == Hot && Hot->next() == Cold,
+ "block placement must put the highest-probability successor on the trace");
+}
+
+void testFallthroughNormalization(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "fallthrough", 2);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *False = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *True = appendBlock(Context, *Function);
+ if (Entry == nullptr || False == nullptr || True == nullptr)
+ return;
+ auto Half = rcc::BranchProbability::fromFraction(1, 2);
+ auto CFG = Function->cfgUpdater();
+ Context.expect(Half && CFG.addSuccessor(*Entry, *True, *Half).has_value() &&
+ CFG.addSuccessor(*Entry, *False, *Half).has_value(),
+ "fallthrough CFG construction must succeed");
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ Context.expect(Builder.setInsertPoint(*Entry).has_value(),
+ "fallthrough insertion point must be established");
+ rcc::MachineInstr *Conditional =
+ buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BEQ, *True);
+ rcc::MachineInstr *Jump = buildJAL(Context, Builder, *False);
+ if (Conditional == nullptr || Jump == nullptr)
+ return;
+ setPostRAProperties(*Function);
+ auto Liveness = rcc::rebuildMachineLiveness(*Function, Target);
+ Context.expect(Liveness.has_value(), "fallthrough fixture liveness must be rebuilt");
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineFallthroughPass Pass(rcc::riscv::branchInfo());
+ auto Result = Pass.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "fallthrough normalization failure: " << Result.error().message() << '\n';
+ Context.expect(Result && Result->Changed && Conditional->next() == nullptr &&
+ Jump->parent() == nullptr,
+ "fallthrough normalization must remove a jump to the next block");
+}
+
+void testFallthroughMaterialization(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "fallthrough_materialization", 8);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Padding = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *True = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *False = appendBlock(Context, *Function);
+ if (Entry == nullptr || Padding == nullptr || True == nullptr || False == nullptr)
+ return;
+ auto Half = rcc::BranchProbability::fromFraction(1, 2);
+ auto CFG = Function->cfgUpdater();
+ Context.expect(Half && CFG.addSuccessor(*Entry, *True, *Half).has_value() &&
+ CFG.addSuccessor(*Entry, *False, *Half).has_value(),
+ "materialized fallthrough CFG construction must succeed");
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ Context.expect(Builder.setInsertPoint(*Entry).has_value(),
+ "materialized fallthrough insertion point must be established");
+ rcc::MachineInstr *Conditional =
+ buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BNE, *True);
+ if (Conditional == nullptr)
+ return;
+ setPostRAProperties(*Function);
+ Context.expect(rcc::rebuildMachineLiveness(*Function, Target).has_value(),
+ "materialized fallthrough liveness must be rebuilt");
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineFallthroughPass Pass(rcc::riscv::branchInfo());
+ auto Result = Pass.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "fallthrough materialization failure: " << Result.error().message() << '\n';
+ Context.expect(Result && Result->Changed && Conditional->next() != nullptr &&
+ Conditional->next()->opcode() == opcode(rcc::riscv::InstructionOpcode::JAL) &&
+ branchTarget(*Conditional->next()) == False,
+ "fallthrough normalization must materialize a displaced false edge");
+}
+
+void testBranchOptimization(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "branch_optimization", 9);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Forward = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Exit = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Dead = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *AddressTaken = appendBlock(Context, *Function);
+ if (Entry == nullptr || Forward == nullptr || Exit == nullptr || Dead == nullptr ||
+ AddressTaken == nullptr) {
+ return;
+ }
+ AddressTaken->attributes().set(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken);
+ auto CFG = Function->cfgUpdater();
+ Context.expect(CFG.addSuccessor(*Entry, *Forward, rcc::BranchProbability::one()).has_value() &&
+ CFG.addSuccessor(*Forward, *Exit, rcc::BranchProbability::one()).has_value(),
+ "branch-optimization CFG construction must succeed");
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ if (!Builder.setInsertPoint(*Entry) || buildJAL(Context, Builder, *Forward) == nullptr ||
+ !Builder.setInsertPoint(*Forward) || buildJAL(Context, Builder, *Exit) == nullptr ||
+ !Builder.setInsertPoint(*Dead) ||
+ buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X12),
+ physical(rcc::riscv::PhysicalRegisterID::X11), 1) == nullptr ||
+ !Builder.setInsertPoint(*AddressTaken) ||
+ buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X13),
+ physical(rcc::riscv::PhysicalRegisterID::X11), 2) == nullptr) {
+ return;
+ }
+ setSelectedProperties(*Function);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineBranchOptimizationPass Pass(rcc::riscv::branchInfo());
+ auto Result = Pass.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "branch optimization failure: " << Result.error().message() << '\n';
+ Context.expect(Result && Result->Changed && !Forward->isLinked() && !Dead->isLinked() &&
+ AddressTaken->isLinked() && Entry->successors().size() == 1 &&
+ Entry->successors().front().Block == Exit &&
+ branchTarget(*Entry->lastInstruction()) == Exit,
+ "branch optimization must thread empty blocks and delete unreachable code");
+
+ auto ConstantFunction = makeFunction(Context, "constant_branch", 10);
+ if (!ConstantFunction)
+ return;
+ rcc::MachineBasicBlock *ConstantEntry = appendBlock(Context, *ConstantFunction);
+ rcc::MachineBasicBlock *Taken = appendBlock(Context, *ConstantFunction);
+ rcc::MachineBasicBlock *NotTaken = appendBlock(Context, *ConstantFunction);
+ if (ConstantEntry == nullptr || Taken == nullptr || NotTaken == nullptr)
+ return;
+ auto Half = rcc::BranchProbability::fromFraction(1, 2);
+ auto ConstantCFG = ConstantFunction->cfgUpdater();
+ Context.expect(Half && ConstantCFG.addSuccessor(*ConstantEntry, *Taken, *Half).has_value() &&
+ ConstantCFG.addSuccessor(*ConstantEntry, *NotTaken, *Half).has_value(),
+ "constant-branch CFG construction must succeed");
+ rcc::MachineIRBuilder ConstantBuilder(*ConstantFunction, Target);
+ if (!ConstantBuilder.setInsertPoint(*ConstantEntry))
+ return;
+ auto Pending = ConstantBuilder.buildInstr(opcode(rcc::riscv::InstructionOpcode::BEQ));
+ if (!Pending || !Pending->addUse(physical(rcc::riscv::PhysicalRegisterID::X0)) ||
+ !Pending->addUse(physical(rcc::riscv::PhysicalRegisterID::X0)) ||
+ !Pending->addBasicBlock(*Taken) || !Pending->build() ||
+ buildJAL(Context, ConstantBuilder, *NotTaken) == nullptr) {
+ Context.expect(false, "constant branch construction must succeed");
+ return;
+ }
+ setSelectedProperties(*ConstantFunction);
+ rcc::MachineAnalysisManager ConstantAnalyses;
+ auto ConstantResult = Pass.run(*ConstantFunction, Target, ConstantAnalyses);
+ if (!ConstantResult)
+ std::cerr << "constant branch folding failure: " << ConstantResult.error().message() << '\n';
+ Context.expect(
+ ConstantResult && ConstantResult->Changed && ConstantEntry->successors().size() == 1 &&
+ ConstantEntry->successors().front().Block == Taken &&
+ branchTarget(*ConstantEntry->lastInstruction()) == Taken && !NotTaken->isLinked(),
+ "branch optimization must fold an always-true RV64GC branch");
+}
+
+void testTailDuplication(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "tail_duplication", 11);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Left = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Right = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Tail = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Exit = appendBlock(Context, *Function);
+ if (Entry == nullptr || Left == nullptr || Right == nullptr || Tail == nullptr ||
+ Exit == nullptr) {
+ return;
+ }
+ auto Half = rcc::BranchProbability::fromFraction(1, 2);
+ auto CFG = Function->cfgUpdater();
+ Context.expect(Half && CFG.addSuccessor(*Entry, *Left, *Half).has_value() &&
+ CFG.addSuccessor(*Entry, *Right, *Half).has_value() &&
+ CFG.addSuccessor(*Left, *Tail, rcc::BranchProbability::one()).has_value() &&
+ CFG.addSuccessor(*Right, *Tail, rcc::BranchProbability::one()).has_value() &&
+ CFG.addSuccessor(*Tail, *Exit, rcc::BranchProbability::one()).has_value(),
+ "tail-duplication CFG construction must succeed");
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ if (!Builder.setInsertPoint(*Entry) ||
+ buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BNE, *Left) == nullptr ||
+ buildJAL(Context, Builder, *Right) == nullptr || !Builder.setInsertPoint(*Left) ||
+ buildJAL(Context, Builder, *Tail) == nullptr || !Builder.setInsertPoint(*Right) ||
+ buildJAL(Context, Builder, *Tail) == nullptr || !Builder.setInsertPoint(*Tail) ||
+ buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X12),
+ physical(rcc::riscv::PhysicalRegisterID::X11), 1) == nullptr ||
+ buildJAL(Context, Builder, *Exit) == nullptr) {
+ return;
+ }
+ setRelaxationProperties(Context, *Function, Target);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineTailDuplicationPass Pass(rcc::riscv::branchInfo());
+ Tail->attributes().set(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken);
+ auto Protected = Pass.run(*Function, Target, Analyses);
+ Context.expect(Protected && !Protected->Changed && Tail->isLinked(),
+ "tail duplication must preserve an address-taken tail");
+ Tail->attributes().clear(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken);
+ auto Result = Pass.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "tail duplication failure: " << Result.error().message() << '\n';
+ Context.expect(
+ Result && Result->Changed && !Tail->isLinked() && Left->instructionCount() == 2 &&
+ Right->instructionCount() == 2 &&
+ Left->firstInstruction()->opcode() == opcode(rcc::riscv::InstructionOpcode::ADDI) &&
+ Right->firstInstruction()->opcode() == opcode(rcc::riscv::InstructionOpcode::ADDI) &&
+ branchTarget(*Left->lastInstruction()) == Exit &&
+ branchTarget(*Right->lastInstruction()) == Exit &&
+ rcc::verifyTrackedMachineLiveness(*Function, Target).has_value(),
+ "tail duplication must clone a small shared tail and repair liveness");
+}
+
+void testTailMerge(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "tail_merge", 12);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *First = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Second = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Exit = appendBlock(Context, *Function);
+ if (Entry == nullptr || First == nullptr || Second == nullptr || Exit == nullptr)
+ return;
+ auto Half = rcc::BranchProbability::fromFraction(1, 2);
+ auto CFG = Function->cfgUpdater();
+ Context.expect(Half && CFG.addSuccessor(*Entry, *First, *Half).has_value() &&
+ CFG.addSuccessor(*Entry, *Second, *Half).has_value() &&
+ CFG.addSuccessor(*First, *Exit, rcc::BranchProbability::one()).has_value() &&
+ CFG.addSuccessor(*Second, *Exit, rcc::BranchProbability::one()).has_value(),
+ "tail-merge CFG construction must succeed");
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ if (!Builder.setInsertPoint(*Entry) ||
+ buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BNE, *First) == nullptr ||
+ buildJAL(Context, Builder, *Second) == nullptr || !Builder.setInsertPoint(*First) ||
+ buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X12),
+ physical(rcc::riscv::PhysicalRegisterID::X11), 7) == nullptr ||
+ buildJAL(Context, Builder, *Exit) == nullptr || !Builder.setInsertPoint(*Second) ||
+ buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X12),
+ physical(rcc::riscv::PhysicalRegisterID::X11), 7) == nullptr ||
+ buildJAL(Context, Builder, *Exit) == nullptr) {
+ return;
+ }
+ setRelaxationProperties(Context, *Function, Target);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineTailMergePass Pass(rcc::riscv::branchInfo());
+ Second->attributes().set(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken);
+ auto Protected = Pass.run(*Function, Target, Analyses);
+ Context.expect(Protected && !Protected->Changed && Second->isLinked(),
+ "tail merge must preserve an address-taken duplicate block");
+ Second->attributes().clear(rcc::MachineBasicBlockAttribute::MachineBlockAddressTaken);
+ auto Result = Pass.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "tail merge failure: " << Result.error().message() << '\n';
+ Context.expect(Result && Result->Changed && !Second->isLinked() &&
+ Entry->successors().size() == 1 &&
+ Entry->successors().front().Block == First &&
+ branchTarget(*Entry->firstInstruction()) == First &&
+ branchTarget(*Entry->lastInstruction()) == First &&
+ rcc::verifyTrackedMachineLiveness(*Function, Target).has_value(),
+ "tail merge must redirect identical tails and repair liveness");
+}
+
+void testCopyPropagation(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "pre_ra_copy_propagation", 13);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+ auto I64 = rcc::LLT::makeInteger(64);
+ if (Block == nullptr || !I64)
+ return;
+ auto Source = Function->registerInfo().createVirtualRegister(*I64);
+ auto CopyResult = Function->registerInfo().createVirtualRegister(*I64);
+ auto ResultReg = Function->registerInfo().createVirtualRegister(*I64);
+ if (!Source || !CopyResult || !ResultReg || !constrainGPR64(Context, *Function, *Source) ||
+ !constrainGPR64(Context, *Function, *CopyResult) ||
+ !constrainGPR64(Context, *Function, *ResultReg)) {
+ return;
+ }
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ if (!Builder.setInsertPoint(*Block))
+ return;
+ auto Copy = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY));
+ if (!Copy || !Copy->addDef(*CopyResult) || !Copy->addUse(*Source) || !Copy->build())
+ return;
+ rcc::MachineInstr *Add = buildADDI(Context, Builder, *ResultReg, *CopyResult, 3);
+ if (Add == nullptr)
+ return;
+ setSelectedProperties(*Function);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineCopyPropagationPass PreRA(rcc::riscv::ssaInfo(),
+ rcc::MachineCopyPropagationStage::PreRegisterAllocation);
+ auto PreResult = PreRA.run(*Function, Target, Analyses);
+ if (!PreResult)
+ std::cerr << "pre-RA COPY propagation failure: " << PreResult.error().message() << '\n';
+ Context.expect(PreResult && PreResult->Changed && Block->instructionCount() == 1 &&
+ Add->operandAt(1)->registerValue() == *Source &&
+ !Function->registerInfo().isKnownVirtualRegister(*CopyResult),
+ "pre-RA COPY propagation must replace uses and retire the COPY result");
+
+ auto PostFunction = makeFunction(Context, "post_ra_copy_propagation", 14);
+ if (!PostFunction)
+ return;
+ rcc::MachineBasicBlock *PostBlock = appendBlock(Context, *PostFunction);
+ if (PostBlock == nullptr)
+ return;
+ rcc::MachineIRBuilder PostBuilder(*PostFunction, Target);
+ if (!PostBuilder.setInsertPoint(*PostBlock))
+ return;
+ auto PhysicalCopy = PostBuilder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY));
+ if (!PhysicalCopy || !PhysicalCopy->addDef(physical(rcc::riscv::PhysicalRegisterID::X11)) ||
+ !PhysicalCopy->addUse(physical(rcc::riscv::PhysicalRegisterID::X10)) ||
+ !PhysicalCopy->build()) {
+ return;
+ }
+ rcc::MachineInstr *PhysicalAdd =
+ buildADDI(Context, PostBuilder, physical(rcc::riscv::PhysicalRegisterID::X12),
+ physical(rcc::riscv::PhysicalRegisterID::X11), 5);
+ rcc::MachineInstr *SourceClobber =
+ buildADDI(Context, PostBuilder, physical(rcc::riscv::PhysicalRegisterID::X10),
+ physical(rcc::riscv::PhysicalRegisterID::X0), 9);
+ rcc::MachineInstr *AfterClobber =
+ buildADDI(Context, PostBuilder, physical(rcc::riscv::PhysicalRegisterID::X13),
+ physical(rcc::riscv::PhysicalRegisterID::X11), 6);
+ if (PhysicalAdd == nullptr || SourceClobber == nullptr || AfterClobber == nullptr)
+ return;
+ setPostRAProperties(*PostFunction);
+ Context.expect(rcc::rebuildMachineLiveness(*PostFunction, Target).has_value(),
+ "post-RA COPY propagation liveness must be rebuilt");
+ rcc::MachineAnalysisManager PostAnalyses;
+ rcc::MachineCopyPropagationPass PostRA(rcc::riscv::ssaInfo(),
+ rcc::MachineCopyPropagationStage::PostRegisterAllocation);
+ auto PostResult = PostRA.run(*PostFunction, Target, PostAnalyses);
+ if (!PostResult)
+ std::cerr << "post-RA COPY propagation failure: " << PostResult.error().message() << '\n';
+ Context.expect(PostResult && PostResult->Changed &&
+ PhysicalAdd->operandAt(1)->registerValue() ==
+ physical(rcc::riscv::PhysicalRegisterID::X10) &&
+ AfterClobber->operandAt(1)->registerValue() ==
+ physical(rcc::riscv::PhysicalRegisterID::X11) &&
+ rcc::verifyTrackedMachineLiveness(*PostFunction, Target).has_value(),
+ "post-RA COPY propagation must stop at clobbers and repair liveness");
+}
+
+void testStackSlotColoring(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "stack_slot_coloring", 15);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+ auto Align = rcc::Alignment::create(8);
+ if (Block == nullptr || !Align)
+ return;
+ rcc::StackObjectDesc Description;
+ Description.Kind = rcc::StackObjectKind::SpillSlot;
+ Description.Size = 8;
+ Description.Align = *Align;
+ auto FirstSlot = Function->frameInfo().createObject(Description);
+ auto SecondSlot = Function->frameInfo().createObject(Description);
+ if (!FirstSlot || !SecondSlot)
+ return;
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ if (!Builder.setInsertPoint(*Block) ||
+ buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::SD,
+ physical(rcc::riscv::PhysicalRegisterID::X10), *FirstSlot,
+ rcc::MemoryAccessFlag::Store) == nullptr ||
+ buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::LD,
+ physical(rcc::riscv::PhysicalRegisterID::X11), *FirstSlot,
+ rcc::MemoryAccessFlag::Load) == nullptr ||
+ buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::SD,
+ physical(rcc::riscv::PhysicalRegisterID::X12), *SecondSlot,
+ rcc::MemoryAccessFlag::Store) == nullptr ||
+ buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::LD,
+ physical(rcc::riscv::PhysicalRegisterID::X13), *SecondSlot,
+ rcc::MemoryAccessFlag::Load) == nullptr) {
+ return;
+ }
+ setPostRAProperties(*Function);
+ Context.expect(rcc::rebuildMachineLiveness(*Function, Target).has_value(),
+ "stack-slot coloring liveness must be rebuilt");
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineStackSlotColoringPass Pass;
+ auto Result = Pass.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "stack-slot coloring failure: " << Result.error().message() << '\n';
+ const rcc::StackObject *Colored = Function->frameInfo().object(*SecondSlot);
+ rcc::MachineInstr *SecondStore = Block->firstInstruction();
+ SecondStore = SecondStore == nullptr ? nullptr : SecondStore->next();
+ SecondStore = SecondStore == nullptr ? nullptr : SecondStore->next();
+ const auto *Frame =
+ SecondStore == nullptr ? nullptr : SecondStore->operandAt(1)->getIf<rcc::FrameIndexOperand>();
+ Context.expect(
+ Result && Result->Changed && Colored != nullptr && Colored->description() != nullptr &&
+ Colored->description()->IsDead && Frame != nullptr && Frame->Index == *FirstSlot &&
+ SecondStore->memOperands().front()->description().Base.frameIndex() == *FirstSlot,
+ "stack-slot coloring must coalesce disjoint spills and rewrite MMOs");
+
+ auto OverlapFunction = makeFunction(Context, "stack_slot_overlap", 17);
+ if (!OverlapFunction)
+ return;
+ rcc::MachineBasicBlock *OverlapBlock = appendBlock(Context, *OverlapFunction);
+ auto Left = OverlapFunction->frameInfo().createObject(Description);
+ auto Right = OverlapFunction->frameInfo().createObject(Description);
+ if (OverlapBlock == nullptr || !Left || !Right)
+ return;
+ rcc::MachineIRBuilder OverlapBuilder(*OverlapFunction, Target);
+ if (!OverlapBuilder.setInsertPoint(*OverlapBlock) ||
+ buildStackAccess(Context, OverlapBuilder, rcc::riscv::InstructionOpcode::SD,
+ physical(rcc::riscv::PhysicalRegisterID::X10), *Left,
+ rcc::MemoryAccessFlag::Store) == nullptr ||
+ buildStackAccess(Context, OverlapBuilder, rcc::riscv::InstructionOpcode::SD,
+ physical(rcc::riscv::PhysicalRegisterID::X11), *Right,
+ rcc::MemoryAccessFlag::Store) == nullptr ||
+ buildStackAccess(Context, OverlapBuilder, rcc::riscv::InstructionOpcode::LD,
+ physical(rcc::riscv::PhysicalRegisterID::X12), *Left,
+ rcc::MemoryAccessFlag::Load) == nullptr ||
+ buildStackAccess(Context, OverlapBuilder, rcc::riscv::InstructionOpcode::LD,
+ physical(rcc::riscv::PhysicalRegisterID::X13), *Right,
+ rcc::MemoryAccessFlag::Load) == nullptr) {
+ return;
+ }
+ setPostRAProperties(*OverlapFunction);
+ Context.expect(rcc::rebuildMachineLiveness(*OverlapFunction, Target).has_value(),
+ "overlapping stack-slot fixture liveness must be rebuilt");
+ auto OverlapResult = Pass.run(*OverlapFunction, Target, Analyses);
+ const rcc::StackObject *RightObject = OverlapFunction->frameInfo().object(*Right);
+ Context.expect(OverlapResult && !OverlapResult->Changed && RightObject != nullptr &&
+ RightObject->description() != nullptr && !RightObject->description()->IsDead,
+ "stack-slot coloring must preserve overlapping spill lifetimes");
+}
+
+void testRISCVMemoryFoldingBoundary(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "riscv_memory_folding", 16);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+ auto I64 = rcc::LLT::makeInteger(64);
+ auto Align = rcc::Alignment::create(8);
+ if (Block == nullptr || !I64 || !Align)
+ return;
+ auto Loaded = Function->registerInfo().createVirtualRegister(*I64);
+ auto ResultReg = Function->registerInfo().createVirtualRegister(*I64);
+ if (!Loaded || !ResultReg || !constrainGPR64(Context, *Function, *Loaded) ||
+ !constrainGPR64(Context, *Function, *ResultReg)) {
+ return;
+ }
+ rcc::StackObjectDesc Description;
+ Description.Size = 8;
+ Description.Align = *Align;
+ auto Slot = Function->frameInfo().createObject(std::move(Description));
+ if (!Slot)
+ return;
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ rcc::MachineInstr *Load = nullptr;
+ rcc::MachineInstr *Consumer = nullptr;
+ if (Builder.setInsertPoint(*Block)) {
+ Load = buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::LD, *Loaded, *Slot,
+ rcc::MemoryAccessFlag::Load);
+ Consumer = buildADDI(Context, Builder, *ResultReg, *Loaded, 1);
+ }
+ if (Load == nullptr || Consumer == nullptr)
+ return;
+ setSelectedProperties(*Function);
+ Function->properties().set(rcc::MachineFunctionProperty::IsSSA);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineMemoryFoldingPass Pass(rcc::riscv::memoryFoldingInfo());
+ auto Result = Pass.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "RISC-V memory folding boundary failure: " << Result.error().message() << '\n';
+ Context.expect(Result && !Result->Changed && Load->parent() == Block &&
+ Load->memOperands().size() == 1 && Consumer->parent() == Block,
+ "RV64GC must explicitly decline unsupported folded-memory ALU forms");
+}
+
+void testShrinkWrappedFrameRegion(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "shrink_wrapped_frame", 20);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Cold = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Hot = appendBlock(Context, *Function);
+ auto Align = rcc::Alignment::create(8);
+ auto ColdProbability = rcc::BranchProbability::fromFraction(1, 10);
+ auto HotProbability = rcc::BranchProbability::fromFraction(9, 10);
+ if (Entry == nullptr || Cold == nullptr || Hot == nullptr || !Align || !ColdProbability ||
+ !HotProbability) {
+ return;
+ }
+ auto CFG = Function->cfgUpdater();
+ if (!CFG.addSuccessor(*Entry, *Cold, *ColdProbability) ||
+ !CFG.addSuccessor(*Entry, *Hot, *HotProbability)) {
+ return;
+ }
+ rcc::StackObjectDesc SpillDescription;
+ SpillDescription.Kind = rcc::StackObjectKind::SpillSlot;
+ SpillDescription.Size = 8;
+ SpillDescription.Align = *Align;
+ auto Spill = Function->frameInfo().createObject(SpillDescription);
+ if (!Spill)
+ return;
+
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ if (!Builder.setInsertPoint(*Cold) ||
+ buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::SD,
+ physical(rcc::riscv::PhysicalRegisterID::X10), *Spill,
+ rcc::MemoryAccessFlag::Store) == nullptr) {
+ return;
+ }
+ auto ColdReturn = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoRET));
+ if (!ColdReturn || !ColdReturn->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) ||
+ !ColdReturn->build() || !Builder.setInsertPoint(*Hot)) {
+ return;
+ }
+ auto HotReturn = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoRET));
+ if (!HotReturn || !HotReturn->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) ||
+ !HotReturn->build()) {
+ return;
+ }
+
+ setPostRAProperties(*Function);
+ Context.expect(rcc::rebuildMachineLiveness(*Function, Target).has_value(),
+ "shrink-wrapping fixture liveness must be rebuilt");
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineShrinkWrappingPass Shrink{rcc::riscv::frameLowering()};
+ auto Shrunk = Shrink.run(*Function, Target, Analyses);
+ Context.expect(Shrunk && Shrunk->Changed &&
+ Function->frameInfo().state().SavePoints.size() == 1 &&
+ Function->frameInfo().state().SavePoints.front().Block == Cold->stableID() &&
+ Function->frameInfo().state().RestorePoints.size() == 1 &&
+ Function->frameInfo().state().RestorePoints.front().Block == Cold->stableID(),
+ "shrink wrapping must select the cold frame region only");
+ if (!Shrunk)
+ return;
+
+ rcc::MachinePassManager FramePipeline;
+ Context.expect(FramePipeline
+ .addPass(std::make_unique<rcc::PrologueEpilogueInsertionPass>(
+ rcc::riscv::frameLowering()))
+ .has_value(),
+ "shrink-wrapped PEI pass must register");
+ auto Lowered = FramePipeline.run(*Function, Target, Analyses);
+ if (!Lowered)
+ std::cerr << "shrink-wrapped PEI failure: " << Lowered.error().message() << '\n';
+ const bool EntryHasFrameCode = std::ranges::any_of(
+ Function->allInstructions(), [Entry](const rcc::MachineInstr *Instruction) {
+ return Instruction != nullptr && Instruction->parent() == Entry &&
+ (Instruction->flags().has(rcc::MachineInstrFlag::FrameSetup) ||
+ Instruction->flags().has(rcc::MachineInstrFlag::FrameDestroy));
+ });
+ const bool ColdHasSetup = std::ranges::any_of(
+ Function->allInstructions(), [Cold](const rcc::MachineInstr *Instruction) {
+ return Instruction != nullptr && Instruction->parent() == Cold &&
+ Instruction->flags().has(rcc::MachineInstrFlag::FrameSetup);
+ });
+ const bool ColdHasDestroy = std::ranges::any_of(
+ Function->allInstructions(), [Cold](const rcc::MachineInstr *Instruction) {
+ return Instruction != nullptr && Instruction->parent() == Cold &&
+ Instruction->flags().has(rcc::MachineInstrFlag::FrameDestroy);
+ });
+ Context.expect(
+ Lowered && !EntryHasFrameCode && ColdHasSetup && ColdHasDestroy &&
+ Function->properties().has(rcc::MachineFunctionProperty::FrameFinalized),
+ "PEI must materialize a shrink-wrapped prologue and epilogue only in the frame region");
+
+ rcc::MachineBlockPlacementPass Placement;
+ auto Placed = Placement.run(*Function, Target, Analyses);
+ Context.expect(Placed && Cold->next() == nullptr,
+ "block placement must keep a shrink-wrapped frame region as a layout suffix");
+}
+
+void testCompressedAndConditionalRelaxation(TestContext &Context,
+ const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "branch_relaxation", 3);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Padding = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *Far = appendBlock(Context, *Function);
+ if (Entry == nullptr || Padding == nullptr || Far == nullptr)
+ return;
+ auto Half = rcc::BranchProbability::fromFraction(1, 2);
+ auto CFG = Function->cfgUpdater();
+ Context.expect(Half && CFG.addSuccessor(*Entry, *Far, *Half).has_value() &&
+ CFG.addSuccessor(*Entry, *Padding, *Half).has_value() &&
+ CFG.addSuccessor(*Padding, *Far, rcc::BranchProbability::one()).has_value(),
+ "relaxation CFG construction must succeed");
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ Context.expect(Builder.setInsertPoint(*Entry).has_value(),
+ "relaxation entry insertion point must be established");
+ rcc::MachineInstr *Conditional =
+ buildConditional(Context, Builder, rcc::riscv::InstructionOpcode::BEQ, *Far);
+ Context.expect(Builder.setInsertPoint(*Padding).has_value(),
+ "relaxation padding insertion point must be established");
+ for (std::size_t Index = 0; Index < 1100; ++Index) {
+ if (buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X0),
+ physical(rcc::riscv::PhysicalRegisterID::X0), 0) == nullptr) {
+ return;
+ }
+ }
+ (void)buildJAL(Context, Builder, *Far);
+ if (Conditional == nullptr)
+ return;
+ setRelaxationProperties(Context, *Function, Target);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::MachineBranchRelaxationPass Pass(rcc::riscv::branchInfo());
+ auto Result = Pass.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "conditional relaxation failure: " << Result.error().message() << '\n';
+ Context.expect(Result && Result->Changed &&
+ Conditional->opcode() == opcode(rcc::riscv::InstructionOpcode::BNE) &&
+ branchTarget(*Conditional) == Padding,
+ "out-of-range conditional branch must invert onto the local fallthrough");
+ rcc::MachineInstr *LongJump = Conditional->next();
+ Context.expect(LongJump != nullptr &&
+ LongJump->opcode() == opcode(rcc::riscv::InstructionOpcode::JAL) &&
+ branchTarget(*LongJump) == Far,
+ "conditional relaxation must insert a reachable long jump to the old target");
+
+ auto CompressedFunction = makeFunction(Context, "compressed_relaxation", 4);
+ if (!CompressedFunction)
+ return;
+ rcc::MachineBasicBlock *CompressedEntry = appendBlock(Context, *CompressedFunction);
+ rcc::MachineBasicBlock *CompressedTarget = appendBlock(Context, *CompressedFunction);
+ rcc::MachineIRBuilder CompressedBuilder(*CompressedFunction, Target);
+ Context.expect(CompressedEntry != nullptr && CompressedTarget != nullptr &&
+ CompressedBuilder.setInsertPoint(*CompressedEntry).has_value(),
+ "compressed relaxation fixture must be valid");
+ if (CompressedEntry == nullptr || CompressedTarget == nullptr)
+ return;
+ auto Pending = CompressedBuilder.buildInstr(opcode(rcc::riscv::InstructionOpcode::C_J));
+ if (!Pending || !Pending->addBasicBlock(*CompressedTarget)) {
+ Context.expect(false, "compressed jump construction must succeed");
+ return;
+ }
+ auto Compressed = Pending->build();
+ Context.expect(Compressed.has_value(), "compressed jump insertion must succeed");
+ if (!Compressed)
+ return;
+ auto Expanded =
+ rcc::riscv::branchInfo().relaxBranch(**Compressed, 4096, *CompressedFunction, Target);
+ Context.expect(Expanded && *Expanded && CompressedEntry->firstInstruction() != nullptr &&
+ CompressedEntry->firstInstruction()->opcode() ==
+ opcode(rcc::riscv::InstructionOpcode::JAL) &&
+ (*Compressed)->parent() == nullptr,
+ "compressed jump relaxation must monotonically expand to JAL");
+}
+
+void testLongJALRelaxation(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "long_jal", 5);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Entry = appendBlock(Context, *Function);
+ rcc::MachineBasicBlock *TargetBlock = appendBlock(Context, *Function);
+ if (Entry == nullptr || TargetBlock == nullptr)
+ return;
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ Context.expect(Builder.setInsertPoint(*Entry).has_value(),
+ "long JAL insertion point must be established");
+ rcc::MachineInstr *JAL = buildJAL(Context, Builder, *TargetBlock);
+ if (JAL == nullptr)
+ return;
+ auto X5 = rcc::riscv::physicalRegister(rcc::riscv::PhysicalRegisterID::X5);
+ auto X6 = rcc::riscv::physicalRegister(rcc::riscv::PhysicalRegisterID::X6);
+ Context.expect(X5 && X6 && TargetBlock->addLiveIn(*X5).has_value() &&
+ TargetBlock->addLiveIn(*X6).has_value() &&
+ JAL->setPreInstrSymbol(".Lpreserved_long_branch").has_value() &&
+ JAL->setPostInstrSymbol(".Lafter_long_branch").has_value(),
+ "long JAL live-in and annotation fixture must be valid");
+ if (!X5 || !X6)
+ return;
+ auto TooFar = rcc::riscv::branchInfo().relaxBranch(*JAL, std::numeric_limits<std::int64_t>::max(),
+ *Function, Target);
+ Context.expect(!TooFar && TooFar.error().code() == rcc::ErrorCode::BranchRelaxationFailed &&
+ JAL->parent() == Entry,
+ "long JAL relaxation must reject an unencodable PC-relative distance atomically");
+ auto Relaxed = rcc::riscv::branchInfo().relaxBranch(*JAL, 2 * 1024 * 1024, *Function, Target);
+ if (!Relaxed)
+ std::cerr << "long JAL relaxation failure: " << Relaxed.error().message() << '\n';
+ rcc::MachineInstr *High = Entry->firstInstruction();
+ rcc::MachineInstr *Low = High == nullptr ? nullptr : High->next();
+ const rcc::Register *Scratch = High == nullptr || High->operandAt(0) == nullptr
+ ? nullptr
+ : High->operandAt(0)->getIf<rcc::Register>();
+ const std::optional<rcc::PhysicalRegister> ScratchPhysical =
+ Scratch == nullptr ? std::nullopt : Scratch->physicalRegister();
+ Context.expect(Relaxed && *Relaxed && High != nullptr && Low != nullptr &&
+ High->opcode() == opcode(rcc::riscv::InstructionOpcode::AUIPC) &&
+ Low->opcode() == opcode(rcc::riscv::InstructionOpcode::JALR) &&
+ ScratchPhysical && *ScratchPhysical != *X5 && *ScratchPhysical != *X6 &&
+ High->operandAt(1)->targetFlags() == 2 &&
+ Low->operandAt(2)->targetFlags() == 4 &&
+ High->flags().has(rcc::MachineInstrFlag::BundledWithSucc) &&
+ Low->flags().has(rcc::MachineInstrFlag::BundledWithPred) &&
+ High->preInstrSymbol() == ".Lpreserved_long_branch" &&
+ Low->postInstrSymbol() == ".Lafter_long_branch" && JAL->parent() == nullptr,
+ "out-of-range JAL must use a dead scratch and preserve its paired annotations");
+}
+
+void testPseudoPreparation(TestContext &Context, const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "pseudo_preparation", 6);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+ auto I64 = rcc::LLT::makeInteger(64);
+ auto F64 = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEEDouble);
+ if (Block == nullptr || !I64 || !F64)
+ return;
+ auto Source = Function->registerInfo().createVirtualRegister(*I64);
+ auto Destination = Function->registerInfo().createVirtualRegister(*F64);
+ const auto GPRBank =
+ rcc::RegisterBankID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterBank::GPR));
+ const auto FPRBank =
+ rcc::RegisterBankID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterBank::FPR));
+ const auto GPRClass =
+ rcc::RegisterClassID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterClass::GPR));
+ const auto FPRClass =
+ rcc::RegisterClassID::fromValue(static_cast<std::uint32_t>(rcc::riscv::RegisterClass::FPR64));
+ Context.expect(Source && Destination &&
+ Function->registerInfo().setRegisterBank(*Source, GPRBank).has_value() &&
+ Function->registerInfo().setRegisterClass(*Source, GPRClass).has_value() &&
+ Function->registerInfo().setRegisterBank(*Destination, FPRBank).has_value() &&
+ Function->registerInfo().setRegisterClass(*Destination, FPRClass).has_value(),
+ "pseudo preparation virtual-register constraints must be valid");
+ if (!Source || !Destination)
+ return;
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ if (!Builder.setInsertPoint(*Block))
+ return;
+ auto Pending = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY));
+ if (!Pending || !Pending->addDef(*Destination) || !Pending->addUse(*Source))
+ return;
+ auto Copy = Pending->build();
+ if (!Copy)
+ return;
+ setSelectedProperties(*Function);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::PreparePostRAPseudosPass Pass(rcc::riscv::pseudoExpansionInfo());
+ auto Result = Pass.run(*Function, Target, Analyses);
+ Context.expect(Result && Result->Changed && (*Copy)->operandAt(0)->targetFlags() == 16 &&
+ (*Copy)->operandAt(1)->targetFlags() == 16,
+ "pre-RA pseudo preparation must retain a 64-bit cross-bank COPY width");
+}
+
+void testPostRAPseudoExpansionAndFinalization(TestContext &Context,
+ const rcc::TargetDescription &Target) {
+ auto Function = makeFunction(Context, "post_ra_pseudos", 7);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+ if (Block == nullptr)
+ return;
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ if (!Builder.setInsertPoint(*Block))
+ return;
+
+ auto GPRCopy = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY));
+ if (!GPRCopy || !GPRCopy->addDef(physical(rcc::riscv::PhysicalRegisterID::X11)) ||
+ !GPRCopy->addUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || !GPRCopy->build()) {
+ Context.expect(false, "post-RA GPR copy construction must succeed");
+ return;
+ }
+ auto CrossCopy = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::COPY));
+ if (!CrossCopy || !CrossCopy->addDef(physical(rcc::riscv::PhysicalRegisterID::F10), {}, {}, 16) ||
+ !CrossCopy->addUse(physical(rcc::riscv::PhysicalRegisterID::X10), {}, {}, 16) ||
+ !CrossCopy->build()) {
+ Context.expect(false, "post-RA cross-bank copy construction must succeed");
+ return;
+ }
+ auto Call = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoCALL));
+ auto Preserved = rcc::riscv::createCallPreservedMask();
+ if (!Call || !Call->addExternalSymbol("callee", 0, 1) || !Call->addImmediate(11) ||
+ !Call->addRegisterMask(Preserved) ||
+ !Call->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) || !Call->build()) {
+ Context.expect(false, "post-RA call pseudo construction must succeed");
+ return;
+ }
+ auto Return = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoRET));
+ if (!Return || !Return->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) ||
+ !Return->build()) {
+ Context.expect(false, "post-RA return pseudo construction must succeed");
+ return;
+ }
+
+ setRelaxationProperties(Context, *Function, Target);
+ Function->properties().clear(rcc::MachineFunctionProperty::NoExecutablePseudos);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::ExpandPostRAPseudosPass Expansion(rcc::riscv::pseudoExpansionInfo());
+ auto Expanded = Expansion.run(*Function, Target, Analyses);
+ if (!Expanded)
+ std::cerr << "post-RA pseudo expansion failure: " << Expanded.error().message() << '\n';
+ std::size_t ADDICount = 0;
+ std::size_t CrossMoveCount = 0;
+ std::size_t CallCount = 0;
+ std::size_t ReturnCount = 0;
+ for (rcc::MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ ADDICount += Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::ADDI);
+ CrossMoveCount += Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::FMV_D_X);
+ CallCount += Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::JALR_CALL);
+ ReturnCount += Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::JALR_RET);
+ }
+ Context.expect(Expanded && Expanded->Changed && ADDICount == 1 && CrossMoveCount == 1 &&
+ CallCount == 1 && ReturnCount == 1,
+ "post-RA expansion must lower copies, calls, and returns to real RV64GC code");
+ if (!Expanded)
+ return;
+
+ rcc::MachinePassManager Pipeline;
+ Context.expect(Pipeline.addPass(std::make_unique<rcc::FinalizeEmissionPseudosPass>()) &&
+ Pipeline.addPass(std::make_unique<rcc::FinalEncodabilityVerifierPass>(
+ rcc::riscv::branchInfo())),
+ "finalization passes must register");
+ auto Finalized = Pipeline.run(*Function, Target, Analyses);
+ if (!Finalized)
+ std::cerr << "final encodability failure: " << Finalized.error().message() << '\n';
+ Context.expect(Finalized &&
+ Function->properties().has(rcc::MachineFunctionProperty::NoExecutablePseudos),
+ "emission finalization must exclusively publish NoExecutablePseudos");
+}
+
+void testCanonicalOptimizationPipelines(TestContext &Context,
+ const rcc::TargetDescription &Target) {
+ const rcc::MachineOptimizationPipelineTargetInfo TargetInfo =
+ rcc::riscv::optimizationPipelineInfo();
+ rcc::MachinePassManager SelectedPipeline;
+ rcc::MachinePassManager PreRAPipeline;
+ Context.expect(rcc::addSelectedSSAOptimizationPipeline(SelectedPipeline, TargetInfo) &&
+ SelectedPipeline.size() == 6,
+ "canonical selected-SSA optimization pipeline must register completely");
+ Context.expect(rcc::addPreRegisterAllocationOptimizationPipeline(PreRAPipeline, TargetInfo) &&
+ PreRAPipeline.size() == 5,
+ "canonical pre-RA optimization pipeline must register completely");
+
+ auto Function = makeFunction(Context, "canonical_post_ra_pipeline", 21);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+ if (Block == nullptr)
+ return;
+ auto SlotAlignment = rcc::Alignment::create(8);
+ if (!SlotAlignment)
+ return;
+ rcc::StackObjectDesc SpillDescription;
+ SpillDescription.Kind = rcc::StackObjectKind::SpillSlot;
+ SpillDescription.Size = 8;
+ SpillDescription.Align = *SlotAlignment;
+ auto Spill = Function->frameInfo().createObject(SpillDescription);
+ if (!Spill)
+ return;
+ rcc::MachineIRBuilder Builder(*Function, Target);
+ if (!Builder.setInsertPoint(*Block) ||
+ buildADDI(Context, Builder, physical(rcc::riscv::PhysicalRegisterID::X10),
+ physical(rcc::riscv::PhysicalRegisterID::X0), 42) == nullptr ||
+ buildStackAccess(Context, Builder, rcc::riscv::InstructionOpcode::SD,
+ physical(rcc::riscv::PhysicalRegisterID::X10), *Spill,
+ rcc::MemoryAccessFlag::Store) == nullptr) {
+ return;
+ }
+ auto Return = Builder.buildInstr(opcode(rcc::riscv::InstructionOpcode::PseudoRET));
+ if (!Return || !Return->addImplicitUse(physical(rcc::riscv::PhysicalRegisterID::X10)) ||
+ !Return->build()) {
+ Context.expect(false, "canonical post-RA return construction must succeed");
+ return;
+ }
+ setPostRAProperties(*Function);
+ if (!rcc::rebuildMachineLiveness(*Function, Target)) {
+ Context.expect(false, "canonical post-RA input liveness must be valid");
+ return;
+ }
+
+ rcc::MachinePassManager Pipeline;
+ auto Added = rcc::addPostRegisterAllocationPipeline(Pipeline, TargetInfo);
+ Context.expect(Added && Pipeline.size() == 20,
+ "canonical post-RA pipeline must register every required pass");
+ if (!Added)
+ return;
+ rcc::MachineAnalysisManager Analyses;
+ auto Registered = rcc::registerMachineOptimizationAnalyses(Analyses);
+ Context.expect(Registered.has_value(),
+ "canonical optimization analyses must register completely");
+ if (!Registered)
+ return;
+ auto Result = Pipeline.run(*Function, Target, Analyses);
+ if (!Result)
+ std::cerr << "canonical post-RA pipeline failure: " << Result.error().message() << '\n';
+ const bool HasReturn =
+ std::ranges::any_of(Function->allInstructions(), [](const rcc::MachineInstr *Instruction) {
+ return Instruction != nullptr &&
+ Instruction->opcode() == opcode(rcc::riscv::InstructionOpcode::JALR_RET);
+ });
+ const bool HasFrameSetup =
+ std::ranges::any_of(Function->allInstructions(), [](const rcc::MachineInstr *Instruction) {
+ return Instruction != nullptr &&
+ Instruction->flags().has(rcc::MachineInstrFlag::FrameSetup);
+ });
+ Context.expect(Result && HasReturn && HasFrameSetup &&
+ Function->properties().has(rcc::MachineFunctionProperty::FrameFinalized) &&
+ Function->properties().has(rcc::MachineFunctionProperty::NoFrameIndices) &&
+ Function->properties().has(rcc::MachineFunctionProperty::NoExecutablePseudos),
+ "canonical post-RA pipeline must reach final encodable RV64GC MachineIR");
+}
+
+void testTargetLateExpansionRejectsSizeGrowth(TestContext &Context) {
+ auto Target = createLateExpansionSizeTarget();
+ Context.expect(Target.has_value(), "target-late size-contract target construction must succeed");
+ if (!Target)
+ return;
+ auto Function = makeFunction(Context, "late_expansion_size_growth", 22);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+ if (Block == nullptr)
+ return;
+ rcc::MachineIRBuilder Builder(*Function, *Target);
+ if (!Builder.setInsertPoint(*Block))
+ return;
+ auto Pending = Builder.buildInstr(GrowingLatePseudoOpcode);
+ if (!Pending)
+ return;
+ auto Pseudo = Pending->build();
+ Context.expect(Pseudo.has_value(), "target-late growth pseudo construction must succeed");
+ if (!Pseudo)
+ return;
+
+ setPostRAProperties(*Function);
+ auto StackAlignment = rcc::Alignment::create(16);
+ if (!StackAlignment || !Function->frameInfo().freeze(0, *StackAlignment))
+ return;
+ Function->properties().set(rcc::MachineFunctionProperty::FrameFinalized);
+ Function->properties().set(rcc::MachineFunctionProperty::NoFrameIndices);
+ if (!rcc::rebuildMachineLiveness(*Function, *Target))
+ return;
+
+ GrowingLatePseudoInfo ExpansionInfo;
+ FixedSizeBranchInfo BranchInfo;
+ rcc::TargetLatePseudoExpansionPass Pass(ExpansionInfo, BranchInfo);
+ rcc::MachineAnalysisManager Analyses;
+ auto Result = Pass.run(*Function, *Target, Analyses);
+ Context.expect(!Result && Result.error().code() == rcc::ErrorCode::PseudoExpansionFailed &&
+ (*Pseudo)->parent() == Block && Block->firstInstruction() == *Pseudo &&
+ Block->lastInstruction() == *Pseudo,
+ "target-late expansion must reject block-size growth atomically");
+}
+
+void testTargetLateExpansionRejectsCompensatedGrowth(TestContext &Context) {
+ auto Target = createLateExpansionSizeTarget();
+ Context.expect(Target.has_value(), "target-late compensation target construction must succeed");
+ if (!Target)
+ return;
+ auto Function = makeFunction(Context, "late_expansion_compensated_growth", 23);
+ if (!Function)
+ return;
+ rcc::MachineBasicBlock *Block = appendBlock(Context, *Function);
+ if (Block == nullptr)
+ return;
+ rcc::MachineIRBuilder Builder(*Function, *Target);
+ if (!Builder.setInsertPoint(*Block))
+ return;
+ auto Shrinking = Builder.buildInstr(ShrinkingLatePseudoOpcode);
+ auto ShrinkingPseudo =
+ Shrinking ? Shrinking->build()
+ : rcc::Expected<rcc::MachineInstr *>(std::unexpected(
+ rcc::Error(rcc::ErrorCode::InternalError, "no shrinking pseudo builder")));
+ auto Growing = Builder.buildInstr(GrowingLatePseudoOpcode);
+ auto GrowingPseudo = Growing ? Growing->build()
+ : rcc::Expected<rcc::MachineInstr *>(std::unexpected(rcc::Error(
+ rcc::ErrorCode::InternalError, "no growing pseudo builder")));
+ Context.expect(ShrinkingPseudo.has_value() && GrowingPseudo.has_value(),
+ "compensated target-late pseudos must be constructed");
+ if (!ShrinkingPseudo || !GrowingPseudo)
+ return;
+
+ setPostRAProperties(*Function);
+ auto StackAlignment = rcc::Alignment::create(16);
+ if (!StackAlignment || !Function->frameInfo().freeze(0, *StackAlignment))
+ return;
+ Function->properties().set(rcc::MachineFunctionProperty::FrameFinalized);
+ Function->properties().set(rcc::MachineFunctionProperty::NoFrameIndices);
+ if (!rcc::rebuildMachineLiveness(*Function, *Target))
+ return;
+
+ GrowingLatePseudoInfo ExpansionInfo;
+ FixedSizeBranchInfo BranchInfo;
+ rcc::TargetLatePseudoExpansionPass Pass(ExpansionInfo, BranchInfo);
+ rcc::MachineAnalysisManager Analyses;
+ auto Result = Pass.run(*Function, *Target, Analyses);
+ Context.expect(
+ !Result && Result.error().code() == rcc::ErrorCode::PseudoExpansionFailed &&
+ (*ShrinkingPseudo)->parent() == Block && (*GrowingPseudo)->parent() == Block &&
+ Block->firstInstruction() == *ShrinkingPseudo &&
+ Block->lastInstruction() == *GrowingPseudo,
+ "one growing target-late expansion must not be hidden by another pseudo shrinking");
+}
+
+void testFinalEncodabilityRejectsInvalidInstruction(TestContext &Context,
+ const rcc::TargetDescription &Target) {
+ auto ImmediateFunction = makeFunction(Context, "final_invalid_immediate", 18);
+ if (!ImmediateFunction)
+ return;
+ rcc::MachineBasicBlock *ImmediateBlock = appendBlock(Context, *ImmediateFunction);
+ if (ImmediateBlock == nullptr)
+ return;
+ rcc::MachineIRBuilder ImmediateBuilder(*ImmediateFunction, Target);
+ if (!ImmediateBuilder.setInsertPoint(*ImmediateBlock))
+ return;
+ rcc::MachineInstr *InvalidImmediate =
+ buildADDI(Context, ImmediateBuilder, physical(rcc::riscv::PhysicalRegisterID::X10),
+ physical(rcc::riscv::PhysicalRegisterID::X0), 0);
+ if (InvalidImmediate == nullptr || !InvalidImmediate->eraseOperand(2) ||
+ !InvalidImmediate->insertOperand(2, rcc::MachineOperand::immediate(4096))) {
+ Context.expect(false, "invalid final immediate fixture must be constructed");
+ return;
+ }
+ setRelaxationProperties(Context, *ImmediateFunction, Target);
+ rcc::MachineAnalysisManager Analyses;
+ rcc::FinalEncodabilityVerifierPass Verifier(rcc::riscv::branchInfo());
+ auto ImmediateResult = Verifier.run(*ImmediateFunction, Target, Analyses);
+ Context.expect(!ImmediateResult &&
+ ImmediateResult.error().code() == rcc::ErrorCode::FinalEncodabilityFailed,
+ "final encodability must reject an out-of-range target immediate");
+
+ auto BundleFunction = makeFunction(Context, "final_invalid_bundle", 19);
+ if (!BundleFunction)
+ return;
+ rcc::MachineBasicBlock *BundleBlock = appendBlock(Context, *BundleFunction);
+ if (BundleBlock == nullptr)
+ return;
+ rcc::MachineIRBuilder BundleBuilder(*BundleFunction, Target);
+ if (!BundleBuilder.setInsertPoint(*BundleBlock))
+ return;
+ rcc::MachineInstr *InvalidBundle =
+ buildADDI(Context, BundleBuilder, physical(rcc::riscv::PhysicalRegisterID::X10),
+ physical(rcc::riscv::PhysicalRegisterID::X0), 0);
+ if (InvalidBundle == nullptr)
+ return;
+ InvalidBundle->setFlags({rcc::MachineInstrFlag::BundledWithSucc});
+ setRelaxationProperties(Context, *BundleFunction, Target);
+ auto BundleResult = Verifier.run(*BundleFunction, Target, Analyses);
+ Context.expect(!BundleResult &&
+ BundleResult.error().code() == rcc::ErrorCode::FinalEncodabilityFailed,
+ "final encodability must reject an inconsistent bundle boundary");
+}
+
+} // namespace
+
+int main() {
+ TestContext Context;
+ auto Target = rcc::riscv::createTargetDescription();
+ Context.expect(Target.has_value(), "RV64GC target construction must succeed");
+ if (!Target)
+ return Context.result();
+ testBlockPlacement(Context, *Target);
+ testFallthroughNormalization(Context, *Target);
+ testFallthroughMaterialization(Context, *Target);
+ testBranchOptimization(Context, *Target);
+ testTailDuplication(Context, *Target);
+ testTailMerge(Context, *Target);
+ testCopyPropagation(Context, *Target);
+ testStackSlotColoring(Context, *Target);
+ testRISCVMemoryFoldingBoundary(Context, *Target);
+ testShrinkWrappedFrameRegion(Context, *Target);
+ testCompressedAndConditionalRelaxation(Context, *Target);
+ testLongJALRelaxation(Context, *Target);
+ testPseudoPreparation(Context, *Target);
+ testPostRAPseudoExpansionAndFinalization(Context, *Target);
+ testCanonicalOptimizationPipelines(Context, *Target);
+ testTargetLateExpansionRejectsSizeGrowth(Context);
+ testTargetLateExpansionRejectsCompensatedGrowth(Context);
+ testFinalEncodabilityRejectsInvalidInstruction(Context, *Target);
+ return Context.result();
+}
diff --git a/backend/unittests/Target/RISCVTargetTest.cpp b/backend/unittests/Target/RISCVTargetTest.cpp
index 98525e9..6203461 100644
--- a/backend/unittests/Target/RISCVTargetTest.cpp
+++ b/backend/unittests/Target/RISCVTargetTest.cpp
@@ -149,7 +149,7 @@ void testInstructionFacts(TestContext &Context) {
rcc::Opcode::fromValue(static_cast<std::uint32_t>(rcc::riscv::InstructionOpcode::FENCE_I)));
Context.expect(
Target->instructions().descriptors().size() ==
- 201 + rcc::genericInstructionDescriptors().size() &&
+ 204 + rcc::genericInstructionDescriptors().size() &&
Addi != nullptr && Addi->Name == "ADDI" && Addi->NumExplicitDefs == 1 &&
Addi->Operands.size() == 3 && Addi->Operands.back().ImmediatePredicate &&
Load != nullptr && Load->Flags.has(rcc::InstructionFlag::MayLoad) &&
@@ -173,7 +173,7 @@ void testInstructionFacts(TestContext &Context) {
const auto Encodings = rcc::riscv::encodingCategories();
const auto Formats = rcc::riscv::assemblyFormats();
Context.expect(Schedules.size() == 15 && Encodings.size() == 10 && Formats.size() == 34 &&
- rcc::riscv::assemblyInstructions().size() == 192 && FenceAssembly != nullptr &&
+ rcc::riscv::assemblyInstructions().size() == 195 && FenceAssembly != nullptr &&
FenceAssembly->Mnemonic == "fence.i" &&
std::ranges::any_of(Schedules,
[](const rcc::riscv::SchedulingClassInfo &Class) {
generated by cgit v1.2.3 (git 2.46.0) at 2026-10-29 04:00:32 +0000