diff options
Diffstat (limited to 'backend')
27 files changed, 4427 insertions, 3229 deletions
diff --git a/backend/include/rcc/ADT/APInt.h b/backend/include/rcc/ADT/APInt.h index abd10f9..22ecf62 100644 --- a/backend/include/rcc/ADT/APInt.h +++ b/backend/include/rcc/ADT/APInt.h @@ -26,6 +26,8 @@ public: std::span<const std::uint64_t> Words); [[nodiscard]] static Expected<APInt> fromUnsigned(std::uint32_t BitWidth, std::uint64_t Value); [[nodiscard]] static Expected<APInt> fromSigned(std::uint32_t BitWidth, std::int64_t Value); + [[nodiscard]] static Expected<APInt> getLowBitsSet(std::uint32_t BitWidth, + std::uint32_t SetBitCount); [[nodiscard]] static Expected<APInt> parseDecimal(std::uint32_t BitWidth, std::string_view Text); [[nodiscard]] static Expected<APInt> parseLLVMHex(std::uint32_t BitWidth, std::string_view Text); @@ -42,6 +44,9 @@ public: [[nodiscard]] std::size_t activeBitWidth() const noexcept; [[nodiscard]] std::size_t countrZero() const noexcept; [[nodiscard]] std::uint64_t word(std::size_t Index) const noexcept; + [[nodiscard]] Expected<APInt> trunc(std::uint32_t NewBitWidth) const; + [[nodiscard]] Expected<APInt> zext(std::uint32_t NewBitWidth) const; + [[nodiscard]] Expected<APInt> sext(std::uint32_t NewBitWidth) const; [[nodiscard]] int compare(const APInt &Other) const noexcept; [[nodiscard]] int compareToShifted(const APInt &Other, std::size_t OtherShift) const noexcept; diff --git a/backend/include/rcc/CodeGen/GlobalISel/GISelChangeObserver.h b/backend/include/rcc/CodeGen/GlobalISel/GISelChangeObserver.h new file mode 100644 index 0000000..9538f20 --- /dev/null +++ b/backend/include/rcc/CodeGen/GlobalISel/GISelChangeObserver.h @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H +#define RCC_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H + +#include "rcc/CodeGen/MachineFunction.h" + +#include <span> +#include <vector> + +namespace rcc { + +class MachineInstr; + +class GISelChangeObserver { +public: + virtual ~GISelChangeObserver() = default; + + virtual void erasingInstr(MachineInstr &Instruction) = 0; + virtual void createdInstr(MachineInstr &Instruction) = 0; + virtual void changingInstr(MachineInstr &Instruction) = 0; + virtual void changedInstr(MachineInstr &Instruction) = 0; +}; + +class GISelObserverWrapper final : public MachineFunction::Delegate, + public GISelChangeObserver { +public: + GISelObserverWrapper() = default; + explicit GISelObserverWrapper(GISelChangeObserver &Observer) { + Observers.push_back(&Observer); + } + explicit GISelObserverWrapper(std::span<GISelChangeObserver *const> InitialObservers) + : Observers(InitialObservers.begin(), InitialObservers.end()) {} + + void addObserver(GISelChangeObserver &Observer); + void removeObserver(GISelChangeObserver &Observer); + void clearObservers() noexcept; + + void erasingInstr(MachineInstr &Instruction) override; + void createdInstr(MachineInstr &Instruction) override; + void changingInstr(MachineInstr &Instruction) override; + void changedInstr(MachineInstr &Instruction) override; + + void handleInsertion(MachineInstr &Instruction) override; + void handleRemoval(MachineInstr &Instruction) override; + +private: + std::vector<GISelChangeObserver *> Observers; +}; + +class MachineFunctionDelegateInstaller final { +public: + MachineFunctionDelegateInstaller(MachineFunction &Function, + MachineFunction::Delegate &Delegate); + MachineFunctionDelegateInstaller(const MachineFunctionDelegateInstaller &) = delete; + MachineFunctionDelegateInstaller &operator=(const MachineFunctionDelegateInstaller &) = delete; + ~MachineFunctionDelegateInstaller(); + +private: + MachineFunction *Function; + MachineFunction::Delegate *Delegate; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H diff --git a/backend/include/rcc/CodeGen/GlobalISel/GISelWorkList.h b/backend/include/rcc/CodeGen/GlobalISel/GISelWorkList.h new file mode 100644 index 0000000..faeafab --- /dev/null +++ b/backend/include/rcc/CodeGen/GlobalISel/GISelWorkList.h @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_GLOBALISEL_GISELWORKLIST_H +#define RCC_CODEGEN_GLOBALISEL_GISELWORKLIST_H + +#include <cstddef> +#include <unordered_map> +#include <vector> + +namespace rcc { + +class MachineInstr; + +template <std::size_t InlineCapacity> class GISelWorkList final { +public: + GISelWorkList() { + Worklist.reserve(InlineCapacity); + WorklistMap.reserve(InlineCapacity); + } + + [[nodiscard]] bool empty() const noexcept { + return WorklistMap.empty(); + } + + [[nodiscard]] std::size_t size() const noexcept { + return WorklistMap.size(); + } + + void deferredInsert(MachineInstr *Instruction) { + Worklist.push_back(Instruction); + } + + void finalize() { + WorklistMap.reserve(Worklist.size()); + for (std::size_t Index = 0; Index < Worklist.size(); ++Index) + WorklistMap.emplace(Worklist[Index], Index); + } + + void insert(MachineInstr *Instruction) { + const auto [Position, Inserted] = WorklistMap.emplace(Instruction, Worklist.size()); + if (Inserted) + Worklist.push_back(Instruction); + } + + void remove(const MachineInstr *Instruction) { + const auto Position = WorklistMap.find(Instruction); + if (Position == WorklistMap.end()) + return; + Worklist[Position->second] = nullptr; + WorklistMap.erase(Position); + } + + void clear() { + Worklist.clear(); + WorklistMap.clear(); + } + + [[nodiscard]] MachineInstr *popBack() { + MachineInstr *Instruction = nullptr; + do { + Instruction = Worklist.back(); + Worklist.pop_back(); + } while (Instruction == nullptr); + WorklistMap.erase(Instruction); + return Instruction; + } + +private: + std::vector<MachineInstr *> Worklist; + std::unordered_map<const MachineInstr *, std::size_t> WorklistMap; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_GLOBALISEL_GISELWORKLIST_H diff --git a/backend/include/rcc/CodeGen/GlobalISel/GenericMachineInstrs.h b/backend/include/rcc/CodeGen/GlobalISel/GenericMachineInstrs.h new file mode 100644 index 0000000..d0a176d --- /dev/null +++ b/backend/include/rcc/CodeGen/GlobalISel/GenericMachineInstrs.h @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H +#define RCC_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/Register.h" + +#include <cassert> +#include <cstddef> + +namespace rcc { + +class GenericMachineInstr : public MachineInstr { +public: + GenericMachineInstr() = delete; + + [[nodiscard]] Register getReg(std::size_t Index) const { + const MachineOperand *Operand = operandAt(Index); + assert(Operand != nullptr && Operand->isRegister()); + return *Operand->registerValue(); + } + + [[nodiscard]] static bool classof(const MachineInstr *Instruction) noexcept { + return Instruction != nullptr && isGenericOpcode(Instruction->opcode()); + } +}; + +class GUnmerge : public GenericMachineInstr { +public: + [[nodiscard]] std::size_t getNumDefs() const noexcept { + return operandCount() - 1; + } + + [[nodiscard]] Register getSourceReg() const { + return getReg(getNumDefs()); + } + + [[nodiscard]] static bool classof(const MachineInstr *Instruction) noexcept { + return Instruction != nullptr && + Instruction->opcode() == rcc::opcode(GenericOpcode::G_UNMERGE_VALUES); + } +}; + +class GMergeLikeInstr : public GenericMachineInstr { +public: + [[nodiscard]] std::size_t getNumSources() const noexcept { + return operandCount() - 1; + } + + [[nodiscard]] Register getSourceReg(std::size_t Index) const { + return getReg(Index + 1); + } + + [[nodiscard]] static bool classof(const MachineInstr *Instruction) noexcept { + if (Instruction == nullptr) + return false; + switch (static_cast<GenericOpcode>(Instruction->opcode().value())) { + case GenericOpcode::G_MERGE_VALUES: + case GenericOpcode::G_CONCAT_VECTORS: + case GenericOpcode::G_BUILD_VECTOR: + return true; + default: + return false; + } + } +}; + +class GMerge : public GMergeLikeInstr { +public: + [[nodiscard]] static bool classof(const MachineInstr *Instruction) noexcept { + return Instruction != nullptr && + Instruction->opcode() == rcc::opcode(GenericOpcode::G_MERGE_VALUES); + } +}; + +class GConcatVectors : public GMergeLikeInstr { +public: + [[nodiscard]] static bool classof(const MachineInstr *Instruction) noexcept { + return Instruction != nullptr && + Instruction->opcode() == rcc::opcode(GenericOpcode::G_CONCAT_VECTORS); + } +}; + +class GBuildVector : public GMergeLikeInstr { +public: + [[nodiscard]] static bool classof(const MachineInstr *Instruction) noexcept { + return Instruction != nullptr && + Instruction->opcode() == rcc::opcode(GenericOpcode::G_BUILD_VECTOR); + } +}; + +template <typename InstructionT> +[[nodiscard]] InstructionT &castGenericMachineInstr(MachineInstr &Instruction) { + assert(InstructionT::classof(&Instruction)); + return static_cast<InstructionT &>(Instruction); +} + +template <typename InstructionT> +[[nodiscard]] const InstructionT & +castGenericMachineInstr(const MachineInstr &Instruction) { + assert(InstructionT::classof(&Instruction)); + return static_cast<const InstructionT &>(Instruction); +} + +} // namespace rcc + +#endif // RCC_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H diff --git a/backend/include/rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/backend/include/rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h new file mode 100644 index 0000000..c3f8d0b --- /dev/null +++ b/backend/include/rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -0,0 +1,1802 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_GLOBALISEL_LEGALIZATIONARTIFACTCOMBINER_H +#define RCC_CODEGEN_GLOBALISEL_LEGALIZATIONARTIFACTCOMBINER_H + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/GenericMachineInstrs.h" +#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/Support/Error.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <span> +#include <unordered_set> +#include <utility> +#include <vector> + +namespace rcc { + +class LegalizationArtifactCombiner final { + MachineIRBuilder &Builder; + MachineRegisterInfo &MRI; + const LegalizerInfo &LI; + + [[nodiscard]] static bool isArtifactCast(Opcode InstructionOpcode) noexcept { + switch (static_cast<GenericOpcode>(InstructionOpcode.value())) { + case GenericOpcode::G_TRUNC: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_ANYEXT: + return true; + default: + return false; + } + } + +public: + LegalizationArtifactCombiner(MachineIRBuilder &Builder, MachineRegisterInfo &MRI, + const LegalizerInfo &Info) + : Builder(Builder), MRI(MRI), LI(Info) {} + + [[nodiscard]] Expected<bool> + tryCombineInstruction(MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + GISelObserverWrapper &Observer) { + if (!DeadInstructions.empty()) { + auto Deleted = deleteMarkedDeadInsts(DeadInstructions, Observer); + if (!Deleted) + return std::unexpected(Deleted.error()); + } + + std::vector<Register> UpdatedDefs; + Expected<bool> Changed = false; + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + default: + return false; + case GenericOpcode::G_ANYEXT: + Changed = tryCombineAnyExt(Instruction, DeadInstructions, UpdatedDefs, Observer); + break; + case GenericOpcode::G_ZEXT: + Changed = tryCombineZExt(Instruction, DeadInstructions, UpdatedDefs, Observer); + break; + case GenericOpcode::G_SEXT: + Changed = tryCombineSExt(Instruction, DeadInstructions, UpdatedDefs, Observer); + break; + case GenericOpcode::G_UNMERGE_VALUES: + Changed = tryCombineUnmergeValues(Instruction, DeadInstructions, UpdatedDefs, Observer); + break; + case GenericOpcode::G_MERGE_VALUES: + case GenericOpcode::G_BUILD_VECTOR: + case GenericOpcode::G_CONCAT_VECTORS: + Changed = tryCombineMergeLike(Instruction, DeadInstructions, UpdatedDefs, Observer); + break; + case GenericOpcode::G_EXTRACT: + Changed = tryCombineExtract(Instruction, DeadInstructions, UpdatedDefs); + break; + case GenericOpcode::G_TRUNC: + Changed = tryCombineTrunc(Instruction, DeadInstructions, UpdatedDefs, Observer); + if (Changed && !*Changed) + UpdatedDefs.push_back(*Instruction.operandAt(0)->registerValue()); + break; + } + if (!Changed) + return std::unexpected(Changed.error()); + + while (!UpdatedDefs.empty()) { + const Register NewDef = UpdatedDefs.back(); + UpdatedDefs.pop_back(); + std::vector<MachineInstr *> Uses; + for (MachineOperand &Use : MRI.useOperands(NewDef)) { + if (Use.parent() != nullptr) + Uses.push_back(Use.parent()); + } + for (MachineInstr *Use : Uses) { + switch (static_cast<GenericOpcode>(Use->opcode().value())) { + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_UNMERGE_VALUES: + case GenericOpcode::G_EXTRACT: + case GenericOpcode::G_TRUNC: + case GenericOpcode::G_BUILD_VECTOR: + Observer.changedInstr(*Use); + break; + case GenericOpcode::G_ASSERT_SEXT: + case GenericOpcode::G_ASSERT_ZEXT: + case GenericOpcode::G_ASSERT_ALIGN: + if (const auto Copy = Use->operandAt(0)->registerValue(); + Copy && Copy->isVirtual()) + UpdatedDefs.push_back(*Copy); + break; + default: + if (Use->opcode() == opcode(GenericOpcode::COPY)) { + if (const auto Copy = Use->operandAt(0)->registerValue(); + Copy && Copy->isVirtual()) + UpdatedDefs.push_back(*Copy); + } + break; + } + } + } + return *Changed; + } + +private: + class ArtifactValueFinder final { + public: + explicit ArtifactValueFinder(LegalizationArtifactCombiner &Combiner) + : Combiner(Combiner) {} + + [[nodiscard]] Expected<Register> findValueFromDef(Register DefReg, + std::uint64_t StartBit, + std::uint64_t Size, + LowLevelType DestinationType); + [[nodiscard]] Expected<bool> + tryCombineUnmergeDefs(GUnmerge &Instruction, + GISelChangeObserver &Observer, + std::vector<Register> &UpdatedDefs); + [[nodiscard]] Expected<GUnmerge *> + findUnmergeThatDefinesReg(Register Reg, std::uint64_t Size, + std::size_t &DefOperandIndex); + [[nodiscard]] Expected<bool> + isSequenceFromUnmerge(GMergeLikeInstr &Instruction, + std::size_t MergeStartIndex, GUnmerge *Unmerge, + std::size_t UnmergeStartIndex, + std::size_t ElementCount, std::uint64_t ElementSize, + bool AllowUndef); + [[nodiscard]] Expected<bool> + tryCombineMergeLike(GMergeLikeInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, + GISelChangeObserver &Observer); + + private: + [[nodiscard]] Expected<Register> + findValueFromDefImpl(Register DefReg, std::uint64_t StartBit, + std::uint64_t Size, LowLevelType DestinationType); + [[nodiscard]] Expected<Register> + findValueFromConcat(GConcatVectors &Instruction, + std::uint64_t StartBit, std::uint64_t Size, + LowLevelType DestinationType); + [[nodiscard]] Expected<Register> + findValueFromBuildVector(GBuildVector &Instruction, + std::uint64_t StartBit, std::uint64_t Size); + [[nodiscard]] Expected<Register> + findValueFromInsert(MachineInstr &Instruction, std::uint64_t StartBit, + std::uint64_t Size, LowLevelType DestinationType); + [[nodiscard]] Expected<Register> + findValueFromExtension(MachineInstr &Instruction, + std::uint64_t StartBit, std::uint64_t Size, + LowLevelType DestinationType); + [[nodiscard]] Expected<Register> + findValueFromTrunc(MachineInstr &Instruction, std::uint64_t StartBit, + std::uint64_t Size, LowLevelType DestinationType); + + LegalizationArtifactCombiner &Combiner; + Register CurrentBest; + std::unordered_set<Register> Visited; + }; + + [[nodiscard]] Expected<bool> + tryCombineAnyExt(MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, + GISelObserverWrapper &Observer); + [[nodiscard]] Expected<bool> + tryCombineZExt(MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, + GISelObserverWrapper &Observer); + [[nodiscard]] Expected<bool> + tryCombineSExt(MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, + GISelObserverWrapper &Observer); + [[nodiscard]] Expected<bool> + tryCombineTrunc(MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, + GISelObserverWrapper &Observer); + [[nodiscard]] Expected<bool> + tryCombineUnmergeValues(MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, + GISelObserverWrapper &Observer); + [[nodiscard]] Expected<bool> + tryCombineExtract(MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs); + [[nodiscard]] Expected<bool> + tryCombineMergeLike(MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, + GISelObserverWrapper &Observer); + [[nodiscard]] Expected<bool> tryFoldImplicitDef( + MachineInstr &Instruction, std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, GISelObserverWrapper &Observer); + + [[nodiscard]] Expected<MachineInstr *> buildUnary(GenericOpcode InstructionOpcode, + Register Destination, + Register Source); + [[nodiscard]] Expected<MachineInstr *> + buildMergeLike(Register Destination, const std::vector<Register> &Sources); + [[nodiscard]] Expected<void> buildConstant(Register Destination, APInt Value); + [[nodiscard]] Expected<Register> buildConstant(LowLevelType Type, APInt Value); + [[nodiscard]] Expected<bool> isConstantUnsupported(LowLevelType Type) const; + [[nodiscard]] static std::optional<std::int64_t> + immediateOperand(const MachineInstr &Instruction, std::size_t Index); + [[nodiscard]] static std::size_t getDefIndex(const MachineInstr &Instruction, + Register SearchDef); + [[nodiscard]] Expected<std::pair<MachineInstr *, Register>> + definitionIgnoringCopies(Register Reg) const; + [[nodiscard]] Expected<std::vector<Register>> + buildOrFindUnmerge(LowLevelType DestinationType, Register Source); + [[nodiscard]] Expected<LowLevelType> divideType(LowLevelType Type, + std::size_t Parts) const; + [[nodiscard]] Expected<bool> + tryFoldUnmergeCast(MachineInstr &Instruction, MachineInstr &Cast, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs); + [[nodiscard]] static bool canFoldMergeOpcode(Opcode MergeOpcode, + std::optional<Opcode> ConvertOpcode, + LowLevelType OperandType, + LowLevelType DestinationType); + + [[nodiscard]] Expected<void> + replaceRegOrBuildCopy(Register Destination, Register Source, + std::vector<Register> &UpdatedDefs, + GISelChangeObserver &Observer) { + auto DestinationType = MRI.getType(Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto SourceType = MRI.getType(Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (*DestinationType != *SourceType) { + auto Copy = Builder.buildCopy(Destination, Source); + if (!Copy) + return std::unexpected(Copy.error()); + UpdatedDefs.push_back(Destination); + return {}; + } + + std::vector<MachineInstr *> Uses; + for (MachineOperand &Use : MRI.useOperands(Destination)) { + if (Use.parent() == nullptr) + continue; + Uses.push_back(Use.parent()); + Observer.changingInstr(*Use.parent()); + } + auto Replaced = MRI.replaceRegWith(Destination, Source); + if (!Replaced) + return std::unexpected(Replaced.error()); + UpdatedDefs.push_back(Source); + for (MachineInstr *Use : Uses) + Observer.changedInstr(*Use); + return {}; + } + + [[nodiscard]] Register lookThroughCopyInstrs(Register Reg) const { + while (MachineInstr *Definition = MRI.getVRegDef(Reg)) { + if (Definition->opcode() != opcode(GenericOpcode::COPY) || + Definition->operandCount() < 2) + break; + const auto Source = Definition->operandAt(1)->registerValue(); + if (!Source || !Source->isVirtual()) + break; + Reg = *Source; + } + return Reg; + } + + [[nodiscard]] static std::optional<Register> + registerOperand(const MachineInstr &Instruction, std::size_t Index) { + const MachineOperand *Operand = Instruction.operandAt(Index); + return Operand == nullptr ? std::nullopt : Operand->registerValue(); + } + + [[nodiscard]] static bool hasOpcode(const MachineInstr *Instruction, + GenericOpcode ExpectedOpcode) noexcept { + return Instruction != nullptr && Instruction->opcode() == opcode(ExpectedOpcode); + } + + void markDefDead(MachineInstr &Instruction, MachineInstr &Definition, + std::vector<MachineInstr *> &DeadInstructions, + std::size_t DefIndex = 0) { + MachineInstr *Previous = &Instruction; + while (Previous != &Definition) { + const auto Source = artifactSourceRegister(*Previous); + if (!Source) + break; + MachineInstr *SourceDefinition = MRI.getVRegDef(*Source); + if (SourceDefinition == nullptr || !MRI.hasOneUse(*Source)) + break; + if (SourceDefinition != &Definition) + DeadInstructions.push_back(SourceDefinition); + Previous = SourceDefinition; + } + if (Previous != &Definition) + return; + + std::size_t CurrentDef = 0; + for (const MachineOperand &Operand : Definition.operands()) { + if (!Operand.isDef()) + continue; + const auto Reg = Operand.registerValue(); + if (!Reg) + return; + if (CurrentDef == DefIndex) { + if (!MRI.hasOneUse(*Reg)) + return; + } else if (!MRI.useEmpty(*Reg)) { + return; + } + ++CurrentDef; + } + DeadInstructions.push_back(&Definition); + } + + void markInstAndDefDead(MachineInstr &Instruction, MachineInstr &Definition, + std::vector<MachineInstr *> &DeadInstructions, + std::size_t DefIndex = 0) { + DeadInstructions.push_back(&Instruction); + markDefDead(Instruction, Definition, DeadInstructions, DefIndex); + } + + [[nodiscard]] Expected<void> + deleteMarkedDeadInsts(std::vector<MachineInstr *> &DeadInstructions, + GISelObserverWrapper &) { + for (MachineInstr *Dead : DeadInstructions) { + if (Dead == nullptr || Dead->parent() == nullptr) + continue; + auto Removed = Dead->parent()->removeInstruction(*Dead); + if (!Removed) + return std::unexpected(Removed.error()); + } + DeadInstructions.clear(); + return {}; + } + + [[nodiscard]] static std::optional<Register> + artifactSourceRegister(const MachineInstr &Instruction) { + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + case GenericOpcode::G_TRUNC: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_EXTRACT: + case GenericOpcode::G_ASSERT_SEXT: + case GenericOpcode::G_ASSERT_ZEXT: + case GenericOpcode::G_ASSERT_ALIGN: + return registerOperand(Instruction, 1); + case GenericOpcode::G_UNMERGE_VALUES: + return registerOperand(Instruction, Instruction.operandCount() - 1); + default: + if (Instruction.opcode() == opcode(GenericOpcode::COPY)) + return registerOperand(Instruction, 1); + return std::nullopt; + } + } + + [[nodiscard]] Expected<bool> isInstUnsupported(LegalizationQuery Query) const { + return LI.isUnsupported(Query); + } + + [[nodiscard]] Expected<bool> isInstLegal(LegalizationQuery Query) const { + return LI.isLegal(Query); + } + +}; + +inline Expected<MachineInstr *> +LegalizationArtifactCombiner::buildUnary(GenericOpcode InstructionOpcode, + Register Destination, + Register Source) { + auto Pending = Builder.buildInstr(opcode(InstructionOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +inline Expected<MachineInstr *> LegalizationArtifactCombiner::buildMergeLike( + Register Destination, const std::vector<Register> &Sources) { + if (Sources.empty()) + return makeUnexpected(ErrorCode::InvalidArgument, + "merge-like instruction requires source registers"); + auto DestinationType = MRI.getType(Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto SourceType = MRI.getType(Sources.front()); + if (!SourceType) + return std::unexpected(SourceType.error()); + + GenericOpcode InstructionOpcode = GenericOpcode::G_MERGE_VALUES; + if (DestinationType->isVector()) + InstructionOpcode = SourceType->isVector() ? GenericOpcode::G_CONCAT_VECTORS + : GenericOpcode::G_BUILD_VECTOR; + auto Pending = Builder.buildInstr(opcode(InstructionOpcode)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + for (Register Source : Sources) { + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + } + return Pending->build(); +} + +inline Expected<void> +LegalizationArtifactCombiner::buildConstant(Register Destination, APInt Value) { + auto Built = Builder.buildConstant(Destination, std::move(Value)); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +inline Expected<Register> +LegalizationArtifactCombiner::buildConstant(LowLevelType Type, APInt Value) { + auto Destination = MRI.createVirtualRegister(Type); + if (!Destination) + return std::unexpected(Destination.error()); + auto Built = buildConstant(*Destination, std::move(Value)); + if (!Built) + return std::unexpected(Built.error()); + return *Destination; +} + +inline Expected<bool> +LegalizationArtifactCombiner::isConstantUnsupported(LowLevelType Type) const { + if (!Type.isVector()) + return isInstUnsupported( + LegalizationQuery(opcode(GenericOpcode::G_CONSTANT), {Type})); + const auto ElementType = Type.scalarElementType(); + if (!ElementType) + return true; + auto ConstantUnsupported = isInstUnsupported( + LegalizationQuery(opcode(GenericOpcode::G_CONSTANT), {*ElementType})); + if (!ConstantUnsupported || *ConstantUnsupported) + return ConstantUnsupported; + return isInstUnsupported(LegalizationQuery( + opcode(GenericOpcode::G_BUILD_VECTOR), {Type, *ElementType})); +} + +inline std::optional<std::int64_t> +LegalizationArtifactCombiner::immediateOperand(const MachineInstr &Instruction, + std::size_t Index) { + const MachineOperand *Operand = Instruction.operandAt(Index); + if (Operand == nullptr) + return std::nullopt; + if (const auto *Value = Operand->getIf<std::int64_t>()) + return *Value; + return std::nullopt; +} + +inline std::size_t +LegalizationArtifactCombiner::getDefIndex(const MachineInstr &Instruction, + Register SearchDef) { + std::size_t Index = 0; + for (const MachineOperand &Operand : Instruction.operands()) { + if (!Operand.isDef()) + continue; + if (Operand.registerValue() == SearchDef) + return Index; + ++Index; + } + return Index; +} + +inline Expected<std::pair<MachineInstr *, Register>> +LegalizationArtifactCombiner::definitionIgnoringCopies(Register Reg) const { + std::unordered_set<Register> Visited; + while (Reg.isVirtual() && Visited.insert(Reg).second) { + MachineInstr *Definition = MRI.getVRegDef(Reg); + if (Definition == nullptr) + return std::pair<MachineInstr *, Register>{nullptr, Reg}; + if (Definition->opcode() != opcode(GenericOpcode::COPY)) + return std::pair{Definition, Reg}; + const auto Source = registerOperand(*Definition, 1); + if (!Source) + return std::pair{Definition, Reg}; + Reg = *Source; + } + return std::pair<MachineInstr *, Register>{nullptr, Reg}; +} + +inline Expected<LowLevelType> +LegalizationArtifactCombiner::divideType(LowLevelType Type, + std::size_t Parts) const { + if (Parts == 0 || Type.minimumBitWidth() % Parts != 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "LowLevelType cannot be divided into equal parts"); + if (!Type.isVector()) + return LowLevelType::makeCategoryNeutralScalar(Type.scalarBitWidth() / Parts); + if (Type.minimumElementCount() % Parts != 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "vector element count cannot be divided into equal parts"); + const auto ElementType = Type.scalarElementType(); + if (!ElementType) + return makeUnexpected(ErrorCode::InvalidArgument, + "vector has no scalar element type"); + const std::uint32_t Elements = + Type.minimumElementCount() / static_cast<std::uint32_t>(Parts); + if (Elements == 1) + return *ElementType; + return Type.isScalableVector() + ? LowLevelType::makeScalableVector(Elements, *ElementType) + : LowLevelType::makeFixedVector(Elements, *ElementType); +} + +inline Expected<std::vector<Register>> +LegalizationArtifactCombiner::buildOrFindUnmerge(LowLevelType DestinationType, + Register Source) { + auto SourceType = MRI.getType(Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (DestinationType.minimumBitWidth() == 0 || + SourceType->minimumBitWidth() % DestinationType.minimumBitWidth() != 0) + return makeUnexpected(ErrorCode::InvalidArgument, + "unmerge destination does not tile the source type"); + const std::size_t Count = static_cast<std::size_t>( + SourceType->minimumBitWidth() / DestinationType.minimumBitWidth()); + + for (MachineOperand &Use : MRI.useOperands(Source)) { + MachineInstr *Candidate = Use.parent(); + if (!GUnmerge::classof(Candidate) || Candidate->operandCount() != Count + 1) + continue; + std::vector<Register> Existing; + Existing.reserve(Count); + bool Matches = true; + for (std::size_t Index = 0; Index < Count; ++Index) { + const auto Reg = registerOperand(*Candidate, Index); + if (!Reg) { + Matches = false; + break; + } + auto Type = MRI.getType(*Reg); + if (!Type || *Type != DestinationType) { + Matches = false; + break; + } + Existing.push_back(*Reg); + } + if (Matches) + return Existing; + } + + std::vector<Register> Destinations; + Destinations.reserve(Count); + for (std::size_t Index = 0; Index < Count; ++Index) { + auto Destination = MRI.createVirtualRegister(DestinationType); + if (!Destination) + return std::unexpected(Destination.error()); + Destinations.push_back(*Destination); + } + auto Built = Builder.buildUnmergeValues(Destinations, Source); + if (!Built) + return std::unexpected(Built.error()); + return Destinations; +} + +inline bool LegalizationArtifactCombiner::canFoldMergeOpcode( + Opcode MergeOpcode, std::optional<Opcode> ConvertOpcode, + LowLevelType OperandType, LowLevelType DestinationType) { + const auto Generic = static_cast<GenericOpcode>(MergeOpcode.value()); + if (Generic == GenericOpcode::G_BUILD_VECTOR || + Generic == GenericOpcode::G_MERGE_VALUES) { + if (!ConvertOpcode) + return true; + const auto ElementType = OperandType.scalarElementType(); + return !DestinationType.isVector() && OperandType.isVector() && + ElementType && DestinationType == *ElementType; + } + if (Generic != GenericOpcode::G_CONCAT_VECTORS) + return false; + if (!ConvertOpcode) + return true; + if (!DestinationType.isVector()) + return false; + const auto ElementType = OperandType.scalarElementType(); + if (!ElementType) + return false; + if (*ConvertOpcode == opcode(GenericOpcode::G_TRUNC)) + return DestinationType.minimumBitWidth() <= ElementType->minimumBitWidth(); + return DestinationType.minimumBitWidth() >= ElementType->minimumBitWidth(); +} + +inline Expected<bool> LegalizationArtifactCombiner::tryCombineAnyExt( + MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, GISelObserverWrapper &Observer) { + const auto Destination = registerOperand(Instruction, 0); + const auto OriginalSource = registerOperand(Instruction, 1); + if (!Destination || !OriginalSource) + return makeUnexpected(ErrorCode::VerificationError, + "G_ANYEXT has invalid register operands"); + const Register Source = lookThroughCopyInstrs(*OriginalSource); + MachineInstr *SourceDefinition = MRI.getVRegDef(Source); + if (SourceDefinition == nullptr) + return false; + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + if (hasOpcode(SourceDefinition, GenericOpcode::G_TRUNC)) { + const auto TruncSource = registerOperand(*SourceDefinition, 1); + if (!TruncSource) + return false; + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto TruncSourceType = MRI.getType(*TruncSource); + if (!TruncSourceType) + return std::unexpected(TruncSourceType.error()); + if (*DestinationType == *TruncSourceType) { + auto Replaced = replaceRegOrBuildCopy(*Destination, *TruncSource, + UpdatedDefs, Observer); + if (!Replaced) + return std::unexpected(Replaced.error()); + } else { + auto Built = Builder.buildAnyExtOrTrunc(*Destination, *TruncSource); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + } + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + + if (hasOpcode(SourceDefinition, GenericOpcode::G_ANYEXT) || + hasOpcode(SourceDefinition, GenericOpcode::G_SEXT) || + hasOpcode(SourceDefinition, GenericOpcode::G_ZEXT)) { + const auto ExtensionSource = registerOperand(*SourceDefinition, 1); + if (!ExtensionSource) + return false; + auto Built = buildUnary( + static_cast<GenericOpcode>(SourceDefinition->opcode().value()), + *Destination, *ExtensionSource); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + + if (hasOpcode(SourceDefinition, GenericOpcode::G_CONSTANT)) { + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto Legal = isInstLegal(LegalizationQuery( + opcode(GenericOpcode::G_CONSTANT), {*DestinationType})); + if (!Legal) + return std::unexpected(Legal.error()); + const auto *Immediate = + SourceDefinition->operandAt(1)->getIf<IntegerImmediateOperand>(); + if (*Legal && Immediate != nullptr) { + auto Extended = Immediate->Value.sext(DestinationType->scalarBitWidth()); + if (!Extended) + return std::unexpected(Extended.error()); + auto Built = buildConstant(*Destination, std::move(*Extended)); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + } + return tryFoldImplicitDef(Instruction, DeadInstructions, UpdatedDefs, + Observer); +} + +inline Expected<bool> LegalizationArtifactCombiner::tryCombineZExt( + MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, GISelObserverWrapper &Observer) { + const auto Destination = registerOperand(Instruction, 0); + const auto OriginalSource = registerOperand(Instruction, 1); + if (!Destination || !OriginalSource) + return makeUnexpected(ErrorCode::VerificationError, + "G_ZEXT has invalid register operands"); + const Register Source = lookThroughCopyInstrs(*OriginalSource); + MachineInstr *SourceDefinition = MRI.getVRegDef(Source); + if (SourceDefinition == nullptr) + return false; + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + if (hasOpcode(SourceDefinition, GenericOpcode::G_TRUNC) || + hasOpcode(SourceDefinition, GenericOpcode::G_SEXT)) { + const auto NarrowSource = registerOperand(*SourceDefinition, 1); + if (!NarrowSource) + return false; + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto SourceType = MRI.getType(Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + auto AndUnsupported = isInstUnsupported( + LegalizationQuery(opcode(GenericOpcode::G_AND), {*DestinationType})); + if (!AndUnsupported) + return std::unexpected(AndUnsupported.error()); + auto ConstantUnsupported = isConstantUnsupported(*DestinationType); + if (!ConstantUnsupported) + return std::unexpected(ConstantUnsupported.error()); + if (*AndUnsupported || *ConstantUnsupported) + return false; + + Register WideSource = *NarrowSource; + auto WideSourceType = MRI.getType(WideSource); + if (!WideSourceType) + return std::unexpected(WideSourceType.error()); + if (*WideSourceType != *DestinationType) { + auto Wide = MRI.createVirtualRegister(*DestinationType); + if (!Wide) + return std::unexpected(Wide.error()); + auto Built = hasOpcode(SourceDefinition, GenericOpcode::G_SEXT) + ? Builder.buildSExtOrTrunc(*Wide, WideSource) + : Builder.buildAnyExtOrTrunc(*Wide, WideSource); + if (!Built) + return std::unexpected(Built.error()); + WideSource = *Wide; + } + auto Mask = APInt::getLowBitsSet(DestinationType->scalarBitWidth(), + SourceType->scalarBitWidth()); + if (!Mask) + return std::unexpected(Mask.error()); + auto MaskRegister = buildConstant(*DestinationType, std::move(*Mask)); + if (!MaskRegister) + return std::unexpected(MaskRegister.error()); + auto Built = Builder.buildAnd(*Destination, WideSource, *MaskRegister); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + + if (hasOpcode(SourceDefinition, GenericOpcode::G_ZEXT)) { + const auto ExtensionSource = registerOperand(*SourceDefinition, 1); + if (!ExtensionSource) + return false; + Observer.changingInstr(Instruction); + auto Updated = Instruction.operandAt(1)->setRegister(*ExtensionSource); + if (!Updated) + return std::unexpected(Updated.error()); + Observer.changedInstr(Instruction); + UpdatedDefs.push_back(*Destination); + markDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + + if (hasOpcode(SourceDefinition, GenericOpcode::G_CONSTANT)) { + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto Legal = isInstLegal(LegalizationQuery( + opcode(GenericOpcode::G_CONSTANT), {*DestinationType})); + if (!Legal) + return std::unexpected(Legal.error()); + const auto *Immediate = + SourceDefinition->operandAt(1)->getIf<IntegerImmediateOperand>(); + if (*Legal && Immediate != nullptr) { + auto Extended = Immediate->Value.zext(DestinationType->scalarBitWidth()); + if (!Extended) + return std::unexpected(Extended.error()); + auto Built = buildConstant(*Destination, std::move(*Extended)); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + } + return tryFoldImplicitDef(Instruction, DeadInstructions, UpdatedDefs, + Observer); +} + +inline Expected<bool> LegalizationArtifactCombiner::tryCombineSExt( + MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, GISelObserverWrapper &Observer) { + const auto Destination = registerOperand(Instruction, 0); + const auto OriginalSource = registerOperand(Instruction, 1); + if (!Destination || !OriginalSource) + return makeUnexpected(ErrorCode::VerificationError, + "G_SEXT has invalid register operands"); + const Register Source = lookThroughCopyInstrs(*OriginalSource); + MachineInstr *SourceDefinition = MRI.getVRegDef(Source); + if (SourceDefinition == nullptr) + return false; + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + if (hasOpcode(SourceDefinition, GenericOpcode::G_TRUNC)) { + const auto TruncSource = registerOperand(*SourceDefinition, 1); + if (!TruncSource) + return false; + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto SourceType = MRI.getType(Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + auto Unsupported = isInstUnsupported(LegalizationQuery( + opcode(GenericOpcode::G_SEXT_INREG), {*DestinationType})); + if (!Unsupported) + return std::unexpected(Unsupported.error()); + if (*Unsupported) + return false; + Register WideSource = *TruncSource; + auto WideSourceType = MRI.getType(WideSource); + if (!WideSourceType) + return std::unexpected(WideSourceType.error()); + if (*WideSourceType != *DestinationType) { + auto Wide = MRI.createVirtualRegister(*DestinationType); + if (!Wide) + return std::unexpected(Wide.error()); + auto Built = Builder.buildAnyExtOrTrunc(*Wide, WideSource); + if (!Built) + return std::unexpected(Built.error()); + WideSource = *Wide; + } + auto Built = Builder.buildSExtInReg(*Destination, WideSource, + SourceType->scalarBitWidth()); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + + if (hasOpcode(SourceDefinition, GenericOpcode::G_ZEXT) || + hasOpcode(SourceDefinition, GenericOpcode::G_SEXT)) { + const auto ExtensionSource = registerOperand(*SourceDefinition, 1); + if (!ExtensionSource) + return false; + auto Built = buildUnary( + static_cast<GenericOpcode>(SourceDefinition->opcode().value()), + *Destination, *ExtensionSource); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + + if (hasOpcode(SourceDefinition, GenericOpcode::G_CONSTANT)) { + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto Legal = isInstLegal(LegalizationQuery( + opcode(GenericOpcode::G_CONSTANT), {*DestinationType})); + if (!Legal) + return std::unexpected(Legal.error()); + const auto *Immediate = + SourceDefinition->operandAt(1)->getIf<IntegerImmediateOperand>(); + if (*Legal && Immediate != nullptr) { + auto Extended = Immediate->Value.sext(DestinationType->scalarBitWidth()); + if (!Extended) + return std::unexpected(Extended.error()); + auto Built = buildConstant(*Destination, std::move(*Extended)); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + } + return tryFoldImplicitDef(Instruction, DeadInstructions, UpdatedDefs, + Observer); +} + +inline Expected<bool> LegalizationArtifactCombiner::tryFoldImplicitDef( + MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, GISelObserverWrapper &) { + const auto Destination = registerOperand(Instruction, 0); + const auto Source = registerOperand(Instruction, 1); + if (!Destination || !Source) + return false; + MachineInstr *Definition = MRI.getVRegDef(lookThroughCopyInstrs(*Source)); + if (!hasOpcode(Definition, GenericOpcode::G_IMPLICIT_DEF)) + return false; + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT)) { + auto Legal = isInstLegal(LegalizationQuery( + opcode(GenericOpcode::G_IMPLICIT_DEF), {*DestinationType})); + if (!Legal) + return std::unexpected(Legal.error()); + if (!*Legal) + return false; + auto Built = Builder.buildImplicitDef(*Destination); + if (!Built) + return std::unexpected(Built.error()); + } else { + auto Unsupported = isConstantUnsupported(*DestinationType); + if (!Unsupported) + return std::unexpected(Unsupported.error()); + if (*Unsupported) + return false; + auto Built = Builder.buildConstant(*Destination, 0); + if (!Built) + return std::unexpected(Built.error()); + } + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *Definition, DeadInstructions); + return true; +} + +inline Expected<bool> LegalizationArtifactCombiner::tryCombineTrunc( + MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, GISelObserverWrapper &Observer) { + const auto Destination = registerOperand(Instruction, 0); + const auto OriginalSource = registerOperand(Instruction, 1); + if (!Destination || !OriginalSource) + return makeUnexpected(ErrorCode::VerificationError, + "G_TRUNC has invalid register operands"); + const Register Source = lookThroughCopyInstrs(*OriginalSource); + MachineInstr *SourceDefinition = MRI.getVRegDef(Source); + if (SourceDefinition == nullptr) + return false; + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + if (hasOpcode(SourceDefinition, GenericOpcode::G_CONSTANT)) { + auto Legal = isInstLegal(LegalizationQuery( + opcode(GenericOpcode::G_CONSTANT), {*DestinationType})); + if (!Legal) + return std::unexpected(Legal.error()); + const auto *Immediate = + SourceDefinition->operandAt(1)->getIf<IntegerImmediateOperand>(); + if (*Legal && Immediate != nullptr) { + auto Truncated = Immediate->Value.trunc(DestinationType->scalarBitWidth()); + if (!Truncated) + return std::unexpected(Truncated.error()); + auto Built = buildConstant(*Destination, std::move(*Truncated)); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + } + + if (GMerge::classof(SourceDefinition)) { + auto &Merge = castGenericMachineInstr<GMerge>(*SourceDefinition); + if (Merge.getNumSources() == 0) + return false; + const Register FirstSource = Merge.getSourceReg(0); + auto FirstSourceType = MRI.getType(FirstSource); + if (!FirstSourceType) + return std::unexpected(FirstSourceType.error()); + if (!DestinationType->isCategoryNeutralScalar() || + !FirstSourceType->isCategoryNeutralScalar()) + return false; + const std::uint64_t DestinationSize = DestinationType->minimumBitWidth(); + const std::uint64_t SourceSize = FirstSourceType->minimumBitWidth(); + if (DestinationSize < SourceSize) { + auto Unsupported = isInstUnsupported(LegalizationQuery( + opcode(GenericOpcode::G_TRUNC), + {*DestinationType, *FirstSourceType})); + if (!Unsupported) + return std::unexpected(Unsupported.error()); + if (*Unsupported) + return false; + auto Built = Builder.buildTrunc(*Destination, FirstSource); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + } else if (DestinationSize == SourceSize) { + auto Replaced = replaceRegOrBuildCopy(*Destination, FirstSource, + UpdatedDefs, Observer); + if (!Replaced) + return std::unexpected(Replaced.error()); + } else if (DestinationSize % SourceSize == 0) { + const std::size_t SourceCount = + static_cast<std::size_t>(DestinationSize / SourceSize); + if (SourceCount >= Merge.getNumSources()) + return false; + auto Unsupported = isInstUnsupported(LegalizationQuery( + opcode(GenericOpcode::G_MERGE_VALUES), + {*DestinationType, *FirstSourceType})); + if (!Unsupported) + return std::unexpected(Unsupported.error()); + if (*Unsupported) + return false; + std::vector<Register> Sources; + Sources.reserve(SourceCount); + for (std::size_t Index = 0; Index < SourceCount; ++Index) + Sources.push_back(Merge.getSourceReg(Index)); + auto Built = Builder.buildMergeValues(*Destination, Sources); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + } else { + return false; + } + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + + if (hasOpcode(SourceDefinition, GenericOpcode::G_TRUNC)) { + const auto TruncSource = registerOperand(*SourceDefinition, 1); + if (!TruncSource) + return false; + auto Built = Builder.buildTrunc(*Destination, *TruncSource); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + + ArtifactValueFinder Finder(*this); + auto Found = Finder.findValueFromDef(*Destination, 0, + DestinationType->minimumBitWidth(), + *DestinationType); + if (!Found) + return std::unexpected(Found.error()); + if (!Found->isNoRegister()) { + auto FoundType = MRI.getType(*Found); + if (!FoundType) + return std::unexpected(FoundType.error()); + if (*FoundType == *DestinationType) { + auto Replaced = replaceRegOrBuildCopy(*Destination, *Found, UpdatedDefs, + Observer); + if (!Replaced) + return std::unexpected(Replaced.error()); + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions); + return true; + } + } + return false; +} + +inline Expected<bool> +LegalizationArtifactCombiner::ArtifactValueFinder::tryCombineMergeLike( + GMergeLikeInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, GISelChangeObserver &Observer) { + if (Instruction.getNumSources() == 0) + return false; + const Register FirstElement = Instruction.getSourceReg(0); + auto ElementType = Combiner.MRI.getType(FirstElement); + if (!ElementType) + return std::unexpected(ElementType.error()); + const std::uint64_t ElementSize = ElementType->minimumBitWidth(); + std::size_t FirstUnmergeIndex = 0; + auto UnmergeResult = + findUnmergeThatDefinesReg(FirstElement, ElementSize, FirstUnmergeIndex); + if (!UnmergeResult) + return std::unexpected(UnmergeResult.error()); + GUnmerge *Unmerge = *UnmergeResult; + if (Unmerge == nullptr) + return false; + + const Register Destination = Instruction.getReg(0); + const Register UnmergeSource = Unmerge->getSourceReg(); + auto DestinationType = Combiner.MRI.getType(Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto UnmergeSourceType = Combiner.MRI.getType(UnmergeSource); + if (!UnmergeSourceType) + return std::unexpected(UnmergeSourceType.error()); + const std::size_t SourceCount = Instruction.getNumSources(); + + if (*DestinationType == *UnmergeSourceType && FirstUnmergeIndex == 0) { + auto Sequence = isSequenceFromUnmerge( + Instruction, 0, Unmerge, 0, SourceCount, ElementSize, + DestinationType->isVector()); + if (!Sequence) + return std::unexpected(Sequence.error()); + if (!*Sequence) + return false; + auto Replaced = Combiner.replaceRegOrBuildCopy( + Destination, UnmergeSource, UpdatedDefs, Observer); + if (!Replaced) + return std::unexpected(Replaced.error()); + DeadInstructions.push_back(&Instruction); + return true; + } + + const bool MatchingDomains = + (!DestinationType->isVector() && !UnmergeSourceType->isVector()) || + (DestinationType->isVector() && UnmergeSourceType->isVector() && + DestinationType->scalarElementType() == + UnmergeSourceType->scalarElementType()); + if (MatchingDomains && FirstUnmergeIndex % SourceCount == 0 && + UnmergeSourceType->minimumBitWidth() % + DestinationType->minimumBitWidth() == + 0) { + auto Sequence = isSequenceFromUnmerge( + Instruction, 0, Unmerge, FirstUnmergeIndex, SourceCount, ElementSize, + false); + if (!Sequence) + return std::unexpected(Sequence.error()); + if (*Sequence) { + auto NewUnmerge = Combiner.buildOrFindUnmerge(*DestinationType, + UnmergeSource); + if (!NewUnmerge) + return std::unexpected(NewUnmerge.error()); + const std::size_t DestinationIndex = static_cast<std::size_t>( + (FirstUnmergeIndex * ElementSize) / + DestinationType->minimumBitWidth()); + if (DestinationIndex >= NewUnmerge->size()) + return false; + auto Replaced = Combiner.replaceRegOrBuildCopy( + Destination, (*NewUnmerge)[DestinationIndex], UpdatedDefs, Observer); + if (!Replaced) + return std::unexpected(Replaced.error()); + DeadInstructions.push_back(&Instruction); + return true; + } + } + + if (DestinationType->isVector() == UnmergeSourceType->isVector() && + DestinationType->minimumBitWidth() % + UnmergeSourceType->minimumBitWidth() == + 0) { + const std::size_t UnmergeElementCount = Unmerge->getNumDefs(); + if (UnmergeElementCount == 0 || SourceCount % UnmergeElementCount != 0) + return false; + std::vector<Register> MergeSources; + for (std::size_t Start = 0; Start < SourceCount; + Start += UnmergeElementCount) { + std::size_t SourceDefIndex = 0; + auto SourceUnmerge = findUnmergeThatDefinesReg( + Instruction.getSourceReg(Start), ElementSize, SourceDefIndex); + if (!SourceUnmerge) + return std::unexpected(SourceUnmerge.error()); + if (*SourceUnmerge == nullptr || + (*SourceUnmerge)->getNumDefs() != UnmergeElementCount || + SourceDefIndex != 0) + return false; + auto Sequence = isSequenceFromUnmerge( + Instruction, Start, *SourceUnmerge, 0, UnmergeElementCount, + ElementSize, false); + if (!Sequence) + return std::unexpected(Sequence.error()); + if (!*Sequence) + return false; + MergeSources.push_back((*SourceUnmerge)->getSourceReg()); + } + auto Built = Combiner.buildMergeLike(Destination, MergeSources); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(Destination); + DeadInstructions.push_back(&Instruction); + return true; + } + return false; +} + +inline Expected<bool> LegalizationArtifactCombiner::tryCombineMergeLike( + MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, GISelObserverWrapper &Observer) { + if (!GMergeLikeInstr::classof(&Instruction)) + return false; + ArtifactValueFinder Finder(*this); + return Finder.tryCombineMergeLike( + castGenericMachineInstr<GMergeLikeInstr>(Instruction), DeadInstructions, + UpdatedDefs, Observer); +} + +inline Expected<bool> LegalizationArtifactCombiner::tryFoldUnmergeCast( + MachineInstr &Instruction, MachineInstr &Cast, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs) { + if (!isArtifactCast(Cast.opcode()) || + Cast.opcode() != opcode(GenericOpcode::G_TRUNC)) + return false; + const auto CastSource = registerOperand(Cast, 1); + if (!CastSource) + return false; + auto CastSourceType = MRI.getType(*CastSource); + if (!CastSourceType) + return std::unexpected(CastSourceType.error()); + const std::size_t OriginalDefCount = Instruction.operandCount() - 1; + if (OriginalDefCount == 0) + return false; + const auto FirstDestination = registerOperand(Instruction, 0); + if (!FirstDestination) + return false; + auto DestinationType = MRI.getType(*FirstDestination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (DestinationType->isVector() || !CastSourceType->isCategoryNeutralScalar() || + CastSourceType->minimumBitWidth() % + DestinationType->minimumBitWidth() != + 0) + return false; + auto Unsupported = isInstUnsupported(LegalizationQuery( + opcode(GenericOpcode::G_UNMERGE_VALUES), + {*DestinationType, *CastSourceType})); + if (!Unsupported) + return std::unexpected(Unsupported.error()); + if (*Unsupported) + return false; + const std::size_t NewDefCount = static_cast<std::size_t>( + CastSourceType->minimumBitWidth() / + DestinationType->minimumBitWidth()); + std::vector<Register> Destinations; + Destinations.reserve(NewDefCount); + for (std::size_t Index = 0; Index < NewDefCount; ++Index) { + if (Index < OriginalDefCount) { + const auto Destination = registerOperand(Instruction, Index); + if (!Destination) + return false; + Destinations.push_back(*Destination); + } else { + auto Destination = MRI.createVirtualRegister(*DestinationType); + if (!Destination) + return std::unexpected(Destination.error()); + Destinations.push_back(*Destination); + } + } + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto Built = Builder.buildUnmergeValues(Destinations, *CastSource); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.insert(UpdatedDefs.end(), Destinations.begin(), + Destinations.begin() + + static_cast<std::ptrdiff_t>(OriginalDefCount)); + markInstAndDefDead(Instruction, Cast, DeadInstructions); + return true; +} + +inline Expected<bool> LegalizationArtifactCombiner::tryCombineUnmergeValues( + MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs, GISelObserverWrapper &Observer) { + if (!GUnmerge::classof(&Instruction)) + return false; + auto &Unmerge = castGenericMachineInstr<GUnmerge>(Instruction); + if (Unmerge.getNumDefs() == 0) + return false; + const Register Source = Unmerge.getSourceReg(); + auto Definition = definitionIgnoringCopies(Source); + if (!Definition) + return std::unexpected(Definition.error()); + MachineInstr *SourceDefinition = Definition->first; + if (SourceDefinition == nullptr) + return false; + const std::size_t SourceDefIndex = getDefIndex(*SourceDefinition, + Definition->second); + auto OperandType = MRI.getType(Source); + if (!OperandType) + return std::unexpected(OperandType.error()); + auto DestinationType = MRI.getType(Unmerge.getReg(0)); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + ArtifactValueFinder Finder(*this); + auto AllCombined = Finder.tryCombineUnmergeDefs(Unmerge, Observer, + UpdatedDefs); + if (!AllCombined) + return std::unexpected(AllCombined.error()); + if (*AllCombined) { + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions, + SourceDefIndex); + return true; + } + + if (GUnmerge::classof(SourceDefinition)) { + auto &SourceUnmerge = castGenericMachineInstr<GUnmerge>(*SourceDefinition); + const Register OriginalSource = SourceUnmerge.getSourceReg(); + auto OriginalSourceType = MRI.getType(OriginalSource); + if (!OriginalSourceType) + return std::unexpected(OriginalSourceType.error()); + auto NewUnmerge = buildOrFindUnmerge(*DestinationType, OriginalSource); + if (!NewUnmerge) + return std::unexpected(NewUnmerge.error()); + const std::size_t BaseIndex = SourceDefIndex * Unmerge.getNumDefs(); + if (BaseIndex + Unmerge.getNumDefs() > NewUnmerge->size()) + return false; + for (std::size_t Index = 0; Index < Unmerge.getNumDefs(); ++Index) { + auto Replaced = replaceRegOrBuildCopy( + Unmerge.getReg(Index), (*NewUnmerge)[BaseIndex + Index], UpdatedDefs, + Observer); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + markInstAndDefDead(Instruction, *SourceDefinition, DeadInstructions, + SourceDefIndex); + return true; + } + + MachineInstr *Merge = SourceDefinition; + std::optional<Opcode> ConvertOpcode; + if (isArtifactCast(SourceDefinition->opcode())) { + ConvertOpcode = SourceDefinition->opcode(); + const auto CastSource = registerOperand(*SourceDefinition, 1); + if (!CastSource) + return false; + auto CastDefinition = definitionIgnoringCopies(*CastSource); + if (!CastDefinition) + return std::unexpected(CastDefinition.error()); + Merge = CastDefinition->first; + } + if (Merge == nullptr || + !canFoldMergeOpcode(Merge->opcode(), ConvertOpcode, *OperandType, + *DestinationType)) + return tryFoldUnmergeCast(Instruction, *SourceDefinition, + DeadInstructions, UpdatedDefs); + + const std::size_t MergeSourceCount = Merge->operandCount() - 1; + const std::size_t DefCount = Unmerge.getNumDefs(); + if (MergeSourceCount < DefCount) { + if (DefCount % MergeSourceCount != 0) + return false; + const std::size_t NewDefCount = DefCount / MergeSourceCount; + for (std::size_t SourceIndex = 0; SourceIndex < MergeSourceCount; + ++SourceIndex) { + std::vector<Register> Destinations; + Destinations.reserve(NewDefCount); + for (std::size_t Index = 0; Index < NewDefCount; ++Index) + Destinations.push_back( + Unmerge.getReg(SourceIndex * NewDefCount + Index)); + Register SplitSource = *registerOperand(*Merge, SourceIndex + 1); + if (ConvertOpcode) { + auto ConvertedType = divideType(*OperandType, MergeSourceCount); + if (!ConvertedType) + return std::unexpected(ConvertedType.error()); + auto Converted = MRI.createVirtualRegister(*ConvertedType); + if (!Converted) + return std::unexpected(Converted.error()); + auto Built = buildUnary( + static_cast<GenericOpcode>(ConvertOpcode->value()), *Converted, + SplitSource); + if (!Built) + return std::unexpected(Built.error()); + SplitSource = *Converted; + } + auto Built = Builder.buildUnmergeValues(Destinations, SplitSource); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.insert(UpdatedDefs.end(), Destinations.begin(), + Destinations.end()); + } + } else if (MergeSourceCount > DefCount) { + if (ConvertOpcode || MergeSourceCount % DefCount != 0) + return false; + const std::size_t SourcesPerDef = MergeSourceCount / DefCount; + for (std::size_t DefIndex = 0; DefIndex < DefCount; ++DefIndex) { + std::vector<Register> Sources; + Sources.reserve(SourcesPerDef); + for (std::size_t Index = 0; Index < SourcesPerDef; ++Index) { + const auto SourceReg = registerOperand( + *Merge, 1 + DefIndex * SourcesPerDef + Index); + if (!SourceReg) + return false; + Sources.push_back(*SourceReg); + } + auto Built = buildMergeLike(Unmerge.getReg(DefIndex), Sources); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(Unmerge.getReg(DefIndex)); + } + } else { + const auto FirstMergeSource = registerOperand(*Merge, 1); + if (!FirstMergeSource) + return false; + auto MergeSourceType = MRI.getType(*FirstMergeSource); + if (!MergeSourceType) + return std::unexpected(MergeSourceType.error()); + if (!ConvertOpcode && *DestinationType != *MergeSourceType) { + if (DestinationType->isPointer()) + ConvertOpcode = opcode(GenericOpcode::G_INTTOPTR); + else if (MergeSourceType->isPointer()) + ConvertOpcode = opcode(GenericOpcode::G_PTRTOINT); + else + ConvertOpcode = opcode(GenericOpcode::G_BITCAST); + } + for (std::size_t Index = 0; Index < DefCount; ++Index) { + const auto MergeSource = registerOperand(*Merge, Index + 1); + if (!MergeSource) + return false; + if (ConvertOpcode) { + if (!MRI.useEmpty(Unmerge.getReg(Index))) { + auto Built = buildUnary( + static_cast<GenericOpcode>(ConvertOpcode->value()), + Unmerge.getReg(Index), *MergeSource); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(Unmerge.getReg(Index)); + } + } else { + auto Replaced = replaceRegOrBuildCopy( + Unmerge.getReg(Index), *MergeSource, UpdatedDefs, Observer); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + } + } + markInstAndDefDead(Instruction, *Merge, DeadInstructions); + return true; +} + +inline Expected<bool> LegalizationArtifactCombiner::tryCombineExtract( + MachineInstr &Instruction, + std::vector<MachineInstr *> &DeadInstructions, + std::vector<Register> &UpdatedDefs) { + const auto Destination = registerOperand(Instruction, 0); + const auto OriginalSource = registerOperand(Instruction, 1); + const auto OffsetValue = immediateOperand(Instruction, 2); + if (!Destination || !OriginalSource || !OffsetValue || *OffsetValue < 0) + return false; + const Register Source = lookThroughCopyInstrs(*OriginalSource); + MachineInstr *Merge = MRI.getVRegDef(Source); + if (Merge == nullptr) + return false; + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + if (hasOpcode(Merge, GenericOpcode::G_IMPLICIT_DEF)) { + auto Built = Builder.buildImplicitDef(*Destination); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *Merge, DeadInstructions); + return true; + } + if (!GMergeLikeInstr::classof(Merge)) + return false; + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto SourceType = MRI.getType(Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + const std::size_t SourceCount = Merge->operandCount() - 1; + if (SourceCount == 0) + return false; + const std::uint64_t MergeSourceSize = + SourceType->minimumBitWidth() / SourceCount; + const std::uint64_t Offset = static_cast<std::uint64_t>(*OffsetValue); + const std::size_t FirstSource = + static_cast<std::size_t>(Offset / MergeSourceSize); + const std::size_t LastSource = static_cast<std::size_t>( + (Offset + DestinationType->minimumBitWidth() - 1) / MergeSourceSize); + if (FirstSource != LastSource || FirstSource >= SourceCount) + return false; + const auto MergeSource = registerOperand(*Merge, FirstSource + 1); + if (!MergeSource) + return false; + auto Built = Builder.buildExtract( + *Destination, *MergeSource, + Offset - FirstSource * MergeSourceSize); + if (!Built) + return std::unexpected(Built.error()); + UpdatedDefs.push_back(*Destination); + markInstAndDefDead(Instruction, *Merge, DeadInstructions); + return true; +} + +inline Expected<Register> +LegalizationArtifactCombiner::ArtifactValueFinder::findValueFromDef( + Register DefReg, std::uint64_t StartBit, std::uint64_t Size, + LowLevelType DestinationType) { + CurrentBest = {}; + Visited.clear(); + auto Found = findValueFromDefImpl(DefReg, StartBit, Size, DestinationType); + if (!Found) + return std::unexpected(Found.error()); + return *Found == DefReg ? Register{} : *Found; +} + +inline Expected<Register> +LegalizationArtifactCombiner::ArtifactValueFinder::findValueFromConcat( + GConcatVectors &Instruction, std::uint64_t StartBit, std::uint64_t Size, + LowLevelType DestinationType) { + if (Instruction.getNumSources() == 0) + return CurrentBest; + const Register FirstSource = Instruction.getSourceReg(0); + auto FirstType = Combiner.MRI.getType(FirstSource); + if (!FirstType) + return std::unexpected(FirstType.error()); + const std::uint64_t SourceSize = FirstType->minimumBitWidth(); + if (SourceSize == 0) + return CurrentBest; + const std::size_t SourceIndex = static_cast<std::size_t>(StartBit / SourceSize); + const std::uint64_t SourceOffset = StartBit % SourceSize; + if (SourceIndex >= Instruction.getNumSources() || + SourceOffset + Size > SourceSize) + return CurrentBest; + const Register Source = Instruction.getSourceReg(SourceIndex); + if (SourceOffset == 0 && Size == SourceSize) + CurrentBest = Source; + return findValueFromDefImpl(Source, SourceOffset, Size, DestinationType); +} + +inline Expected<Register> +LegalizationArtifactCombiner::ArtifactValueFinder::findValueFromBuildVector( + GBuildVector &Instruction, std::uint64_t StartBit, std::uint64_t Size) { + if (Instruction.getNumSources() == 0) + return CurrentBest; + const Register FirstSource = Instruction.getSourceReg(0); + auto FirstType = Combiner.MRI.getType(FirstSource); + if (!FirstType) + return std::unexpected(FirstType.error()); + const std::uint64_t SourceSize = FirstType->minimumBitWidth(); + if (SourceSize == 0 || StartBit % SourceSize != 0 || Size < SourceSize || + Size % SourceSize != 0) + return CurrentBest; + const std::size_t FirstIndex = static_cast<std::size_t>(StartBit / SourceSize); + const std::size_t SourceCount = static_cast<std::size_t>(Size / SourceSize); + if (FirstIndex + SourceCount > Instruction.getNumSources()) + return CurrentBest; + if (SourceCount == 1) + return Instruction.getSourceReg(FirstIndex); + if (FirstIndex == 0 && SourceCount == Instruction.getNumSources()) + return Instruction.getReg(0); + + auto VectorType = LowLevelType::makeFixedVector(SourceCount, *FirstType); + if (!VectorType) + return std::unexpected(VectorType.error()); + auto Legal = Combiner.isInstLegal(LegalizationQuery( + opcode(GenericOpcode::G_BUILD_VECTOR), {*VectorType, *FirstType})); + if (!Legal) + return std::unexpected(Legal.error()); + if (!*Legal) + return CurrentBest; + + std::vector<Register> Sources; + Sources.reserve(SourceCount); + for (std::size_t Index = FirstIndex; Index < FirstIndex + SourceCount; + ++Index) + Sources.push_back(Instruction.getSourceReg(Index)); + auto Destination = Combiner.MRI.createVirtualRegister(*VectorType); + if (!Destination) + return std::unexpected(Destination.error()); + auto Built = Combiner.buildMergeLike(*Destination, Sources); + if (!Built) + return std::unexpected(Built.error()); + return *Destination; +} + +inline Expected<Register> +LegalizationArtifactCombiner::ArtifactValueFinder::findValueFromInsert( + MachineInstr &Instruction, std::uint64_t StartBit, std::uint64_t Size, + LowLevelType DestinationType) { + const auto Container = registerOperand(Instruction, 1); + const auto Inserted = registerOperand(Instruction, 2); + const auto InsertOffset = immediateOperand(Instruction, 3); + if (!Container || !Inserted || !InsertOffset || *InsertOffset < 0) + return CurrentBest; + auto InsertedType = Combiner.MRI.getType(*Inserted); + if (!InsertedType) + return std::unexpected(InsertedType.error()); + const std::uint64_t Offset = static_cast<std::uint64_t>(*InsertOffset); + const std::uint64_t InsertedEnd = Offset + InsertedType->minimumBitWidth(); + const std::uint64_t End = StartBit + Size; + if (End <= Offset || InsertedEnd <= StartBit) + return findValueFromDefImpl(*Container, StartBit, Size, DestinationType); + if (Offset <= StartBit && End <= InsertedEnd) { + const std::uint64_t NewStart = StartBit - Offset; + if (NewStart == 0 && Size == InsertedType->minimumBitWidth()) + CurrentBest = *Inserted; + return findValueFromDefImpl(*Inserted, NewStart, Size, DestinationType); + } + return Register{}; +} + +inline Expected<Register> +LegalizationArtifactCombiner::ArtifactValueFinder::findValueFromExtension( + MachineInstr &Instruction, std::uint64_t StartBit, std::uint64_t Size, + LowLevelType DestinationType) { + const auto Source = registerOperand(Instruction, 1); + if (!Source) + return CurrentBest; + auto SourceType = Combiner.MRI.getType(*Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (!SourceType->isCategoryNeutralScalar() || + StartBit + Size > SourceType->minimumBitWidth()) + return CurrentBest; + if (StartBit == 0 && SourceType->minimumBitWidth() == Size) + CurrentBest = *Source; + return findValueFromDefImpl(*Source, StartBit, Size, DestinationType); +} + +inline Expected<Register> +LegalizationArtifactCombiner::ArtifactValueFinder::findValueFromTrunc( + MachineInstr &Instruction, std::uint64_t StartBit, std::uint64_t Size, + LowLevelType DestinationType) { + const auto Source = registerOperand(Instruction, 1); + if (!Source) + return CurrentBest; + auto SourceType = Combiner.MRI.getType(*Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + if (!SourceType->isCategoryNeutralScalar()) + return CurrentBest; + return findValueFromDefImpl(*Source, StartBit, Size, DestinationType); +} + +inline Expected<Register> +LegalizationArtifactCombiner::ArtifactValueFinder::findValueFromDefImpl( + Register DefReg, std::uint64_t StartBit, std::uint64_t Size, + LowLevelType DestinationType) { + if (!DefReg.isVirtual() || !Visited.insert(DefReg).second) + return CurrentBest; + auto Definition = Combiner.definitionIgnoringCopies(DefReg); + if (!Definition) + return std::unexpected(Definition.error()); + MachineInstr *Instruction = Definition->first; + DefReg = Definition->second; + if (Instruction == nullptr) + return CurrentBest; + + switch (static_cast<GenericOpcode>(Instruction->opcode().value())) { + case GenericOpcode::G_CONCAT_VECTORS: + return findValueFromConcat( + castGenericMachineInstr<GConcatVectors>(*Instruction), StartBit, Size, + DestinationType); + case GenericOpcode::G_UNMERGE_VALUES: { + auto DefType = Combiner.MRI.getType(DefReg); + if (!DefType) + return std::unexpected(DefType.error()); + const std::size_t DefIndex = getDefIndex(*Instruction, DefReg); + const auto Source = registerOperand(*Instruction, + Instruction->operandCount() - 1); + if (!Source) + return CurrentBest; + auto Found = findValueFromDefImpl( + *Source, StartBit + DefIndex * DefType->minimumBitWidth(), Size, + DestinationType); + if (!Found) + return std::unexpected(Found.error()); + if (!Found->isNoRegister()) + return *Found; + if (StartBit == 0 && Size == DefType->minimumBitWidth()) + return DefReg; + return CurrentBest; + } + case GenericOpcode::G_BUILD_VECTOR: + return findValueFromBuildVector( + castGenericMachineInstr<GBuildVector>(*Instruction), StartBit, Size); + case GenericOpcode::G_INSERT: + return findValueFromInsert(*Instruction, StartBit, Size, DestinationType); + case GenericOpcode::G_TRUNC: + return findValueFromTrunc(*Instruction, StartBit, Size, DestinationType); + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_ANYEXT: + return findValueFromExtension(*Instruction, StartBit, Size, + DestinationType); + case GenericOpcode::G_IMPLICIT_DEF: { + auto Type = Combiner.MRI.getType(DefReg); + if (!Type) + return std::unexpected(Type.error()); + if (*Type == DestinationType) + return DefReg; + auto Undef = Combiner.MRI.createVirtualRegister(DestinationType); + if (!Undef) + return std::unexpected(Undef.error()); + auto Built = Combiner.Builder.buildImplicitDef(*Undef); + if (!Built) + return std::unexpected(Built.error()); + return *Undef; + } + default: + return CurrentBest; + } +} + +inline Expected<bool> +LegalizationArtifactCombiner::ArtifactValueFinder::tryCombineUnmergeDefs( + GUnmerge &Instruction, GISelChangeObserver &Observer, + std::vector<Register> &UpdatedDefs) { + if (Instruction.getNumDefs() == 0) + return false; + auto DestinationType = Combiner.MRI.getType(Instruction.getReg(0)); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + bool AllDead = true; + for (std::size_t Index = 0; Index < Instruction.getNumDefs(); ++Index) { + const Register Def = Instruction.getReg(Index); + if (Combiner.MRI.useEmpty(Def)) + continue; + auto Found = findValueFromDef(Def, 0, DestinationType->minimumBitWidth(), + *DestinationType); + if (!Found) + return std::unexpected(Found.error()); + if (Found->isNoRegister()) { + AllDead = false; + continue; + } + auto FoundType = Combiner.MRI.getType(*Found); + if (!FoundType) + return std::unexpected(FoundType.error()); + if (*FoundType != *DestinationType) { + AllDead = false; + continue; + } + auto Replaced = Combiner.replaceRegOrBuildCopy(Def, *Found, UpdatedDefs, + Observer); + if (!Replaced) + return std::unexpected(Replaced.error()); + Observer.changingInstr(Instruction); + auto Restored = Instruction.operandAt(Index)->setRegister(Def); + if (!Restored) + return std::unexpected(Restored.error()); + Observer.changedInstr(Instruction); + } + return AllDead; +} + +inline Expected<GUnmerge *> +LegalizationArtifactCombiner::ArtifactValueFinder::findUnmergeThatDefinesReg( + Register Reg, std::uint64_t Size, std::size_t &DefOperandIndex) { + auto Type = Combiner.MRI.getType(Reg); + if (!Type) + return std::unexpected(Type.error()); + CurrentBest = {}; + Visited.clear(); + auto Found = findValueFromDefImpl(Reg, 0, Size, *Type); + if (!Found) + return std::unexpected(Found.error()); + if (Found->isNoRegister()) + return nullptr; + auto Definition = Combiner.definitionIgnoringCopies(*Found); + if (!Definition) + return std::unexpected(Definition.error()); + if (!GUnmerge::classof(Definition->first)) + return nullptr; + DefOperandIndex = getDefIndex(*Definition->first, Definition->second); + return &castGenericMachineInstr<GUnmerge>(*Definition->first); +} + +inline Expected<bool> +LegalizationArtifactCombiner::ArtifactValueFinder::isSequenceFromUnmerge( + GMergeLikeInstr &Instruction, std::size_t MergeStartIndex, + GUnmerge *Unmerge, std::size_t UnmergeStartIndex, + std::size_t ElementCount, std::uint64_t ElementSize, bool AllowUndef) { + for (std::size_t Index = 0; Index < ElementCount; ++Index) { + const Register Source = Instruction.getSourceReg(MergeStartIndex + Index); + std::size_t SourceDefIndex = 0; + auto SourceUnmerge = + findUnmergeThatDefinesReg(Source, ElementSize, SourceDefIndex); + if (!SourceUnmerge) + return std::unexpected(SourceUnmerge.error()); + if (*SourceUnmerge == Unmerge) { + if (SourceDefIndex != UnmergeStartIndex + Index) + return false; + continue; + } + MachineInstr *Definition = Combiner.MRI.getVRegDef(Source); + if (!AllowUndef || + !hasOpcode(Definition, GenericOpcode::G_IMPLICIT_DEF)) + return false; + } + return true; +} + +} // namespace rcc + +#endif // RCC_CODEGEN_GLOBALISEL_LEGALIZATIONARTIFACTCOMBINER_H diff --git a/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h b/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h new file mode 100644 index 0000000..9a34898 --- /dev/null +++ b/backend/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H +#define RCC_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/Support/Error.h" + +namespace rcc { + +class MachineFunction; +class MachineInstr; +class GISelChangeObserver; + +class LegalizerHelper final { +public: + MachineIRBuilder &MIRBuilder; + GISelChangeObserver &Observer; + + LegalizerHelper(MachineFunction &Function, const LegalizerInfo &Info, + GISelChangeObserver &Observer, MachineIRBuilder &MIRBuilder); + + [[nodiscard]] Expected<void> legalize(MachineInstr &Instruction, + const LegalizationDecision &Decision); + [[nodiscard]] Expected<void> lower(MachineInstr &Instruction, + const LegalizationDecision &Decision); + [[nodiscard]] Expected<void> lowerVAArg(MachineInstr &Instruction); + [[nodiscard]] Expected<void> widenScalar(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType WideType); + [[nodiscard]] Expected<void> widenScalarSrc(MachineInstr &Instruction, LowLevelType WideType, + std::size_t OperandIndex, + GenericOpcode ExtensionOpcode); + [[nodiscard]] Expected<void> widenScalarDst(MachineInstr &Instruction, LowLevelType WideType, + std::size_t OperandIndex = 0, + GenericOpcode TruncOpcode = GenericOpcode::G_TRUNC); + +private: + MachineFunction &Function; + MachineRegisterInfo &MRI; + const LegalizerInfo &Info; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H diff --git a/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h b/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h index 218a4c6..d3515f3 100644 --- a/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h +++ b/backend/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h @@ -3,15 +3,27 @@ #ifndef RCC_CODEGEN_GLOBALISEL_LEGALIZERINFO_H #define RCC_CODEGEN_GLOBALISEL_LEGALIZERINFO_H +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineIRTypes.h" #include "rcc/Support/Error.h" +#include <cstddef> #include <cstdint> +#include <functional> +#include <initializer_list> +#include <optional> +#include <tuple> +#include <unordered_map> +#include <utility> +#include <vector> namespace rcc { class MachineFunction; class MachineInstr; class TargetDescription; +class LegalizerHelper; enum class LegalizationAction : std::uint8_t { Legal, @@ -28,25 +40,170 @@ enum class LegalizationAction : std::uint8_t { ChangeAddressSpace, }; +struct LegalizationMemoryDesc final { + std::optional<LowLevelType> Type; + MemorySize Size; + std::uint64_t AlignmentInBits = 0; + std::uint32_t AddressSpace = 0; + AtomicOperation Operation = AtomicOperation::None; + AtomicOrdering SuccessOrdering = AtomicOrdering::NotAtomic; + AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic; + SyncScopeID SyncScope; +}; + +struct LegalizeMemoryRule final { + LowLevelType ValueType; + LowLevelType AddressType; + LowLevelType MemoryType; + std::uint64_t MinimumAlignmentInBits = 0; +}; + +struct LegalizationQuery final { + Opcode InstructionOpcode; + std::vector<LowLevelType> Types; + std::vector<std::vector<std::size_t>> TypeOperands; + std::vector<LegalizationMemoryDesc> MemoryOperands; + + LegalizationQuery(Opcode InstructionOpcode, std::vector<LowLevelType> Types, + std::vector<LegalizationMemoryDesc> MemoryOperands = {}) + : InstructionOpcode(InstructionOpcode), Types(std::move(Types)), + MemoryOperands(std::move(MemoryOperands)) {} +}; + +struct LegalizationDecision final { + LegalizationAction Action = LegalizationAction::Unsupported; + std::size_t TypeIndex = 0; + LowLevelType NewType; +}; + +using LegalityPredicate = std::function<bool(const LegalizationQuery &)>; +using LegalizeMutation = + std::function<std::pair<std::size_t, LowLevelType>(const LegalizationQuery &)>; + +class LegalizeRule final { +public: + LegalizeRule(LegalityPredicate Predicate, LegalizationAction Action, + LegalizeMutation Mutation = {}); + + [[nodiscard]] bool matches(const LegalizationQuery &Query) const; + [[nodiscard]] LegalizationDecision apply(const LegalizationQuery &Query) const; + +private: + LegalityPredicate Predicate; + LegalizationAction Action; + LegalizeMutation Mutation; +}; + +class LegalizeRuleSet final { +public: + LegalizeRuleSet &actionIf(LegalizationAction Action, LegalityPredicate Predicate, + LegalizeMutation Mutation = {}); + LegalizeRuleSet &actionFor(LegalizationAction Action, + std::initializer_list<LowLevelType> Types); + LegalizeRuleSet &actionFor( + LegalizationAction Action, + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &actionFor( + LegalizationAction Action, + std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &legalIf(LegalityPredicate Predicate); + LegalizeRuleSet &customIf(LegalityPredicate Predicate); + LegalizeRuleSet &lowerIf(LegalityPredicate Predicate); + LegalizeRuleSet &libcallIf(LegalityPredicate Predicate); + LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate); + LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation); + LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation); + LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation); + LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation); + LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate, LegalizeMutation Mutation); + + LegalizeRuleSet &legalFor(std::initializer_list<LowLevelType> Types); + LegalizeRuleSet &legalFor(bool Enabled, std::initializer_list<LowLevelType> Types); + LegalizeRuleSet &legalFor( + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &legalFor( + bool Enabled, std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &legalFor( + std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &customFor(std::initializer_list<LowLevelType> Types); + LegalizeRuleSet &customFor(bool Enabled, std::initializer_list<LowLevelType> Types); + LegalizeRuleSet &customFor( + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &customFor( + bool Enabled, std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &lowerFor(std::initializer_list<LowLevelType> Types); + LegalizeRuleSet &lowerFor( + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &libcallFor(std::initializer_list<LowLevelType> Types); + LegalizeRuleSet &libcallFor(bool Enabled, std::initializer_list<LowLevelType> Types); + LegalizeRuleSet &libcallFor( + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &libcallFor( + bool Enabled, std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types); + LegalizeRuleSet &legalForTypesWithMemDesc( + std::initializer_list<LegalizeMemoryRule> Rules); + + LegalizeRuleSet &widenScalarToNextPow2(std::size_t TypeIndex, + std::uint32_t MinimumBitWidth = 0); + LegalizeRuleSet &minScalar(std::size_t TypeIndex, LowLevelType MinimumType); + LegalizeRuleSet &minScalar(bool Enabled, std::size_t TypeIndex, + LowLevelType MinimumType); + LegalizeRuleSet &maxScalar(std::size_t TypeIndex, LowLevelType MaximumType); + LegalizeRuleSet &maxScalar(bool Enabled, std::size_t TypeIndex, + LowLevelType MaximumType); + LegalizeRuleSet &clampScalar(std::size_t TypeIndex, LowLevelType MinimumType, + LowLevelType MaximumType); + LegalizeRuleSet &scalarSameSizeAs(std::size_t TypeIndex, std::size_t ReferenceTypeIndex); + LegalizeRuleSet &lowerIfMemSizeNotByteSizePow2(std::size_t MemoryIndex = 0); + LegalizeRuleSet &alwaysLegal(); + LegalizeRuleSet &lower(); + LegalizeRuleSet &libcall(); + + [[nodiscard]] std::optional<LegalizationDecision> + apply(const LegalizationQuery &Query) const; + +private: + std::vector<LegalizeRule> Rules; +}; + class LegalizerInfo { public: virtual ~LegalizerInfo() = default; - [[nodiscard]] virtual Expected<LegalizationAction> - classify(const MachineInstr &Instruction, const MachineFunction &Function, - const TargetDescription &Target) const = 0; + [[nodiscard]] static Expected<LegalizationQuery> + buildQuery(const MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target); + + [[nodiscard]] Expected<LegalizationDecision> + getDecision(const MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target) const; + + [[nodiscard]] Expected<LegalizationDecision> + getDecision(const LegalizationQuery &Query) const; - [[nodiscard]] virtual Expected<void> lower(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target, - LegalizationAction Action) const = 0; + [[nodiscard]] Expected<bool> isLegal(const LegalizationQuery &Query) const; + [[nodiscard]] Expected<bool> isUnsupported(const LegalizationQuery &Query) const; + + [[nodiscard]] virtual Expected<void> + legalizeCustom(LegalizerHelper &Helper, MachineInstr &Instruction, + const LegalizationDecision &Decision) const = 0; // This hook performs read-only target checks before a lowering mutation is committed. [[nodiscard]] virtual Expected<void> validateLowering(const MachineInstr &, const MachineFunction &, const TargetDescription &, - LegalizationAction) const { + const LegalizationDecision &) const { return {}; } + +protected: + [[nodiscard]] LegalizeRuleSet &getActionDefinitionsBuilder(Opcode InstructionOpcode); + [[nodiscard]] LegalizeRuleSet & + getActionDefinitionsBuilder(std::initializer_list<Opcode> InstructionOpcodes); + +private: + std::unordered_map<Opcode, LegalizeRuleSet> RuleSets; + std::unordered_map<Opcode, Opcode> RuleAliases; }; } // namespace rcc diff --git a/backend/include/rcc/CodeGen/InstructionDescriptor.h b/backend/include/rcc/CodeGen/InstructionDescriptor.h index ddc42e8..ccd1b63 100644 --- a/backend/include/rcc/CodeGen/InstructionDescriptor.h +++ b/backend/include/rcc/CodeGen/InstructionDescriptor.h @@ -267,6 +267,10 @@ struct InstructionDescriptor final { InstructionSemanticVerifier VerifyInstructionSemantics = nullptr; }; +[[nodiscard]] Expected<std::vector<const OperandDescriptor *>> +resolveExplicitOperandDescriptors(const InstructionDescriptor &Descriptor, + std::size_t OperandCount); + struct ImmediatePredicate final { ImmediatePredicateID ID; std::string Name; diff --git a/backend/include/rcc/CodeGen/LowLevelType.h b/backend/include/rcc/CodeGen/LowLevelType.h index d35d30d..1915ccd 100644 --- a/backend/include/rcc/CodeGen/LowLevelType.h +++ b/backend/include/rcc/CodeGen/LowLevelType.h @@ -98,6 +98,11 @@ public: return static_cast<std::uint32_t>((RawValue >> ElementCountShift) & ElementCountMask); } + [[nodiscard]] constexpr std::uint64_t minimumBitWidth() const noexcept { + return isVector() ? static_cast<std::uint64_t>(scalarBitWidth()) * minimumElementCount() + : scalarBitWidth(); + } + [[nodiscard]] std::optional<std::uint32_t> addressSpace() const noexcept; [[nodiscard]] std::optional<LowLevelType> scalarElementType() const noexcept; diff --git a/backend/include/rcc/CodeGen/MachineFunction.h b/backend/include/rcc/CodeGen/MachineFunction.h index 563aa4e..501f334 100644 --- a/backend/include/rcc/CodeGen/MachineFunction.h +++ b/backend/include/rcc/CodeGen/MachineFunction.h @@ -37,6 +37,13 @@ class Opcode; class MachineFunction final { public: + class Delegate { + public: + virtual ~Delegate() = default; + virtual void handleInsertion(MachineInstr &Instruction) = 0; + virtual void handleRemoval(MachineInstr &Instruction) = 0; + }; + [[nodiscard]] static Expected<std::unique_ptr<MachineFunction>> create(std::string Name, MachineFunctionStableID StableID); @@ -154,6 +161,14 @@ public: [[nodiscard]] Expected<const MachineMemOperand *> createMemOperand(MachineMemOperandDesc Description); + void setDelegate(Delegate &NewDelegate) noexcept { + TheDelegate = &NewDelegate; + } + void resetDelegate(Delegate &CurrentDelegate) noexcept { + if (TheDelegate == &CurrentDelegate) + TheDelegate = nullptr; + } + [[nodiscard]] Expected<void> appendBlock(MachineBasicBlock &Block); [[nodiscard]] Expected<void> insertBlockBefore(MachineBasicBlock &Position, MachineBasicBlock &Block); @@ -224,6 +239,7 @@ private: StableIDGenerator<MachineInstrStableID> InstrIDGenerator; std::uint32_t DebugInstrNumberingCount = 0; std::unique_ptr<MachineArena> Arena; + Delegate *TheDelegate = nullptr; friend class MachineBasicBlock; friend class MachineCFGUpdater; diff --git a/backend/include/rcc/CodeGen/MachineIRBuilder.h b/backend/include/rcc/CodeGen/MachineIRBuilder.h index b8ff248..01cd648 100644 --- a/backend/include/rcc/CodeGen/MachineIRBuilder.h +++ b/backend/include/rcc/CodeGen/MachineIRBuilder.h @@ -178,8 +178,42 @@ public: } [[nodiscard]] Expected<MachineInstrBuilder> buildInstr(Opcode InstructionOpcode) const; + [[nodiscard]] Expected<MachineInstr *> buildTrunc(Register Destination, Register Source) const; + [[nodiscard]] Expected<MachineInstr *> buildCopy(Register Destination, Register Source) const; + [[nodiscard]] Expected<MachineInstr *> buildImplicitDef(Register Destination) const; + [[nodiscard]] Expected<MachineInstr *> buildAnyExt(Register Destination, + Register Source) const; + [[nodiscard]] Expected<MachineInstr *> buildZExt(Register Destination, Register Source) const; + [[nodiscard]] Expected<MachineInstr *> buildSExt(Register Destination, Register Source) const; + [[nodiscard]] Expected<MachineInstr *> buildAnyExtOrTrunc(Register Destination, + Register Source) const; + [[nodiscard]] Expected<MachineInstr *> buildSExtOrTrunc(Register Destination, + Register Source) const; + [[nodiscard]] Expected<MachineInstr *> buildConstant(Register Destination, APInt Value) const; + [[nodiscard]] Expected<MachineInstr *> buildConstant(Register Destination, + std::int64_t Value) const; + [[nodiscard]] Expected<MachineInstr *> buildSExtInReg(Register Destination, Register Source, + std::uint32_t Width) const; + [[nodiscard]] Expected<MachineInstr *> buildAnd(Register Destination, Register Left, + Register Right) const; + [[nodiscard]] Expected<MachineInstr *> buildMergeValues( + Register Destination, const std::vector<Register> &Sources) const; + [[nodiscard]] Expected<MachineInstr *> buildUnmergeValues( + const std::vector<Register> &Destinations, Register Source) const; + [[nodiscard]] Expected<MachineInstr *> buildExtract(Register Destination, Register Source, + std::uint64_t Offset) const; private: + [[nodiscard]] Expected<MachineInstr *> buildUnary(Opcode InstructionOpcode, + Register Destination, + Register Source) const; + [[nodiscard]] Expected<MachineInstr *> buildBinary(Opcode InstructionOpcode, + Register Destination, Register Left, + Register Right) const; + [[nodiscard]] Expected<MachineInstr *> buildExtOrTrunc(Opcode ExtensionOpcode, + Register Destination, + Register Source) const; + [[nodiscard]] Expected<LowLevelType> registerType(Register Reg) const; [[nodiscard]] Expected<void> validateBlock(const MachineBasicBlock &Block) const; [[nodiscard]] Expected<void> validateInstruction(const MachineInstr &Instruction) const; diff --git a/backend/include/rcc/CodeGen/MachineInstr.h b/backend/include/rcc/CodeGen/MachineInstr.h index 84d2f0e..60f8014 100644 --- a/backend/include/rcc/CodeGen/MachineInstr.h +++ b/backend/include/rcc/CodeGen/MachineInstr.h @@ -25,7 +25,7 @@ class MachineFunction; class MachineMemOperand; class MachineRegisterInfo; -class MachineInstr final { +class MachineInstr { public: MachineInstr(const MachineInstr &) = delete; MachineInstr &operator=(const MachineInstr &) = delete; diff --git a/backend/include/rcc/CodeGen/MachineRegisterInfo.h b/backend/include/rcc/CodeGen/MachineRegisterInfo.h index e0538d3..456115f 100644 --- a/backend/include/rcc/CodeGen/MachineRegisterInfo.h +++ b/backend/include/rcc/CodeGen/MachineRegisterInfo.h @@ -21,6 +21,7 @@ namespace rcc { class MachineOperand; +class MachineInstr; class RegOperandIterator final { public: @@ -153,6 +154,9 @@ public: [[nodiscard]] bool useEmpty(Register Reg, bool IncludeDebug = true) const noexcept; [[nodiscard]] bool hasOneUse(Register Reg, bool IncludeDebug = true) const noexcept; [[nodiscard]] const MachineOperand *getUniqueVRegDef(Register Reg) const noexcept; + [[nodiscard]] MachineInstr *getVRegDef(Register Reg) noexcept; + [[nodiscard]] const MachineInstr *getVRegDef(Register Reg) const noexcept; + [[nodiscard]] Expected<LowLevelType> getType(Register Reg) const; [[nodiscard]] Expected<std::size_t> replaceRegWith(Register OldReg, Register NewReg); [[nodiscard]] Expected<void> verifyDefUseChains() const; diff --git a/backend/lib/CodeGen/GenericOpcodes.cpp b/backend/lib/CodeGen/GenericOpcodes.cpp index f3ff5f1..ef51147 100644 --- a/backend/lib/CodeGen/GenericOpcodes.cpp +++ b/backend/lib/CodeGen/GenericOpcodes.cpp @@ -451,7 +451,13 @@ private: if (Predicate == nullptr || ((Generic == GO::G_ICMP) != Predicate->isInteger())) { return semanticError("generic comparison predicate kind does not match its opcode"); } - return requireI1WithShape(TypeAt(0), TypeAt(2), false); + const LowLevelType Result = TypeAt(0); + const LowLevelType Compared = TypeAt(2); + if (!Compared.isVector()) + return Result.isCategoryNeutralScalar() + ? Expected<void>{} + : semanticError("scalar generic comparison requires a scalar integer result"); + return requireI1WithShape(Result, Compared, false); } case GO::G_SCMP: case GO::G_UCMP: diff --git a/backend/lib/CodeGen/GlobalISel/CMakeLists.txt b/backend/lib/CodeGen/GlobalISel/CMakeLists.txt index 8a37497..b7064e3 100644 --- a/backend/lib/CodeGen/GlobalISel/CMakeLists.txt +++ b/backend/lib/CodeGen/GlobalISel/CMakeLists.txt @@ -2,16 +2,24 @@ set( RCCBackendGlobalISelSources + GISelChangeObserver.cpp InstructionSelect.cpp Legalizer.cpp + LegalizerHelper.cpp + LegalizerInfo.cpp RegBankSelect.cpp ) set( RCCBackendGlobalISelHeaders + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelChangeObserver.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelWorkList.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GenericMachineInstrs.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/InstructionSelect.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/InstructionSelector.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/Legalizer.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/RegBankSelect.h" ) @@ -19,14 +27,20 @@ set( target_sources(RCCBackendCodeGen PRIVATE ${RCCBackendGlobalISelSources}) rcc_backend_register_format_files( + "${CMAKE_CURRENT_SOURCE_DIR}/GISelChangeObserver.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/InstructionSelect.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/Legalizer.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerHelper.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RegBankSelect.cpp" ${RCCBackendGlobalISelHeaders} ) rcc_backend_register_tidy_files( + "${CMAKE_CURRENT_SOURCE_DIR}/GISelChangeObserver.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/InstructionSelect.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/Legalizer.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerHelper.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RegBankSelect.cpp" ) diff --git a/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp b/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp new file mode 100644 index 0000000..4ce1473 --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h" + +#include <algorithm> + +namespace rcc { + +void GISelObserverWrapper::addObserver(GISelChangeObserver &Observer) { + Observers.push_back(&Observer); +} + +void GISelObserverWrapper::removeObserver(GISelChangeObserver &Observer) { + const auto Position = std::ranges::find(Observers, &Observer); + if (Position != Observers.end()) + Observers.erase(Position); +} + +void GISelObserverWrapper::clearObservers() noexcept { + Observers.clear(); +} + +void GISelObserverWrapper::erasingInstr(MachineInstr &Instruction) { + for (GISelChangeObserver *Observer : Observers) + Observer->erasingInstr(Instruction); +} + +void GISelObserverWrapper::createdInstr(MachineInstr &Instruction) { + for (GISelChangeObserver *Observer : Observers) + Observer->createdInstr(Instruction); +} + +void GISelObserverWrapper::changingInstr(MachineInstr &Instruction) { + for (GISelChangeObserver *Observer : Observers) + Observer->changingInstr(Instruction); +} + +void GISelObserverWrapper::changedInstr(MachineInstr &Instruction) { + for (GISelChangeObserver *Observer : Observers) + Observer->changedInstr(Instruction); +} + +void GISelObserverWrapper::handleInsertion(MachineInstr &Instruction) { + createdInstr(Instruction); +} + +void GISelObserverWrapper::handleRemoval(MachineInstr &Instruction) { + erasingInstr(Instruction); +} + +MachineFunctionDelegateInstaller::MachineFunctionDelegateInstaller( + MachineFunction &Function, MachineFunction::Delegate &Delegate) + : Function(&Function), Delegate(&Delegate) { + Function.setDelegate(Delegate); +} + +MachineFunctionDelegateInstaller::~MachineFunctionDelegateInstaller() { + Function->resetDelegate(*Delegate); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp index 83efb6a..d971b42 100644 --- a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp +++ b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp @@ -2,6 +2,11 @@ #include "rcc/CodeGen/GlobalISel/Legalizer.h" +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "rcc/CodeGen/GlobalISel/GISelWorkList.h" +#include "rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h" +#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h" #include "rcc/CodeGen/LiveVariables.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFunction.h" @@ -10,8 +15,10 @@ #include "rcc/CodeGen/MachineRegisterInfo.h" #include "rcc/CodeGen/TargetDescription.h" -#include <cstdint> +#include <algorithm> +#include <cstddef> #include <optional> +#include <ranges> #include <string> #include <unordered_set> #include <utility> @@ -34,60 +41,122 @@ namespace { return Contract; } -struct OpcodeState final { - MachineInstrStableID Instruction; - Opcode OpcodeValue; - std::uint64_t OperandFingerprint = 0; +[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) { + std::vector<MachineInstr *> Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) + Result.push_back(Instruction); + return Result; +} - friend bool operator==(const OpcodeState &Left, const OpcodeState &Right) noexcept = default; -}; +[[nodiscard]] bool isArtifact(const MachineInstr &Instruction) noexcept { + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + default: + return false; + case GenericOpcode::G_TRUNC: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_MERGE_VALUES: + case GenericOpcode::G_UNMERGE_VALUES: + case GenericOpcode::G_CONCAT_VECTORS: + case GenericOpcode::G_BUILD_VECTOR: + case GenericOpcode::G_EXTRACT: + case GenericOpcode::G_INSERT: + return true; + } +} + +using InstList = GISelWorkList<256>; +using ArtifactList = GISelWorkList<128>; + +class LegalizerWorkListManager final : public GISelChangeObserver { +public: + LegalizerWorkListManager(InstList &Instructions, ArtifactList &Artifacts) + : Instructions(Instructions), Artifacts(Artifacts) {} -struct OpcodeStateHash final { - [[nodiscard]] std::size_t operator()(const OpcodeState &State) const noexcept { - return std::hash<std::uint64_t>{}(State.Instruction.value()) ^ - (std::hash<std::uint32_t>{}(State.OpcodeValue.value()) << 1U) ^ - (std::hash<std::uint64_t>{}(State.OperandFingerprint) << 2U); + void erasingInstr(MachineInstr &Instruction) override { + Instructions.remove(&Instruction); + Artifacts.remove(&Instruction); } -}; -void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept { - State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U); -} + void createdInstr(MachineInstr &Instruction) override { + createdOrChangedInstr(Instruction); + } + + void changingInstr(MachineInstr &) override {} -[[nodiscard]] std::uint64_t operandFingerprint(const MachineInstr &Instruction) noexcept { - std::uint64_t Result = 0xcbf29ce484222325ULL; - combineFingerprint(Result, Instruction.flags().rawValue()); - combineFingerprint(Result, Instruction.operandCount()); - for (const MachineOperand &Operand : Instruction.operands()) { - combineFingerprint(Result, static_cast<std::uint64_t>(Operand.kind())); - combineFingerprint(Result, Operand.targetFlags()); - combineFingerprint(Result, Operand.registerFlags().rawValue()); - combineFingerprint(Result, Operand.subReg().value()); - if (const Register *Reg = Operand.getIf<Register>()) - combineFingerprint(Result, Reg->opaqueValue()); - else if (const std::int64_t *Immediate = Operand.getIf<std::int64_t>()) - combineFingerprint(Result, static_cast<std::uint64_t>(*Immediate)); - else if (const auto *Integer = Operand.getIf<IntegerImmediateOperand>()) - combineFingerprint(Result, Integer->Value.bitWidth()); - else if (const auto *Floating = Operand.getIf<FloatingImmediateOperand>()) - combineFingerprint(Result, static_cast<std::uint64_t>(Floating->Bits.semantics())); - else if (const auto *Block = Operand.getIf<BasicBlockOperand>()) - combineFingerprint(Result, Block->Block == nullptr ? 0 : Block->Block->stableID().value()); + void changedInstr(MachineInstr &Instruction) override { + createdOrChangedInstr(Instruction); } - return Result; -} -[[nodiscard]] OpcodeState stateOf(const MachineInstr &Instruction) noexcept { - return {Instruction.stableID(), Instruction.opcode(), operandFingerprint(Instruction)}; -} +private: + void createdOrChangedInstr(MachineInstr &Instruction) { + if (!isGenericOpcode(Instruction.opcode())) + return; + if (isArtifact(Instruction)) + Artifacts.insert(&Instruction); + else + Instructions.insert(&Instruction); + } -[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) { - std::vector<MachineInstr *> Result; + InstList &Instructions; + ArtifactList &Artifacts; +}; + +[[nodiscard]] std::vector<MachineBasicBlock *> reversePostOrderBlocks(MachineFunction &Function) { + struct TraversalFrame final { + MachineBasicBlock *Block = nullptr; + std::vector<MachineBasicBlock *> Successors; + std::size_t NextSuccessor = 0; + }; + + const auto makeFrame = [](MachineBasicBlock &Block) { + TraversalFrame Frame; + Frame.Block = &Block; + for (const MachineSuccessor &Successor : Block.successors()) { + if (Successor.Block != nullptr) + Frame.Successors.push_back(Successor.Block); + } + std::ranges::sort(Frame.Successors, {}, [](const MachineBasicBlock *Successor) { + return Successor->stableID().value(); + }); + return Frame; + }; + + std::vector<MachineBasicBlock *> Roots; for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) - for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; - Instruction = Instruction->next()) - Result.push_back(Instruction); - return Result; + Roots.push_back(Block); + std::ranges::sort(Roots, {}, + [](const MachineBasicBlock *Block) { return Block->stableID().value(); }); + if (MachineBasicBlock *Entry = Function.firstBlock(); Entry != nullptr) { + const auto Position = std::ranges::find(Roots, Entry); + if (Position != Roots.end()) + std::rotate(Roots.begin(), Position, std::next(Position)); + } + + std::vector<MachineBasicBlock *> PostOrder; + std::unordered_set<MachineBasicBlock *> Visited; + std::vector<TraversalFrame> Stack; + for (MachineBasicBlock *Root : Roots) { + if (!Visited.insert(Root).second) + continue; + Stack.push_back(makeFrame(*Root)); + while (!Stack.empty()) { + TraversalFrame &Frame = Stack.back(); + if (Frame.NextSuccessor < Frame.Successors.size()) { + MachineBasicBlock *Successor = Frame.Successors[Frame.NextSuccessor++]; + if (Visited.insert(Successor).second) + Stack.push_back(makeFrame(*Successor)); + continue; + } + PostOrder.push_back(Frame.Block); + Stack.pop_back(); + } + } + std::ranges::reverse(PostOrder); + return PostOrder; } [[nodiscard]] Expected<void> requireProperties(const MachineFunction &Function, @@ -202,91 +271,136 @@ Expected<MachinePassResult> Legalizer::run(MachineFunction &Function, Expected<MachinePassResult> Legalizer::runImplementation(MachineFunction &Function, const TargetDescription &Target, MachineActionReplay &Actions) { - MachinePassResult Result; - std::unordered_set<OpcodeState, OpcodeStateHash> SeenStates; - - std::vector<MachineInstr *> Worklist = layoutInstructions(Function); - std::unordered_set<MachineInstrStableID> PendingInstructions; - for (const MachineInstr *Instruction : Worklist) - PendingInstructions.insert(Instruction->stableID()); - for (std::size_t Index = 0; Index < Worklist.size(); ++Index) { - MachineInstr *Instruction = Worklist[Index]; - if (Instruction != nullptr) - PendingInstructions.erase(Instruction->stableID()); - if (Instruction == nullptr || Instruction->parent() == nullptr) - continue; + InstList InstructionWorklist; + ArtifactList ArtifactWorklist; + std::vector<MachineInstr *> RetryList; - const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); - if (Descriptor == nullptr) - return std::unexpected( - passFailure("legalizer encountered an unknown instruction descriptor")); - if (!Descriptor->Flags.has(InstructionFlag::Generic)) - continue; + for (MachineBasicBlock *Block : reversePostOrderBlocks(Function)) { + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) { + if (!isGenericOpcode(Instruction->opcode())) + continue; + if (isArtifact(*Instruction)) + ArtifactWorklist.deferredInsert(Instruction); + else + InstructionWorklist.deferredInsert(Instruction); + } + } + ArtifactWorklist.finalize(); + InstructionWorklist.finalize(); - auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); - if (!Validation) - return std::unexpected(withErrorContext(Validation.error(), "legalizer input is invalid: ")); + LegalizerWorkListManager WorkListObserver(InstructionWorklist, ArtifactWorklist); + GISelObserverWrapper Observer(WorkListObserver); + MachineFunctionDelegateInstaller ObserverInstaller(Function, Observer); + MachineIRBuilder Builder(Function, Target); + LegalizerHelper Helper(Function, *Info, Observer, Builder); + LegalizationArtifactCombiner ArtifactCombiner(Builder, Function.registerInfo(), *Info); - auto Action = Info->classify(*Instruction, Function, Target); - if (!Action) - return std::unexpected(Action.error()); - if (*Action == LegalizationAction::Legal) - continue; - if (*Action == LegalizationAction::Unsupported) { - return std::unexpected( - passFailure("target legalizer rejected generic instruction '" + Descriptor->Name + "'", - ErrorCode::Unsupported)); - } + do { + while (!InstructionWorklist.empty()) { + MachineInstr *Instruction = InstructionWorklist.popBack(); + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; - const OpcodeState Before = stateOf(*Instruction); - if (!SeenStates.insert(Before).second) - return std::unexpected( - passFailure("target legalizer entered a repeated state for '" + Descriptor->Name + "'", - ErrorCode::InternalError)); + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return std::unexpected( + passFailure("legalizer encountered an unknown instruction descriptor")); + if (!Descriptor->Flags.has(InstructionFlag::Generic)) + continue; - auto LoweringValidation = Info->validateLowering(*Instruction, Function, Target, *Action); - if (!LoweringValidation) - return std::unexpected( - withErrorContext(LoweringValidation.error(), "legalizer lowering precondition failed: ")); - std::string Rule = "legalize."; - Rule.append(legalizationActionName(*Action)); - Rule.push_back('.'); - Rule.append(Descriptor->Name); - auto Accepted = Actions.request(Rule, Function, Instruction->stableID()); - if (!Accepted) - return std::unexpected(Accepted.error()); - if (!*Accepted) { - return std::unexpected( - passFailure("required legalizer action was rejected for '" + Descriptor->Name + "'", - ErrorCode::VerificationError)); + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) { + return std::unexpected( + withErrorContext(Validation.error(), "legalizer input is invalid: ")); + } + + auto Decision = Info->getDecision(*Instruction, Function, Target); + if (!Decision) + return std::unexpected(Decision.error()); + if (Decision->Action == LegalizationAction::Legal) + continue; + if (Decision->Action == LegalizationAction::Unsupported) { + if (isArtifact(*Instruction)) { + RetryList.push_back(Instruction); + continue; + } + return std::unexpected( + passFailure("target legalizer rejected generic instruction '" + Descriptor->Name + "'", + ErrorCode::Unsupported)); + } + + auto LoweringValidation = Info->validateLowering(*Instruction, Function, Target, *Decision); + if (!LoweringValidation) { + return std::unexpected(withErrorContext(LoweringValidation.error(), + "legalizer lowering precondition failed: ")); + } + std::string Rule = "legalize."; + Rule.append(legalizationActionName(Decision->Action)); + Rule.push_back('.'); + Rule.append(Descriptor->Name); + auto Accepted = Actions.request(Rule, Function, Instruction->stableID()); + if (!Accepted) + return std::unexpected(Accepted.error()); + if (!*Accepted) { + return std::unexpected( + passFailure("required legalizer action was rejected for '" + Descriptor->Name + "'", + ErrorCode::VerificationError)); + } + Expected<void> Lowered = Helper.legalize(*Instruction, *Decision); + if (!Lowered) { + if (Lowered.error().code() == ErrorCode::Unsupported && Instruction->parent() != nullptr && + isArtifact(*Instruction)) { + RetryList.push_back(Instruction); + continue; + } + return std::unexpected(Lowered.error()); + } + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); } - auto Lowered = Info->lower(*Instruction, Function, Target, *Action); - if (!Lowered) - return std::unexpected(Lowered.error()); - const bool InstructionWasErased = Instruction->parent() == nullptr; - if (!InstructionWasErased) { - const OpcodeState After = stateOf(*Instruction); - if (After == Before) - return std::unexpected(passFailure("target legalizer made no observable progress for '" + - Descriptor->Name + "'", - ErrorCode::InternalError)); + + if (!RetryList.empty()) { + if (!ArtifactWorklist.empty()) { + for (MachineInstr *Instruction : RetryList) + ArtifactWorklist.insert(Instruction); + RetryList.clear(); + } else { + MachineInstr *Instruction = RetryList.front(); + const InstructionDescriptor *Descriptor = + Instruction == nullptr ? nullptr : Target.instructionDescriptor(Instruction->opcode()); + return std::unexpected(passFailure( + "target legalizer rejected generic artifact '" + + std::string(Descriptor == nullptr ? "<unknown>" : Descriptor->Name) + "'", + ErrorCode::Unsupported)); + } } - Result.Changed = true; - Result.Changes.add(MachinePassChange::Instructions); - - const std::vector<MachineInstr *> Current = layoutInstructions(Function); - for (MachineInstr *Candidate : Current) { - const InstructionDescriptor *CandidateDescriptor = - Target.instructionDescriptor(Candidate->opcode()); - if (CandidateDescriptor == nullptr) - return std::unexpected(passFailure("legalizer produced an unknown instruction descriptor")); - if (CandidateDescriptor->Flags.has(InstructionFlag::Generic) && - PendingInstructions.insert(Candidate->stableID()).second) { - Worklist.push_back(Candidate); + + while (!ArtifactWorklist.empty()) { + MachineInstr *Instruction = ArtifactWorklist.popBack(); + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + std::vector<MachineInstr *> DeadInstructions; + auto Combined = + ArtifactCombiner.tryCombineInstruction(*Instruction, DeadInstructions, Observer); + if (!Combined) + return std::unexpected(Combined.error()); + if (*Combined) { + for (MachineInstr *Dead : DeadInstructions) { + if (Dead == nullptr || Dead->parent() == nullptr) + continue; + auto Removed = Dead->parent()->removeInstruction(*Dead); + if (!Removed) + return std::unexpected(Removed.error()); + } + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + continue; } + InstructionWorklist.insert(Instruction); } - } + } while (!InstructionWorklist.empty()); for (MachineInstr *Instruction : layoutInstructions(Function)) { const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); @@ -294,15 +408,16 @@ Expected<MachinePassResult> Legalizer::runImplementation(MachineFunction &Functi return std::unexpected(passFailure("legalizer produced an unknown instruction descriptor")); if (!Descriptor->Flags.has(InstructionFlag::Generic)) continue; - auto Action = Info->classify(*Instruction, Function, Target); - if (!Action) - return std::unexpected(Action.error()); - if (*Action != LegalizationAction::Legal) - return std::unexpected( - passFailure("legalizer reached a fixed point with an illegal generic instruction '" + - Descriptor->Name + "'", - *Action == LegalizationAction::Unsupported ? ErrorCode::Unsupported - : ErrorCode::InternalError)); + auto Decision = Info->getDecision(*Instruction, Function, Target); + if (!Decision) + return std::unexpected(Decision.error()); + if (Decision->Action != LegalizationAction::Legal && + Decision->Action != LegalizationAction::Custom) + return std::unexpected(passFailure( + "legalizer reached a fixed point with an illegal generic instruction '" + + Descriptor->Name + "'", + Decision->Action == LegalizationAction::Unsupported ? ErrorCode::Unsupported + : ErrorCode::InternalError)); auto FinalValidation = Target.validateInstruction(*Instruction, Function.registerInfo()); if (!FinalValidation) return std::unexpected( diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp new file mode 100644 index 0000000..1e52214 --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -0,0 +1,590 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h" + +#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" + +#include <algorithm> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <utility> + +namespace rcc { +namespace { + +struct ScalarLayout final { + std::uint64_t StoreSize = 0; + std::uint64_t AllocationSize = 0; + std::uint64_t ABIAlignment = 0; +}; + +[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) { + if (const std::int64_t *Value = Operand.getIf<std::int64_t>()) + return *Value; + const auto *Integer = Operand.getIf<IntegerImmediateOperand>(); + if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) { + return makeUnexpected(ErrorCode::VerificationError, + "legalizer requires an immediate no wider than 64 bits"); + } + std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front(); + const std::uint32_t Width = Integer->Value.bitWidth(); + if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0) + Raw |= ~std::uint64_t{0} << Width; + return std::bit_cast<std::int64_t>(Raw); +} + +[[nodiscard]] std::optional<ScalarLayout> scalarLayout(LowLevelType Type) noexcept { + if (!Type.isScalarValue() || Type.scalarBitWidth() == 0) + return std::nullopt; + const std::uint64_t StoreSize = (Type.scalarBitWidth() + 7U) / 8U; + const std::uint64_t AllocationSize = std::bit_ceil(StoreSize); + return ScalarLayout{StoreSize, AllocationSize, std::min<std::uint64_t>(AllocationSize, 8)}; +} + +[[nodiscard]] Expected<const MachineMemOperand *> createMemoryAccess(MachineFunction &Function, + MemoryAccessFlag Access, + LowLevelType Type, + const ScalarLayout &Layout) { + auto Size = MemorySize::fixed(Layout.StoreSize); + auto Align = Alignment::create(Layout.ABIAlignment); + if (!Size || !Align) + return makeUnexpected(ErrorCode::InternalError, + "legalizer cannot represent scalar memory layout"); + MachineMemOperandDesc Description; + Description.Accesses = {Access}; + Description.Size = *Size; + Description.MemoryType = Type; + Description.Align = *Align; + return Function.createMemOperand(std::move(Description)); +} + +[[nodiscard]] Expected<Register> buildConstantBefore(MachineIRBuilder &Builder, + MachineInstr &Anchor, LowLevelType Type, + std::uint64_t Value) { + auto Result = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + auto Immediate = APInt::fromUnsigned(Type.scalarBitWidth(), Value); + if (!Immediate) + return std::unexpected(Immediate.error()); + if (auto Point = Builder.setInsertPointBefore(Anchor); !Point) + return std::unexpected(Point.error()); + auto Built = Builder.buildConstant(*Result, std::move(*Immediate)); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LowLevelType Type, + Register Left, Register Right) { + auto Result = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + if (auto Point = Builder.setInsertPointBefore(Anchor); !Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(Generic)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Left); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Right); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +} // namespace + +LegalizerHelper::LegalizerHelper(MachineFunction &Function, const LegalizerInfo &Info, + GISelChangeObserver &Observer, MachineIRBuilder &MIRBuilder) + : MIRBuilder(MIRBuilder), Observer(Observer), Function(Function), MRI(Function.registerInfo()), + Info(Info) {} + +Expected<void> LegalizerHelper::widenScalarSrc(MachineInstr &Instruction, LowLevelType WideType, + std::size_t OperandIndex, + GenericOpcode ExtensionOpcode) { + MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Source == nullptr || !Source->isVirtual() || !Operand->isUse()) + return makeUnexpected(ErrorCode::VerificationError, + "scalar widening source is not a virtual-register use"); + + auto WideSource = MRI.createVirtualRegister(WideType); + if (!WideSource) + return std::unexpected(WideSource.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto Extension = MIRBuilder.buildInstr(opcode(ExtensionOpcode)); + if (!Extension) + return std::unexpected(Extension.error()); + if (auto Added = Extension->addDef(*WideSource); !Added) + return std::unexpected(Added.error()); + if (auto Added = Extension->addUse(*Source); !Added) + return std::unexpected(Added.error()); + if (auto Built = Extension->build(); !Built) + return std::unexpected(Built.error()); + return Operand->setRegister(*WideSource); +} + +Expected<void> LegalizerHelper::widenScalarDst(MachineInstr &Instruction, LowLevelType WideType, + std::size_t OperandIndex, + GenericOpcode TruncOpcode) { + MachineOperand *Operand = Instruction.operandAt(OperandIndex); + const Register *Destination = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Destination == nullptr || !Destination->isVirtual() || !Operand->isDef()) + return makeUnexpected(ErrorCode::VerificationError, + "scalar widening destination is not a virtual-register definition"); + + auto WideDestination = MRI.createVirtualRegister(WideType); + if (!WideDestination) + return std::unexpected(WideDestination.error()); + if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point) + return std::unexpected(Point.error()); + auto Truncation = MIRBuilder.buildInstr(opcode(TruncOpcode)); + if (!Truncation) + return std::unexpected(Truncation.error()); + if (auto Added = Truncation->addDef(*Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Truncation->addUse(*WideDestination); !Added) + return std::unexpected(Added.error()); + if (auto Built = Truncation->build(); !Built) + return std::unexpected(Built.error()); + return Operand->setRegister(*WideDestination); +} + +Expected<void> LegalizerHelper::widenScalar(MachineInstr &Instruction, std::size_t TypeIndex, + LowLevelType WideType) { + auto Query = LegalizerInfo::buildQuery(Instruction, Function, MIRBuilder.target()); + if (!Query) + return std::unexpected(Query.error()); + if (!WideType.isCategoryNeutralScalar() || TypeIndex >= Query->Types.size() || + !Query->Types[TypeIndex].isCategoryNeutralScalar() || + WideType.scalarBitWidth() <= Query->Types[TypeIndex].scalarBitWidth()) { + return makeUnexpected(ErrorCode::VerificationError, + "scalar widening has an invalid type index or target type"); + } + + const auto widenSource = [&](std::size_t OperandIndex, GenericOpcode ExtensionOpcode) { + return widenScalarSrc(Instruction, WideType, OperandIndex, ExtensionOpcode); + }; + const auto widenDestination = [&](std::size_t OperandIndex = 0, + GenericOpcode TruncOpcode = GenericOpcode::G_TRUNC) { + return widenScalarDst(Instruction, WideType, OperandIndex, TruncOpcode); + }; + const auto apply = [](Expected<void> Result) -> Expected<void> { + if (!Result) + return std::unexpected(Result.error()); + return {}; + }; + + Observer.changingInstr(Instruction); + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + case GenericOpcode::G_ATOMICRMW_XCHG: + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + case GenericOpcode::G_ATOMICRMW_AND: + case GenericOpcode::G_ATOMICRMW_OR: + case GenericOpcode::G_ATOMICRMW_XOR: + case GenericOpcode::G_ATOMICRMW_MIN: + case GenericOpcode::G_ATOMICRMW_MAX: + case GenericOpcode::G_ATOMICRMW_UMIN: + case GenericOpcode::G_ATOMICRMW_UMAX: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "atomic RMW value widening has an invalid type index"); + if (auto Result = apply(widenSource(2, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_ATOMIC_CMPXCHG: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "compare-exchange widening has an invalid type index"); + if (auto Result = apply(widenSource(2, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenSource(3, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: + if (TypeIndex == 0) { + if (auto Result = apply(widenSource(3, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenSource(4, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "compare-exchange success widening has an invalid type index"); + if (auto Result = apply(widenDestination(1)); !Result) + return Result; + break; + + case GenericOpcode::G_ADD: + case GenericOpcode::G_SUB: + case GenericOpcode::G_MUL: + case GenericOpcode::G_AND: + case GenericOpcode::G_OR: + case GenericOpcode::G_XOR: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "binary operation widening has an invalid type index"); + if (auto Result = apply(widenSource(1, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenSource(2, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_SDIV: + case GenericOpcode::G_SREM: + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_ABDS: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "signed operation widening has an invalid type index"); + if (auto Result = apply(widenSource(1, GenericOpcode::G_SEXT)); !Result) + return Result; + if (auto Result = apply(widenSource(2, GenericOpcode::G_SEXT)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_UDIV: + case GenericOpcode::G_UREM: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: + case GenericOpcode::G_ABDU: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "unsigned operation widening has an invalid type index"); + if (auto Result = apply(widenSource(1, GenericOpcode::G_ZEXT)); !Result) + return Result; + if (auto Result = apply(widenSource(2, GenericOpcode::G_ZEXT)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_SHL: + case GenericOpcode::G_LSHR: + case GenericOpcode::G_ASHR: + if (TypeIndex == 0) { + GenericOpcode ExtensionOpcode = GenericOpcode::G_ANYEXT; + if (Instruction.opcode() == opcode(GenericOpcode::G_LSHR)) + ExtensionOpcode = GenericOpcode::G_ZEXT; + else if (Instruction.opcode() == opcode(GenericOpcode::G_ASHR)) + ExtensionOpcode = GenericOpcode::G_SEXT; + if (auto Result = apply(widenSource(1, ExtensionOpcode)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "shift widening has an invalid type index"); + if (auto Result = apply(widenSource(2, GenericOpcode::G_ZEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_ROTL: + case GenericOpcode::G_ROTR: + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::Unsupported, "rotate value widening is not defined"); + if (auto Result = apply(widenSource(2, GenericOpcode::G_ZEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_FREEZE: + case GenericOpcode::G_CONSTANT_FOLD_BARRIER: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "unary operation widening has an invalid type index"); + if (auto Result = apply(widenSource(1, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_ABS: + if (TypeIndex != 0) + return makeUnexpected(ErrorCode::VerificationError, + "absolute-value widening has an invalid type index"); + if (auto Result = apply(widenSource(1, GenericOpcode::G_SEXT)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + + case GenericOpcode::G_SELECT: + if (TypeIndex == 0) { + if (auto Result = apply(widenSource(2, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenSource(3, GenericOpcode::G_ANYEXT)); !Result) + return Result; + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "select widening has an invalid type index"); + if (auto Result = apply(widenSource(1, GenericOpcode::G_ZEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_FPEXT: + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::Unsupported, + "floating-point extension destination widening is not defined"); + if (auto Result = apply(widenSource(1, GenericOpcode::G_FPEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_FPTOSI: + case GenericOpcode::G_FPTOUI: + case GenericOpcode::G_INTRINSIC_LRINT: + case GenericOpcode::G_INTRINSIC_LLRINT: + case GenericOpcode::G_IS_FPCLASS: + if (TypeIndex == 0) { + if (auto Result = apply(widenDestination()); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "floating-to-integer widening has an invalid type index"); + if (auto Result = apply(widenSource(1, GenericOpcode::G_FPEXT)); !Result) + return Result; + break; + + case GenericOpcode::G_SITOFP: + case GenericOpcode::G_UITOFP: + if (TypeIndex == 0) { + if (auto Result = apply(widenDestination(0, GenericOpcode::G_FPTRUNC)); !Result) + return Result; + break; + } + if (TypeIndex != 1) + return makeUnexpected(ErrorCode::VerificationError, + "integer-to-floating widening has an invalid type index"); + if (auto Result = apply(widenSource(1, Instruction.opcode() == opcode(GenericOpcode::G_SITOFP) + ? GenericOpcode::G_SEXT + : GenericOpcode::G_ZEXT)); + !Result) + return Result; + break; + + default: + return makeUnexpected(ErrorCode::Unsupported, + "scalar widening is not implemented for this opcode"); + } + + Observer.changedInstr(Instruction); + return {}; +} + +Expected<void> LegalizerHelper::lowerVAArg(MachineInstr &Instruction) { + if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 || + !Instruction.memOperands().empty()) { + return makeUnexpected(ErrorCode::VerificationError, "G_VAARG has invalid operands"); + } + + MachineOperand *DestinationOperand = Instruction.operandAt(0); + MachineOperand *ListOperand = Instruction.operandAt(1); + MachineOperand *AlignmentOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>(); + if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() || + !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() || + AlignmentOperand == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "G_VAARG requires virtual-register operands"); + } + + auto DestinationInfo = MRI.virtualRegisterInfo(*Destination); + auto PointerInfo = MRI.virtualRegisterInfo(*ListPointer); + auto RequestedAlignment = signedImmediate(*AlignmentOperand); + if (!DestinationInfo || !PointerInfo || !DestinationInfo->get().Type || + !PointerInfo->get().Type || !RequestedAlignment || *RequestedAlignment <= 0 || + !std::has_single_bit(static_cast<std::uint64_t>(*RequestedAlignment))) { + return makeUnexpected(ErrorCode::VerificationError, + "G_VAARG has invalid type or alignment information"); + } + + const LowLevelType DestinationType = *DestinationInfo->get().Type; + const LowLevelType PointerType = *PointerInfo->get().Type; + const auto DestinationLayout = scalarLayout(DestinationType); + const auto PointerLayout = scalarLayout(PointerType); + if (!DestinationLayout || !PointerLayout || !PointerType.isPointer()) { + return makeUnexpected(ErrorCode::Unsupported, + "G_VAARG requires scalar result and pointer list types"); + } + + auto ListLoadMemory = + createMemoryAccess(Function, MemoryAccessFlag::Load, PointerType, *PointerLayout); + auto ListStoreMemory = + createMemoryAccess(Function, MemoryAccessFlag::Store, PointerType, *PointerLayout); + if (!ListLoadMemory) + return std::unexpected(ListLoadMemory.error()); + if (!ListStoreMemory) + return std::unexpected(ListStoreMemory.error()); + + auto CurrentList = MRI.createVirtualRegister(PointerType); + if (!CurrentList) + return std::unexpected(CurrentList.error()); + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto ListLoad = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!ListLoad) + return std::unexpected(ListLoad.error()); + if (auto Added = ListLoad->addDef(*CurrentList); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListLoad->addUse(*ListPointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ListLoad->build(); !Built) + return std::unexpected(Built.error()); + + Register ValueAddress = *CurrentList; + const std::uint64_t AlignmentValue = static_cast<std::uint64_t>(*RequestedAlignment); + auto PointerScalarType = LowLevelType::makeCategoryNeutralScalar(PointerType.scalarBitWidth()); + if (!PointerScalarType) + return std::unexpected(PointerScalarType.error()); + if (AlignmentValue > 1) { + auto AlignmentOffset = + buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, AlignmentValue - 1U); + if (!AlignmentOffset) + return std::unexpected(AlignmentOffset.error()); + auto BiasedAddress = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTR_ADD, + PointerType, ValueAddress, *AlignmentOffset); + if (!BiasedAddress) + return std::unexpected(BiasedAddress.error()); + auto AlignmentMask = + buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, ~(AlignmentValue - 1U)); + if (!AlignmentMask) + return std::unexpected(AlignmentMask.error()); + auto AlignedAddress = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTRMASK, + PointerType, *BiasedAddress, *AlignmentMask); + if (!AlignedAddress) + return std::unexpected(AlignedAddress.error()); + ValueAddress = *AlignedAddress; + } + + auto Increment = buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, + DestinationLayout->AllocationSize); + if (!Increment) + return std::unexpected(Increment.error()); + auto NextList = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTR_ADD, PointerType, + ValueAddress, *Increment); + if (!NextList) + return std::unexpected(NextList.error()); + + auto ListStore = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!ListStore) + return std::unexpected(ListStore.error()); + if (auto Added = ListStore->addUse(*NextList); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListStore->addUse(*ListPointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ListStore->build(); !Built) + return std::unexpected(Built.error()); + + if (!MRI.useEmpty(*Destination)) { + auto ValueLoadMemory = + createMemoryAccess(Function, MemoryAccessFlag::Load, DestinationType, *DestinationLayout); + if (!ValueLoadMemory) + return std::unexpected(ValueLoadMemory.error()); + auto ValueLoad = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!ValueLoad) + return std::unexpected(ValueLoad.error()); + if (auto Added = ValueLoad->addDef(*Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = ValueLoad->addUse(ValueAddress); !Added) + return std::unexpected(Added.error()); + if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ValueLoad->build(); !Built) + return std::unexpected(Built.error()); + } + + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "G_VAARG is detached during legalization"); + return Block->removeInstruction(Instruction); +} + +Expected<void> LegalizerHelper::lower(MachineInstr &Instruction, + const LegalizationDecision &Decision) { + if (Decision.Action != LegalizationAction::Lower) + return makeUnexpected(ErrorCode::InvalidArgument, + "lower received a non-lower legalization action"); + switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { + case GenericOpcode::G_VAARG: + return lowerVAArg(Instruction); + default: { + const InstructionDescriptor *Descriptor = + MIRBuilder.target().instructionDescriptor(Instruction.opcode()); + const std::string_view Name = + Descriptor == nullptr ? std::string_view{"<unknown>"} : Descriptor->Name; + return makeUnexpected(ErrorCode::Unsupported, + "generic lowering is not implemented for opcode '" + std::string(Name) + + "'"); + } + } +} + +Expected<void> LegalizerHelper::legalize(MachineInstr &Instruction, + const LegalizationDecision &Decision) { + switch (Decision.Action) { + case LegalizationAction::WidenScalar: + return widenScalar(Instruction, Decision.TypeIndex, Decision.NewType); + case LegalizationAction::Custom: + return Info.legalizeCustom(*this, Instruction, Decision); + case LegalizationAction::Lower: + return lower(Instruction, Decision); + case LegalizationAction::Legal: + return {}; + case LegalizationAction::Unsupported: + case LegalizationAction::Libcall: + case LegalizationAction::NarrowScalar: + case LegalizationAction::MoreElements: + case LegalizationAction::FewerElements: + case LegalizationAction::Bitcast: + case LegalizationAction::ChangeElementType: + case LegalizationAction::ChangeAddressSpace: + return makeUnexpected(ErrorCode::Unsupported, + "legalizer helper does not implement the requested action"); + } + return makeUnexpected(ErrorCode::InternalError, "unknown legalization action"); +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp new file mode 100644 index 0000000..7d9e6ff --- /dev/null +++ b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp @@ -0,0 +1,477 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/InstructionDescriptor.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include <algorithm> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <ranges> +#include <tuple> +#include <utility> +#include <vector> + +namespace rcc { +namespace { + +[[nodiscard]] LegalityPredicate typeInSet(std::size_t TypeIndex, std::vector<LowLevelType> Types) { + return [TypeIndex, Types = std::move(Types)](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && + std::ranges::find(Types, Query.Types[TypeIndex]) != Types.end(); + }; +} + +[[nodiscard]] LegalityPredicate +typePairInSet(std::vector<std::pair<LowLevelType, LowLevelType>> Types) { + return [Types = std::move(Types)](const LegalizationQuery &Query) { + return Query.Types.size() >= 2 && + std::ranges::find(Types, std::pair{Query.Types[0], Query.Types[1]}) != Types.end(); + }; +} + +[[nodiscard]] LegalityPredicate +typeTripleInSet(std::vector<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) { + return [Types = std::move(Types)](const LegalizationQuery &Query) { + return Query.Types.size() >= 3 && + std::ranges::find(Types, std::tuple{Query.Types[0], Query.Types[1], Query.Types[2]}) != + Types.end(); + }; +} + +[[nodiscard]] LowLevelType scalarType(std::uint32_t BitWidth) { + auto Type = LowLevelType::makeCategoryNeutralScalar(BitWidth); + return Type ? *Type : LowLevelType{}; +} + +[[nodiscard]] std::uint32_t nextPowerOfTwo(std::uint32_t Width, std::uint32_t MinimumBitWidth) { + return std::max(std::bit_ceil(Width), MinimumBitWidth); +} + +[[nodiscard]] bool memoryRuleMatches(const LegalizationQuery &Query, + const LegalizeMemoryRule &Rule) { + if (Query.Types.size() < 2 || Query.MemoryOperands.size() != 1 || + Query.Types[0] != Rule.ValueType || Query.Types[1] != Rule.AddressType) + return false; + const LegalizationMemoryDesc &Memory = Query.MemoryOperands.front(); + if (Memory.AlignmentInBits < Rule.MinimumAlignmentInBits) + return false; + if (Memory.Type) + return Memory.Type->minimumBitWidth() == Rule.MemoryType.minimumBitWidth(); + return Memory.Size.isKnown() && !Memory.Size.isScalable() && + Memory.Size.minimumBytes() * 8U == Rule.MemoryType.scalarBitWidth(); +} + +} // namespace + +LegalizeRule::LegalizeRule(LegalityPredicate Predicate, LegalizationAction Action, + LegalizeMutation Mutation) + : Predicate(std::move(Predicate)), Action(Action), Mutation(std::move(Mutation)) {} + +bool LegalizeRule::matches(const LegalizationQuery &Query) const { + return Predicate(Query); +} + +LegalizationDecision LegalizeRule::apply(const LegalizationQuery &Query) const { + if (!Mutation) + return {.Action = Action, .TypeIndex = 0, .NewType = {}}; + auto [TypeIndex, NewType] = Mutation(Query); + return {.Action = Action, .TypeIndex = TypeIndex, .NewType = NewType}; +} + +LegalizeRuleSet &LegalizeRuleSet::actionIf(LegalizationAction Action, LegalityPredicate Predicate, + LegalizeMutation Mutation) { + Rules.emplace_back(std::move(Predicate), Action, std::move(Mutation)); + return *this; +} + +LegalizeRuleSet &LegalizeRuleSet::actionFor(LegalizationAction Action, + std::initializer_list<LowLevelType> Types) { + return actionIf(Action, typeInSet(0, std::vector<LowLevelType>(Types))); +} + +LegalizeRuleSet & +LegalizeRuleSet::actionFor(LegalizationAction Action, + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionIf(Action, typePairInSet(std::vector<std::pair<LowLevelType, LowLevelType>>(Types))); +} + +LegalizeRuleSet &LegalizeRuleSet::actionFor( + LegalizationAction Action, + std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) { + return actionIf( + Action, + typeTripleInSet(std::vector<std::tuple<LowLevelType, LowLevelType, LowLevelType>>(Types))); +} + +LegalizeRuleSet &LegalizeRuleSet::legalIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Legal, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::customIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Custom, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::lowerIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Lower, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::libcallIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Libcall, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::unsupportedIf(LegalityPredicate Predicate) { + return actionIf(LegalizationAction::Unsupported, std::move(Predicate)); +} + +LegalizeRuleSet &LegalizeRuleSet::widenScalarIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::WidenScalar, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::narrowScalarIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::NarrowScalar, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::moreElementsIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::MoreElements, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::fewerElementsIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::FewerElements, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::bitcastIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizationAction::Bitcast, std::move(Predicate), std::move(Mutation)); +} + +LegalizeRuleSet &LegalizeRuleSet::legalFor(std::initializer_list<LowLevelType> Types) { + return actionFor(LegalizationAction::Legal, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::legalFor(bool Enabled, + std::initializer_list<LowLevelType> Types) { + return Enabled ? legalFor(Types) : *this; +} + +LegalizeRuleSet & +LegalizeRuleSet::legalFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Legal, Types); +} + +LegalizeRuleSet & +LegalizeRuleSet::legalFor(bool Enabled, + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return Enabled ? legalFor(Types) : *this; +} + +LegalizeRuleSet &LegalizeRuleSet::legalFor( + std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Legal, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::customFor(std::initializer_list<LowLevelType> Types) { + return actionFor(LegalizationAction::Custom, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::customFor(bool Enabled, + std::initializer_list<LowLevelType> Types) { + return Enabled ? customFor(Types) : *this; +} + +LegalizeRuleSet & +LegalizeRuleSet::customFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Custom, Types); +} + +LegalizeRuleSet & +LegalizeRuleSet::customFor(bool Enabled, + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return Enabled ? customFor(Types) : *this; +} + +LegalizeRuleSet &LegalizeRuleSet::lowerFor(std::initializer_list<LowLevelType> Types) { + return actionFor(LegalizationAction::Lower, Types); +} + +LegalizeRuleSet & +LegalizeRuleSet::lowerFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Lower, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::libcallFor(std::initializer_list<LowLevelType> Types) { + return actionFor(LegalizationAction::Libcall, Types); +} + +LegalizeRuleSet &LegalizeRuleSet::libcallFor(bool Enabled, + std::initializer_list<LowLevelType> Types) { + return Enabled ? libcallFor(Types) : *this; +} + +LegalizeRuleSet & +LegalizeRuleSet::libcallFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return actionFor(LegalizationAction::Libcall, Types); +} + +LegalizeRuleSet & +LegalizeRuleSet::libcallFor(bool Enabled, + std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) { + return Enabled ? libcallFor(Types) : *this; +} + +LegalizeRuleSet & +LegalizeRuleSet::legalForTypesWithMemDesc(std::initializer_list<LegalizeMemoryRule> MemoryRules) { + const std::vector<LegalizeMemoryRule> MemoryRuleList(MemoryRules); + return legalIf([MemoryRuleList](const LegalizationQuery &Query) { + return std::ranges::any_of(MemoryRuleList, [&Query](const LegalizeMemoryRule &Rule) { + return memoryRuleMatches(Query, Rule); + }); + }); +} + +LegalizeRuleSet &LegalizeRuleSet::widenScalarToNextPow2(std::size_t TypeIndex, + std::uint32_t MinimumBitWidth) { + return widenScalarIf( + [TypeIndex, MinimumBitWidth](const LegalizationQuery &Query) { + if (TypeIndex >= Query.Types.size() || !Query.Types[TypeIndex].isCategoryNeutralScalar()) + return false; + const std::uint32_t Width = Query.Types[TypeIndex].scalarBitWidth(); + return nextPowerOfTwo(Width, MinimumBitWidth) > Width; + }, + [TypeIndex, MinimumBitWidth](const LegalizationQuery &Query) { + return std::pair{TypeIndex, scalarType(nextPowerOfTwo( + Query.Types[TypeIndex].scalarBitWidth(), MinimumBitWidth))}; + }); +} + +LegalizeRuleSet &LegalizeRuleSet::minScalar(std::size_t TypeIndex, LowLevelType MinimumType) { + return widenScalarIf( + [TypeIndex, MinimumType](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isCategoryNeutralScalar() && + Query.Types[TypeIndex].scalarBitWidth() < MinimumType.scalarBitWidth(); + }, + [TypeIndex, MinimumType](const LegalizationQuery &) { + return std::pair{TypeIndex, MinimumType}; + }); +} + +LegalizeRuleSet &LegalizeRuleSet::minScalar(bool Enabled, std::size_t TypeIndex, + LowLevelType MinimumType) { + return Enabled ? minScalar(TypeIndex, MinimumType) : *this; +} + +LegalizeRuleSet &LegalizeRuleSet::maxScalar(std::size_t TypeIndex, LowLevelType MaximumType) { + return narrowScalarIf( + [TypeIndex, MaximumType](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isCategoryNeutralScalar() && + Query.Types[TypeIndex].scalarBitWidth() > MaximumType.scalarBitWidth(); + }, + [TypeIndex, MaximumType](const LegalizationQuery &) { + return std::pair{TypeIndex, MaximumType}; + }); +} + +LegalizeRuleSet &LegalizeRuleSet::maxScalar(bool Enabled, std::size_t TypeIndex, + LowLevelType MaximumType) { + return Enabled ? maxScalar(TypeIndex, MaximumType) : *this; +} + +LegalizeRuleSet &LegalizeRuleSet::clampScalar(std::size_t TypeIndex, LowLevelType MinimumType, + LowLevelType MaximumType) { + return minScalar(TypeIndex, MinimumType).maxScalar(TypeIndex, MaximumType); +} + +LegalizeRuleSet &LegalizeRuleSet::scalarSameSizeAs(std::size_t TypeIndex, + std::size_t ReferenceTypeIndex) { + const auto changeToReference = [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) { + return std::pair{TypeIndex, scalarType(Query.Types[ReferenceTypeIndex].scalarBitWidth())}; + }; + widenScalarIf( + [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && ReferenceTypeIndex < Query.Types.size() && + Query.Types[TypeIndex].isCategoryNeutralScalar() && + Query.Types[ReferenceTypeIndex].isCategoryNeutralScalar() && + Query.Types[TypeIndex].scalarBitWidth() < + Query.Types[ReferenceTypeIndex].scalarBitWidth(); + }, + changeToReference); + return narrowScalarIf( + [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) { + return TypeIndex < Query.Types.size() && ReferenceTypeIndex < Query.Types.size() && + Query.Types[TypeIndex].isCategoryNeutralScalar() && + Query.Types[ReferenceTypeIndex].isCategoryNeutralScalar() && + Query.Types[TypeIndex].scalarBitWidth() > + Query.Types[ReferenceTypeIndex].scalarBitWidth(); + }, + changeToReference); +} + +LegalizeRuleSet &LegalizeRuleSet::lowerIfMemSizeNotByteSizePow2(std::size_t MemoryIndex) { + return lowerIf([MemoryIndex](const LegalizationQuery &Query) { + if (MemoryIndex >= Query.MemoryOperands.size()) + return false; + const MemorySize Size = Query.MemoryOperands[MemoryIndex].Size; + return !Size.isKnown() || Size.isScalable() || !std::has_single_bit(Size.minimumBytes()); + }); +} + +LegalizeRuleSet &LegalizeRuleSet::alwaysLegal() { + return legalIf([](const LegalizationQuery &) { return true; }); +} + +LegalizeRuleSet &LegalizeRuleSet::lower() { + return lowerIf([](const LegalizationQuery &) { return true; }); +} + +LegalizeRuleSet &LegalizeRuleSet::libcall() { + return libcallIf([](const LegalizationQuery &) { return true; }); +} + +std::optional<LegalizationDecision> LegalizeRuleSet::apply(const LegalizationQuery &Query) const { + for (const LegalizeRule &Rule : Rules) { + if (Rule.matches(Query)) + return Rule.apply(Query); + } + return std::nullopt; +} + +LegalizeRuleSet &LegalizerInfo::getActionDefinitionsBuilder(Opcode InstructionOpcode) { + return RuleSets[InstructionOpcode]; +} + +LegalizeRuleSet & +LegalizerInfo::getActionDefinitionsBuilder(std::initializer_list<Opcode> InstructionOpcodes) { + const Opcode Representative = *InstructionOpcodes.begin(); + for (const Opcode InstructionOpcode : InstructionOpcodes) { + if (InstructionOpcode != Representative) + RuleAliases.emplace(InstructionOpcode, Representative); + } + return getActionDefinitionsBuilder(Representative); +} + +Expected<LegalizationQuery> LegalizerInfo::buildQuery(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode()); + if (Descriptor == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "legalization query requires a registered instruction opcode"); + + std::size_t ExplicitCount = 0; + while (ExplicitCount < Instruction.operandCount() && + !Instruction.operandAt(ExplicitCount)->isImplicit()) { + ++ExplicitCount; + } + auto OperandDescriptors = resolveExplicitOperandDescriptors(*Descriptor, ExplicitCount); + if (!OperandDescriptors) + return std::unexpected(OperandDescriptors.error()); + + std::size_t TypeCount = 0; + for (const OperandDescriptor *Operand : *OperandDescriptors) { + if (Operand->TypeConstraint && Operand->TypeConstraint->EqualityGroup) { + TypeCount = std::max(TypeCount, + static_cast<std::size_t>(*Operand->TypeConstraint->EqualityGroup) + 1U); + } + } + + std::vector<std::optional<LowLevelType>> OptionalTypes(TypeCount); + std::vector<std::vector<std::size_t>> TypeOperands(TypeCount); + for (std::size_t Index = 0; Index < ExplicitCount; ++Index) { + const OperandDescriptor &OperandDescriptor = *(*OperandDescriptors)[Index]; + if (!OperandDescriptor.TypeConstraint || !OperandDescriptor.TypeConstraint->EqualityGroup) { + continue; + } + const std::size_t TypeIndex = *OperandDescriptor.TypeConstraint->EqualityGroup; + const Register *Reg = Instruction.operandAt(Index)->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return makeUnexpected(ErrorCode::VerificationError, + "legalization type operand must name a virtual register"); + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "legalization type operand has no LowLevelType"); + if (OptionalTypes[TypeIndex] && *OptionalTypes[TypeIndex] != *Info->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "legalization type operands disagree within one type index"); + OptionalTypes[TypeIndex] = *Info->get().Type; + TypeOperands[TypeIndex].push_back(Index); + } + + std::vector<LowLevelType> Types; + Types.reserve(TypeCount); + for (const std::optional<LowLevelType> &Type : OptionalTypes) { + if (!Type) + return makeUnexpected(ErrorCode::VerificationError, + "legalization query contains a sparse type index"); + Types.push_back(*Type); + } + + std::vector<LegalizationMemoryDesc> MemoryOperands; + MemoryOperands.reserve(Instruction.memOperands().size()); + for (const MachineMemOperand *Memory : Instruction.memOperands()) { + const MachineMemOperandDesc &Description = Memory->description(); + MemoryOperands.push_back({Description.MemoryType, Description.Size, + Description.Align.bytes() * 8U, Description.AddressSpace, + Description.AtomicKind, Description.SuccessOrdering, + Description.FailureOrdering, Description.SyncScope}); + } + + LegalizationQuery Query(Instruction.opcode(), std::move(Types), std::move(MemoryOperands)); + Query.TypeOperands = std::move(TypeOperands); + return Query; +} + +Expected<LegalizationDecision> LegalizerInfo::getDecision(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) const { + auto Query = buildQuery(Instruction, Function, Target); + if (!Query) + return std::unexpected(Query.error()); + return getDecision(*Query); +} + +Expected<LegalizationDecision> LegalizerInfo::getDecision(const LegalizationQuery &Query) const { + if (!isGenericOpcode(Query.InstructionOpcode)) + return LegalizationDecision{.Action = LegalizationAction::Legal, .TypeIndex = 0, .NewType = {}}; + + Opcode RuleOpcode = Query.InstructionOpcode; + if (const auto Alias = RuleAliases.find(RuleOpcode); Alias != RuleAliases.end()) + RuleOpcode = Alias->second; + const auto RuleSet = RuleSets.find(RuleOpcode); + if (RuleSet == RuleSets.end()) + return LegalizationDecision{ + .Action = LegalizationAction::Unsupported, .TypeIndex = 0, .NewType = {}}; + const std::optional<LegalizationDecision> Decision = RuleSet->second.apply(Query); + return Decision.value_or(LegalizationDecision{ + .Action = LegalizationAction::Unsupported, .TypeIndex = 0, .NewType = {}}); +} + +Expected<bool> LegalizerInfo::isLegal(const LegalizationQuery &Query) const { + auto Decision = getDecision(Query); + if (!Decision) + return std::unexpected(Decision.error()); + return Decision->Action == LegalizationAction::Legal; +} + +Expected<bool> LegalizerInfo::isUnsupported(const LegalizationQuery &Query) const { + auto Decision = getDecision(Query); + if (!Decision) + return std::unexpected(Decision.error()); + return Decision->Action == LegalizationAction::Unsupported; +} + +} // namespace rcc diff --git a/backend/lib/CodeGen/InstructionDescriptor.cpp b/backend/lib/CodeGen/InstructionDescriptor.cpp index d164cde..027f738 100644 --- a/backend/lib/CodeGen/InstructionDescriptor.cpp +++ b/backend/lib/CodeGen/InstructionDescriptor.cpp @@ -302,6 +302,19 @@ struct OperandShape final { } // namespace +Expected<std::vector<const OperandDescriptor *>> +resolveExplicitOperandDescriptors(const InstructionDescriptor &Descriptor, + std::size_t OperandCount) { + auto Shape = resolveOperandShape(Descriptor, OperandCount); + if (!Shape) + return std::unexpected(Shape.error()); + std::vector<const OperandDescriptor *> Result; + Result.reserve(OperandCount); + for (std::size_t Index = 0; Index < OperandCount; ++Index) + Result.push_back(&descriptorForOperand(Descriptor, *Shape, Index)); + return Result; +} + OperandKindSet::OperandKindSet(std::initializer_list<MachineOperand::Kind> Kinds) noexcept { for (const MachineOperand::Kind Kind : Kinds) add(Kind); diff --git a/backend/lib/CodeGen/MachineBasicBlock.cpp b/backend/lib/CodeGen/MachineBasicBlock.cpp index a8d69f6..763a577 100644 --- a/backend/lib/CodeGen/MachineBasicBlock.cpp +++ b/backend/lib/CodeGen/MachineBasicBlock.cpp @@ -40,6 +40,8 @@ Expected<void> MachineBasicBlock::appendInstruction(MachineInstr &Instruction) { else FirstInstruction = &Instruction; LastInstruction = &Instruction; + if (OwnerFunction->TheDelegate != nullptr) + OwnerFunction->TheDelegate->handleInsertion(Instruction); return {}; } @@ -65,6 +67,8 @@ Expected<void> MachineBasicBlock::insertInstructionBefore(MachineInstr &Position else FirstInstruction = &Instruction; Position.PreviousInstruction = &Instruction; + if (OwnerFunction->TheDelegate != nullptr) + OwnerFunction->TheDelegate->handleInsertion(Instruction); return {}; } @@ -90,6 +94,8 @@ Expected<void> MachineBasicBlock::insertInstructionAfter(MachineInstr &Position, else LastInstruction = &Instruction; Position.NextInstruction = &Instruction; + if (OwnerFunction->TheDelegate != nullptr) + OwnerFunction->TheDelegate->handleInsertion(Instruction); return {}; } @@ -98,6 +104,9 @@ Expected<void> MachineBasicBlock::removeInstruction(MachineInstr &Instruction) { return makeUnexpected(ErrorCode::InvalidArgument, "instruction does not belong to this block"); } + if (OwnerFunction->TheDelegate != nullptr) + OwnerFunction->TheDelegate->handleRemoval(Instruction); + if (Instruction.PreviousInstruction != nullptr) Instruction.PreviousInstruction->NextInstruction = Instruction.NextInstruction; else diff --git a/backend/lib/CodeGen/MachineIRBuilder.cpp b/backend/lib/CodeGen/MachineIRBuilder.cpp index e59f613..9195d1a 100644 --- a/backend/lib/CodeGen/MachineIRBuilder.cpp +++ b/backend/lib/CodeGen/MachineIRBuilder.cpp @@ -2,6 +2,7 @@ #include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFrameInfo.h" #include "rcc/CodeGen/MachineFunction.h" @@ -707,4 +708,196 @@ Expected<MachineInstrBuilder> MachineIRBuilder::buildInstr(Opcode InstructionOpc return MachineInstrBuilder(*Function, *Target, *Descriptor, *InsertBlock, InsertBefore); } +Expected<LowLevelType> MachineIRBuilder::registerType(Register Reg) const { + if (!Reg.isVirtual()) + return makeUnexpected(ErrorCode::InvalidArgument, + "generic builder operand is not a virtual register"); + auto Info = Function->registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Type) + return makeUnexpected(ErrorCode::InvalidArgument, + "generic builder operand has no LowLevelType"); + return *Info->get().Type; +} + +Expected<MachineInstr *> MachineIRBuilder::buildUnary(Opcode InstructionOpcode, + Register Destination, + Register Source) const { + auto Pending = buildInstr(InstructionOpcode); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildBinary(Opcode InstructionOpcode, + Register Destination, Register Left, + Register Right) const { + auto Pending = buildInstr(InstructionOpcode); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Left); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Right); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildTrunc(Register Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_TRUNC), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildCopy(Register Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::COPY), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildImplicitDef(Register Destination) const { + auto Pending = buildInstr(opcode(GenericOpcode::G_IMPLICIT_DEF)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildAnyExt(Register Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_ANYEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildZExt(Register Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_ZEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildSExt(Register Destination, + Register Source) const { + return buildUnary(opcode(GenericOpcode::G_SEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildExtOrTrunc(Opcode ExtensionOpcode, + Register Destination, + Register Source) const { + auto DestinationType = registerType(Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + auto SourceType = registerType(Source); + if (!SourceType) + return std::unexpected(SourceType.error()); + const std::uint64_t DestinationWidth = DestinationType->minimumBitWidth(); + const std::uint64_t SourceWidth = SourceType->minimumBitWidth(); + if (DestinationWidth > SourceWidth) + return buildUnary(ExtensionOpcode, Destination, Source); + if (DestinationWidth < SourceWidth) + return buildTrunc(Destination, Source); + return buildUnary(opcode(GenericOpcode::COPY), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildAnyExtOrTrunc(Register Destination, + Register Source) const { + return buildExtOrTrunc(opcode(GenericOpcode::G_ANYEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildSExtOrTrunc(Register Destination, + Register Source) const { + return buildExtOrTrunc(opcode(GenericOpcode::G_SEXT), Destination, Source); +} + +Expected<MachineInstr *> MachineIRBuilder::buildConstant(Register Destination, + APInt Value) const { + auto Pending = buildInstr(opcode(GenericOpcode::G_CONSTANT)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addIntegerImmediate(std::move(Value)); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildConstant(Register Destination, + std::int64_t Value) const { + auto Type = registerType(Destination); + if (!Type) + return std::unexpected(Type.error()); + auto Immediate = APInt::fromSigned(Type->scalarBitWidth(), Value); + if (!Immediate) + return std::unexpected(Immediate.error()); + return buildConstant(Destination, std::move(*Immediate)); +} + +Expected<MachineInstr *> MachineIRBuilder::buildSExtInReg(Register Destination, + Register Source, + std::uint32_t Width) const { + auto Pending = buildInstr(opcode(GenericOpcode::G_SEXT_INREG)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addImmediate(static_cast<std::int64_t>(Width)); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildAnd(Register Destination, Register Left, + Register Right) const { + return buildBinary(opcode(GenericOpcode::G_AND), Destination, Left, Right); +} + +Expected<MachineInstr *> MachineIRBuilder::buildMergeValues( + Register Destination, const std::vector<Register> &Sources) const { + auto Pending = buildInstr(opcode(GenericOpcode::G_MERGE_VALUES)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + for (Register Source : Sources) { + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + } + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildUnmergeValues( + const std::vector<Register> &Destinations, Register Source) const { + auto Pending = buildInstr(opcode(GenericOpcode::G_UNMERGE_VALUES)); + if (!Pending) + return std::unexpected(Pending.error()); + for (Register Destination : Destinations) { + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + } + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + +Expected<MachineInstr *> MachineIRBuilder::buildExtract(Register Destination, + Register Source, + std::uint64_t Offset) const { + if (Offset > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) + return makeUnexpected(ErrorCode::InvalidArgument, + "generic extract offset does not fit a machine immediate"); + auto Pending = buildInstr(opcode(GenericOpcode::G_EXTRACT)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addImmediate(static_cast<std::int64_t>(Offset)); !Added) + return std::unexpected(Added.error()); + return Pending->build(); +} + } // namespace rcc diff --git a/backend/lib/CodeGen/MachineRegisterInfo.cpp b/backend/lib/CodeGen/MachineRegisterInfo.cpp index a88bfb0..4627f9e 100644 --- a/backend/lib/CodeGen/MachineRegisterInfo.cpp +++ b/backend/lib/CodeGen/MachineRegisterInfo.cpp @@ -311,6 +311,27 @@ const MachineOperand *MachineRegisterInfo::getUniqueVRegDef(Register Reg) const return UniqueDef; } +MachineInstr *MachineRegisterInfo::getVRegDef(Register Reg) noexcept { + const MachineOperand *Definition = getUniqueVRegDef(Reg); + return Definition == nullptr ? nullptr + : const_cast<MachineInstr *>(Definition->parent()); +} + +const MachineInstr *MachineRegisterInfo::getVRegDef(Register Reg) const noexcept { + const MachineOperand *Definition = getUniqueVRegDef(Reg); + return Definition == nullptr ? nullptr : Definition->parent(); +} + +Expected<LowLevelType> MachineRegisterInfo::getType(Register Reg) const { + auto Info = virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Type) + return makeUnexpected(ErrorCode::InvalidArgument, + "virtual register has no LowLevelType"); + return *Info->get().Type; +} + Expected<std::size_t> MachineRegisterInfo::replaceRegWith(Register OldReg, Register NewReg) { if (OldReg.isNoRegister() || NewReg.isNoRegister()) { return makeUnexpected(ErrorCode::InvalidArgument, diff --git a/backend/lib/Support/APInt.cpp b/backend/lib/Support/APInt.cpp index 898deee..d54a4c2 100644 --- a/backend/lib/Support/APInt.cpp +++ b/backend/lib/Support/APInt.cpp @@ -138,6 +138,23 @@ Expected<APInt> APInt::fromSigned(std::uint32_t BitWidth, std::int64_t Value) { return create(BitWidth, Words); } +Expected<APInt> APInt::getLowBitsSet(std::uint32_t BitWidth, + std::uint32_t SetBitCount) { + if (BitWidth == 0 || BitWidth > MaxBitWidth || SetBitCount > BitWidth) + return makeUnexpected(ErrorCode::InvalidArgument, + "APInt low-bit mask has an invalid width"); + std::vector<std::uint64_t> Words(wordCount(BitWidth), 0); + const std::size_t FullWords = SetBitCount / 64U; + std::ranges::fill_n( + Words.begin(), + static_cast<std::vector<std::uint64_t>::difference_type>(FullWords), + std::numeric_limits<std::uint64_t>::max()); + const std::uint32_t RemainingBits = SetBitCount % 64U; + if (RemainingBits != 0) + Words[FullWords] = (std::uint64_t{1} << RemainingBits) - 1U; + return create(BitWidth, Words); +} + Expected<APInt> APInt::parseDecimal(std::uint32_t BitWidth, std::string_view Text) { if (BitWidth == 0 || BitWidth > MaxBitWidth) return makeUnexpected(ErrorCode::ParseError, "APInt bit width is outside the supported range"); @@ -243,6 +260,43 @@ std::uint64_t APInt::word(std::size_t Index) const noexcept { return Index < Words.size() ? Words[Index] : 0; } +Expected<APInt> APInt::trunc(std::uint32_t NewBitWidth) const { + if (NewBitWidth == 0 || NewBitWidth >= BitWidth) + return makeUnexpected(ErrorCode::InvalidArgument, + "APInt truncation requires a smaller nonzero width"); + std::vector<std::uint64_t> ResultWords(wordCount(NewBitWidth), 0); + std::ranges::copy_n( + Words.begin(), + static_cast<std::vector<std::uint64_t>::difference_type>( + ResultWords.size()), + ResultWords.begin()); + ResultWords.back() &= highWordMask(NewBitWidth); + return create(NewBitWidth, ResultWords); +} + +Expected<APInt> APInt::zext(std::uint32_t NewBitWidth) const { + if (NewBitWidth <= BitWidth || NewBitWidth > MaxBitWidth) + return makeUnexpected(ErrorCode::InvalidArgument, + "APInt zero extension requires a larger supported width"); + std::vector<std::uint64_t> ResultWords(wordCount(NewBitWidth), 0); + std::ranges::copy(Words, ResultWords.begin()); + return create(NewBitWidth, ResultWords); +} + +Expected<APInt> APInt::sext(std::uint32_t NewBitWidth) const { + if (NewBitWidth <= BitWidth || NewBitWidth > MaxBitWidth) + return makeUnexpected(ErrorCode::InvalidArgument, + "APInt sign extension requires a larger supported width"); + std::vector<std::uint64_t> ResultWords( + wordCount(NewBitWidth), + isNegative() ? std::numeric_limits<std::uint64_t>::max() : std::uint64_t{0}); + std::ranges::copy(Words, ResultWords.begin()); + if (isNegative() && BitWidth % 64U != 0) + ResultWords[wordCount(BitWidth) - 1U] |= ~highWordMask(BitWidth); + ResultWords.back() &= highWordMask(NewBitWidth); + return create(NewBitWidth, ResultWords); +} + int APInt::compare(const APInt &Other) const noexcept { const std::size_t LeftCount = significantWordCount(Words); const std::size_t RightCount = significantWordCount(Other.Words); diff --git a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp index e4c937f..59c992e 100644 --- a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp +++ b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp @@ -177,10 +177,10 @@ public: const std::optional<LowLevelType> Type = resultType(Function, Instruction); const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); }; const auto Float = [Type] { return Type && isSupportedFloatingScalar(*Type); }; - auto Classification = Subtarget.getLegalizerInfo()->classify(Instruction, Function, Target); + auto Classification = Subtarget.getLegalizerInfo()->getDecision(Instruction, Function, Target); if (!Classification) return std::unexpected(Classification.error()); - if (*Classification != LegalizationAction::Legal) { + if (Classification->Action != LegalizationAction::Legal) { std::string Message = "RISC-V selector cannot select opcode '" + std::string(getGenericOpcodeName(Generic)) + "'"; if (Type) diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp index 1a29ebf..4d81404 100644 --- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp +++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -2,9 +2,9 @@ #include "RISCVLegalizerInfo.h" -#include "RISCVCallLowering.h" #include "RISCVImmediateMaterialization.h" #include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h" #include "rcc/CodeGen/LowLevelType.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFunction.h" @@ -59,45 +59,6 @@ operandType(const MachineFunction &Function, const MachineInstr &Instruction, st return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); } -[[nodiscard]] bool isSupportedFloatingScalar(LowLevelType Type) noexcept { - return Type.isCategoryNeutralScalar() && - (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); -} - -[[nodiscard]] bool isSupportedIntegerType(LowLevelType Type) noexcept { - return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 && - Type.scalarBitWidth() <= 64) || - (Type.isPointer() && Type.scalarBitWidth() == 64); -} - -[[nodiscard]] bool isSupportedIntegerScalar(LowLevelType Type) noexcept { - return Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 && - Type.scalarBitWidth() <= 64; -} - -[[nodiscard]] bool isNativeIntegerScalar(LowLevelType Type) noexcept { - return Type.isCategoryNeutralScalar() && - (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); -} - -[[nodiscard]] bool isScalarGPRType(LowLevelType Type) noexcept { - return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 && - Type.scalarBitWidth() <= 64) || - (Type.isPointer() && Type.scalarBitWidth() == 64); -} - -[[nodiscard]] bool isI1Type(LowLevelType Type) noexcept { - return Type.isCategoryNeutralScalar() && Type.scalarBitWidth() == 1; -} - -[[nodiscard]] bool isScalarIntegerExtension(LowLevelType Destination, - LowLevelType Source) noexcept { - return Destination.isCategoryNeutralScalar() && Source.isCategoryNeutralScalar() && - Destination.scalarBitWidth() != 0 && Source.scalarBitWidth() != 0 && - Source.scalarBitWidth() < Destination.scalarBitWidth() && - Destination.scalarBitWidth() <= 64; -} - [[nodiscard]] Expected<APInt> extendIntegerImmediate(const APInt &Value, std::uint32_t DestinationWidth, bool SignExtend) { if (DestinationWidth == 0 || DestinationWidth > 64) @@ -193,13 +154,6 @@ foldConstantExtension(MachineInstr &Instruction, MachineFunction &Function, Gene return {}; } -struct WideConstantUnmergePlan final { - MachineInstr *Extension = nullptr; - MachineInstr *Unmerge = nullptr; - std::vector<Register> Outputs; - bool SignExtend = false; -}; - [[nodiscard]] MachineInstr *singleRegisterUse(const MachineFunction &Function, Register Reg) { MachineInstr *UseInstruction = nullptr; auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo()); @@ -213,189 +167,6 @@ struct WideConstantUnmergePlan final { return UseInstruction; } -[[nodiscard]] std::optional<WideConstantUnmergePlan> -wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Function) { - const MachineOperand *ResultOperand = Constant.operandAt(0); - const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>(); - if (Result == nullptr || !Result->isVirtual()) - return std::nullopt; - - MachineInstr *FirstUse = singleRegisterUse(Function, *Result); - if (FirstUse == nullptr) - return std::nullopt; - MachineInstr *Extension = nullptr; - MachineInstr *Unmerge = FirstUse; - if (FirstUse->opcode() == opcode(GenericOpcode::G_ANYEXT) || - FirstUse->opcode() == opcode(GenericOpcode::G_SEXT) || - FirstUse->opcode() == opcode(GenericOpcode::G_ZEXT)) { - Extension = FirstUse; - const MachineOperand *ExtendedOperand = Extension->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); - if (Extended == nullptr || !Extended->isVirtual()) - return std::nullopt; - Unmerge = singleRegisterUse(Function, *Extended); - if (Unmerge == nullptr) - return std::nullopt; - } - if (Unmerge->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES) || Unmerge->operandCount() < 3) - return std::nullopt; - - WideConstantUnmergePlan Plan; - Plan.Extension = Extension; - Plan.Unmerge = Unmerge; - Plan.SignExtend = Extension != nullptr && Extension->opcode() != opcode(GenericOpcode::G_ZEXT); - for (std::size_t Index = 0; Index + 1U < Unmerge->operandCount(); ++Index) { - const MachineOperand *Operand = Unmerge->operandAt(Index); - const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); - if (Reg == nullptr || !Reg->isVirtual() || !Operand->isDef()) - return std::nullopt; - const auto Type = registerType(Function, *Reg); - if (!Type || !isSupportedIntegerScalar(*Type)) - return std::nullopt; - Plan.Outputs.push_back(*Reg); - } - const MachineOperand *SourceOperand = Unmerge->operandAt(Unmerge->operandCount() - 1U); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - const Register *ExpectedSource = Result; - if (Extension != nullptr) { - const MachineOperand *ExtendedOperand = Extension->operandAt(0); - ExpectedSource = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); - } - if (Source == nullptr || ExpectedSource == nullptr || *Source != *ExpectedSource) - return std::nullopt; - return Plan; -} - -[[nodiscard]] Expected<APInt> wideConstantWord(const APInt &Value, std::size_t WordIndex, - std::uint32_t Width, bool SignExtend) { - std::uint64_t Raw = 0; - if (WordIndex < Value.words().size()) - Raw = Value.words()[WordIndex]; - else if (SignExtend && Value.isNegative()) - Raw = std::numeric_limits<std::uint64_t>::max(); - const std::size_t WordBitOffset = WordIndex * 64U; - if (SignExtend && Value.isNegative() && WordBitOffset < Value.bitWidth()) { - const std::uint32_t Remaining = Value.bitWidth() - static_cast<std::uint32_t>(WordBitOffset); - if (Remaining < 64) - Raw |= ~std::uint64_t{0} << Remaining; - } - if (Width < 64) - Raw &= (std::uint64_t{1} << Width) - 1U; - return APInt::create(Width, std::span<const std::uint64_t>(&Raw, 1)); -} - -[[nodiscard]] Expected<void> foldWideConstant(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const auto Plan = wideConstantUnmergePlan(Instruction, Function); - if (!Plan) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide constant requires a G_UNMERGE_VALUES use"); - const MachineOperand *ImmediateOperand = Instruction.operandAt(1); - const auto *Integer = - ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>(); - if (Integer == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide constant has no integer immediate"); - - std::vector<APInt> Words; - Words.reserve(Plan->Outputs.size()); - for (std::size_t Index = 0; Index < Plan->Outputs.size(); ++Index) { - const auto Type = registerType(Function, Plan->Outputs[Index]); - if (!Type) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide constant output has no virtual-register type"); - auto Word = wideConstantWord(Integer->Value, Index, Type->scalarBitWidth(), Plan->SignExtend); - if (!Word) - return std::unexpected(Word.error()); - Words.push_back(std::move(*Word)); - } - - std::vector<std::size_t> Representatives(Plan->Outputs.size()); - for (std::size_t Index = 0; Index < Representatives.size(); ++Index) { - Representatives[Index] = Index; - for (std::size_t Previous = 0; Previous < Index; ++Previous) { - if (Words[Previous] == Words[Index]) { - Representatives[Index] = Previous; - break; - } - } - } - - const Register FirstOutput = Plan->Outputs.front(); - auto Retargeted = Instruction.operandAt(0)->setRegister(FirstOutput); - if (!Retargeted) - return std::unexpected(Retargeted.error()); - auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - while (Instruction.operandCount() > 1) { - auto Erased = Instruction.eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - } - auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(Words.front()))); - if (!Added) - return std::unexpected(Added.error()); - - MachineBasicBlock *Block = Instruction.parent(); - if (Block == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide constant is detached during legalization"); - if (Plan->Unmerge != &Instruction) { - auto RemovedUnmerge = Plan->Unmerge->parent()->removeInstruction(*Plan->Unmerge); - if (!RemovedUnmerge) - return std::unexpected(RemovedUnmerge.error()); - } - if (Plan->Extension != nullptr) { - auto RemovedExtension = Plan->Extension->parent()->removeInstruction(*Plan->Extension); - if (!RemovedExtension) - return std::unexpected(RemovedExtension.error()); - } - - for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { - if (Representatives[Index] == Index) - continue; - auto Replaced = Function.registerInfo().replaceRegWith(Plan->Outputs[Index], - Plan->Outputs[Representatives[Index]]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - - MachineInstr *Last = &Instruction; - for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { - if (Representatives[Index] != Index) - continue; - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointAfter(*Last); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); - if (!Pending) - return std::unexpected(Pending.error()); - auto Def = Pending->addDef(Plan->Outputs[Index]); - if (!Def) - return std::unexpected(Def.error()); - auto Value = Pending->addIntegerImmediate(std::move(Words[Index])); - if (!Value) - return std::unexpected(Value.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - Last = *Built; - } - return {}; -} - -[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder, - MachineInstr &Anchor, - std::uint32_t Width, - std::uint64_t Value); - -[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LowLevelType Type, - Register Left, Register Right); - [[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LowLevelType Type, Register Source); @@ -405,123 +176,6 @@ wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Fun [[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, Register Destination, Register Source); -struct NarrowAnyExtBinaryPlan final { - MachineInstr *Binary = nullptr; - GenericOpcode Generic = GenericOpcode::Invalid; - std::array<Register, 2> WideInputs{}; - std::vector<MachineInstr *> Artifacts; - std::vector<MachineInstr *> Constants; - std::uint32_t SourceWidth = 0; -}; - -[[nodiscard]] std::optional<NarrowAnyExtBinaryPlan> -narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Function) { - const MachineOperand *DestinationOperand = Extension.operandAt(0); - const MachineOperand *SourceOperand = Extension.operandAt(1); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() || - !Source->isVirtual() || singleRegisterUse(Function, *Source) != &Extension) - return std::nullopt; - const auto DestinationType = registerType(Function, *Destination); - const auto SourceType = registerType(Function, *Source); - if (!DestinationType || !SourceType) - return std::nullopt; - if (!isScalarIntegerExtension(*DestinationType, *SourceType) || - DestinationType->scalarBitWidth() != 64) - return std::nullopt; - - const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(*Source); - MachineInstr *Binary = SourceDefinition == nullptr - ? nullptr - : const_cast<MachineInstr *>(SourceDefinition->parent()); - if (Binary == nullptr || Binary->operandCount() != 3) - return std::nullopt; - const GenericOpcode Generic = static_cast<GenericOpcode>(Binary->opcode().value()); - if (Generic != GenericOpcode::G_ADD && Generic != GenericOpcode::G_SUB && - Generic != GenericOpcode::G_MUL && Generic != GenericOpcode::G_AND && - Generic != GenericOpcode::G_OR && Generic != GenericOpcode::G_XOR && - Generic != GenericOpcode::G_SHL) - return std::nullopt; - - NarrowAnyExtBinaryPlan Plan; - Plan.Binary = Binary; - Plan.Generic = Generic; - Plan.SourceWidth = SourceType->scalarBitWidth(); - const auto AddArtifact = [&Plan](MachineInstr *Artifact) { - if (std::ranges::find(Plan.Artifacts, Artifact) == Plan.Artifacts.end()) - Plan.Artifacts.push_back(Artifact); - }; - const auto TruncatedWideSource = [&](MachineInstr *Truncation, - MachineInstr *OnlyUse) -> std::optional<Register> { - if (Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || - Truncation->operandCount() != 2) - return std::nullopt; - const MachineOperand *NarrowOperand = Truncation->operandAt(0); - const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>(); - const MachineOperand *WideOperand = Truncation->operandAt(1); - const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>(); - const std::optional<LowLevelType> WideType = - WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); - if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType || - *WideType != *DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse) - return std::nullopt; - return *Wide; - }; - for (std::size_t Index = 0; Index != Plan.WideInputs.size(); ++Index) { - const MachineOperand *Operand = Binary->operandAt(Index + 1U); - const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); - if (Reg == nullptr || !Reg->isVirtual()) - return std::nullopt; - const auto Type = registerType(Function, *Reg); - if (!Type) - return std::nullopt; - if (*Type == *DestinationType) { - Plan.WideInputs[Index] = *Reg; - continue; - } - if (*Type != *SourceType) - return std::nullopt; - - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg); - MachineInstr *Def = - Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); - if (auto Wide = TruncatedWideSource(Def, Binary)) { - Plan.WideInputs[Index] = *Wide; - AddArtifact(Def); - continue; - } - if (Def != nullptr && - (Def->opcode() == opcode(GenericOpcode::COPY) || - Def->opcode() == opcode(GenericOpcode::G_ANYEXT)) && - Def->operandCount() == 2 && singleRegisterUse(Function, *Reg) == Binary) { - const MachineOperand *NarrowOperand = Def->operandAt(1); - const Register *Narrow = - NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>(); - const MachineOperand *NarrowDefinition = - Narrow == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Narrow); - MachineInstr *Truncation = NarrowDefinition == nullptr - ? nullptr - : const_cast<MachineInstr *>(NarrowDefinition->parent()); - if (auto Wide = TruncatedWideSource(Truncation, Def)) { - Plan.WideInputs[Index] = *Wide; - AddArtifact(Def); - AddArtifact(Truncation); - continue; - } - } - if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && - singleRegisterUse(Function, *Reg) == Binary) { - Plan.WideInputs[Index] = *Reg; - Plan.Constants.push_back(Def); - continue; - } - return std::nullopt; - } - return Plan; -} - [[nodiscard]] Expected<void> widenConstantToRV64(MachineInstr &Instruction, MachineFunction &Function) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); @@ -893,127 +547,6 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction return {}; } -[[nodiscard]] Expected<void> foldNarrowAnyExtBinary(MachineInstr &Extension, - MachineFunction &Function, - const TargetDescription &Target) { - const auto Plan = narrowAnyExtBinaryPlan(Extension, Function); - if (!Plan) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V narrow any-extension does not have a legal binary chain"); - for (MachineInstr *Constant : Plan->Constants) { - auto Widened = widenConstantToRV64(*Constant, Function); - if (!Widened) - return std::unexpected(Widened.error()); - } - for (std::size_t Index = 0; Index != Plan->WideInputs.size(); ++Index) { - auto Replaced = Plan->Binary->operandAt(Index + 1U)->setRegister(Plan->WideInputs[Index]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - for (MachineInstr *Artifact : Plan->Artifacts) { - if (Artifact->parent() == nullptr) - continue; - auto Removed = Artifact->parent()->removeInstruction(*Artifact); - if (!Removed) - return std::unexpected(Removed.error()); - } - - if (Plan->Generic == GenericOpcode::G_SHL) { - Register Amount = Plan->WideInputs[1]; - if (Plan->SourceWidth < 32 || Plan->SourceWidth > 32) { - const std::uint64_t Mask = Plan->SourceWidth == 64 - ? std::numeric_limits<std::uint64_t>::max() - : (std::uint64_t{1} << Plan->SourceWidth) - 1U; - auto WideType = LowLevelType::makeCategoryNeutralScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - auto MaskValue = buildIntegerConstantBefore(Builder, Extension, 64, Mask); - if (!MaskValue) - return std::unexpected(MaskValue.error()); - auto MaskedAmount = buildBinaryBefore(Builder, Extension, GenericOpcode::G_AND, *WideType, - Amount, *MaskValue); - if (!MaskedAmount) - return std::unexpected(MaskedAmount.error()); - Amount = *MaskedAmount; - } - auto SetOpcode = - Extension.setOpcode(Plan->SourceWidth == 32 ? instructionOpcode(InstructionOpcode::G_SLLW) - : opcode(GenericOpcode::G_SHL)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - while (Extension.operandCount() > 1) { - auto Erased = Extension.eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - } - for (Register Input : {Plan->WideInputs[0], Amount}) { - auto Operand = - MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use}); - if (!Operand) - return std::unexpected(Operand.error()); - auto Added = Extension.addOperand(std::move(*Operand)); - if (!Added) - return std::unexpected(Added.error()); - } - MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); - if (BinaryBlock == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow shift is detached during legalization"); - return BinaryBlock->removeInstruction(*Plan->Binary); - } - - if (Plan->SourceWidth == 32 && - (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) { - auto WideType = LowLevelType::makeCategoryNeutralScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - auto Temporary = Function.registerInfo().createVirtualRegister(*WideType); - if (!Temporary) - return std::unexpected(Temporary.error()); - auto Retargeted = Plan->Binary->operandAt(0)->setRegister(*Temporary); - if (!Retargeted) - return std::unexpected(Retargeted.error()); - auto SetOpcode = Extension.setOpcode(opcode(GenericOpcode::G_SEXT_INREG)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - auto ReplacedSource = Extension.operandAt(1)->setRegister(*Temporary); - if (!ReplacedSource) - return std::unexpected(ReplacedSource.error()); - auto Width = Extension.addOperand(MachineOperand::immediate(32)); - if (!Width) - return std::unexpected(Width.error()); - return {}; - } - - auto SetOpcode = Extension.setOpcode(opcode(Plan->Generic)); - if (!SetOpcode) - return std::unexpected(SetOpcode.error()); - while (Extension.operandCount() > 1) { - auto Erased = Extension.eraseOperand(1); - if (!Erased) - return std::unexpected(Erased.error()); - } - for (Register Input : Plan->WideInputs) { - auto Operand = - MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use}); - if (!Operand) - return std::unexpected(Operand.error()); - auto Added = Extension.addOperand(std::move(*Operand)); - if (!Added) - return std::unexpected(Added.error()); - } - MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); - if (BinaryBlock == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow binary instruction is detached during legalization"); - auto Removed = BinaryBlock->removeInstruction(*Plan->Binary); - if (!Removed) - return std::unexpected(Removed.error()); - (void)Target; - return {}; -} - [[nodiscard]] Expected<void> lowerConstantToPool(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { @@ -1097,63 +630,6 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction return {}; } -[[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept { - if (Instruction.memOperands().size() != 1) - return false; - const MachineMemOperand &Memory = *Instruction.memOperands().front(); - const MachineMemOperandDesc &Description = Memory.description(); - return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() && - !Description.Size.isScalable() && Description.Size.minimumBytes() != 0; -} - -[[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction, - AtomicOperation Operation, - std::uint64_t Bytes) noexcept { - if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8)) - return false; - const MachineMemOperand &Memory = *Instruction.memOperands().front(); - const MachineMemOperandDesc &Description = Memory.description(); - return Description.AtomicKind == Operation && Description.AddressSpace == 0 && - Description.Size.isKnown() && !Description.Size.isScalable() && - Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes && - Description.SyncScope.value() == 0; -} - -[[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction, - AtomicOperation Operation, - std::uint64_t ContainerBytes) noexcept { - if (Instruction.memOperands().size() != 1 || ContainerBytes == 0) - return false; - const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); - if (Description.AtomicKind != Operation || Description.AddressSpace != 0 || - !Description.Size.isKnown() || Description.Size.isScalable() || - Description.SyncScope.value() != 0) - return false; - const std::uint64_t MemoryBytes = Description.Size.minimumBytes(); - return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) && - MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes; -} - -[[nodiscard]] std::optional<std::uint64_t> -memoryAccessBytes(const MachineInstr &Instruction) noexcept { - if (Instruction.memOperands().size() != 1) - return std::nullopt; - const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); - if (!Description.Size.isKnown() || Description.Size.isScalable()) - return std::nullopt; - return Description.Size.minimumBytes(); -} - -[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LowLevelType Type) noexcept { - const auto Bytes = memoryAccessBytes(Instruction); - if (!Bytes || (!isSupportedIntegerType(Type) && !isSupportedFloatingScalar(Type))) - return false; - if (isSupportedFloatingScalar(Type)) - return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8; - const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U; - return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes; -} - [[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) { if (const std::int64_t *Value = Operand.getIf<std::int64_t>()) return *Value; @@ -1168,19 +644,6 @@ memoryAccessBytes(const MachineInstr &Instruction) noexcept { return std::bit_cast<std::int64_t>(Raw); } -[[nodiscard]] Expected<void> replaceRegisterUses(MachineFunction &Function, Register OldReg, - Register NewReg) { - std::vector<MachineOperand *> Uses; - for (MachineOperand &Operand : Function.registerInfo().useOperands(OldReg)) - Uses.push_back(&Operand); - for (MachineOperand *Operand : Uses) { - auto Replaced = Operand->setRegister(NewReg); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - return {}; -} - [[nodiscard]] Expected<void> eraseInstruction(MachineInstr &Instruction) { MachineBasicBlock *Block = Instruction.parent(); if (Block == nullptr) @@ -1189,67 +652,6 @@ memoryAccessBytes(const MachineInstr &Instruction) noexcept { return Block->removeInstruction(Instruction); } -[[nodiscard]] bool allDefsUnused(const MachineInstr &Instruction, const MachineFunction &Function) { - for (const MachineOperand &Operand : Instruction.operands()) { - const Register *Reg = Operand.getIf<Register>(); - if (Reg != nullptr && Operand.isDef() && Reg->isVirtual() && - !Function.registerInfo().useEmpty(*Reg)) - return false; - } - return true; -} - -void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, - const MachineFunction &Function) { - bool Changed = false; - do { - Changed = false; - for (auto It = Candidates.rbegin(); It != Candidates.rend(); ++It) { - MachineInstr *Instruction = *It; - if (Instruction == nullptr || Instruction->parent() == nullptr || - !allDefsUnused(*Instruction, Function)) - continue; - const Opcode Value = Instruction->opcode(); - if (Value != opcode(GenericOpcode::G_TRUNC) && Value != opcode(GenericOpcode::G_ANYEXT) && - Value != opcode(GenericOpcode::G_SEXT) && Value != opcode(GenericOpcode::G_ZEXT) && - Value != opcode(GenericOpcode::G_MERGE_VALUES) && - Value != opcode(GenericOpcode::G_UNMERGE_VALUES) && Value != opcode(GenericOpcode::COPY)) - continue; - if (eraseInstruction(*Instruction)) - Changed = true; - } - } while (Changed); -} - -[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder, - MachineInstr &Anchor, - std::uint32_t Width, - std::uint64_t Value) { - auto Type = LowLevelType::makeCategoryNeutralScalar(Width); - if (!Type) - return std::unexpected(Type.error()); - auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); - if (!Result) - return std::unexpected(Result.error()); - auto Immediate = APInt::fromUnsigned(Width, Value); - if (!Immediate) - return std::unexpected(Immediate.error()); - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addIntegerImmediate(std::move(*Immediate)); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - [[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LowLevelType Type, Register Source) { @@ -1272,30 +674,6 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, return *Result; } -[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - GenericOpcode Generic, LowLevelType Type, - Register Left, Register Right) { - auto Result = Builder.function().registerInfo().createVirtualRegister(Type); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(Generic)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Left); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Right); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - [[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, Register Destination, Register Source) { auto SetOpcode = Instruction.setOpcode(opcode(Generic)); @@ -1319,2469 +697,415 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, return Instruction.addOperand(std::move(*Use)); } -[[nodiscard]] Expected<void> widenNarrowMultiply(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const std::optional<LowLevelType> Type = resultType(Function, Instruction); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(1); - const MachineOperand *RightOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); - if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || - Type->scalarBitWidth() >= 64) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow multiply has invalid operands"); - } - - MachineInstr *Extension = singleRegisterUse(Function, *Destination); - if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT)) { - const auto Plan = narrowAnyExtBinaryPlan(*Extension, Function); - if (Plan && Plan->Binary == &Instruction) - return foldNarrowAnyExtBinary(*Extension, Function, Target); - } - - auto WideType = LowLevelType::makeCategoryNeutralScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - auto WideLeft = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Left); - if (!WideLeft) - return std::unexpected(WideLeft.error()); - auto WideRight = - buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Right); - if (!WideRight) - return std::unexpected(WideRight.error()); - auto Product = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_MUL, *WideType, *WideLeft, - *WideRight); - if (!Product) - return std::unexpected(Product.error()); - return rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Product); -} - -[[nodiscard]] Expected<void> lowerMultiplyLibcall(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target, - CallingConvention Convention, - const TargetCallLowering &CallLowering) { - const std::optional<LowLevelType> Type = resultType(Function, Instruction); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(1); - const MachineOperand *RightOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); - if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || - Type->scalarBitWidth() != 64) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V multiply libcall requires three s64 registers"); - } - auto ABIAlignment = Alignment::create(8); - auto Callee = MachineCallTarget::externalSymbol("__muldi3"); - if (!ABIAlignment || !Callee) - return makeUnexpected(ErrorCode::InternalError, - "RISC-V multiply libcall ABI facts are unavailable"); - const auto Value = [Type = *Type, Alignment = *ABIAlignment](Register Reg, std::uint32_t Number) { - return ABIValue{ - .Value = Reg, - .Type = Type, - .Category = ABIValueCategory::Integer, - .OriginalValue = Number, - .FragmentIndex = 0, - .FragmentCount = 1, - .FragmentOffsetBytes = 0, - .OriginalAlignment = Alignment, - .Extension = ABIExtensionKind::None, - .Flags = {}, - .ByValSizeBytes = 0, - .ByValAlignment = Alignment, - }; - }; - - CallLoweringRequest Request; - Request.Convention = callingConventionID(Convention); - Request.Callee = std::move(*Callee); - Request.Arguments = {Value(*Left, 0), Value(*Right, 1)}; - Request.Results = {Value(*Destination, 0)}; - Request.FixedArgumentCount = Request.Arguments.size(); - - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Lowered = CallLowering.lowerCall(Builder, Request); - if (!Lowered) - return std::unexpected(Lowered.error()); - return eraseInstruction(Instruction); -} - -[[nodiscard]] Expected<void> rewriteBinary(MachineInstr &Instruction, GenericOpcode Generic, - Register Destination, Register Left, Register Right) { - auto Rewritten = rewriteUnary(Instruction, Generic, Destination, Left); - if (!Rewritten) - return std::unexpected(Rewritten.error()); - auto Use = MachineOperand::registerOperand(Right, RegisterOperandFlags{RegisterOperandFlag::Use}); - if (!Use) - return std::unexpected(Use.error()); - return Instruction.addOperand(std::move(*Use)); -} - -[[nodiscard]] bool isKnownZeroExtended(const MachineFunction &Function, Register Reg, - std::uint32_t ActiveBits, unsigned Depth = 0) { - if (Depth > 8 || !Reg.isVirtual()) - return false; - const auto Type = registerType(Function, Reg); - if (!Type) - return false; - const std::uint32_t Width = Type->scalarBitWidth(); - if (Width <= ActiveBits) - return true; - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg); - const MachineInstr *Def = Definition == nullptr ? nullptr : Definition->parent(); - if (Def == nullptr) - return false; - if (Def->opcode() == opcode(GenericOpcode::G_ASSERT_ZEXT)) { - const MachineOperand *BitsOperand = Def->operandAt(2); - auto Bits = BitsOperand == nullptr - ? Expected<std::int64_t>(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing asserted extension width"))) - : signedImmediate(*BitsOperand); - return Bits && *Bits >= 0 && static_cast<std::uint64_t>(*Bits) <= ActiveBits; - } - if (Def->opcode() == opcode(GenericOpcode::G_ZEXT)) { - const MachineOperand *SourceOperand = Def->operandAt(1); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - if (Source == nullptr) - return false; - const auto SourceType = registerType(Function, *Source); - return SourceType && SourceType->scalarBitWidth() <= ActiveBits; - } - if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { - const MachineOperand *SourceOperand = Def->operandAt(1); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - return Source != nullptr && isKnownZeroExtended(Function, *Source, ActiveBits, Depth + 1U); - } - return false; -} - -[[nodiscard]] Expected<Register> zeroExtendToRV64(MachineIRBuilder &Builder, MachineInstr &Anchor, - MachineFunction &Function, Register Source, - std::vector<MachineInstr *> &DeadCandidates) { - const auto SourceType = registerType(Function, Source); - if (!SourceType) - return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); - if (SourceType->scalarBitWidth() == 64) - return Source; - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); - MachineInstr *Def = - Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); - if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { - const MachineOperand *WideOperand = Def->operandAt(1); - const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>(); - if (Wide != nullptr) { - const auto WideType = registerType(Function, *Wide); - if (WideType && WideType->scalarBitWidth() == 64 && - isKnownZeroExtended(Function, *Wide, SourceType->scalarBitWidth())) { - DeadCandidates.push_back(Def); - return *Wide; - } - } - } - auto WideType = LowLevelType::makeCategoryNeutralScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source); -} - -[[nodiscard]] MachineInstr *singleUseInstruction(const MachineFunction &Function, Register Reg) { - MachineInstr *Result = nullptr; - auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo()); - for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { - MachineInstr *Instruction = Operand.parent(); - if (Instruction == nullptr || Instruction->parent() == nullptr) - continue; - if (Result != nullptr && Result != Instruction) - return nullptr; - Result = Instruction; - } - return Result; -} - -[[nodiscard]] Expected<void> lowerWideMergeValues(MachineInstr &Instruction, - MachineFunction &Function) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - const MachineOperand *LowOperand = Instruction.operandAt(1); - const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>(); - if (Destination == nullptr || Low == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide merge has invalid register operands"); - - std::vector<MachineInstr *> Truncations; - auto &RegisterInfo = Function.registerInfo(); - std::vector<MachineOperand *> Uses; - for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) - Uses.push_back(&Use); - for (MachineOperand *Use : Uses) { - MachineInstr *Trunc = Use->parent(); - if (Trunc == nullptr || Trunc->parent() == nullptr || - Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide merge is only representable through low-part truncation"); - const MachineOperand *TruncResultOperand = Trunc->operandAt(0); - const Register *TruncResult = - TruncResultOperand == nullptr ? nullptr : TruncResultOperand->getIf<Register>(); - if (TruncResult == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide merge truncation has no result"); - const auto LowType = registerType(Function, *Low); - const auto ResultType = registerType(Function, *TruncResult); - if (!LowType || !ResultType || ResultType->scalarBitWidth() > LowType->scalarBitWidth()) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide merge truncation spans more than its low source"); - if (*ResultType == *LowType) { - auto Replaced = replaceRegisterUses(Function, *TruncResult, *Low); - if (!Replaced) - return std::unexpected(Replaced.error()); - Truncations.push_back(Trunc); - } else { - auto Replaced = Use->setRegister(*Low); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - } - for (MachineInstr *Trunc : Truncations) { - auto Erased = eraseInstruction(*Trunc); - if (!Erased) - return std::unexpected(Erased.error()); - } - return eraseInstruction(Instruction); -} - -[[nodiscard]] Expected<void> lowerMergeValues(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const std::optional<LowLevelType> DestinationType = resultType(Function, Instruction); - if (!DestinationType || DestinationType->scalarBitWidth() == 0) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V merge destination has no scalar type"); - if (DestinationType->scalarBitWidth() > 64) - return lowerWideMergeValues(Instruction, Function); - - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - if (Destination == nullptr || Instruction.operandCount() < 3) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V merge has invalid register operands"); - - Register Result = *Destination; - MachineInstr *FoldedExtension = nullptr; - if (DestinationType->scalarBitWidth() < 64) { - MachineInstr *Use = singleUseInstruction(Function, *Destination); - if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) && - operandType(Function, *Use, 0) - .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; }) - .value_or(false)) { - const MachineOperand *ExtendedOperand = Use->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); - if (Extended != nullptr) { - Result = *Extended; - FoldedExtension = Use; - } - } - } - - auto WideType = LowLevelType::makeCategoryNeutralScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - std::vector<MachineInstr *> DeadCandidates; - std::vector<Register> Terms; - Terms.reserve(Instruction.operandCount() - 1U); - for (std::size_t Index = 1; Index < Instruction.operandCount(); ++Index) { - const MachineOperand *Operand = Instruction.operandAt(Index); - const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>(); - if (Source == nullptr) - return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source is not a register"); - auto WideSource = zeroExtendToRV64(Builder, Instruction, Function, *Source, DeadCandidates); - if (!WideSource) - return std::unexpected(WideSource.error()); - if (Index == 1) { - Terms.push_back(*WideSource); - continue; - } - const std::optional<LowLevelType> SourceType = registerType(Function, *Operand); - if (!SourceType) - return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); - const std::uint64_t Shift = - static_cast<std::uint64_t>(SourceType->scalarBitWidth()) * (Index - 1U); - auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Shift); - if (!Amount) - return std::unexpected(Amount.error()); - auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_SHL, *WideType, - *WideSource, *Amount); - if (!Shifted) - return std::unexpected(Shifted.error()); - Terms.push_back(*Shifted); - } - - Register Accumulator = Terms.front(); - for (std::size_t Index = 1; Index + 1U < Terms.size(); ++Index) { - auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, - Accumulator, Terms[Index]); - if (!Combined) - return std::unexpected(Combined.error()); - Accumulator = *Combined; - } - if (DestinationType->scalarBitWidth() == 64 || FoldedExtension != nullptr) { - auto Rewritten = - rewriteBinary(Instruction, GenericOpcode::G_OR, Result, Accumulator, Terms.back()); - if (!Rewritten) - return std::unexpected(Rewritten.error()); - } else { - auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, - Accumulator, Terms.back()); - if (!Combined) - return std::unexpected(Combined.error()); - auto Rewritten = rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Combined); - if (!Rewritten) - return std::unexpected(Rewritten.error()); - } - if (FoldedExtension != nullptr) { - auto Erased = eraseInstruction(*FoldedExtension); - if (!Erased) - return std::unexpected(Erased.error()); - } - eraseDeadArtifacts(DeadCandidates, Function); - return {}; -} - -[[nodiscard]] Expected<void> foldWideExtendedUnmerge(MachineInstr &Instruction, - MachineFunction &Function) { - const std::size_t OutputCount = Instruction.operandCount() - 1U; - const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - const MachineOperand *Definition = - Source == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Source); - MachineInstr *Extension = - Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); - if (Extension == nullptr || (Extension->opcode() != opcode(GenericOpcode::G_ZEXT) && - Extension->opcode() != opcode(GenericOpcode::G_ANYEXT) && - Extension->opcode() != opcode(GenericOpcode::G_SEXT))) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide unmerge requires an extension-defined source"); - const MachineOperand *NarrowOperand = Extension->operandAt(1); - const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>(); - if (Narrow == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide unmerge extension has no register source"); - const auto NarrowType = registerType(Function, *Narrow); - if (!NarrowType) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide unmerge extension source has no type"); - const std::uint32_t NarrowWidth = NarrowType->scalarBitWidth(); - - std::vector<MachineInstr *> DeadUsers; - for (std::size_t Index = 0; Index < OutputCount; ++Index) { - const MachineOperand *OutputOperand = Instruction.operandAt(Index); - const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>(); - if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) - continue; - const auto OutputType = registerType(Function, *Output); - if (!OutputType) - return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge output has no type"); - const std::uint64_t Offset = static_cast<std::uint64_t>(Index) * OutputType->scalarBitWidth(); - if (Offset != 0) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide unmerge cannot materialize a used high wide part"); - if (OutputType->scalarBitWidth() == NarrowWidth) { - auto Replaced = replaceRegisterUses(Function, *Output, *Narrow); - if (!Replaced) - return std::unexpected(Replaced.error()); - continue; - } - if (OutputType->scalarBitWidth() < NarrowWidth) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide unmerge output unexpectedly narrows its source"); - std::vector<MachineOperand *> Uses; - for (MachineOperand &Use : Function.registerInfo().useOperands(*Output)) - Uses.push_back(&Use); - for (MachineOperand *Use : Uses) { - MachineInstr *Trunc = Use->parent(); - if (Trunc == nullptr || Trunc->parent() == nullptr || - Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) - return makeUnexpected( - ErrorCode::Unsupported, - "RISC-V wide unmerge result is only representable through truncation"); - const MachineOperand *ResultOperand = Trunc->operandAt(0); - const Register *Result = - ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>(); - if (Result == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide unmerge truncation has no result"); - const auto ResultType = registerType(Function, *Result); - if (!ResultType || ResultType->scalarBitWidth() > NarrowWidth) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide unmerge truncation exceeds its source width"); - if (ResultType->scalarBitWidth() == NarrowWidth) { - auto Replaced = replaceRegisterUses(Function, *Result, *Narrow); - if (!Replaced) - return std::unexpected(Replaced.error()); - DeadUsers.push_back(Trunc); - } else { - auto Replaced = Use->setRegister(*Narrow); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - } - } - for (MachineInstr *User : DeadUsers) { - auto Erased = eraseInstruction(*User); - if (!Erased) - return std::unexpected(Erased.error()); - } - auto Erased = eraseInstruction(Instruction); - if (!Erased) - return std::unexpected(Erased.error()); - if (Function.registerInfo().useEmpty(*Source)) - return eraseInstruction(*Extension); - return {}; -} - -[[nodiscard]] Expected<void> lowerUnmergeValues(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - if (Instruction.operandCount() < 3) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V unmerge has invalid register operands"); - const std::size_t OutputCount = Instruction.operandCount() - 1U; - const std::optional<LowLevelType> OutputType = operandType(Function, Instruction, 0); - const std::optional<LowLevelType> SourceType = operandType(Function, Instruction, OutputCount); - if (!OutputType || !SourceType) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V unmerge operands have no scalar types"); - if (SourceType->scalarBitWidth() > 64 || OutputType->scalarBitWidth() > 64) - return foldWideExtendedUnmerge(Instruction, Function); - - const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - if (Source == nullptr) - return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge source is not a register"); - auto WideType = LowLevelType::makeCategoryNeutralScalar(64); - if (!WideType) - return std::unexpected(WideType.error()); - MachineIRBuilder Builder(Function, Target); - std::vector<MachineInstr *> FoldedExtensions; - for (std::size_t Index = 0; Index < OutputCount; ++Index) { - const MachineOperand *OutputOperand = Instruction.operandAt(Index); - const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>(); - if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) - continue; - Register Extracted = *Source; - const std::uint64_t Offset = static_cast<std::uint64_t>(Index) * OutputType->scalarBitWidth(); - if (Offset != 0) { - auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Offset); - if (!Amount) - return std::unexpected(Amount.error()); - auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_LSHR, *WideType, - *Source, *Amount); - if (!Shifted) - return std::unexpected(Shifted.error()); - Extracted = *Shifted; - } - - MachineInstr *Extension = singleUseInstruction(Function, *Output); - const bool FoldZExt = - Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ZEXT) && - operandType(Function, *Extension, 0) - .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; }) - .value_or(false); - if (FoldZExt) { - const MachineOperand *ExtendedOperand = Extension->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); - if (Extended == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V unmerge extension has no destination"); - const std::uint64_t Mask = OutputType->scalarBitWidth() == 64 - ? std::numeric_limits<std::uint64_t>::max() - : (std::uint64_t{1} << OutputType->scalarBitWidth()) - 1U; - auto MaskReg = buildIntegerConstantBefore(Builder, Instruction, 64, Mask); - if (!MaskReg) - return std::unexpected(MaskReg.error()); - auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *WideType, - Extracted, *MaskReg); - if (!Masked) - return std::unexpected(Masked.error()); - auto Replaced = replaceRegisterUses(Function, *Extended, *Masked); - if (!Replaced) - return std::unexpected(Replaced.error()); - FoldedExtensions.push_back(Extension); - continue; - } - auto Truncated = - buildUnaryBefore(Builder, Instruction, GenericOpcode::G_TRUNC, *OutputType, Extracted); - if (!Truncated) - return std::unexpected(Truncated.error()); - auto Replaced = replaceRegisterUses(Function, *Output, *Truncated); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - for (MachineInstr *Extension : FoldedExtensions) { - auto Erased = eraseInstruction(*Extension); - if (!Erased) - return std::unexpected(Erased.error()); - } - return eraseInstruction(Instruction); -} - -struct WideValueParts final { - std::uint32_t BitWidth = 0; - std::vector<Register> Parts; - std::vector<MachineInstr *> Artifacts; -}; - -[[nodiscard]] Expected<WideValueParts> -collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth = 0) { - if (Depth > 16) - return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value artifact chain is too deep"); - const auto Type = registerType(Function, Value); - if (!Type) - return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide value has no scalar type"); - if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide-value decomposition requires an integer scalar"); - if (Type->scalarBitWidth() <= 64) - return WideValueParts{.BitWidth = Type->scalarBitWidth(), .Parts = {Value}, .Artifacts = {}}; - - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Value); - MachineInstr *Def = - Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); - if (Def == nullptr) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide value has no decomposable definition"); - if (Def->opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) { - WideValueParts Result; - Result.BitWidth = Type->scalarBitWidth(); - for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) { - const MachineOperand *Operand = Def->operandAt(Index); - const Register *Part = Operand == nullptr ? nullptr : Operand->getIf<Register>(); - const std::optional<LowLevelType> PartType = - Operand == nullptr ? std::nullopt : registerType(Function, *Operand); - if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide merge must already use 64-bit limbs"); - Result.Parts.push_back(*Part); - } - Result.Artifacts.push_back(Def); - return Result; - } - if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { - const MachineOperand *SourceOperand = Def->operandAt(1); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - if (Source == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide truncation has no source register"); - auto Result = collectWideValueParts(*Source, Function, Depth + 1U); - if (!Result) - return std::unexpected(Result.error()); - Result->BitWidth = Type->scalarBitWidth(); - const std::size_t RequiredParts = (Type->scalarBitWidth() + 63U) / 64U; - if (Result->Parts.size() < RequiredParts) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide truncation source has too few limbs"); - Result->Parts.resize(RequiredParts); - Result->Artifacts.push_back(Def); - return Result; - } - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide value is not defined by merge/trunc artifacts"); -} - -struct WideResultPlan final { - std::vector<Register> Outputs; - MachineInstr *Extension = nullptr; - MachineInstr *Unmerge = nullptr; -}; - -[[nodiscard]] Expected<WideResultPlan> wideResultPlan(Register Result, MachineFunction &Function, - std::size_t RequiredParts) { - MachineInstr *Use = singleUseInstruction(Function, Result); - WideResultPlan Plan; - if (Use != nullptr && (Use->opcode() == opcode(GenericOpcode::G_ANYEXT) || - Use->opcode() == opcode(GenericOpcode::G_SEXT) || - Use->opcode() == opcode(GenericOpcode::G_ZEXT))) { - Plan.Extension = Use; - const MachineOperand *ExtendedOperand = Use->operandAt(0); - const Register *Extended = - ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); - Use = Extended == nullptr ? nullptr : singleUseInstruction(Function, *Extended); - } - if (Use == nullptr || Use->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide result must feed an unmerge artifact"); - Plan.Unmerge = Use; - const std::size_t OutputCount = Use->operandCount() - 1U; - if (OutputCount < RequiredParts) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide result unmerge has too few outputs"); - Plan.Outputs.reserve(RequiredParts); - for (std::size_t Index = 0; Index < RequiredParts; ++Index) { - const MachineOperand *Operand = Use->operandAt(Index); - const Register *Output = Operand == nullptr ? nullptr : Operand->getIf<Register>(); - const std::optional<LowLevelType> Type = - Operand == nullptr ? std::nullopt : registerType(Function, *Operand); - if (Output == nullptr || !Type || Type->scalarBitWidth() != 64) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V wide result must unmerge to 64-bit limbs"); - Plan.Outputs.push_back(*Output); - } - return Plan; -} - -[[nodiscard]] Expected<Register> buildIntegerCompareBefore(MachineIRBuilder &Builder, - MachineInstr &Anchor, - IntegerPredicate Predicate, - Register Left, Register Right) { - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addIntegerPredicate(Predicate); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Left); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Right); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - -[[nodiscard]] Expected<Register> buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, - LowLevelType Type, Register Condition, - Register TrueValue, Register FalseValue) { - auto Result = Builder.function().registerInfo().createVirtualRegister(Type); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SELECT)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Condition); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(TrueValue); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(FalseValue); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - -struct TwoLimbShiftResult final { - Register Low; - Register High; -}; - -[[nodiscard]] Expected<TwoLimbShiftResult> -emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, - Register High, Register Amount, Register AmountExcess, Register AmountLack, - Register Zero, Register IsShort, Register IsZero) { - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount); - if (!LowShort) - return std::unexpected(LowShort.error()); - auto CrossLow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, AmountLack); - if (!CrossLow) - return std::unexpected(CrossLow.error()); - auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, Amount); - if (!HighShort) - return std::unexpected(HighShort.error()); - auto ShortHigh = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *CrossLow, *HighShort); - if (!ShortHigh) - return std::unexpected(ShortHigh.error()); - auto LongHigh = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, AmountExcess); - if (!LongHigh) - return std::unexpected(LongHigh.error()); - auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, IsShort, *LowShort, Zero); - if (!ResultLow) - return std::unexpected(ResultLow.error()); - auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, IsShort, *ShortHigh, *LongHigh); - if (!ShiftedHigh) - return std::unexpected(ShiftedHigh.error()); - auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, IsZero, High, *ShiftedHigh); - if (!ResultHigh) - return std::unexpected(ResultHigh.error()); - return TwoLimbShiftResult{.Low = *ResultLow, .High = *ResultHigh}; -} - -[[nodiscard]] Expected<TwoLimbShiftResult> emitTwoLimbLeftShift(MachineIRBuilder &Builder, - MachineInstr &Anchor, Register Low, - Register High, Register Amount) { - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); - if (!Boundary) - return std::unexpected(Boundary.error()); - auto AmountExcess = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *Boundary); - if (!AmountExcess) - return std::unexpected(AmountExcess.error()); - auto AmountLack = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Boundary, Amount); - if (!AmountLack) - return std::unexpected(AmountLack.error()); - auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); - if (!Zero) - return std::unexpected(Zero.error()); - auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - Amount, *Boundary); - if (!IsShort) - return std::unexpected(IsShort.error()); - auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); - if (!IsZero) - return std::unexpected(IsZero.error()); - return emitTwoLimbLeftShiftBody(Builder, Anchor, Low, High, Amount, *AmountExcess, *AmountLack, - *Zero, *IsShort, *IsZero); -} - -[[nodiscard]] Expected<Register> -emitLowLimbOfTwoLimbLogicalRightShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, - Register High, Register Amount, Register Boundary, - Register Zero) { - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - auto AmountExcess = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, Boundary); - if (!AmountExcess) - return std::unexpected(AmountExcess.error()); - auto AmountLack = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Boundary, Amount); - if (!AmountLack) - return std::unexpected(AmountLack.error()); - auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - Amount, Boundary); - if (!IsShort) - return std::unexpected(IsShort.error()); - auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, Zero); - if (!IsZero) - return std::unexpected(IsZero.error()); - auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, Amount); - if (!LowShort) - return std::unexpected(LowShort.error()); - auto HighShort = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, *AmountLack); - if (!HighShort) - return std::unexpected(HighShort.error()); - auto ShortValue = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *LowShort, *HighShort); - if (!ShortValue) - return std::unexpected(ShortValue.error()); - auto LongValue = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, High, *AmountExcess); - if (!LongValue) - return std::unexpected(LongValue.error()); - auto Shifted = buildSelectBefore(Builder, Anchor, *Type, *IsShort, *ShortValue, *LongValue); - if (!Shifted) - return std::unexpected(Shifted.error()); - return buildSelectBefore(Builder, Anchor, *Type, *IsZero, Low, *Shifted); -} - -[[nodiscard]] Expected<std::vector<Register>> -emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, - std::span<const Register> Parts, Register Amount) { - if (Parts.size() != 3) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V three-limb shift requires exactly three limbs"); - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - - auto OuterBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 128); - if (!OuterBoundary) - return std::unexpected(OuterBoundary.error()); - auto OuterExcess = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *OuterBoundary); - if (!OuterExcess) - return std::unexpected(OuterExcess.error()); - auto OuterLack = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *OuterBoundary, Amount); - if (!OuterLack) - return std::unexpected(OuterLack.error()); - auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); - if (!Zero) - return std::unexpected(Zero.error()); - auto OuterIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - Amount, *OuterBoundary); - if (!OuterIsShort) - return std::unexpected(OuterIsShort.error()); - auto OuterIsZero = - buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); - if (!OuterIsZero) - return std::unexpected(OuterIsZero.error()); - - auto InnerBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); - if (!InnerBoundary) - return std::unexpected(InnerBoundary.error()); - auto InnerExcess = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *InnerBoundary); - if (!InnerExcess) - return std::unexpected(InnerExcess.error()); - auto InnerLack = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *InnerBoundary, Amount); - if (!InnerLack) - return std::unexpected(InnerLack.error()); - auto InnerIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - Amount, *InnerBoundary); - if (!InnerIsShort) - return std::unexpected(InnerIsShort.error()); - auto InnerIsZero = - buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); - if (!InnerIsZero) - return std::unexpected(InnerIsZero.error()); - auto Lower = emitTwoLimbLeftShiftBody(Builder, Anchor, Parts[0], Parts[1], Amount, *InnerExcess, - *InnerLack, *Zero, *InnerIsShort, *InnerIsZero); - if (!Lower) - return std::unexpected(Lower.error()); - - auto Cross = emitLowLimbOfTwoLimbLogicalRightShift(Builder, Anchor, Parts[0], Parts[1], - *OuterLack, *InnerBoundary, *Zero); - if (!Cross) - return std::unexpected(Cross.error()); - auto HighIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - Amount, *InnerBoundary); - if (!HighIsShort) - return std::unexpected(HighIsShort.error()); - auto HighShort = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[2], Amount); - if (!HighShort) - return std::unexpected(HighShort.error()); - auto BoundedHigh = buildSelectBefore(Builder, Anchor, *Type, *HighIsShort, *HighShort, *Zero); - if (!BoundedHigh) - return std::unexpected(BoundedHigh.error()); - auto ShortHigh = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *Cross, *BoundedHigh); - if (!ShortHigh) - return std::unexpected(ShortHigh.error()); - - auto LongIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - *OuterExcess, *InnerBoundary); - if (!LongIsShort) - return std::unexpected(LongIsShort.error()); - auto LongShift = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[0], *OuterExcess); - if (!LongShift) - return std::unexpected(LongShift.error()); - auto LongHigh = buildSelectBefore(Builder, Anchor, *Type, *LongIsShort, *LongShift, *Zero); - if (!LongHigh) - return std::unexpected(LongHigh.error()); - - auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->Low, *Zero); - if (!ResultLow) - return std::unexpected(ResultLow.error()); - auto ResultMiddle = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->High, *Zero); - if (!ResultMiddle) - return std::unexpected(ResultMiddle.error()); - auto ShiftedHigh = - buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, *ShortHigh, *LongHigh); - if (!ShiftedHigh) - return std::unexpected(ShiftedHigh.error()); - auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsZero, Parts[2], *ShiftedHigh); - if (!ResultHigh) - return std::unexpected(ResultHigh.error()); - return std::vector<Register>{*ResultLow, *ResultMiddle, *ResultHigh}; -} - -[[nodiscard]] Expected<void> lowerWideShiftLeft(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *ValueOperand = Instruction.operandAt(1); - const MachineOperand *AmountOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); - const Register *Amount = AmountOperand == nullptr ? nullptr : AmountOperand->getIf<Register>(); - if (Destination == nullptr || Value == nullptr || Amount == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide shift has invalid register operands"); - auto ValueParts = collectWideValueParts(*Value, Function); - if (!ValueParts) - return std::unexpected(ValueParts.error()); - auto AmountParts = collectWideValueParts(*Amount, Function); - if (!AmountParts) - return std::unexpected(AmountParts.error()); - if (ValueParts->Parts.size() < 2 || AmountParts->Parts.empty()) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide shift has an invalid limb decomposition"); - auto ResultPlan = wideResultPlan(*Destination, Function, ValueParts->Parts.size()); - if (!ResultPlan) - return std::unexpected(ResultPlan.error()); - - MachineIRBuilder Builder(Function, Target); - Expected<std::vector<Register>> Results = - ValueParts->Parts.size() == 2 - ? emitTwoLimbLeftShift(Builder, Instruction, ValueParts->Parts[0], ValueParts->Parts[1], - AmountParts->Parts[0]) - .transform([](TwoLimbShiftResult Result) { - return std::vector<Register>{Result.Low, Result.High}; - }) - : ValueParts->Parts.size() == 3 - ? emitThreeLimbLeftShift(Builder, Instruction, ValueParts->Parts, AmountParts->Parts[0]) - : Expected<std::vector<Register>>(std::unexpected( - Error(ErrorCode::Unsupported, - "RISC-V wide shift currently requires two or three 64-bit limbs"))); - if (!Results) - return std::unexpected(Results.error()); - for (std::size_t Index = 0; Index < Results->size(); ++Index) { - auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) - return std::unexpected(Erased.error()); - if (ResultPlan->Extension != nullptr) { - if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) - return std::unexpected(Erased.error()); - } - if (auto Erased = eraseInstruction(Instruction); !Erased) - return std::unexpected(Erased.error()); - for (std::size_t Index = 1; Index < AmountParts->Parts.size(); ++Index) { - const MachineOperand *Definition = - Function.registerInfo().getUniqueVRegDef(AmountParts->Parts[Index]); - if (Definition != nullptr && Definition->parent() != nullptr) - AmountParts->Artifacts.push_back(const_cast<MachineInstr *>(Definition->parent())); - } - ValueParts->Artifacts.insert(ValueParts->Artifacts.end(), AmountParts->Artifacts.begin(), - AmountParts->Artifacts.end()); - eraseDeadArtifacts(ValueParts->Artifacts, Function); - return {}; -} - -[[nodiscard]] Expected<std::vector<Register>> emitWideAdd(MachineIRBuilder &Builder, - MachineInstr &Anchor, - std::span<const Register> Left, - std::span<const Register> Right) { - if (Left.size() != Right.size() || Left.size() < 2) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide add requires matching multi-limb operands"); - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - std::vector<Register> Results; - Results.reserve(Left.size()); - auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[0], Right[0]); - if (!Low) - return std::unexpected(Low.error()); - auto Carry = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Low, - Right[0]); - if (!Carry) - return std::unexpected(Carry.error()); - auto LowCopy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Low); - if (!LowCopy) - return std::unexpected(LowCopy.error()); - Results.push_back(*LowCopy); - for (std::size_t Index = 1; Index < Left.size(); ++Index) { - auto Base = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[Index], Right[Index]); - if (!Base) - return std::unexpected(Base.error()); - Register BaseCarry; - if (Index + 1U < Left.size()) { - auto Compared = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - *Base, Left[Index]); - if (!Compared) - return std::unexpected(Compared.error()); - BaseCarry = *Compared; - } - auto Sum = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, *Base, *Carry); - if (!Sum) - return std::unexpected(Sum.error()); - if (Index + 1U < Left.size()) { - auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); - if (!Zero) - return std::unexpected(Zero.error()); - auto IsZero = - buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Sum, *Zero); - if (!IsZero) - return std::unexpected(IsZero.error()); - auto CarryFromIncoming = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Carry); - if (!CarryFromIncoming) - return std::unexpected(CarryFromIncoming.error()); - auto NextCarry = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, BaseCarry, - *CarryFromIncoming); - if (!NextCarry) - return std::unexpected(NextCarry.error()); - Carry = *NextCarry; - } - auto Copy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Sum); - if (!Copy) - return std::unexpected(Copy.error()); - Results.push_back(*Copy); - } - return Results; -} - -[[nodiscard]] Expected<std::vector<Register>> emitWideSub(MachineIRBuilder &Builder, - MachineInstr &Anchor, - std::span<const Register> Left, - std::span<const Register> Right) { - if (Left.size() != Right.size() || Left.size() < 2) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide sub requires matching multi-limb operands"); - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - std::vector<Register> Results; - Results.reserve(Left.size()); - auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[0], Right[0]); - if (!Low) - return std::unexpected(Low.error()); - Results.push_back(*Low); - auto Borrow = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, - Left[0], Right[0]); - if (!Borrow) - return std::unexpected(Borrow.error()); - for (std::size_t Index = 1; Index < Left.size(); ++Index) { - auto Base = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[Index], Right[Index]); - if (!Base) - return std::unexpected(Base.error()); - Register BaseBorrow; - if (Index + 1U < Left.size()) { - auto Compared = buildIntegerCompareBefore( - Builder, Anchor, IntegerPredicate::UnsignedGreaterThan, *Base, Left[Index]); - if (!Compared) - return std::unexpected(Compared.error()); - BaseBorrow = *Compared; - } - auto Difference = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Base, *Borrow); - if (!Difference) - return std::unexpected(Difference.error()); - Results.push_back(*Difference); - if (Index + 1U < Left.size()) { - auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); - if (!Zero) - return std::unexpected(Zero.error()); - auto IsZero = - buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Base, *Zero); - if (!IsZero) - return std::unexpected(IsZero.error()); - auto BorrowFromIncoming = - buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Borrow); - if (!BorrowFromIncoming) - return std::unexpected(BorrowFromIncoming.error()); - auto NextBorrow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, BaseBorrow, - *BorrowFromIncoming); - if (!NextBorrow) - return std::unexpected(NextBorrow.error()); - Borrow = *NextBorrow; - } - } - return Results; -} - -[[nodiscard]] Expected<std::vector<Register>> -emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, - std::span<const Register> Left, std::span<const Register> Right) { - if (Left.size() != Right.size() || Left.size() < 2) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide bitwise operation requires matching multi-limb operands"); - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - std::vector<Register> Results; - Results.reserve(Left.size()); - for (std::size_t Index = 0; Index < Left.size(); ++Index) { - auto Part = buildBinaryBefore(Builder, Anchor, Generic, *Type, Left[Index], Right[Index]); - if (!Part) - return std::unexpected(Part.error()); - Results.push_back(*Part); - } - return Results; -} - -[[nodiscard]] Expected<void> lowerWideBinary(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target, - GenericOpcode Generic) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *LeftOperand = Instruction.operandAt(1); - const MachineOperand *RightOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); - const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); - if (Destination == nullptr || Left == nullptr || Right == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide binary instruction has invalid operands"); - auto LeftParts = collectWideValueParts(*Left, Function); - if (!LeftParts) - return std::unexpected(LeftParts.error()); - auto RightParts = collectWideValueParts(*Right, Function); - if (!RightParts) - return std::unexpected(RightParts.error()); - if (LeftParts->Parts.size() != RightParts->Parts.size()) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide binary operands have different limb counts"); - auto ResultPlan = wideResultPlan(*Destination, Function, LeftParts->Parts.size()); - if (!ResultPlan) - return std::unexpected(ResultPlan.error()); - - MachineIRBuilder Builder(Function, Target); - Expected<std::vector<Register>> Results = - Generic == GenericOpcode::G_ADD - ? emitWideAdd(Builder, Instruction, LeftParts->Parts, RightParts->Parts) - : Generic == GenericOpcode::G_SUB - ? emitWideSub(Builder, Instruction, LeftParts->Parts, RightParts->Parts) - : emitWideBitwise(Builder, Instruction, Generic, LeftParts->Parts, RightParts->Parts); - if (!Results) - return std::unexpected(Results.error()); - for (std::size_t Index = 0; Index < Results->size(); ++Index) { - auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) - return std::unexpected(Erased.error()); - if (ResultPlan->Extension != nullptr) { - if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) - return std::unexpected(Erased.error()); - } - if (auto Erased = eraseInstruction(Instruction); !Erased) - return std::unexpected(Erased.error()); - LeftParts->Artifacts.insert(LeftParts->Artifacts.end(), RightParts->Artifacts.begin(), - RightParts->Artifacts.end()); - eraseDeadArtifacts(LeftParts->Artifacts, Function); - return {}; -} - -[[nodiscard]] bool canLowerNarrowBranchCondition(const MachineInstr &Instruction, - const MachineFunction &Function) { - if (Instruction.opcode() != opcode(GenericOpcode::G_BRCOND) || Instruction.operandCount() < 2) - return false; - const MachineOperand *ConditionOperand = Instruction.operandAt(0); - const Register *Condition = - ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf<Register>(); - if (Condition == nullptr || !Condition->isVirtual() || - singleRegisterUse(Function, *Condition) != &Instruction) - return false; - const auto ConditionType = registerType(Function, *Condition); - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Condition); - const MachineInstr *Truncation = Definition == nullptr ? nullptr : Definition->parent(); - if (!ConditionType || ConditionType->scalarBitWidth() != 1 || Truncation == nullptr || - Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2) - return false; - return operandType(Function, *Truncation, 1) - .transform( - [](LowLevelType Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; }) - .value_or(false); -} - -[[nodiscard]] Expected<void> lowerNarrowBranchCondition(MachineInstr &Instruction, - MachineFunction &Function, - const TargetDescription &Target) { - if (!canLowerNarrowBranchCondition(Instruction, Function)) - return makeUnexpected(ErrorCode::Unsupported, - "RISC-V branch condition is not a narrow truncation"); - const MachineOperand *ConditionOperand = Instruction.operandAt(0); - const Register Condition = *ConditionOperand->getIf<Register>(); - const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Condition); - MachineInstr *Truncation = const_cast<MachineInstr *>(Definition->parent()); - const Register Source = *Truncation->operandAt(1)->getIf<Register>(); - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - MachineIRBuilder Builder(Function, Target); - auto One = buildIntegerConstantBefore(Builder, Instruction, 64, 1); - if (!One) - return std::unexpected(One.error()); - auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *Type, Source, *One); - if (!Masked) - return std::unexpected(Masked.error()); - auto Replaced = Instruction.operandAt(0)->setRegister(*Masked); - if (!Replaced) - return std::unexpected(Replaced.error()); - if (Function.registerInfo().useEmpty(Condition)) - return eraseInstruction(*Truncation); - return {}; -} - -struct NarrowPhiIncoming final { - Register Value; - MachineBasicBlock *Block = nullptr; - MachineInstr *ReusableTruncation = nullptr; -}; - -[[nodiscard]] MachineInstr *firstInstructionAfterLeadingCopies(MachineBasicBlock &Block) { - MachineInstr *Instruction = Block.firstInstruction(); - while (Instruction != nullptr && Instruction->opcode() == opcode(GenericOpcode::COPY)) - Instruction = Instruction->next(); - return Instruction; -} - -[[nodiscard]] Expected<Register> -buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target, - MachineBasicBlock &Block, MachineInstr *Anchor, GenericOpcode Generic, - LowLevelType Type, Register Source) { - auto Result = Function.registerInfo().createVirtualRegister(Type); - if (!Result) - return std::unexpected(Result.error()); - MachineIRBuilder Builder(Function, Target); - auto Point = - Anchor == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Anchor); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(Generic)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addUse(Source); !Added) - return std::unexpected(Added.error()); - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - return *Result; -} - -[[nodiscard]] Expected<void> lowerNarrowPhi(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const std::optional<LowLevelType> OriginalType = resultType(Function, Instruction); - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 || - Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0) - return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has invalid operands"); - auto LegalType = LowLevelType::makeCategoryNeutralScalar(32); - if (!LegalType) - return std::unexpected(LegalType.error()); - - std::vector<NarrowPhiIncoming> Incoming; - Incoming.reserve((Instruction.operandCount() - 1U) / 2U); - for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { - const MachineOperand *ValueOperand = Instruction.operandAt(Index); - const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); - const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); - const auto *Block = - BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>(); - if (Value == nullptr || Block == nullptr || Block->Block == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow PHI has an invalid incoming edge"); - MachineInstr *Reusable = nullptr; - const MachineOperand *Definition = - Value->isVirtual() ? Function.registerInfo().getUniqueVRegDef(*Value) : nullptr; - MachineInstr *Def = - Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); - if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && - Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction && - operandType(Function, *Def, 1) - .transform([](LowLevelType Type) { return Type.scalarBitWidth() >= 32; }) - .value_or(false)) { - Reusable = Def; - } - Incoming.push_back( - NarrowPhiIncoming{.Value = *Value, .Block = Block->Block, .ReusableTruncation = Reusable}); - } - for (NarrowPhiIncoming &Entry : Incoming) { - if (Entry.ReusableTruncation == nullptr) - continue; - const std::size_t Count = static_cast<std::size_t>(std::ranges::count( - Incoming, Entry.ReusableTruncation, &NarrowPhiIncoming::ReusableTruncation)); - if (Count != 1) - Entry.ReusableTruncation = nullptr; - } - for (NarrowPhiIncoming &Entry : Incoming) { - if (Entry.ReusableTruncation == nullptr) - continue; - MachineBasicBlock *OriginalBlock = Entry.ReusableTruncation->parent(); - if (OriginalBlock == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow PHI input truncation is detached"); - auto Removed = OriginalBlock->removeInstruction(*Entry.ReusableTruncation); - if (!Removed) - return std::unexpected(Removed.error()); - } - - std::vector<std::pair<MachineBasicBlock *, MachineInstr *>> Anchors; - for (const NarrowPhiIncoming &Entry : Incoming) { - if (std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first) == - Anchors.end()) { - Anchors.emplace_back(Entry.Block, firstInstructionAfterLeadingCopies(*Entry.Block)); - } - } - std::vector<Register> LegalIncoming; - LegalIncoming.reserve(Incoming.size()); - for (NarrowPhiIncoming &Entry : Incoming) { - const auto AnchorPosition = - std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first); - MachineInstr *Anchor = AnchorPosition->second; - if (Entry.ReusableTruncation != nullptr) { - auto SetType = Function.registerInfo().setType(Entry.Value, *LegalType); - if (!SetType) - return std::unexpected(SetType.error()); - auto Inserted = - Anchor == nullptr - ? Entry.Block->appendInstruction(*Entry.ReusableTruncation) - : Entry.Block->insertInstructionBefore(*Anchor, *Entry.ReusableTruncation); - if (!Inserted) - return std::unexpected(Inserted.error()); - LegalIncoming.push_back(Entry.Value); - continue; - } - const auto ValueType = registerType(Function, Entry.Value); - if (!ValueType) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V narrow PHI input has no scalar type"); - const std::uint32_t Width = ValueType->scalarBitWidth(); - const GenericOpcode Conversion = Width < 32 ? GenericOpcode::G_ANYEXT - : Width > 32 ? GenericOpcode::G_TRUNC - : GenericOpcode::COPY; - auto Converted = buildIncomingConversion(Function, Target, *Entry.Block, Anchor, Conversion, - *LegalType, Entry.Value); - if (!Converted) - return std::unexpected(Converted.error()); - LegalIncoming.push_back(*Converted); - } - - auto LegalResult = Function.registerInfo().createVirtualRegister(*LegalType); - if (!LegalResult) - return std::unexpected(LegalResult.error()); - MachineIRBuilder Builder(Function, Target); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*LegalResult); !Added) - return std::unexpected(Added.error()); - for (std::size_t Index = 0; Index < Incoming.size(); ++Index) { - if (auto Added = Pending->addUse(LegalIncoming[Index]); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addBasicBlock(*Incoming[Index].Block); !Added) - return std::unexpected(Added.error()); - } - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - auto Replaced = replaceRegisterUses(Function, *Destination, *LegalResult); - if (!Replaced) - return std::unexpected(Replaced.error()); - return eraseInstruction(Instruction); -} - -[[nodiscard]] Expected<void> lowerWidePhi(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - if (Destination == nullptr || Instruction.operandCount() < 5 || - (Instruction.operandCount() % 2U) == 0) - return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has invalid operands"); - - std::vector<WideValueParts> IncomingParts; - std::vector<MachineBasicBlock *> IncomingBlocks; - for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { - const MachineOperand *ValueOperand = Instruction.operandAt(Index); - const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); - const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); - const auto *Block = - BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>(); - if (Value == nullptr || Block == nullptr || Block->Block == nullptr) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide PHI has an invalid incoming edge"); - auto Parts = collectWideValueParts(*Value, Function); - if (!Parts) - return std::unexpected(Parts.error()); - if (!IncomingParts.empty() && Parts->Parts.size() != IncomingParts.front().Parts.size()) - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V wide PHI incoming values have different limb counts"); - IncomingParts.push_back(std::move(*Parts)); - IncomingBlocks.push_back(Block->Block); - } - const std::size_t PartCount = IncomingParts.front().Parts.size(); - auto ResultPlan = wideResultPlan(*Destination, Function, PartCount); - if (!ResultPlan) - return std::unexpected(ResultPlan.error()); - auto Type = LowLevelType::makeCategoryNeutralScalar(64); - if (!Type) - return std::unexpected(Type.error()); - MachineIRBuilder Builder(Function, Target); - std::vector<Register> Results; - Results.reserve(PartCount); - for (std::size_t PartIndex = 0; PartIndex < PartCount; ++PartIndex) { - auto Result = Function.registerInfo().createVirtualRegister(*Type); - if (!Result) - return std::unexpected(Result.error()); - auto Point = Builder.setInsertPointBefore(Instruction); - if (!Point) - return std::unexpected(Point.error()); - auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); - if (!Pending) - return std::unexpected(Pending.error()); - if (auto Added = Pending->addDef(*Result); !Added) - return std::unexpected(Added.error()); - for (std::size_t IncomingIndex = 0; IncomingIndex < IncomingParts.size(); ++IncomingIndex) { - if (auto Added = Pending->addUse(IncomingParts[IncomingIndex].Parts[PartIndex]); !Added) - return std::unexpected(Added.error()); - if (auto Added = Pending->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added) - return std::unexpected(Added.error()); - } - auto Built = Pending->build(); - if (!Built) - return std::unexpected(Built.error()); - Results.push_back(*Result); - } - for (std::size_t Index = 0; Index < PartCount; ++Index) { - auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], Results[Index]); - if (!Replaced) - return std::unexpected(Replaced.error()); - } - if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) - return std::unexpected(Erased.error()); - if (ResultPlan->Extension != nullptr) { - if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) - return std::unexpected(Erased.error()); - } - if (auto Erased = eraseInstruction(Instruction); !Erased) - return std::unexpected(Erased.error()); - std::vector<MachineInstr *> Artifacts; - for (WideValueParts &Parts : IncomingParts) - Artifacts.insert(Artifacts.end(), Parts.Artifacts.begin(), Parts.Artifacts.end()); - eraseDeadArtifacts(Artifacts, Function); - return {}; -} - -[[nodiscard]] bool canFoldWideMergeImmediately(const MachineInstr &Instruction, - const MachineFunction &Function) { - const MachineOperand *LowOperand = Instruction.operandAt(1); - const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>(); - if (Low == nullptr) - return false; - const auto LowType = registerType(Function, *Low); - if (!LowType) - return false; - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - if (Destination == nullptr) - return false; - auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo()); - for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) { - MachineInstr *Trunc = Use.parent(); - if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || - Trunc->operandAt(1) != &Use) - return false; - const std::optional<LowLevelType> Result = resultType(Function, *Trunc); - if (!Result || Result->scalarBitWidth() > LowType->scalarBitWidth()) - return false; - } - return true; -} - -struct RV64ScalarLayout final { - std::uint64_t StoreSize = 0; - std::uint64_t AllocationSize = 0; - std::uint64_t ABIAlignment = 0; -}; - -[[nodiscard]] std::optional<RV64ScalarLayout> rv64ScalarLayout(LowLevelType Type) noexcept { - if (Type.isPointer()) { - return Type.addressSpace() == 0 && Type.scalarBitWidth() == 64 - ? std::optional<RV64ScalarLayout>{{8, 8, 8}} - : std::nullopt; - } - if ((!isSupportedIntegerType(Type) && !isSupportedFloatingScalar(Type)) || - Type.scalarBitWidth() == 0) { - return std::nullopt; - } - const std::uint64_t StoreBytes = (Type.scalarBitWidth() + 7U) / 8U; - const std::uint64_t AllocationBytes = std::bit_ceil(StoreBytes); - return RV64ScalarLayout{StoreBytes, AllocationBytes, std::min<std::uint64_t>(AllocationBytes, 8)}; -} - -[[nodiscard]] Expected<const MachineMemOperand *> -createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LowLevelType Type, - std::uint64_t SizeBytes, std::uint64_t AlignmentBytes) { - auto Size = MemorySize::fixed(SizeBytes); - auto Align = Alignment::create(AlignmentBytes); - if (!Size || !Align) - return makeUnexpected(ErrorCode::InternalError, "RISC-V memory facts are invalid"); - MachineMemOperandDesc Description; - Description.Accesses = {Access}; - Description.Size = *Size; - Description.MemoryType = Type; - Description.Align = *Align; - return Function.createMemOperand(std::move(Description)); -} - -struct VAArgPlan final { - Register Destination; - Register ListPointer; - LowLevelType DestinationType; - LowLevelType PointerType; - std::uint64_t Alignment = 0; - RV64ScalarLayout ValueLayout; -}; - -[[nodiscard]] std::optional<VAArgPlan> vaargPlan(const MachineInstr &Instruction, - const MachineFunction &Function) { - if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 || - !Instruction.memOperands().empty()) { - return std::nullopt; - } - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const MachineOperand *ListOperand = Instruction.operandAt(1); - const MachineOperand *AlignmentOperand = Instruction.operandAt(2); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>(); - const std::optional<LowLevelType> DestinationType = resultType(Function, Instruction); - const std::optional<LowLevelType> PointerType = operandType(Function, Instruction, 1); - const auto AlignmentValue = AlignmentOperand == nullptr - ? Expected<std::int64_t>(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing alignment"))) - : signedImmediate(*AlignmentOperand); - const std::optional<RV64ScalarLayout> ValueLayout = - DestinationType ? rv64ScalarLayout(*DestinationType) : std::nullopt; - if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() || - !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() || - !DestinationType || !PointerType || !PointerType->isPointer() || - PointerType->addressSpace() != 0 || PointerType->scalarBitWidth() != 64 || !AlignmentValue || - *AlignmentValue <= 0 || !std::has_single_bit(static_cast<std::uint64_t>(*AlignmentValue)) || - !ValueLayout) { - return std::nullopt; - } - - return VAArgPlan{*Destination, - *ListPointer, - *DestinationType, - *PointerType, - static_cast<std::uint64_t>(*AlignmentValue), - *ValueLayout}; -} - -[[nodiscard]] Expected<void> lowerVAArg(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const auto Plan = vaargPlan(Instruction, Function); - if (!Plan) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V va_arg legalization requires a scalar result, p0 list, and " - "power-of-two alignment"); - } - - auto ListLoadMemory = - createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->PointerType, 8, 8); - if (!ListLoadMemory) - return std::unexpected(ListLoadMemory.error()); - auto ListStoreMemory = - createMemoryAccess(Function, MemoryAccessFlag::Store, Plan->PointerType, 8, 8); - if (!ListStoreMemory) - return std::unexpected(ListStoreMemory.error()); - - auto CurrentList = Function.registerInfo().createVirtualRegister(Plan->PointerType); - if (!CurrentList) - return std::unexpected(CurrentList.error()); - MachineIRBuilder Builder(Function, Target); - if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) - return std::unexpected(Point.error()); - auto ListLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); - if (!ListLoad) - return std::unexpected(ListLoad.error()); - if (auto Added = ListLoad->addDef(*CurrentList); !Added) - return std::unexpected(Added.error()); - if (auto Added = ListLoad->addUse(Plan->ListPointer); !Added) - return std::unexpected(Added.error()); - if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = ListLoad->build(); !Built) - return std::unexpected(Built.error()); - - Register ValueAddress = *CurrentList; - if (Plan->Alignment > 1) { - auto AlignmentOffset = - buildIntegerConstantBefore(Builder, Instruction, 64, Plan->Alignment - 1U); - if (!AlignmentOffset) - return std::unexpected(AlignmentOffset.error()); - auto BiasedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, - Plan->PointerType, ValueAddress, *AlignmentOffset); - if (!BiasedAddress) - return std::unexpected(BiasedAddress.error()); - auto AlignmentMask = - buildIntegerConstantBefore(Builder, Instruction, 64, ~(Plan->Alignment - std::uint64_t{1})); - if (!AlignmentMask) - return std::unexpected(AlignmentMask.error()); - auto AlignedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTRMASK, - Plan->PointerType, *BiasedAddress, *AlignmentMask); - if (!AlignedAddress) - return std::unexpected(AlignedAddress.error()); - ValueAddress = *AlignedAddress; - } - - auto Increment = - buildIntegerConstantBefore(Builder, Instruction, 64, Plan->ValueLayout.AllocationSize); - if (!Increment) - return std::unexpected(Increment.error()); - auto NextList = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, - Plan->PointerType, ValueAddress, *Increment); - if (!NextList) - return std::unexpected(NextList.error()); - auto ListStore = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); - if (!ListStore) - return std::unexpected(ListStore.error()); - if (auto Added = ListStore->addUse(*NextList); !Added) - return std::unexpected(Added.error()); - if (auto Added = ListStore->addUse(Plan->ListPointer); !Added) - return std::unexpected(Added.error()); - if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = ListStore->build(); !Built) - return std::unexpected(Built.error()); - - if (!Function.registerInfo().useEmpty(Plan->Destination)) { - auto ValueLoadMemory = - createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->DestinationType, - Plan->ValueLayout.StoreSize, Plan->ValueLayout.ABIAlignment); - if (!ValueLoadMemory) - return std::unexpected(ValueLoadMemory.error()); - auto ValueLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); - if (!ValueLoad) - return std::unexpected(ValueLoad.error()); - if (auto Added = ValueLoad->addDef(Plan->Destination); !Added) - return std::unexpected(Added.error()); - if (auto Added = ValueLoad->addUse(ValueAddress); !Added) - return std::unexpected(Added.error()); - if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = ValueLoad->build(); !Built) - return std::unexpected(Built.error()); - } - return eraseInstruction(Instruction); -} - -struct VACopyPlan final { - Register Destination; - Register Source; - LowLevelType PointerType; -}; - -[[nodiscard]] std::optional<VACopyPlan> vacopyPlan(const MachineInstr &Instruction, - const MachineFunction &Function) { - if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) || - Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) { - return std::nullopt; - } - const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0); - const MachineOperand *DestinationOperand = Instruction.operandAt(1); - const MachineOperand *SourceOperand = Instruction.operandAt(2); - const auto *Intrinsic = - IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf<IntrinsicOperand>(); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - const std::optional<LowLevelType> DestinationType = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> SourceType = operandType(Function, Instruction, 2); - if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) || - Destination == nullptr || Source == nullptr || !Destination->isVirtual() || - !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() || - !DestinationType || !SourceType || *DestinationType != *SourceType || - !DestinationType->isPointer() || DestinationType->addressSpace() != 0 || - DestinationType->scalarBitWidth() != 64) { - return std::nullopt; - } - return VACopyPlan{*Destination, *Source, *DestinationType}; -} - -[[nodiscard]] Expected<void> lowerVACopy(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target) { - const auto Plan = vacopyPlan(Instruction, Function); - if (!Plan) { - return makeUnexpected(ErrorCode::VerificationError, - "RISC-V va_copy legalization requires two p0 operands"); - } - auto Size = MemorySize::fixed(8); - auto Align = Alignment::create(8); - if (!Size || !Align) - return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid"); - - MachineMemOperandDesc LoadDescription; - LoadDescription.Accesses = {MemoryAccessFlag::Load}; - LoadDescription.Size = *Size; - LoadDescription.MemoryType = Plan->PointerType; - LoadDescription.Align = *Align; - auto LoadMemory = Function.createMemOperand(std::move(LoadDescription)); - if (!LoadMemory) - return std::unexpected(LoadMemory.error()); - - MachineMemOperandDesc StoreDescription; - StoreDescription.Accesses = {MemoryAccessFlag::Store}; - StoreDescription.Size = *Size; - StoreDescription.MemoryType = Plan->PointerType; - StoreDescription.Align = *Align; - auto StoreMemory = Function.createMemOperand(std::move(StoreDescription)); - if (!StoreMemory) - return std::unexpected(StoreMemory.error()); - - auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType); - if (!Value) - return std::unexpected(Value.error()); - MachineIRBuilder Builder(Function, Target); - if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) - return std::unexpected(Point.error()); - - auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); - if (!Load) - return std::unexpected(Load.error()); - if (auto Added = Load->addDef(*Value); !Added) - return std::unexpected(Added.error()); - if (auto Added = Load->addUse(Plan->Source); !Added) - return std::unexpected(Added.error()); - if (auto Added = Load->addMemOperand(**LoadMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = Load->build(); !Built) - return std::unexpected(Built.error()); - - auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); - if (!Store) - return std::unexpected(Store.error()); - if (auto Added = Store->addUse(*Value); !Added) - return std::unexpected(Added.error()); - if (auto Added = Store->addUse(Plan->Destination); !Added) - return std::unexpected(Added.error()); - if (auto Added = Store->addMemOperand(**StoreMemory); !Added) - return std::unexpected(Added.error()); - if (auto Built = Store->build(); !Built) - return std::unexpected(Built.error()); - return eraseInstruction(Instruction); -} - } // namespace -Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &Instruction, - const MachineFunction &Function, - const TargetDescription &) const { - if (!Subtarget.features().has(RISCVExtension::I)) { - return makeUnexpected(ErrorCode::InternalError, - "RISC-V legalizer requires the base I extension"); - } - if (!isGenericOpcode(Instruction.opcode())) - return LegalizationAction::Legal; - - const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); - const std::optional<LowLevelType> Type = resultType(Function, Instruction); - switch (Generic) { - case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - case GenericOpcode::G_VAARG: - return vaargPlan(Instruction, Function) ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - case GenericOpcode::G_FRAME_INDEX: - return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_BR: - return LegalizationAction::Legal; - case GenericOpcode::G_BRCOND: { - const std::optional<LowLevelType> Condition = operandType(Function, Instruction, 0); - if (!Condition || !isScalarGPRType(*Condition)) - return LegalizationAction::Unsupported; - return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom - : LegalizationAction::Legal; - } - case GenericOpcode::G_PHI: { - if (!Type) - return LegalizationAction::Unsupported; - for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { - const std::optional<LowLevelType> Incoming = operandType(Function, Instruction, Index); - if (!Incoming || *Incoming != *Type) - return LegalizationAction::Unsupported; - } - if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() > 64) - return LegalizationAction::Custom; - if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() < 32) - return LegalizationAction::Custom; - if (!isSupportedIntegerType(*Type)) - return LegalizationAction::Unsupported; - return LegalizationAction::Legal; - } - case GenericOpcode::G_IMPLICIT_DEF: - return Type && isSupportedIntegerType(*Type) ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_STORE: - if (Instruction.operandCount() == 0) - return LegalizationAction::Unsupported; - if (const std::optional<LowLevelType> ValueType = - registerType(Function, *Instruction.operandAt(0)); - ValueType) { - const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); - if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || - AddressType->scalarBitWidth() != 64) { - return LegalizationAction::Unsupported; - } - if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { - const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U; - return isSupportedIntegerType(*ValueType) && - isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - if (isSupportedIntegerType(*ValueType) && isSupportedNonAtomicMemoryAccess(Instruction) && - memoryMatchesType(Instruction, *ValueType)) { - return LegalizationAction::Legal; - } - } - return LegalizationAction::Unsupported; - case GenericOpcode::G_SEXTLOAD: - case GenericOpcode::G_ZEXTLOAD: { - const auto Bytes = memoryAccessBytes(Instruction); - const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); - return Type && Bytes && AddressType && isSupportedIntegerType(*Type) && - AddressType->isPointer() && AddressType->addressSpace() == 0 && - AddressType->scalarBitWidth() == 64 && - *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) && - (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) && - isSupportedNonAtomicMemoryAccess(Instruction) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FPEXTLOAD: { - const auto Bytes = memoryAccessBytes(Instruction); - const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); - return Type && Bytes && AddressType && Type->isCategoryNeutralScalar() && - Type->scalarBitWidth() == 64 && AddressType->isPointer() && - AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && - *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FPTRUNCSTORE: { - const std::optional<LowLevelType> ValueType = operandType(Function, Instruction, 0); - const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); - const auto Bytes = memoryAccessBytes(Instruction); - return ValueType && AddressType && Bytes && ValueType->isCategoryNeutralScalar() && - ValueType->scalarBitWidth() == 64 && AddressType->isPointer() && - AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && - *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_LOAD: - if (Type) { - const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1); - if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || - AddressType->scalarBitWidth() != 64) { - return LegalizationAction::Unsupported; - } - if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { - const std::uint64_t Bytes = Type->scalarBitWidth() / 8U; - return isSupportedIntegerType(*Type) && - isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - if (isSupportedIntegerType(*Type) && isSupportedNonAtomicMemoryAccess(Instruction) && - memoryMatchesType(Instruction, *Type)) { - return LegalizationAction::Legal; - } - } - return LegalizationAction::Unsupported; - case GenericOpcode::G_FADD: - case GenericOpcode::G_FSUB: - case GenericOpcode::G_FMUL: - case GenericOpcode::G_FDIV: { - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type && - *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FMA: { - const std::optional<LowLevelType> First = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Second = operandType(Function, Instruction, 2); - const std::optional<LowLevelType> Third = operandType(Function, Instruction, 3); - return Type && First && Second && Third && isSupportedFloatingScalar(*Type) && - *First == *Type && *Second == *Type && *Third == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FNEG: - case GenericOpcode::G_FABS: - case GenericOpcode::G_FSQRT: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - return Type && Source && isSupportedFloatingScalar(*Type) && *Source == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FCOPYSIGN: { - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type && - *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FCONSTANT: { - const MachineOperand *Immediate = Instruction.operandAt(1); - const auto *Floating = - Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>(); - return Type && isSupportedFloatingScalar(*Type) && Floating != nullptr && - Floating->Bits.bitWidth() == Type->scalarBitWidth() - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ASSERT_SEXT: - case GenericOpcode::G_ASSERT_ZEXT: - case GenericOpcode::G_ASSERT_ALIGN: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - return Type && Source && *Type == *Source && isScalarGPRType(*Type) && - Instruction.operandCount() == 3 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FREEZE: - case GenericOpcode::G_CONSTANT_FOLD_BARRIER: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - return Type && Source && *Type == *Source && isSupportedIntegerType(*Type) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_PTRTOINT: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - return Type && Source && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64 && - Source->isPointer() && Source->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_INTTOPTR: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 && - Source->isCategoryNeutralScalar() && Source->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_BITCAST: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - return Type && Source && *Type != *Source && Type->scalarBitWidth() == Source->scalarBitWidth() - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ANYEXT: - case GenericOpcode::G_SEXT: - case GenericOpcode::G_ZEXT: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - if (!Type || !Source) - return LegalizationAction::Unsupported; - if (!isScalarIntegerExtension(*Type, *Source)) { - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - MachineInstr *OnlyUse = - Destination == nullptr ? nullptr : singleUseInstruction(Function, *Destination); - return Type->isCategoryNeutralScalar() && Source->isCategoryNeutralScalar() && - Type->scalarBitWidth() > 64 && - Source->scalarBitWidth() < Type->scalarBitWidth() && OnlyUse != nullptr && - OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - const MachineOperand *SourceOperand = Instruction.operandAt(1); - const Register *SourceRegister = - SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); - const MachineOperand *SourceDefinition = - SourceRegister == nullptr || !SourceRegister->isVirtual() - ? nullptr - : Function.registerInfo().getUniqueVRegDef(*SourceRegister); - const MachineInstr *SourceDef = - SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); - if (SourceDef != nullptr && SourceDef->opcode() == opcode(GenericOpcode::G_CONSTANT)) - return LegalizationAction::Custom; - if (Generic == GenericOpcode::G_ANYEXT && - narrowAnyExtBinaryPlan(Instruction, Function).has_value()) - return LegalizationAction::Custom; - return LegalizationAction::Legal; - } - case GenericOpcode::G_TRUNC: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - if (Type && Source && Type->isCategoryNeutralScalar() && Source->isCategoryNeutralScalar() && - Type->scalarBitWidth() > 64 && Source->scalarBitWidth() > Type->scalarBitWidth()) - return LegalizationAction::Legal; - return Type && Source && isSupportedIntegerScalar(*Type) && isSupportedIntegerScalar(*Source) && - Type->scalarBitWidth() != 0 && Source->scalarBitWidth() >= Type->scalarBitWidth() - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_MERGE_VALUES: - case GenericOpcode::G_UNMERGE_VALUES: { - if (Instruction.operandCount() < 3) - return LegalizationAction::Unsupported; - const std::size_t SmallTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0; - const std::size_t BigTypeIndex = - Generic == GenericOpcode::G_MERGE_VALUES ? 0 : Instruction.operandCount() - 1U; - const std::optional<LowLevelType> SmallType = - operandType(Function, Instruction, SmallTypeIndex); - const std::optional<LowLevelType> BigType = operandType(Function, Instruction, BigTypeIndex); - if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES && - BigType->scalarBitWidth() > 64 && !canFoldWideMergeImmediately(Instruction, Function)) - return LegalizationAction::Legal; - return SmallType && BigType && SmallType->isCategoryNeutralScalar() && - BigType->isCategoryNeutralScalar() && SmallType->scalarBitWidth() != 0 && - BigType->scalarBitWidth() != 0 - ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SEXT_INREG: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - const MachineOperand *WidthOperand = Instruction.operandAt(2); - const auto Width = - WidthOperand == nullptr - ? Expected<std::int64_t>(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing in-register extension width"))) - : signedImmediate(*WidthOperand); - return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 && - static_cast<std::uint64_t>(*Width) <= Type->scalarBitWidth() - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ICMP: { - const MachineOperand *Predicate = Instruction.operandAt(1); - const auto *IntegerPredicate = - Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>(); - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); - if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr || - IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right || - !isSupportedIntegerType(*Left)) { - return LegalizationAction::Unsupported; - } - return Type->scalarBitWidth() == 64 && Left->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Custom; - } - case GenericOpcode::G_FCMP: { - const MachineOperand *Predicate = Instruction.operandAt(1); - const auto *FloatingPredicate = - Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>(); - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); - return Type && isI1Type(*Type) && FloatingPredicate != nullptr && - FloatingPredicate->floatingPredicate() != nullptr && Left && Right && - *Left == *Right && isSupportedFloatingScalar(*Left) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FPEXT: - case GenericOpcode::G_FPTRUNC: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - const bool IsExtension = Generic == GenericOpcode::G_FPEXT; - return Type && Source && isSupportedFloatingScalar(*Type) && - isSupportedFloatingScalar(*Source) && - ((IsExtension && Source->scalarBitWidth() == 32 && - Type->scalarBitWidth() == 64) || - (!IsExtension && Source->scalarBitWidth() == 64 && - Type->scalarBitWidth() == 32)) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FPTOSI: - case GenericOpcode::G_FPTOUI: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - return Type && Source && isSupportedIntegerType(*Type) && isSupportedFloatingScalar(*Source) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SITOFP: - case GenericOpcode::G_UITOFP: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - return Type && Source && isSupportedFloatingScalar(*Type) && isSupportedIntegerType(*Source) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FMINNUM: - case GenericOpcode::G_FMAXNUM: { - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type && - *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SELECT: { - const std::optional<LowLevelType> Condition = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> TrueValue = operandType(Function, Instruction, 2); - const std::optional<LowLevelType> FalseValue = operandType(Function, Instruction, 3); - return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) && - *TrueValue == *Type && *FalseValue == *Type && isSupportedIntegerType(*Type) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_PTRMASK: { - const std::optional<LowLevelType> Pointer = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Mask = operandType(Function, Instruction, 2); - return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 && - Type->scalarBitWidth() == 64 && *Pointer == *Type && - isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ABS: { - const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1); - return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SDIVREM: - case GenericOpcode::G_UDIVREM: { - const std::optional<LowLevelType> Remainder = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); - return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) && - *Remainder == *Type && *Left == *Type && *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SMULH: - case GenericOpcode::G_UMULH: { - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type && - *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_SMIN: - case GenericOpcode::G_SMAX: - case GenericOpcode::G_UMIN: - case GenericOpcode::G_UMAX: { - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); - return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type && - *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_UADDO: - case GenericOpcode::G_USUBO: - case GenericOpcode::G_SADDO: - case GenericOpcode::G_SSUBO: - case GenericOpcode::G_UMULO: - case GenericOpcode::G_SMULO: { - const std::optional<LowLevelType> Overflow = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); - return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) && - isI1Type(*Overflow) && *Left == *Type && *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_UADDE: - case GenericOpcode::G_USUBE: - case GenericOpcode::G_SADDE: - case GenericOpcode::G_SSUBE: { - const std::optional<LowLevelType> CarryOut = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3); - const std::optional<LowLevelType> CarryIn = operandType(Function, Instruction, 4); - return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) && - isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ATOMICRMW_XCHG: - case GenericOpcode::G_ATOMICRMW_ADD: - case GenericOpcode::G_ATOMICRMW_SUB: - case GenericOpcode::G_ATOMICRMW_AND: - case GenericOpcode::G_ATOMICRMW_OR: - case GenericOpcode::G_ATOMICRMW_XOR: - case GenericOpcode::G_ATOMICRMW_MAX: - case GenericOpcode::G_ATOMICRMW_MIN: - case GenericOpcode::G_ATOMICRMW_UMAX: - case GenericOpcode::G_ATOMICRMW_UMIN: { - const std::optional<LowLevelType> Address = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Value = operandType(Function, Instruction, 2); - const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0; - return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type && - Address->isPointer() && Address->addressSpace() == 0 && - Address->scalarBitWidth() == 64 && - isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite, - Bytes) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ATOMIC_CMPXCHG: { - const std::optional<LowLevelType> Address = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Compare = operandType(Function, Instruction, 2); - const std::optional<LowLevelType> NewValue = operandType(Function, Instruction, 3); - if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type || - !Type->isCategoryNeutralScalar() || - (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 && - Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) || - !Address->isPointer() || Address->addressSpace() != 0 || Address->scalarBitWidth() != 64 || - !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, - (Type->scalarBitWidth() + 7U) / 8U)) { - return LegalizationAction::Unsupported; - } - return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; - } - case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { - const std::optional<LowLevelType> Success = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Address = operandType(Function, Instruction, 2); - const std::optional<LowLevelType> Compare = operandType(Function, Instruction, 3); - const std::optional<LowLevelType> NewValue = operandType(Function, Instruction, 4); - return Type && Success && Address && Compare && NewValue && isI1Type(*Success) && - *Compare == *Type && *NewValue == *Type && Type->isCategoryNeutralScalar() && - (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 || - Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) && - Address->isPointer() && Address->addressSpace() == 0 && - Address->scalarBitWidth() == 64 && - isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, - (Type->scalarBitWidth() + 7U) / 8U) - ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_FENCE: { - const MachineOperand *OrderingOperand = Instruction.operandAt(0); - const MachineOperand *ScopeOperand = Instruction.operandAt(1); - const auto Ordering = OrderingOperand == nullptr - ? Expected<std::int64_t>(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing fence ordering"))) - : signedImmediate(*OrderingOperand); - const auto Scope = ScopeOperand == nullptr - ? Expected<std::int64_t>(std::unexpected( - Error(ErrorCode::InvalidArgument, "missing fence scope"))) - : signedImmediate(*ScopeOperand); - return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_CONSTANT_POOL: { - const MachineOperand *PoolOperand = Instruction.operandAt(1); - const auto *Pool = PoolOperand == nullptr ? nullptr : PoolOperand->getIf<ConstantPoolOperand>(); - return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && - Pool != nullptr && Function.constantPool().entry(Pool->Index) != nullptr - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_GLOBAL_VALUE: { - const MachineOperand *GlobalOperand = Instruction.operandAt(1); - const auto *Global = - GlobalOperand == nullptr ? nullptr : GlobalOperand->getIf<GlobalSymbolOperand>(); - return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && - Global != nullptr && !Global->Name.empty() - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_BRINDIRECT: - return operandType(Function, Instruction, 0) - .transform([](LowLevelType Value) { - return Value.isPointer() && Value.addressSpace() == 0 && - Value.scalarBitWidth() == 64; - }) - .value_or(false) - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_TRAP: - case GenericOpcode::G_DEBUGTRAP: - return Instruction.operandCount() == 0 ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_UBSANTRAP: - return Instruction.operandCount() == 1 ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - case GenericOpcode::G_CONSTANT: { - const MachineOperand *Immediate = Instruction.operandAt(1); - if (!Type || Immediate == nullptr || - (Immediate->kind() != MachineOperand::Kind::Immediate && - Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) - return LegalizationAction::Unsupported; - if (Type->scalarBitWidth() > 64) - return wideConstantUnmergePlan(Instruction, Function).has_value() - ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64) { - auto Value = signedImmediate(*Immediate); - if (Value && shouldUseRV64IntegerConstantPool(*Value)) - return LegalizationAction::Custom; - } - return isSupportedIntegerType(*Type) ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_ADD: - case GenericOpcode::G_SUB: - case GenericOpcode::G_MUL: - case GenericOpcode::G_SDIV: - case GenericOpcode::G_UDIV: - case GenericOpcode::G_SREM: - case GenericOpcode::G_UREM: - case GenericOpcode::G_AND: - case GenericOpcode::G_OR: - case GenericOpcode::G_XOR: - case GenericOpcode::G_SHL: - case GenericOpcode::G_LSHR: - case GenericOpcode::G_ASHR: { - const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2); - if (!Type || !Left || !Right || *Left != *Type || *Right != *Type || - !Type->isCategoryNeutralScalar()) - return LegalizationAction::Unsupported; - if (Type->scalarBitWidth() > 64) { - return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB || - Generic == GenericOpcode::G_AND || Generic == GenericOpcode::G_OR || - Generic == GenericOpcode::G_XOR || Generic == GenericOpcode::G_SHL - ? LegalizationAction::Custom - : LegalizationAction::Unsupported; - } - if (Generic == GenericOpcode::G_MUL && !Subtarget.features().has(RISCVExtension::M)) { - if (Type->scalarBitWidth() == 64) - return LegalizationAction::Libcall; - const MachineOperand *DestinationOperand = Instruction.operandAt(0); - const Register *Destination = - DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); - MachineInstr *Extension = - Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination); - const std::optional<LowLevelType> ExtensionType = - Extension == nullptr ? std::nullopt : resultType(Function, *Extension); - if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) && - ExtensionType && ExtensionType->scalarBitWidth() == 64) { - return LegalizationAction::Legal; - } - return LegalizationAction::Custom; - } - return isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - case GenericOpcode::G_PTR_ADD: { - const std::optional<LowLevelType> Pointer = operandType(Function, Instruction, 1); - const std::optional<LowLevelType> Offset = operandType(Function, Instruction, 2); - return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 && - Type->scalarBitWidth() == 64 && *Pointer == *Type && - isSupportedIntegerScalar(*Offset) && Offset->scalarBitWidth() == 64 - ? LegalizationAction::Legal - : LegalizationAction::Unsupported; - } - default: - break; - } - - // Keep the unsupported policy mechanically synchronized with the complete - // target-independent opcode inventory. Supported cases return above. - switch (Generic) { -#define RCC_CORE_OPCODE(Name) -#define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name: -#include "rcc/CodeGen/GenericOpcode.def" -#undef RCC_GENERIC_OPCODE -#undef RCC_CORE_OPCODE - return LegalizationAction::Unsupported; - case GenericOpcode::Invalid: - case GenericOpcode::CORE_PSEUDO_END: - case GenericOpcode::G_GENERIC_END: - default: - break; - } - return makeUnexpected(ErrorCode::InternalError, - "RISC-V legalizer received an invalid generic opcode value"); -} - -Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target, - LegalizationAction Action) const { - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) { - return lowerVACopy(Instruction, Function, Target); - } - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_VAARG)) { - return lowerVAArg(Instruction, Function, Target); - } - if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) && - !Subtarget.features().has(RISCVExtension::M)) { - if (Action == LegalizationAction::Custom) - return widenNarrowMultiply(Instruction, Function, Target); - if (Action == LegalizationAction::Libcall) - return lowerMultiplyLibcall(Instruction, Function, Target, - Subtarget.features().has(RISCVExtension::D) - ? CallingConvention::LP64D - : CallingConvention::LP64, - *Subtarget.getCallLowering()); - } - if (Action == LegalizationAction::Custom && - resultType(Function, Instruction) - .transform([](LowLevelType Type) { return Type.scalarBitWidth() > 64; }) - .value_or(false)) { - const Opcode Value = Instruction.opcode(); - if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) || - Value == opcode(GenericOpcode::G_AND) || Value == opcode(GenericOpcode::G_OR) || - Value == opcode(GenericOpcode::G_XOR)) { - return lowerWideBinary(Instruction, Function, Target, - static_cast<GenericOpcode>(Value.value())); - } - if (Value == opcode(GenericOpcode::G_SHL)) - return lowerWideShiftLeft(Instruction, Function, Target); - if (Value == opcode(GenericOpcode::G_PHI)) - return lowerWidePhi(Instruction, Function, Target); - } - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) - return lowerMergeValues(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES)) - return lowerUnmergeValues(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_BRCOND)) - return lowerNarrowBranchCondition(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_PHI)) - return lowerNarrowPhi(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ICMP)) +RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) { + const LowLevelType S8 = *LowLevelType::makeCategoryNeutralScalar(8); + const LowLevelType S16 = *LowLevelType::makeCategoryNeutralScalar(16); + const LowLevelType S32 = *LowLevelType::makeCategoryNeutralScalar(32); + const LowLevelType S64 = *LowLevelType::makeCategoryNeutralScalar(64); + const LowLevelType S128 = *LowLevelType::makeCategoryNeutralScalar(128); + const LowLevelType P0 = *LowLevelType::makePointer(0, 64); + const bool HasM = Subtarget.features().has(RISCVExtension::M); + const bool HasA = Subtarget.features().has(RISCVExtension::A); + const bool HasF = Subtarget.features().has(RISCVExtension::F); + const bool HasD = Subtarget.features().has(RISCVExtension::D); + + getActionDefinitionsBuilder({opcode(GenericOpcode::G_ASSERT_SEXT), + opcode(GenericOpcode::G_ASSERT_ZEXT), + opcode(GenericOpcode::G_ASSERT_ALIGN)}) + .alwaysLegal(); + + getActionDefinitionsBuilder({opcode(GenericOpcode::G_ADD), opcode(GenericOpcode::G_SUB)}) + .legalFor({S64}) + .customFor({S32}) + .widenScalarToNextPow2(0) + .clampScalar(0, S64, S64); + + getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_AND), opcode(GenericOpcode::G_OR), opcode(GenericOpcode::G_XOR)}) + .legalFor({S64}) + .widenScalarToNextPow2(0) + .clampScalar(0, S64, S64); + + getActionDefinitionsBuilder({opcode(GenericOpcode::G_UADDE), opcode(GenericOpcode::G_UADDO), + opcode(GenericOpcode::G_USUBE), opcode(GenericOpcode::G_USUBO), + opcode(GenericOpcode::G_READ_REGISTER), + opcode(GenericOpcode::G_WRITE_REGISTER)}) + .lower(); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_SADDE), opcode(GenericOpcode::G_SADDO), + opcode(GenericOpcode::G_SSUBE), opcode(GenericOpcode::G_SSUBO)}) + .minScalar(0, S64) + .lower(); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_UADDSAT), opcode(GenericOpcode::G_SADDSAT), + opcode(GenericOpcode::G_USUBSAT), opcode(GenericOpcode::G_SSUBSAT), + opcode(GenericOpcode::G_SSHLSAT), opcode(GenericOpcode::G_USHLSAT)}) + .lower(); + + getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_SHL), opcode(GenericOpcode::G_ASHR), opcode(GenericOpcode::G_LSHR)}) + .legalFor({{S64, S64}}) + .customFor({{S32, S32}}) + .widenScalarToNextPow2(0) + .clampScalar(1, S64, S64) + .clampScalar(0, S64, S64); + + getActionDefinitionsBuilder({opcode(GenericOpcode::G_ZEXT), opcode(GenericOpcode::G_SEXT), + opcode(GenericOpcode::G_ANYEXT)}) + .legalFor({{S32, S16}, {S64, S16}, {S64, S32}}) + .maxScalar(0, S64); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_TRUNC)).alwaysLegal(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_SEXT_INREG)) + .customFor({S64}) + .clampScalar(0, S64, S64) + .lower(); + + for (const GenericOpcode Generic : + {GenericOpcode::G_MERGE_VALUES, GenericOpcode::G_UNMERGE_VALUES}) { + const std::size_t BigTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 0 : 1; + const std::size_t LittleTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0; + getActionDefinitionsBuilder(opcode(Generic)) + .widenScalarToNextPow2(LittleTypeIndex, 64) + .widenScalarToNextPow2(BigTypeIndex, 64) + .clampScalar(LittleTypeIndex, S64, S64) + .clampScalar(BigTypeIndex, S64, S64); + } + getActionDefinitionsBuilder({opcode(GenericOpcode::G_FSHL), opcode(GenericOpcode::G_FSHR)}) + .lower(); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_ROTR), opcode(GenericOpcode::G_ROTL)}) + .lower(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_BITREVERSE)).maxScalar(0, S64).lower(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_BSWAP)).maxScalar(0, S64).lower(); + + getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_CTLZ), opcode(GenericOpcode::G_CTTZ), opcode(GenericOpcode::G_CTLS)}) + .maxScalar(0, S64) + .scalarSameSizeAs(1, 0) + .lower(); + getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_CTLZ_ZERO_POISON), opcode(GenericOpcode::G_CTTZ_ZERO_POISON)}) + .maxScalar(0, S64) + .scalarSameSizeAs(1, 0) + .lower(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_CTPOP)) + .widenScalarToNextPow2(0, 8) + .clampScalar(0, S8, S64) + .scalarSameSizeAs(1, 0) + .lower(); + + getActionDefinitionsBuilder(opcode(GenericOpcode::G_CONSTANT)) + .legalFor({P0}) + .customFor({S64}) + .widenScalarToNextPow2(0) + .clampScalar(0, S64, S64); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREEZE)) + .legalFor({S16, S32, S64, P0}) + .widenScalarToNextPow2(0) + .clampScalar(0, S16, S64); + getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_IMPLICIT_DEF), opcode(GenericOpcode::G_CONSTANT_FOLD_BARRIER)}) + .legalFor({S32, S64, P0}) + .widenScalarToNextPow2(0) + .clampScalar(0, S32, S64); + + getActionDefinitionsBuilder(opcode(GenericOpcode::G_ICMP)) + .legalFor({{S64, S64}, {S64, P0}}) + .customIf([](const LegalizationQuery &Query) { + return Query.Types.size() == 2 && Query.Types[0].isCategoryNeutralScalar() && + (Query.Types[1].isCategoryNeutralScalar() || Query.Types[1].isPointer()); + }); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_SELECT)) + .legalFor({{S32, S64}, {S64, S64}, {P0, S64}}) + .widenScalarToNextPow2(0) + .clampScalar(0, S32, S64) + .clampScalar(1, S64, S64); + + auto &LoadRules = getActionDefinitionsBuilder(opcode(GenericOpcode::G_LOAD)); + auto &StoreRules = getActionDefinitionsBuilder(opcode(GenericOpcode::G_STORE)); + auto &ExtLoadRules = getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_SEXTLOAD), opcode(GenericOpcode::G_ZEXTLOAD)}); + const std::uint64_t Align8 = + Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 8; + const std::uint64_t Align16 = + Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 16; + const std::uint64_t Align32 = + Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 32; + const std::uint64_t Align64 = + Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 64; + LoadRules + .legalForTypesWithMemDesc({{S16, P0, S8, Align8}, + {S32, P0, S8, Align8}, + {S16, P0, S16, Align16}, + {S32, P0, S16, Align16}, + {S32, P0, S32, Align32}, + {S64, P0, S8, Align8}, + {S64, P0, S16, Align16}, + {S64, P0, S32, Align32}, + {S64, P0, S64, Align64}, + {P0, P0, S64, Align64}}) + .widenScalarToNextPow2(0, 8) + .lowerIfMemSizeNotByteSizePow2() + .clampScalar(0, S16, S64) + .lower(); + StoreRules + .legalForTypesWithMemDesc({{S16, P0, S8, Align8}, + {S32, P0, S8, Align8}, + {S16, P0, S16, Align16}, + {S32, P0, S16, Align16}, + {S32, P0, S32, Align32}, + {S64, P0, S8, Align8}, + {S64, P0, S16, Align16}, + {S64, P0, S32, Align32}, + {S64, P0, S64, Align64}, + {P0, P0, S64, Align64}}) + .clampScalar(0, S16, S64) + .lowerIfMemSizeNotByteSizePow2() + .lower(); + ExtLoadRules + .legalForTypesWithMemDesc( + {{S64, P0, S8, Align8}, {S64, P0, S16, Align16}, {S64, P0, S32, Align32}}) + .widenScalarToNextPow2(0) + .clampScalar(0, S64, S64) + .lower(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPEXTLOAD)) + .legalForTypesWithMemDesc({{S64, P0, S32, Align32}}) + .lower(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPTRUNCSTORE)) + .legalForTypesWithMemDesc({{S64, P0, S32, Align32}}) + .lower(); + + getActionDefinitionsBuilder({opcode(GenericOpcode::G_PTR_ADD), opcode(GenericOpcode::G_PTRMASK)}) + .legalFor({{P0, S64}}); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_PTRTOINT)) + .legalFor({{S64, P0}}) + .clampScalar(0, S64, S64); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_INTTOPTR)) + .legalFor({{P0, S64}}) + .clampScalar(1, S64, S64); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_BR)).alwaysLegal(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRCOND)).legalFor({S64}).minScalar(0, S64); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRJT)).customFor({{P0, S64}}); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRINDIRECT)).legalFor({P0}); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_PHI)) + .legalFor({P0, S32, S64}) + .widenScalarToNextPow2(0) + .clampScalar(0, S32, S64); + getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_GLOBAL_VALUE), opcode(GenericOpcode::G_JUMP_TABLE), + opcode(GenericOpcode::G_CONSTANT_POOL), opcode(GenericOpcode::G_FRAME_INDEX)}) + .legalFor({P0}); + + if (HasM) { + getActionDefinitionsBuilder(opcode(GenericOpcode::G_MUL)) + .legalFor({S64}) + .widenScalarToNextPow2(0) + .clampScalar(0, S64, S64); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULH), opcode(GenericOpcode::G_UMULH)}) + .legalFor({S64}) + .lower(); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULO), opcode(GenericOpcode::G_UMULO)}) + .minScalar(0, S64) + .lower(); + } else { + getActionDefinitionsBuilder(opcode(GenericOpcode::G_MUL)) + .libcallFor({S64, S128}) + .widenScalarToNextPow2(0) + .clampScalar(0, S64, S128); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULH), opcode(GenericOpcode::G_UMULH)}) + .lowerFor({S64}); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULO), opcode(GenericOpcode::G_UMULO)}) + .minScalar(0, S64) + .lower(); + } + if (HasM) { + getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIV), opcode(GenericOpcode::G_UDIV), + opcode(GenericOpcode::G_UREM)}) + .legalFor({S64}) + .customFor({S32}) + .libcallFor({S128}) + .clampScalar(0, S32, S128) + .widenScalarToNextPow2(0); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_SREM)) + .legalFor({S64}) + .libcallFor({S128}) + .clampScalar(0, S64, S128) + .widenScalarToNextPow2(0); + } else { + getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIV), opcode(GenericOpcode::G_UDIV), + opcode(GenericOpcode::G_SREM), opcode(GenericOpcode::G_UREM)}) + .libcallFor({S64, S128}) + .clampScalar(0, S64, S128) + .widenScalarToNextPow2(0); + } + getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIVREM), opcode(GenericOpcode::G_UDIVREM)}) + .lower(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_ABS)).lower(); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_ABDS), opcode(GenericOpcode::G_ABDU)}) + .lower(); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_UMAX), opcode(GenericOpcode::G_UMIN), + opcode(GenericOpcode::G_SMAX), opcode(GenericOpcode::G_SMIN), + opcode(GenericOpcode::G_SCMP), opcode(GenericOpcode::G_UCMP)}) + .lower(); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_MEMCPY), opcode(GenericOpcode::G_MEMMOVE), + opcode(GenericOpcode::G_MEMSET)}) + .libcall(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_MEMCPY_INLINE)).lower(); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_DYN_STACKALLOC), + opcode(GenericOpcode::G_STACKSAVE), + opcode(GenericOpcode::G_STACKRESTORE)}) + .lower(); + + getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_FADD), opcode(GenericOpcode::G_FSUB), opcode(GenericOpcode::G_FMUL), + opcode(GenericOpcode::G_FDIV), opcode(GenericOpcode::G_FMA), opcode(GenericOpcode::G_FSQRT), + opcode(GenericOpcode::G_FMAXNUM), opcode(GenericOpcode::G_FMINNUM), + opcode(GenericOpcode::G_FMAXIMUMNUM), opcode(GenericOpcode::G_FMINIMUMNUM)}) + .legalFor(HasF, {S32}) + .legalFor(HasD, {S64}) + .libcallFor({S32, S64}) + .libcallFor({S128}); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_FNEG), opcode(GenericOpcode::G_FABS)}) + .legalFor(HasF, {S32}) + .legalFor(HasD, {S64}) + .lowerFor({S32, S64, S128}); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREM)) + .libcallFor({S32, S64, S128}) + .minScalar(0, S32); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCOPYSIGN)) + .legalFor(HasF, {{S32, S32}}) + .legalFor(HasD, {{S64, S64}, {S32, S64}, {S64, S32}}) + .lower(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPTRUNC)) + .legalFor(HasD, {{S32, S64}}) + .libcallFor({{S32, S64}, {S32, S128}, {S64, S128}}); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPEXT)) + .legalFor(HasD, {{S64, S32}}) + .libcallFor({{S64, S32}, {S128, S32}, {S128, S64}}); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCMP)) + .legalFor(HasF, {{S64, S32}}) + .legalFor(HasD, {{S64, S64}}) + .clampScalar(0, S64, S64) + .libcallFor({{S64, S32}, {S64, S64}, {S64, S128}}); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_IS_FPCLASS)).lower(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCONSTANT)) + .legalFor(HasF, {S32}) + .legalFor(HasD, {S64}) + .customFor({S32, S64}) + .lowerFor({S64, S128}); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_FPTOSI), opcode(GenericOpcode::G_FPTOUI)}) + .legalFor(HasF, {{S64, S32}}) + .legalFor(HasD, {{S64, S64}}) + .customFor(HasF, {{S32, S32}}) + .customFor(HasD, {{S32, S64}}) + .widenScalarToNextPow2(0) + .minScalar(0, S32) + .libcallFor({{S32, S32}, + {S64, S32}, + {S32, S64}, + {S64, S64}, + {S32, S128}, + {S64, S128}, + {S128, S32}, + {S128, S64}, + {S128, S128}}); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_SITOFP), opcode(GenericOpcode::G_UITOFP)}) + .legalFor(HasF, {{S32, S64}}) + .legalFor(HasD, {{S64, S64}}) + .widenScalarToNextPow2(1) + .minScalar(1, S32) + .libcallFor({{S32, S32}, + {S64, S32}, + {S32, S64}, + {S64, S64}, + {S128, S32}, + {S128, S64}, + {S32, S128}, + {S64, S128}, + {S128, S128}}); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_FCEIL), opcode(GenericOpcode::G_FFLOOR), + opcode(GenericOpcode::G_FRINT), opcode(GenericOpcode::G_FNEARBYINT), + opcode(GenericOpcode::G_INTRINSIC_TRUNC), + opcode(GenericOpcode::G_INTRINSIC_ROUND), + opcode(GenericOpcode::G_INTRINSIC_ROUNDEVEN)}) + .libcallFor({S32, S64, S128}); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_FCOS), opcode(GenericOpcode::G_FSIN), + opcode(GenericOpcode::G_FTAN), opcode(GenericOpcode::G_FPOW), + opcode(GenericOpcode::G_FLOG), opcode(GenericOpcode::G_FLOG2), + opcode(GenericOpcode::G_FLOG10), opcode(GenericOpcode::G_FEXP), + opcode(GenericOpcode::G_FEXP2), opcode(GenericOpcode::G_FEXP10), + opcode(GenericOpcode::G_FACOS), opcode(GenericOpcode::G_FASIN), + opcode(GenericOpcode::G_FATAN), opcode(GenericOpcode::G_FATAN2), + opcode(GenericOpcode::G_FCOSH), opcode(GenericOpcode::G_FSINH), + opcode(GenericOpcode::G_FTANH), opcode(GenericOpcode::G_FMODF)}) + .libcallFor({S32, S64, S128}); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_FPOWI), opcode(GenericOpcode::G_FLDEXP)}) + .libcallFor({{S32, S32}, {S64, S32}, {S128, S32}}); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCANONICALIZE)) + .legalFor(HasF, {S32}) + .legalFor(HasD, {S64}); + + getActionDefinitionsBuilder(opcode(GenericOpcode::G_VASTART)).customFor({P0}); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_VAARG)).clampScalar(0, S64, S64).lower(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) + .customIf([](const LegalizationQuery &Query) { + return Query.Types.size() >= 3 && Query.Types[0].isCategoryNeutralScalar() && + Query.Types[2].isPointer(); + }); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMIC_CMPXCHG)) + .legalFor(HasA, {{S64, P0}}) + .customIf([](const LegalizationQuery &Query) { + return Query.Types.size() >= 2 && Query.Types[0].isCategoryNeutralScalar() && + Query.Types[1].isPointer(); + }) + .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}}) + .clampScalar(0, S64, S64); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMICRMW_ADD)) + .legalFor(HasA, {{S64, P0}}) + .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}}) + .clampScalar(0, S64, S64); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMICRMW_SUB)) + .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}}) + .clampScalar(0, S64, S64) + .lower(); + getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_ATOMICRMW_XCHG), opcode(GenericOpcode::G_ATOMICRMW_AND), + opcode(GenericOpcode::G_ATOMICRMW_OR), opcode(GenericOpcode::G_ATOMICRMW_XOR), + opcode(GenericOpcode::G_ATOMICRMW_MAX), opcode(GenericOpcode::G_ATOMICRMW_MIN), + opcode(GenericOpcode::G_ATOMICRMW_UMAX), opcode(GenericOpcode::G_ATOMICRMW_UMIN)}) + .legalFor(HasA, {{S64, P0}}) + .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}}) + .clampScalar(0, S64, S64); + getActionDefinitionsBuilder( + {opcode(GenericOpcode::G_INTRINSIC), opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)}) + .alwaysLegal(); + getActionDefinitionsBuilder(opcode(GenericOpcode::G_FENCE)).alwaysLegal(); + getActionDefinitionsBuilder({opcode(GenericOpcode::G_TRAP), opcode(GenericOpcode::G_DEBUGTRAP), + opcode(GenericOpcode::G_UBSANTRAP)}) + .alwaysLegal(); +} + +Expected<void> RISCVLegalizerInfo::legalizeCustom(LegalizerHelper &Helper, + MachineInstr &Instruction, + const LegalizationDecision &Decision) const { + if (Decision.Action != LegalizationAction::Custom) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V custom legalizer received a non-custom action"); + + MachineFunction &Function = Helper.MIRBuilder.function(); + const TargetDescription &Target = Helper.MIRBuilder.target(); + if (Instruction.opcode() == opcode(GenericOpcode::G_ICMP)) return lowerIntegerCompare(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG)) + if (Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG)) return lowerAtomicCompareExchange(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) + if (Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) return lowerAtomicCompareExchangeWithSuccess(Instruction, Function, Target); - if (Action == LegalizationAction::Custom && - (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) || - Instruction.opcode() == opcode(GenericOpcode::G_SEXT) || - Instruction.opcode() == opcode(GenericOpcode::G_ZEXT))) { - if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) && - narrowAnyExtBinaryPlan(Instruction, Function).has_value()) - return foldNarrowAnyExtBinary(Instruction, Function, Target); + if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) || + Instruction.opcode() == opcode(GenericOpcode::G_SEXT) || + Instruction.opcode() == opcode(GenericOpcode::G_ZEXT)) { return foldConstantExtension(Instruction, Function, static_cast<GenericOpcode>(Instruction.opcode().value())); } - if (Action == LegalizationAction::Custom && - Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) { + if (Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) { const MachineOperand *Immediate = Instruction.operandAt(1); const std::optional<LowLevelType> Type = resultType(Function, Instruction); if (Immediate != nullptr && Type && Type->isCategoryNeutralScalar() && @@ -3789,8 +1113,10 @@ Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunct auto Value = signedImmediate(*Immediate); if (Value && shouldUseRV64IntegerConstantPool(*Value)) return lowerConstantToPool(Instruction, Function, Target); + return {}; } - return foldWideConstant(Instruction, Function, Target); + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant custom legalization is unavailable"); } return makeUnexpected(ErrorCode::Unsupported, "RISC-V legalizer has no lowering rule for this instruction"); diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h index f6ba000..d8e5e3e 100644 --- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h +++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h @@ -11,17 +11,11 @@ class RISCVSubtarget; class RISCVLegalizerInfo final : public LegalizerInfo { public: - explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) : Subtarget(Subtarget) {} - - [[nodiscard]] Expected<LegalizationAction> - classify(const MachineInstr &Instruction, const MachineFunction &Function, - const TargetDescription &Target) const override; - [[nodiscard]] Expected<void> lower(MachineInstr &Instruction, MachineFunction &Function, - const TargetDescription &Target, - LegalizationAction Action) const override; + explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget); + [[nodiscard]] Expected<void> legalizeCustom(LegalizerHelper &Helper, MachineInstr &Instruction, + const LegalizationDecision &Decision) const override; private: - const RISCVSubtarget &Subtarget; }; } // namespace rcc::riscv |