aboutsummaryrefslogtreecommitdiff
path: root/backend/lib/CodeGen/GlobalISel
diff options
context:
space:
mode:
Diffstat (limited to 'backend/lib/CodeGen/GlobalISel')
-rw-r--r--backend/lib/CodeGen/GlobalISel/CMakeLists.txt55
-rw-r--r--backend/lib/CodeGen/GlobalISel/CSEInfo.cpp467
-rw-r--r--backend/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp220
-rw-r--r--backend/lib/CodeGen/GlobalISel/Combiner.cpp81
-rw-r--r--backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp1611
-rw-r--r--backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp61
-rw-r--r--backend/lib/CodeGen/GlobalISel/GISelValueTracking.cpp722
-rw-r--r--backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp398
-rw-r--r--backend/lib/CodeGen/GlobalISel/Legalizer.cpp493
-rw-r--r--backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp7120
-rw-r--r--backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp497
-rw-r--r--backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp403
-rw-r--r--backend/lib/CodeGen/GlobalISel/Utils.cpp492
13 files changed, 12620 insertions, 0 deletions
diff --git a/backend/lib/CodeGen/GlobalISel/CMakeLists.txt b/backend/lib/CodeGen/GlobalISel/CMakeLists.txt
new file mode 100644
index 0000000..cd336cb
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/CMakeLists.txt
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set(
+ RCCBackendGlobalISelSources
+ CSEInfo.cpp
+ CSEMIRBuilder.cpp
+ Combiner.cpp
+ GIMatchTableExecutor.cpp
+ GISelChangeObserver.cpp
+ GISelValueTracking.cpp
+ InstructionSelect.cpp
+ Legalizer.cpp
+ LegalizerHelper.cpp
+ LegalizerInfo.cpp
+ RegBankSelect.cpp
+ Utils.cpp
+)
+
+set(
+ RCCBackendGlobalISelHeaders
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/CSEInfo.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/CSEMIRBuilder.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/Combiner.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GIMatchTableExecutor.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelChangeObserver.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelValueTracking.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelWorkList.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GenericMachineInstrs.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/InstructionSelect.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/InstructionSelector.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/Legalizer.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/RegBankSelect.h"
+ "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/Utils.h"
+)
+
+target_sources(RCCBackendCodeGen PRIVATE ${RCCBackendGlobalISelSources})
+
+rcc_backend_register_format_files(
+ "${CMAKE_CURRENT_SOURCE_DIR}/CSEInfo.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/CSEMIRBuilder.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Combiner.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/GIMatchTableExecutor.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/GISelChangeObserver.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/GISelValueTracking.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/InstructionSelect.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Legalizer.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerHelper.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerInfo.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/RegBankSelect.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Utils.cpp"
+ ${RCCBackendGlobalISelHeaders}
+)
diff --git a/backend/lib/CodeGen/GlobalISel/CSEInfo.cpp b/backend/lib/CodeGen/GlobalISel/CSEInfo.cpp
new file mode 100644
index 0000000..201dedd
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/CSEInfo.cpp
@@ -0,0 +1,467 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/CSEInfo.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <utility>
+
+namespace rcc {
+namespace {
+
+enum class ProfileTag : std::uint64_t {
+ Definition,
+ Use,
+ RegisterNumber,
+ PhysicalRegister,
+ Type,
+ RegisterBank,
+ NoRegisterBank,
+ RegisterClass,
+ NoRegisterClass,
+ Immediate,
+ IntegerImmediate,
+ FloatingImmediate,
+ IntegerPredicate,
+ FloatingPredicate,
+ Flags,
+};
+
+void appendTypeProfile(std::vector<std::uint64_t> &Profile, LowLevelType Type,
+ std::optional<RegisterBankID> Bank,
+ std::optional<RegisterClassID> RegClass) {
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Type));
+ Profile.push_back(Type.rawValue());
+ if (Bank) {
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::RegisterBank));
+ Profile.push_back(Bank->value());
+ } else {
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::NoRegisterBank));
+ }
+ if (RegClass) {
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::RegisterClass));
+ Profile.push_back(RegClass->value());
+ } else {
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::NoRegisterClass));
+ }
+}
+
+void appendAPIntProfile(std::vector<std::uint64_t> &Profile, const APInt &Value) {
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::IntegerImmediate));
+ Profile.push_back(Value.bitWidth());
+ Profile.push_back(Value.words().size());
+ Profile.insert(Profile.end(), Value.words().begin(), Value.words().end());
+}
+
+void appendAPFloatProfile(std::vector<std::uint64_t> &Profile, const APFloat &Value) {
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::FloatingImmediate));
+ Profile.push_back(static_cast<std::uint64_t>(Value.semantics()));
+ Profile.push_back(Value.words().size());
+ Profile.insert(Profile.end(), Value.words().begin(), Value.words().end());
+}
+
+void hashCombine(std::size_t &Seed, std::size_t Value) noexcept {
+ Seed ^= Value + 0x9e3779b97f4a7c15ULL + (Seed << 6U) + (Seed >> 2U);
+}
+
+} // namespace
+
+bool CSEConfigFull::shouldCSEOpcode(Opcode InstructionOpcode) const noexcept {
+ switch (static_cast<GenericOpcode>(InstructionOpcode.value())) {
+ default:
+ return false;
+ case GenericOpcode::G_ADD:
+ case GenericOpcode::G_AND:
+ case GenericOpcode::G_ASHR:
+ case GenericOpcode::G_LSHR:
+ case GenericOpcode::G_MUL:
+ case GenericOpcode::G_OR:
+ case GenericOpcode::G_SHL:
+ case GenericOpcode::G_SUB:
+ case GenericOpcode::G_XOR:
+ case GenericOpcode::G_UDIV:
+ case GenericOpcode::G_SDIV:
+ case GenericOpcode::G_UREM:
+ case GenericOpcode::G_SREM:
+ case GenericOpcode::G_CONSTANT:
+ case GenericOpcode::G_FCONSTANT:
+ case GenericOpcode::G_IMPLICIT_DEF:
+ case GenericOpcode::G_ZEXT:
+ case GenericOpcode::G_SEXT:
+ case GenericOpcode::G_ANYEXT:
+ case GenericOpcode::G_UNMERGE_VALUES:
+ case GenericOpcode::G_TRUNC:
+ case GenericOpcode::G_PTR_ADD:
+ case GenericOpcode::G_EXTRACT:
+ case GenericOpcode::G_SELECT:
+ case GenericOpcode::G_BUILD_VECTOR:
+ case GenericOpcode::G_BUILD_VECTOR_TRUNC:
+ case GenericOpcode::G_SEXT_INREG:
+ case GenericOpcode::G_FADD:
+ case GenericOpcode::G_FSUB:
+ case GenericOpcode::G_FMUL:
+ case GenericOpcode::G_FDIV:
+ case GenericOpcode::G_FABS:
+ case GenericOpcode::G_FMAXNUM:
+ case GenericOpcode::G_FMINNUM:
+ case GenericOpcode::G_FMAXNUM_IEEE:
+ case GenericOpcode::G_FMINNUM_IEEE:
+ return true;
+ }
+}
+
+bool CSEConfigConstantOnly::shouldCSEOpcode(Opcode InstructionOpcode) const noexcept {
+ return InstructionOpcode == opcode(GenericOpcode::G_CONSTANT) ||
+ InstructionOpcode == opcode(GenericOpcode::G_FCONSTANT) ||
+ InstructionOpcode == opcode(GenericOpcode::G_IMPLICIT_DEF);
+}
+
+std::unique_ptr<CSEConfigBase> getStandardCSEConfigForOpt(OptimizationLevel Level) {
+ if (Level == OptimizationLevel::O0)
+ return std::make_unique<CSEConfigConstantOnly>();
+ return std::make_unique<CSEConfigFull>();
+}
+
+std::size_t GISelCSEInfo::CSEKeyHash::operator()(const CSEKey &Key) const noexcept {
+ std::size_t Result = std::hash<const MachineBasicBlock *>{}(Key.Block);
+ hashCombine(Result, std::hash<std::uint32_t>{}(Key.InstructionOpcode.value()));
+ for (std::uint64_t Value : Key.Profile)
+ hashCombine(Result, std::hash<std::uint64_t>{}(Value));
+ return Result;
+}
+
+void GISelCSEInfo::setCSEConfig(std::unique_ptr<CSEConfigBase> NewConfig) noexcept {
+ Config = std::move(NewConfig);
+}
+
+bool GISelCSEInfo::shouldCSE(Opcode InstructionOpcode) const noexcept {
+ return Config != nullptr && isGenericOpcode(InstructionOpcode) &&
+ Config->shouldCSEOpcode(InstructionOpcode);
+}
+
+Expected<void> GISelCSEInfo::appendRegisterProfile(std::vector<std::uint64_t> &Profile,
+ Register Reg, bool IsDefinition) const {
+ Profile.push_back(
+ static_cast<std::uint64_t>(IsDefinition ? ProfileTag::Definition : ProfileTag::Use));
+ if (!IsDefinition) {
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::RegisterNumber));
+ Profile.push_back(Reg.opaqueValue());
+ }
+ if (!Reg.isVirtual()) {
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::PhysicalRegister));
+ Profile.push_back(Reg.opaqueValue());
+ return {};
+ }
+ if (Function == nullptr)
+ return makeUnexpected(ErrorCode::InternalError, "GlobalISel CSE has no machine function");
+ auto Info = Function->registerInfo().virtualRegisterInfo(Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ const std::optional<LowLevelType> &Type = Info->get().Type;
+ if (!Type.has_value()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "GlobalISel CSE requires typed virtual registers");
+ }
+ appendTypeProfile(Profile, Type.value(), Info->get().Bank, Info->get().RegClass);
+ return {};
+}
+
+Expected<void> GISelCSEInfo::appendResultProfile(std::vector<std::uint64_t> &Profile,
+ const MachineIRBuilderResult &Result) const {
+ if (Result.kind() == MachineIRBuilderResult::Kind::Register)
+ return appendRegisterProfile(Profile, Result.reg(), true);
+ Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Definition));
+ appendTypeProfile(Profile, Result.type(), std::nullopt, std::nullopt);
+ return {};
+}
+
+Expected<GISelCSEInfo::CSEKey>
+GISelCSEInfo::profileInstruction(const MachineInstr &Instruction) const {
+ if (Instruction.parent() == nullptr)
+ return makeUnexpected(ErrorCode::InvalidArgument, "cannot profile a detached instruction");
+ if (!Instruction.memOperands().empty()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "GlobalISel CSE does not profile memory operands");
+ }
+
+ CSEKey Key{
+ .Block = Instruction.parent(), .InstructionOpcode = Instruction.opcode(), .Profile = {}};
+ for (const MachineOperand &Operand : Instruction.operands()) {
+ switch (Operand.kind()) {
+ case MachineOperand::Kind::Register: {
+ const Register *Reg = Operand.getIf<Register>();
+ if (Reg == nullptr || Operand.isImplicit()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "GlobalISel CSE does not profile implicit register operands");
+ }
+ auto Appended = appendRegisterProfile(Key.Profile, *Reg, Operand.isDef());
+ if (!Appended)
+ return std::unexpected(Appended.error());
+ break;
+ }
+ case MachineOperand::Kind::Immediate: {
+ const std::int64_t *Value = Operand.getIf<std::int64_t>();
+ if (Value == nullptr)
+ return makeUnexpected(ErrorCode::InternalError, "invalid immediate operand storage");
+ Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Immediate));
+ Key.Profile.push_back(std::bit_cast<std::uint64_t>(*Value));
+ break;
+ }
+ case MachineOperand::Kind::IntegerImmediate: {
+ const auto *Value = Operand.getIf<IntegerImmediateOperand>();
+ if (Value == nullptr)
+ return makeUnexpected(ErrorCode::InternalError,
+ "invalid integer immediate operand storage");
+ appendAPIntProfile(Key.Profile, Value->Value);
+ break;
+ }
+ case MachineOperand::Kind::FloatingImmediate: {
+ const auto *Value = Operand.getIf<FloatingImmediateOperand>();
+ if (Value == nullptr)
+ return makeUnexpected(ErrorCode::InternalError,
+ "invalid floating immediate operand storage");
+ appendAPFloatProfile(Key.Profile, Value->Bits);
+ break;
+ }
+ case MachineOperand::Kind::Predicate: {
+ const auto *Predicate = Operand.getIf<PredicateOperand>();
+ if (Predicate == nullptr)
+ return makeUnexpected(ErrorCode::InternalError, "invalid predicate operand storage");
+ if (const IntegerPredicate *Integer = Predicate->integerPredicate()) {
+ Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::IntegerPredicate));
+ Key.Profile.push_back(static_cast<std::uint64_t>(*Integer));
+ } else if (const FloatingPredicate *Floating = Predicate->floatingPredicate()) {
+ Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::FloatingPredicate));
+ Key.Profile.push_back(static_cast<std::uint64_t>(*Floating));
+ } else {
+ return makeUnexpected(ErrorCode::InternalError, "empty predicate operand");
+ }
+ break;
+ }
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "GlobalISel CSE encountered an unsupported operand kind");
+ }
+ }
+ Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Flags));
+ Key.Profile.push_back(Instruction.flags().rawValue());
+ return Key;
+}
+
+Expected<GISelCSEInfo::CSEKey> GISelCSEInfo::profileBuilderInstruction(
+ Opcode InstructionOpcode, std::span<const MachineIRBuilderResult> Destinations,
+ std::span<const Register> Sources, std::span<const std::int64_t> Immediates,
+ MachineInstrFlags Flags, const MachineBasicBlock &Block) const {
+ CSEKey Key{.Block = &Block, .InstructionOpcode = InstructionOpcode, .Profile = {}};
+ for (const MachineIRBuilderResult &Destination : Destinations) {
+ auto Appended = appendResultProfile(Key.Profile, Destination);
+ if (!Appended)
+ return std::unexpected(Appended.error());
+ }
+ for (Register Source : Sources) {
+ auto Appended = appendRegisterProfile(Key.Profile, Source, false);
+ if (!Appended)
+ return std::unexpected(Appended.error());
+ }
+ for (std::int64_t Immediate : Immediates) {
+ Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Immediate));
+ Key.Profile.push_back(std::bit_cast<std::uint64_t>(Immediate));
+ }
+ Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Flags));
+ Key.Profile.push_back(Flags.rawValue());
+ return Key;
+}
+
+Expected<GISelCSEInfo::CSEKey>
+GISelCSEInfo::profileConstant(const MachineIRBuilderResult &Destination, const APInt &Value,
+ const MachineBasicBlock &Block) const {
+ CSEKey Key{
+ .Block = &Block, .InstructionOpcode = opcode(GenericOpcode::G_CONSTANT), .Profile = {}};
+ auto Appended = appendResultProfile(Key.Profile, Destination);
+ if (!Appended)
+ return std::unexpected(Appended.error());
+ appendAPIntProfile(Key.Profile, Value);
+ Key.Profile.push_back(static_cast<std::uint64_t>(ProfileTag::Flags));
+ Key.Profile.push_back(0);
+ return Key;
+}
+
+Expected<void> GISelCSEInfo::insertInstruction(MachineInstr &Instruction) {
+ TemporaryInstructions.remove(&Instruction);
+ auto Key = profileInstruction(Instruction);
+ if (!Key)
+ return std::unexpected(Key.error());
+ auto [Position, Inserted] = CSEMap.emplace(*Key, &Instruction);
+ if (Inserted)
+ InstructionMapping.emplace(&Instruction, std::move(*Key));
+ return {};
+}
+
+void GISelCSEInfo::recordNewInstruction(MachineInstr &Instruction) {
+ if (shouldCSE(Instruction.opcode()))
+ TemporaryInstructions.insert(&Instruction);
+}
+
+Expected<void> GISelCSEInfo::handleRecordedInstruction(MachineInstr &Instruction) {
+ TemporaryInstructions.remove(&Instruction);
+ handleRemoveInstruction(Instruction);
+ if (Instruction.parent() == nullptr || !shouldCSE(Instruction.opcode()))
+ return {};
+ return insertInstruction(Instruction);
+}
+
+Expected<void> GISelCSEInfo::handleRecordedInstructions() {
+ if (HandlingRecordedInstructions)
+ return {};
+ HandlingRecordedInstructions = true;
+ while (!TemporaryInstructions.empty()) {
+ MachineInstr *Instruction = TemporaryInstructions.popBack();
+ if (Instruction == nullptr)
+ continue;
+ auto Handled = handleRecordedInstruction(*Instruction);
+ if (!Handled) {
+ HandlingRecordedInstructions = false;
+ return std::unexpected(Handled.error());
+ }
+ }
+ HandlingRecordedInstructions = false;
+ return {};
+}
+
+void GISelCSEInfo::handleRemoveInstruction(MachineInstr &Instruction) noexcept {
+ const auto Mapping = InstructionMapping.find(&Instruction);
+ if (Mapping != InstructionMapping.end()) {
+ const auto Position = CSEMap.find(Mapping->second);
+ if (Position != CSEMap.end() && Position->second == &Instruction)
+ CSEMap.erase(Position);
+ InstructionMapping.erase(Mapping);
+ }
+ TemporaryInstructions.remove(&Instruction);
+}
+
+Expected<MachineInstr *> GISelCSEInfo::getMachineInstrIfExists(
+ Opcode InstructionOpcode, std::span<const MachineIRBuilderResult> Destinations,
+ std::span<const Register> Sources, std::span<const std::int64_t> Immediates,
+ MachineInstrFlags Flags, MachineBasicBlock &Block) {
+ auto Handled = handleRecordedInstructions();
+ if (!Handled)
+ return std::unexpected(Handled.error());
+ auto Key =
+ profileBuilderInstruction(InstructionOpcode, Destinations, Sources, Immediates, Flags, Block);
+ if (!Key)
+ return std::unexpected(Key.error());
+ const auto Position = CSEMap.find(*Key);
+ if (Position == CSEMap.end())
+ return nullptr;
+ countOpcodeHit(InstructionOpcode);
+ return Position->second;
+}
+
+Expected<MachineInstr *>
+GISelCSEInfo::getConstantIfExists(const MachineIRBuilderResult &Destination, const APInt &Value,
+ MachineBasicBlock &Block) {
+ auto Handled = handleRecordedInstructions();
+ if (!Handled)
+ return std::unexpected(Handled.error());
+ auto Key = profileConstant(Destination, Value, Block);
+ if (!Key)
+ return std::unexpected(Key.error());
+ const auto Position = CSEMap.find(*Key);
+ if (Position == CSEMap.end())
+ return nullptr;
+ countOpcodeHit(opcode(GenericOpcode::G_CONSTANT));
+ return Position->second;
+}
+
+void GISelCSEInfo::countOpcodeHit(Opcode InstructionOpcode) noexcept {
+ ++OpcodeHitTable[InstructionOpcode.value()];
+}
+
+Expected<void> GISelCSEInfo::analyze(MachineFunction &NewFunction) {
+ if (Config == nullptr)
+ return makeUnexpected(ErrorCode::InvalidArgument, "GlobalISel CSE configuration is not set");
+ CSEMap.clear();
+ InstructionMapping.clear();
+ TemporaryInstructions.clear();
+ OpcodeHitTable.clear();
+ HandlingRecordedInstructions = false;
+ Function = &NewFunction;
+ for (MachineBasicBlock *Block = NewFunction.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ if (!shouldCSE(Instruction->opcode()))
+ continue;
+ auto Inserted = insertInstruction(*Instruction);
+ if (!Inserted)
+ return std::unexpected(Inserted.error());
+ }
+ }
+ return {};
+}
+
+Expected<void> GISelCSEInfo::verify() {
+ auto Handled = handleRecordedInstructions();
+ if (!Handled)
+ return std::unexpected(Handled.error());
+ for (const auto &[Instruction, Key] : InstructionMapping) {
+ if (Instruction == nullptr || Instruction->parent() == nullptr)
+ return makeUnexpected(ErrorCode::InternalError, "GlobalISel CSE maps a detached instruction");
+ auto CurrentKey = profileInstruction(*Instruction);
+ if (!CurrentKey)
+ return std::unexpected(CurrentKey.error());
+ const auto Position = CSEMap.find(*CurrentKey);
+ if (*CurrentKey != Key || Position == CSEMap.end() || Position->second != Instruction) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "GlobalISel CSE instruction mapping is inconsistent");
+ }
+ }
+ for (const auto &[Key, Instruction] : CSEMap) {
+ const auto Mapping = InstructionMapping.find(Instruction);
+ if (Mapping == InstructionMapping.end() || Mapping->second != Key) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "GlobalISel CSE table has no instruction mapping");
+ }
+ }
+ return {};
+}
+
+void GISelCSEInfo::releaseMemory() noexcept {
+ CSEMap.clear();
+ InstructionMapping.clear();
+ TemporaryInstructions.clear();
+ OpcodeHitTable.clear();
+ Config.reset();
+ Function = nullptr;
+ HandlingRecordedInstructions = false;
+}
+
+void GISelCSEInfo::erasingInstr(MachineInstr &Instruction) {
+ handleRemoveInstruction(Instruction);
+}
+
+void GISelCSEInfo::createdInstr(MachineInstr &Instruction) {
+ recordNewInstruction(Instruction);
+}
+
+void GISelCSEInfo::changingInstr(MachineInstr &Instruction) {
+ erasingInstr(Instruction);
+ createdInstr(Instruction);
+}
+
+void GISelCSEInfo::changedInstr(MachineInstr &Instruction) {
+ changingInstr(Instruction);
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/backend/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
new file mode 100644
index 0000000..caf9652
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
@@ -0,0 +1,220 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/CSEMIRBuilder.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/CSEInfo.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <limits>
+#include <span>
+#include <utility>
+
+namespace rcc {
+
+bool CSEMIRBuilder::canPerformCSE(Opcode InstructionOpcode) const noexcept {
+ const GISelCSEInfo *Info = getCSEInfo();
+ return Info != nullptr && Info->shouldCSE(InstructionOpcode);
+}
+
+bool CSEMIRBuilder::dominatesInsertionPoint(const MachineInstr &Instruction) const noexcept {
+ const MachineBasicBlock *Block = insertionBlock();
+ const MachineInstr *InsertBefore = insertionInstruction();
+ if (Block == nullptr || Instruction.parent() != Block)
+ return false;
+ if (InsertBefore == nullptr)
+ return true;
+ for (const MachineInstr *Current = Block->firstInstruction(); Current != nullptr;
+ Current = Current->next()) {
+ if (Current == &Instruction)
+ return true;
+ if (Current == InsertBefore)
+ return false;
+ }
+ return false;
+}
+
+Expected<void> CSEMIRBuilder::makeInstructionDominate(MachineInstr &Instruction) const {
+ MachineBasicBlock *Block = insertionBlock();
+ MachineInstr *InsertBefore = insertionInstruction();
+ if (Block == nullptr || Instruction.parent() != Block) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "GlobalISel CSE candidate is not in the insertion block");
+ }
+ if (&Instruction == InsertBefore) {
+ return const_cast<CSEMIRBuilder *>(this)->setInsertPointAfter(Instruction);
+ }
+ if (InsertBefore == nullptr || dominatesInsertionPoint(Instruction))
+ return {};
+ return Block->spliceInstructionBefore(*InsertBefore, Instruction);
+}
+
+Expected<MachineInstr *>
+CSEMIRBuilder::generateCopyIfRequired(const MachineIRBuilderResult &Destination,
+ MachineInstr &Instruction) const {
+ if (Destination.kind() == MachineIRBuilderResult::Kind::Type)
+ return &Instruction;
+ const MachineOperand *Definition = Instruction.operandAt(0);
+ const Register *Source = Definition == nullptr ? nullptr : Definition->getIf<Register>();
+ if (Source == nullptr || !Definition->isDef()) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "GlobalISel CSE candidate has no register definition");
+ }
+ if (Destination.reg() == *Source)
+ return &Instruction;
+ return MachineIRBuilder::buildCopy(Destination, *Source);
+}
+
+Expected<MachineInstr *> CSEMIRBuilder::memoize(MachineInstr &Instruction) const {
+ GISelCSEInfo *Info = getCSEInfo();
+ if (Info == nullptr)
+ return &Instruction;
+ auto Inserted = Info->handleRecordedInstruction(Instruction);
+ if (!Inserted)
+ return std::unexpected(Inserted.error());
+ return &Instruction;
+}
+
+Expected<MachineInstr *>
+CSEMIRBuilder::buildInstr(Opcode InstructionOpcode,
+ std::span<const MachineIRBuilderResult> Destinations,
+ std::span<const Register> Sources) const {
+ if (!canPerformCSE(InstructionOpcode))
+ return MachineIRBuilder::buildInstr(InstructionOpcode, Destinations, Sources);
+ MachineBasicBlock *Block = insertionBlock();
+ if (Block == nullptr)
+ return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point");
+
+ GISelCSEInfo *Info = getCSEInfo();
+ const bool CanGenerateCopies =
+ Destinations.size() == 1 || std::ranges::all_of(Destinations, [](const auto &Destination) {
+ return Destination.kind() == MachineIRBuilderResult::Kind::Type;
+ });
+ if (!CanGenerateCopies) {
+ auto Built = MachineIRBuilder::buildInstr(InstructionOpcode, Destinations, Sources);
+ if (!Built)
+ return std::unexpected(Built.error());
+ Info->handleRemoveInstruction(**Built);
+ return *Built;
+ }
+
+ auto Existing =
+ Info->getMachineInstrIfExists(InstructionOpcode, Destinations, Sources,
+ std::span<const std::int64_t>(), MachineInstrFlags(), *Block);
+ if (!Existing)
+ return std::unexpected(Existing.error());
+ if (*Existing != nullptr) {
+ auto Dominating = makeInstructionDominate(**Existing);
+ if (!Dominating)
+ return std::unexpected(Dominating.error());
+ if (Destinations.size() == 1)
+ return generateCopyIfRequired(Destinations.front(), **Existing);
+ return *Existing;
+ }
+
+ auto Built = MachineIRBuilder::buildInstr(InstructionOpcode, Destinations, Sources);
+ if (!Built)
+ return std::unexpected(Built.error());
+ return memoize(**Built);
+}
+
+Expected<MachineInstr *> CSEMIRBuilder::buildConstant(MachineIRBuilderResult Destination,
+ APInt Value) const {
+ const Opcode ConstantOpcode = opcode(GenericOpcode::G_CONSTANT);
+ if (!canPerformCSE(ConstantOpcode))
+ return MachineIRBuilder::buildConstant(Destination, std::move(Value));
+ MachineBasicBlock *Block = insertionBlock();
+ if (Block == nullptr)
+ return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point");
+
+ GISelCSEInfo *Info = getCSEInfo();
+ auto Existing = Info->getConstantIfExists(Destination, Value, *Block);
+ if (!Existing)
+ return std::unexpected(Existing.error());
+ if (*Existing != nullptr) {
+ auto Dominating = makeInstructionDominate(**Existing);
+ if (!Dominating)
+ return std::unexpected(Dominating.error());
+ return generateCopyIfRequired(Destination, **Existing);
+ }
+
+ auto Built = MachineIRBuilder::buildConstant(Destination, std::move(Value));
+ if (!Built)
+ return std::unexpected(Built.error());
+ return memoize(**Built);
+}
+
+Expected<MachineInstr *> CSEMIRBuilder::buildSExtInReg(MachineIRBuilderResult Destination,
+ Register Source, std::uint32_t Width) const {
+ const Opcode InstructionOpcode = opcode(GenericOpcode::G_SEXT_INREG);
+ if (!canPerformCSE(InstructionOpcode))
+ return MachineIRBuilder::buildSExtInReg(Destination, Source, Width);
+ MachineBasicBlock *Block = insertionBlock();
+ if (Block == nullptr)
+ return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point");
+
+ GISelCSEInfo *Info = getCSEInfo();
+ const std::array Destinations{Destination};
+ const std::array Sources{Source};
+ const std::array Immediates{static_cast<std::int64_t>(Width)};
+ auto Existing = Info->getMachineInstrIfExists(
+ InstructionOpcode, std::span<const MachineIRBuilderResult>(Destinations),
+ std::span<const Register>(Sources), std::span<const std::int64_t>(Immediates),
+ MachineInstrFlags(), *Block);
+ if (!Existing)
+ return std::unexpected(Existing.error());
+ if (*Existing != nullptr) {
+ auto Dominating = makeInstructionDominate(**Existing);
+ if (!Dominating)
+ return std::unexpected(Dominating.error());
+ return generateCopyIfRequired(Destination, **Existing);
+ }
+
+ auto Built = MachineIRBuilder::buildSExtInReg(Destination, Source, Width);
+ if (!Built)
+ return std::unexpected(Built.error());
+ return memoize(**Built);
+}
+
+Expected<MachineInstr *> CSEMIRBuilder::buildExtract(MachineIRBuilderResult Destination,
+ Register Source, std::uint64_t Offset) const {
+ if (Offset > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max()))
+ return MachineIRBuilder::buildExtract(Destination, Source, Offset);
+ const Opcode InstructionOpcode = opcode(GenericOpcode::G_EXTRACT);
+ if (!canPerformCSE(InstructionOpcode))
+ return MachineIRBuilder::buildExtract(Destination, Source, Offset);
+ MachineBasicBlock *Block = insertionBlock();
+ if (Block == nullptr)
+ return makeUnexpected(ErrorCode::InvalidArgument, "machine IR builder has no insertion point");
+
+ GISelCSEInfo *Info = getCSEInfo();
+ const std::array Destinations{Destination};
+ const std::array Sources{Source};
+ const std::array Immediates{static_cast<std::int64_t>(Offset)};
+ auto Existing = Info->getMachineInstrIfExists(
+ InstructionOpcode, std::span<const MachineIRBuilderResult>(Destinations),
+ std::span<const Register>(Sources), std::span<const std::int64_t>(Immediates),
+ MachineInstrFlags(), *Block);
+ if (!Existing)
+ return std::unexpected(Existing.error());
+ if (*Existing != nullptr) {
+ auto Dominating = makeInstructionDominate(**Existing);
+ if (!Dominating)
+ return std::unexpected(Dominating.error());
+ return generateCopyIfRequired(Destination, **Existing);
+ }
+
+ auto Built = MachineIRBuilder::buildExtract(Destination, Source, Offset);
+ if (!Built)
+ return std::unexpected(Built.error());
+ return memoize(**Built);
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/Combiner.cpp b/backend/lib/CodeGen/GlobalISel/Combiner.cpp
new file mode 100644
index 0000000..c0a92cc
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/Combiner.cpp
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/Combiner.h"
+
+#include "rcc/CodeGen/GlobalISel/Utils.h"
+#include "rcc/CodeGen/LiveVariables.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+
+#include <string>
+
+namespace rcc {
+
+const MachinePassContract &GlobalISelCombinerPass::contract() const noexcept {
+ return Contract;
+}
+
+Expected<MachinePassResult> GlobalISelCombinerPass::run(MachineFunction &Function,
+ const TargetDescription &Target,
+ MachineFunctionAnalysisManager &Analyses) {
+ return runImplementation(Function, Target, Analyses);
+}
+
+Expected<MachinePassResult>
+GlobalISelCombinerPass::runImplementation(MachineFunction &Function,
+ const TargetDescription &Target,
+ MachineFunctionAnalysisManager &Analyses) const {
+ if (MaximumRewrites == 0 || MaximumIterations == 0)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "GlobalISel combiner iteration limits must be nonzero");
+ MachinePassResult Result;
+ std::size_t Rewrites = 0;
+ for (std::size_t Iteration = 0; Iteration < MaximumIterations; ++Iteration) {
+ bool IterationChanged = false;
+ for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;) {
+ MachineInstr *Next = Instruction->next();
+ auto Matches = Executor->match(*Table, *Instruction, Function, Target);
+ if (!Matches)
+ return std::unexpected(Matches.error());
+ if (*Matches && Rewrites < MaximumRewrites) {
+ auto Accepted =
+ Analyses.requestAction("gisel-combiner", Function, Instruction->stableID());
+ if (!Accepted)
+ return std::unexpected(Accepted.error());
+ if (*Accepted) {
+ auto Combined = Executor->combine(*Table, *Instruction, Function, Target);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ if (*Combined) {
+ ++Rewrites;
+ IterationChanged = true;
+ Result.Changed = true;
+ }
+ }
+ }
+ Instruction = Next;
+ }
+ }
+ if (!IterationChanged || Rewrites == MaximumRewrites)
+ break;
+ }
+
+ if (!Result.Changed) {
+ Result.Preserved = PreservedAnalyses::all();
+ return Result;
+ }
+ if (Function.properties().has(MachineFunctionProperty::TracksLiveness)) {
+ auto Rebuilt = rebuildMachineLiveness(Function, Target);
+ if (!Rebuilt)
+ return std::unexpected(Rebuilt.error());
+ }
+ Result.Changes = {MachinePassChange::Instructions, MachinePassChange::Operands,
+ MachinePassChange::Registers};
+ Result.Preserved = PreservedAnalyses::none();
+ return Result;
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp b/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp
new file mode 100644
index 0000000..88fcc24
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp
@@ -0,0 +1,1611 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/GIMatchTableExecutor.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/Utils.h"
+#include "rcc/CodeGen/LowLevelType.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <algorithm>
+#include <charconv>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <limits>
+#include <optional>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] Expected<std::optional<LowLevelType>> operandType(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ std::size_t OperandIndex) {
+ const MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ return std::optional<LowLevelType>{};
+ auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ return Info->get().Type;
+}
+
+[[nodiscard]] Expected<bool> matchesType(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const GIMatchTableInstruction &Step) {
+ auto Type = operandType(Instruction, Function, Step.OperandIndex);
+ if (!Type)
+ return std::unexpected(Type.error());
+ if (!*Type)
+ return false;
+
+ const auto Kind = static_cast<GIMatchTypeKind>(Step.Auxiliary);
+ switch (Kind) {
+ case GIMatchTypeKind::Any:
+ return Step.Value == 0 || (*Type)->scalarBitWidth() == Step.Value;
+ case GIMatchTypeKind::Scalar:
+ return (*Type)->isCategoryNeutralScalar() && (*Type)->scalarBitWidth() == Step.Value;
+ case GIMatchTypeKind::Pointer:
+ return (*Type)->isPointer() && (*Type)->scalarBitWidth() == Step.Value;
+ }
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid type predicate");
+}
+
+[[nodiscard]] bool matchesInstructionFlags(const MachineInstr &Instruction,
+ const GIMatchTableInstruction &Step) noexcept {
+ return (Instruction.flags().rawValue() & Step.Value) == Step.Value;
+}
+
+[[nodiscard]] Expected<bool> matchesMemorySize(const MachineInstr &Instruction,
+ const GIMatchTableInstruction &Step) {
+ if (Step.Value == 0) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid memory size");
+ }
+ if (Step.OperandIndex >= Instruction.memOperands().size())
+ return false;
+ const MachineMemOperand *Memory = Instruction.memOperands()[Step.OperandIndex];
+ if (Memory == nullptr)
+ return false;
+ const MemorySize Size = Memory->description().Size;
+ return Size.isKnown() && !Size.isScalable() && Size.minimumBytes() == Step.Value;
+}
+
+[[nodiscard]] Expected<bool> matchesBank(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const GIMatchTableInstruction &Step) {
+ const MachineOperand *Operand = Instruction.operandAt(Step.OperandIndex);
+ const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ return false;
+ auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ const VirtualRegisterInfo &RegisterInfo = Info->get();
+ if (!RegisterInfo.Bank)
+ return false;
+ return RegisterInfo.Bank->value() == Step.Value;
+}
+
+[[nodiscard]] Expected<bool> matchesClass(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const GIMatchTableInstruction &Step) {
+ const MachineOperand *Operand = Instruction.operandAt(Step.OperandIndex);
+ const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ return false;
+ auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ const VirtualRegisterInfo &RegisterInfo = Info->get();
+ if (!RegisterInfo.RegClass)
+ return false;
+ return RegisterInfo.RegClass->value() == Step.Value;
+}
+
+[[nodiscard]] Expected<bool> matchesIntegerPredicate(const MachineInstr &Instruction,
+ std::size_t OperandIndex,
+ std::uint32_t PredicateValue) {
+ if (PredicateValue >= static_cast<std::uint32_t>(IntegerPredicate::Count)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid integer predicate");
+ }
+ const MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const auto *Predicate = Operand == nullptr ? nullptr : Operand->getIf<PredicateOperand>();
+ const IntegerPredicate *Integer = Predicate == nullptr ? nullptr : Predicate->integerPredicate();
+ return Integer != nullptr && static_cast<std::uint32_t>(*Integer) == PredicateValue;
+}
+
+[[nodiscard]] Expected<bool> matchesFloatingPredicate(const MachineInstr &Instruction,
+ std::size_t OperandIndex,
+ std::uint32_t PredicateValue) {
+ if (PredicateValue >= static_cast<std::uint32_t>(FloatingPredicate::Count)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid floating predicate");
+ }
+ const MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const auto *Predicate = Operand == nullptr ? nullptr : Operand->getIf<PredicateOperand>();
+ const FloatingPredicate *Floating =
+ Predicate == nullptr ? nullptr : Predicate->floatingPredicate();
+ return Floating != nullptr && static_cast<std::uint32_t>(*Floating) == PredicateValue;
+}
+
+[[nodiscard]] std::optional<std::int64_t> immediateValue(const MachineOperand &Operand) {
+ if (const std::int64_t *Immediate = Operand.getIf<std::int64_t>())
+ return *Immediate;
+ const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+ if (Integer == nullptr)
+ return std::optional<std::int64_t>{};
+ const std::string Text = Integer->Value.toSignedDecimal();
+ std::int64_t Parsed = 0;
+ const auto [End, Error] = std::from_chars(Text.data(), Text.data() + Text.size(), Parsed);
+ if (Error != std::errc{} || End != Text.data() + Text.size())
+ return std::optional<std::int64_t>{};
+ return Parsed;
+}
+
+[[nodiscard]] Expected<std::optional<std::int64_t>>
+operandDefConstantValue(const MachineInstr &Instruction, const MachineFunction &Function,
+ std::size_t OperandIndex) {
+ const MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ return std::optional<std::int64_t>{};
+ const MachineInstr *Definition = Function.registerInfo().getVRegDef(*Reg);
+ if (Definition == nullptr || Definition->opcode() != opcode(GenericOpcode::G_CONSTANT))
+ return std::optional<std::int64_t>{};
+ const MachineOperand *Value = Definition->operandAt(1);
+ if (Value == nullptr)
+ return std::optional<std::int64_t>{};
+ return immediateValue(*Value);
+}
+
+[[nodiscard]] Expected<std::optional<std::int64_t>>
+operandOrDefConstantValue(const MachineInstr &Instruction, const MachineFunction &Function,
+ std::size_t OperandIndex) {
+ const MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ if (Operand == nullptr)
+ return std::optional<std::int64_t>{};
+ if (const auto Value = immediateValue(*Operand))
+ return Value;
+ return operandDefConstantValue(Instruction, Function, OperandIndex);
+}
+
+[[nodiscard]] Expected<bool> matchesConstantRanges(const GIMatchTable &Table, std::size_t First,
+ std::size_t Count,
+ std::optional<std::int64_t> Value) {
+ if (Count == 0 || First > Table.ConstantRanges.size() ||
+ Count > Table.ConstantRanges.size() - First) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid constant range");
+ }
+ if (!Value)
+ return false;
+ for (std::size_t Index = First; Index < First + Count; ++Index) {
+ const GIMatchTableConstantRange &Range = Table.ConstantRanges[Index];
+ if (Range.Minimum > Range.Maximum) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an inverted constant range");
+ }
+ if (*Value >= Range.Minimum && *Value <= Range.Maximum)
+ return true;
+ }
+ return false;
+}
+
+[[nodiscard]] Expected<bool> matchesOperandImmediateRanges(const GIMatchTable &Table,
+ const MachineInstr &Instruction,
+ const GIMatchTableInstruction &Step) {
+ const MachineOperand *Operand = Instruction.operandAt(Step.OperandIndex);
+ const std::optional<std::int64_t> Value =
+ Operand == nullptr ? std::optional<std::int64_t>{} : immediateValue(*Operand);
+ return matchesConstantRanges(Table, Step.Value, Step.Auxiliary, Value);
+}
+
+[[nodiscard]] Expected<bool> matchesOperandDefConstantRanges(const GIMatchTable &Table,
+ const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const GIMatchTableInstruction &Step) {
+ auto Value = operandDefConstantValue(Instruction, Function, Step.OperandIndex);
+ if (!Value)
+ return std::unexpected(Value.error());
+ return matchesConstantRanges(Table, Step.Value, Step.Auxiliary, *Value);
+}
+
+[[nodiscard]] const MachineInstr *operandDefinition(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ std::size_t OperandIndex) {
+ const MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ return Reg != nullptr && Reg->isVirtual() ? Function.registerInfo().getVRegDef(*Reg) : nullptr;
+}
+
+[[nodiscard]] Expected<bool> matchesOperandDefOpcode(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const GIMatchTableInstruction &Step) {
+ if (Step.Value == 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid def-use opcode");
+ const MachineInstr *Definition = operandDefinition(Instruction, Function, Step.OperandIndex);
+ return Definition != nullptr && Definition->opcode().value() == Step.Value;
+}
+
+[[nodiscard]] Expected<bool>
+matchesOperandDefIntegerPredicate(const MachineInstr &Instruction, const MachineFunction &Function,
+ const GIMatchTableInstruction &Step) {
+ const MachineInstr *Definition = operandDefinition(Instruction, Function, Step.OperandIndex);
+ if (Definition == nullptr)
+ return false;
+ return matchesIntegerPredicate(*Definition, Step.Auxiliary, Step.Value);
+}
+
+[[nodiscard]] Expected<bool>
+matchesOperandDefOperandConstantRanges(const GIMatchTable &Table, const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const GIMatchTableInstruction &Step) {
+ if (Step.Immediate <= 0 ||
+ static_cast<std::uint64_t>(Step.Immediate) > std::numeric_limits<std::uint32_t>::max()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid def-use constant range count");
+ }
+ const std::size_t Count = static_cast<std::size_t>(Step.Immediate);
+ const MachineInstr *Definition = operandDefinition(Instruction, Function, Step.OperandIndex);
+ if (Definition == nullptr)
+ return false;
+ auto Value = operandOrDefConstantValue(*Definition, Function, Step.Auxiliary);
+ if (!Value)
+ return std::unexpected(Value.error());
+ return matchesConstantRanges(Table, Step.Value, Count, *Value);
+}
+
+[[nodiscard]] Expected<RegisterOperandFlags> registerFlagsFromRaw(std::uint32_t Raw) {
+ constexpr std::uint32_t ValidBits =
+ (std::uint32_t{1} << static_cast<std::uint16_t>(RegisterOperandFlag::Count)) - 1U;
+ if ((Raw & ~ValidBits) != 0) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has invalid register flags");
+ }
+ RegisterOperandFlags Flags;
+ for (std::uint16_t Index = 0; Index < static_cast<std::uint16_t>(RegisterOperandFlag::Count);
+ ++Index) {
+ if ((Raw & (std::uint32_t{1} << Index)) != 0)
+ Flags.set(static_cast<RegisterOperandFlag>(Index));
+ }
+ return Flags;
+}
+
+struct MatchedRule final {
+ struct CapturedOperand final {
+ std::uint32_t ID = 0;
+ MachineOperand Operand;
+ };
+
+ struct Temporary final {
+ std::uint32_t ID = 0;
+ LowLevelType Type;
+ };
+
+ struct RenderedOperand final {
+ enum class Kind : std::uint8_t {
+ RootOperand,
+ ComplexOperand,
+ CapturedOperand,
+ PhysicalRegister,
+ Temporary,
+ CapturedImmediate,
+ CapturedTypeBits,
+ Immediate,
+ };
+
+ Kind SourceKind = Kind::RootOperand;
+ std::uint32_t Value = 0;
+ std::uint32_t Flags = 0;
+ std::string_view ComplexRenderer;
+ GIMatchImmediateTransform Transform = GIMatchImmediateTransform::Identity;
+ std::int64_t Immediate = 0;
+ };
+
+ struct Instruction final {
+ Opcode ResultOpcode;
+ bool ReplacesRoot = false;
+ std::vector<RenderedOperand> Operands;
+ };
+
+ std::vector<Temporary> Temporaries;
+ std::vector<CapturedOperand> CapturedOperands;
+ std::vector<Instruction> Instructions;
+ std::vector<Register> EraseOperandDefinitions;
+ std::optional<std::uint32_t> ReplaceRootCapture;
+ std::uint32_t ClearInstructionFlags = 0;
+};
+
+[[nodiscard]] bool hasTemporary(const MatchedRule &Rule, std::uint32_t ID) {
+ return std::ranges::any_of(Rule.Temporaries, [ID](const MatchedRule::Temporary &Temporary) {
+ return Temporary.ID == ID;
+ });
+}
+
+[[nodiscard]] bool hasCapturedOperand(const MatchedRule &Rule, std::uint32_t ID) {
+ return std::ranges::any_of(
+ Rule.CapturedOperands,
+ [ID](const MatchedRule::CapturedOperand &Capture) { return Capture.ID == ID; });
+}
+
+[[nodiscard]] bool hasRootInstruction(const MatchedRule &Rule) {
+ return std::ranges::any_of(Rule.Instructions, &MatchedRule::Instruction::ReplacesRoot);
+}
+
+[[nodiscard]] const MatchedRule::CapturedOperand *
+findCapturedOperand(const std::vector<MatchedRule::CapturedOperand> &Captures, std::uint32_t ID) {
+ const auto Position = std::ranges::find(Captures, ID, &MatchedRule::CapturedOperand::ID);
+ return Position == Captures.end() ? nullptr : &*Position;
+}
+
+template <typename TargetPredicateMatcher>
+[[nodiscard]] Expected<std::optional<MatchedRule>>
+matchRule(const GIMatchTable &Table, const GIMatchTableRule &Rule, const MachineInstr &Instruction,
+ const MachineFunction &Function, const TargetDescription &Target,
+ TargetPredicateMatcher &&MatchesTargetPredicate) {
+ const std::size_t First = Rule.FirstInstruction;
+ const std::size_t Count = Rule.InstructionCount;
+ if (First > Table.Instructions.size() || Count > Table.Instructions.size() - First) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table rule has an invalid instruction range");
+ }
+
+ MatchedRule Result;
+ bool Applying = false;
+ bool Complete = false;
+ for (std::size_t Index = First; Index < First + Count; ++Index) {
+ const GIMatchTableInstruction &Step = Table.Instructions[Index];
+ switch (Step.Opcode) {
+ case GIMatchTableOpcode::CheckOpcode:
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ if (Instruction.opcode().value() != Step.Value)
+ return std::optional<MatchedRule>{};
+ break;
+ case GIMatchTableOpcode::CheckOperandType: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesType(Instruction, Function, Step);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckInstructionFlag:
+ if (Applying || Step.Value == 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid instruction flag predicate");
+ if (!matchesInstructionFlags(Instruction, Step))
+ return std::optional<MatchedRule>{};
+ break;
+ case GIMatchTableOpcode::ClearInstructionFlags:
+ if (!Applying || Complete || Step.Value == 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid flag clear action");
+ if (Result.ClearInstructionFlags != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table clears instruction flags twice");
+ Result.ClearInstructionFlags = Step.Value;
+ break;
+ case GIMatchTableOpcode::CheckOperandBank: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesBank(Instruction, Function, Step);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckOperandClass: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesClass(Instruction, Function, Step);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckIntegerPredicate: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesIntegerPredicate(Instruction, Step.OperandIndex, Step.Value);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckFloatingPredicate: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesFloatingPredicate(Instruction, Step.OperandIndex, Step.Value);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckMemorySize: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesMemorySize(Instruction, Step);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckTargetPredicate: {
+ if (Applying || Step.Symbol.empty()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid target predicate");
+ }
+ auto Matched = MatchesTargetPredicate(Step.Symbol, Instruction, Function, Target);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckOperandImmediateRanges: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesOperandImmediateRanges(Table, Instruction, Step);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckOperandDefConstantRanges: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesOperandDefConstantRanges(Table, Instruction, Function, Step);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckOperandDefOpcode: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesOperandDefOpcode(Instruction, Function, Step);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckOperandDefIntegerPredicate: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesOperandDefIntegerPredicate(Instruction, Function, Step);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CheckOperandDefOperandConstantRanges: {
+ if (Applying) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table performs a predicate after an action");
+ }
+ auto Matched = matchesOperandDefOperandConstantRanges(Table, Instruction, Function, Step);
+ if (!Matched)
+ return std::unexpected(Matched.error());
+ if (!*Matched)
+ return std::optional<MatchedRule>{};
+ break;
+ }
+ case GIMatchTableOpcode::CaptureDefOperand: {
+ if (Applying || Complete || Step.Value == 0 || hasCapturedOperand(Result, Step.Value)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid def-use capture");
+ }
+ const MachineInstr *Definition = operandDefinition(Instruction, Function, Step.OperandIndex);
+ const MachineOperand *Operand =
+ Definition == nullptr ? nullptr : Definition->operandAt(Step.Auxiliary);
+ if (Operand == nullptr || Operand->isImplicit())
+ return std::optional<MatchedRule>{};
+ Result.CapturedOperands.push_back({
+ .ID = Step.Value,
+ .Operand = Operand->cloneDetached(),
+ });
+ break;
+ }
+ case GIMatchTableOpcode::CreateTemporary: {
+ if (Complete || Step.Value == 0 || hasTemporary(Result, Step.Value)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid temporary");
+ }
+ auto Type = operandType(Instruction, Function, Step.OperandIndex);
+ if (!Type)
+ return std::unexpected(Type.error());
+ if (!*Type) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table temporary has no type source");
+ }
+ Applying = true;
+ Result.Temporaries.push_back({
+ .ID = Step.Value,
+ .Type = **Type,
+ });
+ break;
+ }
+ case GIMatchTableOpcode::CreateTemporaryType: {
+ if (Complete || Step.Value == 0 || hasTemporary(Result, Step.Value) ||
+ Step.Auxiliary != static_cast<std::uint32_t>(GIMatchTypeKind::Scalar) ||
+ Step.Immediate <= 0 ||
+ static_cast<std::uint64_t>(Step.Immediate) > LowLevelType::MaxBitWidth) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid explicit temporary type");
+ }
+ auto Type =
+ LowLevelType::makeCategoryNeutralScalar(static_cast<std::uint64_t>(Step.Immediate));
+ if (!Type)
+ return std::unexpected(Type.error());
+ Applying = true;
+ Result.Temporaries.push_back({.ID = Step.Value, .Type = *Type});
+ break;
+ }
+ case GIMatchTableOpcode::MutateOpcode:
+ if (Complete || Step.Value == 0 || hasRootInstruction(Result)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid root mutation");
+ }
+ Applying = true;
+ Result.Instructions.push_back({
+ .ResultOpcode = Opcode::fromValue(Step.Value),
+ .ReplacesRoot = true,
+ .Operands = {},
+ });
+ break;
+ case GIMatchTableOpcode::BuildInstruction:
+ if (Complete || Step.Value == 0) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid instruction action");
+ }
+ Applying = true;
+ Result.Instructions.push_back({
+ .ResultOpcode = Opcode::fromValue(Step.Value),
+ .ReplacesRoot = false,
+ .Operands = {},
+ });
+ break;
+ case GIMatchTableOpcode::RenderOperand:
+ if (Complete || Result.Instructions.empty() ||
+ Instruction.operandAt(Step.OperandIndex) == nullptr ||
+ Instruction.operandAt(Step.OperandIndex)->isImplicit()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid operand renderer");
+ }
+ Result.Instructions.back().Operands.push_back({
+ .SourceKind = MatchedRule::RenderedOperand::Kind::RootOperand,
+ .Value = Step.OperandIndex,
+ .ComplexRenderer = {},
+ });
+ break;
+ case GIMatchTableOpcode::RenderComplexOperand:
+ if (Complete || Result.Instructions.empty() || Step.Symbol.empty() ||
+ Instruction.operandAt(Step.OperandIndex) == nullptr ||
+ Instruction.operandAt(Step.OperandIndex)->isImplicit()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid complex operand renderer");
+ }
+ Result.Instructions.back().Operands.push_back({
+ .SourceKind = MatchedRule::RenderedOperand::Kind::ComplexOperand,
+ .Value = Step.OperandIndex,
+ .ComplexRenderer = Step.Symbol,
+ });
+ break;
+ case GIMatchTableOpcode::RenderCapturedOperand:
+ if (Complete || Result.Instructions.empty() || Step.Value == 0 ||
+ !hasCapturedOperand(Result, Step.Value)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid captured operand renderer");
+ }
+ Result.Instructions.back().Operands.push_back({
+ .SourceKind = MatchedRule::RenderedOperand::Kind::CapturedOperand,
+ .Value = Step.Value,
+ .ComplexRenderer = {},
+ });
+ break;
+ case GIMatchTableOpcode::RenderCapturedTypeBits:
+ if (Complete || Result.Instructions.empty() || Step.Value == 0 ||
+ !hasCapturedOperand(Result, Step.Value)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid captured type renderer");
+ }
+ Result.Instructions.back().Operands.push_back({
+ .SourceKind = MatchedRule::RenderedOperand::Kind::CapturedTypeBits,
+ .Value = Step.Value,
+ .ComplexRenderer = {},
+ });
+ break;
+ case GIMatchTableOpcode::RenderPhysicalRegister:
+ if (Complete || Result.Instructions.empty() || Step.Value == 0) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid physical register renderer");
+ }
+ Result.Instructions.back().Operands.push_back({
+ .SourceKind = MatchedRule::RenderedOperand::Kind::PhysicalRegister,
+ .Value = Step.Value,
+ .Flags = Step.Auxiliary,
+ .ComplexRenderer = {},
+ });
+ break;
+ case GIMatchTableOpcode::RenderTemporary:
+ if (Complete || Result.Instructions.empty() || !hasTemporary(Result, Step.Value)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid temporary renderer");
+ }
+ Result.Instructions.back().Operands.push_back({
+ .SourceKind = MatchedRule::RenderedOperand::Kind::Temporary,
+ .Value = Step.Value,
+ .Flags = Step.Auxiliary,
+ .ComplexRenderer = {},
+ });
+ break;
+ case GIMatchTableOpcode::RenderCapturedImmediate: {
+ if (Complete || Result.Instructions.empty() ||
+ Step.Value > static_cast<std::uint32_t>(GIMatchImmediateTransform::PlusOne)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid captured immediate renderer");
+ }
+ auto Value = operandDefConstantValue(Instruction, Function, Step.OperandIndex);
+ if (!Value)
+ return std::unexpected(Value.error());
+ if (!*Value) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table lost a captured constant");
+ }
+ Result.Instructions.back().Operands.push_back({
+ .SourceKind = MatchedRule::RenderedOperand::Kind::CapturedImmediate,
+ .Value = Step.OperandIndex,
+ .ComplexRenderer = {},
+ .Transform = static_cast<GIMatchImmediateTransform>(Step.Value),
+ });
+ break;
+ }
+ case GIMatchTableOpcode::RenderImmediate:
+ if (Complete || Result.Instructions.empty()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid immediate renderer");
+ }
+ Result.Instructions.back().Operands.push_back({
+ .SourceKind = MatchedRule::RenderedOperand::Kind::Immediate,
+ .ComplexRenderer = {},
+ .Immediate = Step.Immediate,
+ });
+ break;
+ case GIMatchTableOpcode::ReplaceRootWithCapturedOperand:
+ if (Table.Kind != GIMatchTableKind::Combiner || Complete || Applying ||
+ !Result.Instructions.empty() || Step.Value == 0 ||
+ !hasCapturedOperand(Result, Step.Value) || Result.ReplaceRootCapture) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid root replacement action");
+ }
+ Applying = true;
+ Result.ReplaceRootCapture = Step.Value;
+ break;
+ case GIMatchTableOpcode::EraseOperandDefinition: {
+ if (Complete || (Result.Instructions.empty() && !Result.ReplaceRootCapture)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid erased definition");
+ }
+ const MachineOperand *Operand = Instruction.operandAt(Step.OperandIndex);
+ const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table erases a non-virtual operand definition");
+ }
+ Result.EraseOperandDefinitions.push_back(*Reg);
+ break;
+ }
+ case GIMatchTableOpcode::Complete:
+ if (Complete || (Result.Instructions.empty() && !Result.ReplaceRootCapture) ||
+ Index + 1 != First + Count) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid completion action");
+ }
+ Complete = true;
+ break;
+ }
+ }
+
+ if (!Complete ||
+ (Table.Kind == GIMatchTableKind::Selection &&
+ (Result.Instructions.empty() || !hasRootInstruction(Result))) ||
+ (Table.Kind == GIMatchTableKind::Combiner && Result.Instructions.empty() &&
+ !Result.ReplaceRootCapture) ||
+ (Table.Kind == GIMatchTableKind::Combiner && !Result.Instructions.empty() &&
+ Result.ReplaceRootCapture)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table rule does not complete rendering");
+ }
+ return std::optional<MatchedRule>(std::move(Result));
+}
+
+template <typename ComplexRenderer>
+[[nodiscard]] Expected<bool>
+complexRenderersMatch(const MatchedRule &Rule, const MachineInstr &Instruction,
+ const MachineFunction &Function, const TargetDescription &Target,
+ ComplexRenderer &&RenderComplex) {
+ for (const MatchedRule::Instruction &Action : Rule.Instructions) {
+ for (const MatchedRule::RenderedOperand &Rendered : Action.Operands) {
+ if (Rendered.SourceKind != MatchedRule::RenderedOperand::Kind::ComplexOperand)
+ continue;
+ auto Complex =
+ RenderComplex(Rendered.ComplexRenderer, Instruction, Function, Target, Rendered.Value);
+ if (!Complex)
+ return std::unexpected(Complex.error());
+ if (!*Complex)
+ return false;
+ }
+ }
+ return true;
+}
+
+[[nodiscard]] std::size_t explicitOperandCount(const MachineInstr &Instruction) noexcept {
+ std::size_t Count = 0;
+ while (Count < Instruction.operandCount() && !Instruction.operandAt(Count)->isImplicit())
+ ++Count;
+ return Count;
+}
+
+[[nodiscard]] Expected<void>
+restoreInstruction(MachineInstr &Instruction, Opcode OriginalOpcode,
+ const std::vector<MachineOperand> &OriginalOperands,
+ const std::vector<std::pair<std::size_t, std::size_t>> &Ties) {
+ while (Instruction.operandCount() != 0) {
+ auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ }
+ auto RestoredOpcode = Instruction.setOpcode(OriginalOpcode);
+ if (!RestoredOpcode)
+ return std::unexpected(RestoredOpcode.error());
+ for (const MachineOperand &Operand : OriginalOperands) {
+ auto Added = Instruction.addOperand(Operand.cloneDetached());
+ if (!Added)
+ return std::unexpected(Added.error());
+ }
+ for (const auto [DefIndex, UseIndex] : Ties) {
+ auto Tied = Instruction.tieOperands(DefIndex, UseIndex);
+ if (!Tied)
+ return std::unexpected(Tied.error());
+ }
+ return {};
+}
+
+struct MaterializedTemporary final {
+ std::uint32_t ID = 0;
+ Register Reg;
+};
+
+struct RegisterConstraintSnapshot final {
+ Register Reg;
+ std::optional<RegisterBankID> Bank;
+ std::optional<RegisterClassID> RegClass;
+};
+
+[[nodiscard]] const MaterializedTemporary *
+findTemporary(const std::vector<MaterializedTemporary> &Temporaries, std::uint32_t ID) {
+ const auto Position = std::ranges::find(Temporaries, ID, &MaterializedTemporary::ID);
+ return Position == Temporaries.end() ? nullptr : &*Position;
+}
+
+[[nodiscard]] Expected<void>
+eraseTemporaries(MachineRegisterInfo &Registers,
+ const std::vector<MaterializedTemporary> &Temporaries) {
+ for (auto Iterator = Temporaries.rbegin(); Iterator != Temporaries.rend(); ++Iterator) {
+ if (!Registers.isKnownVirtualRegister(Iterator->Reg))
+ continue;
+ auto Erased = Registers.eraseVirtualRegister(Iterator->Reg);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void> eraseRegisters(MachineRegisterInfo &Registers,
+ const std::vector<Register> &RegistersToErase) {
+ for (auto Iterator = RegistersToErase.rbegin(); Iterator != RegistersToErase.rend(); ++Iterator) {
+ if (!Registers.isKnownVirtualRegister(*Iterator))
+ continue;
+ auto Erased = Registers.eraseVirtualRegister(*Iterator);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void>
+restoreRegisterConstraints(MachineRegisterInfo &Registers,
+ const std::vector<RegisterConstraintSnapshot> &Snapshots) {
+ for (auto Iterator = Snapshots.rbegin(); Iterator != Snapshots.rend(); ++Iterator) {
+ if (!Registers.isKnownVirtualRegister(Iterator->Reg))
+ continue;
+ if (Iterator->Bank) {
+ auto Restored = Registers.setRegisterBank(Iterator->Reg, *Iterator->Bank);
+ if (!Restored)
+ return std::unexpected(Restored.error());
+ } else {
+ auto Cleared = Registers.clearRegisterBank(Iterator->Reg);
+ if (!Cleared)
+ return std::unexpected(Cleared.error());
+ }
+ if (Iterator->RegClass) {
+ auto Restored = Registers.setRegisterClass(Iterator->Reg, *Iterator->RegClass);
+ if (!Restored)
+ return std::unexpected(Restored.error());
+ } else {
+ auto Cleared = Registers.clearRegisterClass(Iterator->Reg);
+ if (!Cleared)
+ return std::unexpected(Cleared.error());
+ }
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void>
+constrainRenderedInstruction(const InstructionDescriptor &Descriptor,
+ std::span<const MachineOperand> Operands, MachineFunction &Function,
+ const TargetDescription &Target,
+ std::vector<RegisterConstraintSnapshot> &Snapshots) {
+ if (Descriptor.Operands.empty() && !Operands.empty()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table result has no operand descriptors");
+ }
+
+ MachineRegisterInfo &Registers = Function.registerInfo();
+ const auto Remember = [&Snapshots](Register Reg, const VirtualRegisterInfo &Info) {
+ if (std::ranges::find(Snapshots, Reg, &RegisterConstraintSnapshot::Reg) == Snapshots.end())
+ Snapshots.push_back({.Reg = Reg, .Bank = Info.Bank, .RegClass = Info.RegClass});
+ };
+
+ for (std::size_t Index = 0; Index < Operands.size(); ++Index) {
+ const Register *Reg = Operands[Index].getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ continue;
+
+ const OperandDescriptor &OperandDescriptor = Index < Descriptor.Operands.size()
+ ? Descriptor.Operands[Index]
+ : Descriptor.Operands.back();
+ if (!OperandDescriptor.RegBank && !OperandDescriptor.RegClass)
+ continue;
+
+ auto Info = Registers.virtualRegisterInfo(*Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ if (OperandDescriptor.RegBank) {
+ if (Info->get().Bank && *Info->get().Bank != *OperandDescriptor.RegBank) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table result has an incompatible register bank");
+ }
+ if (!Info->get().Bank) {
+ Remember(*Reg, Info->get());
+ auto SetBank = Registers.setRegisterBank(*Reg, *OperandDescriptor.RegBank);
+ if (!SetBank)
+ return std::unexpected(SetBank.error());
+ }
+ }
+ if (OperandDescriptor.RegClass) {
+ if (Info->get().RegClass &&
+ !Target.registers().isSubclass(*Info->get().RegClass, *OperandDescriptor.RegClass)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table result has an incompatible register class");
+ }
+ if (!Info->get().RegClass) {
+ Remember(*Reg, Info->get());
+ auto SetClass = Registers.setRegisterClass(*Reg, *OperandDescriptor.RegClass);
+ if (!SetClass)
+ return std::unexpected(SetClass.error());
+ }
+ }
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<MachineOperand>
+renderOperand(const MatchedRule::RenderedOperand &Rendered, const MachineInstr &Instruction,
+ const MachineFunction &Function, const TargetDescription &Target,
+ const std::vector<MaterializedTemporary> &Temporaries,
+ const std::vector<MatchedRule::CapturedOperand> &CapturedOperands,
+ std::size_t OriginalExplicitCount) {
+ switch (Rendered.SourceKind) {
+ case MatchedRule::RenderedOperand::Kind::RootOperand:
+ if (Rendered.Value >= OriginalExplicitCount) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table renders an implicit or missing operand");
+ }
+ return Instruction.operandAt(Rendered.Value)->cloneDetached();
+ case MatchedRule::RenderedOperand::Kind::ComplexOperand:
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table did not resolve a complex operand renderer");
+ case MatchedRule::RenderedOperand::Kind::CapturedOperand: {
+ const MatchedRule::CapturedOperand *Captured =
+ findCapturedOperand(CapturedOperands, Rendered.Value);
+ if (Captured == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table renders an unknown captured operand");
+ }
+ return Captured->Operand.cloneDetached();
+ }
+ case MatchedRule::RenderedOperand::Kind::CapturedTypeBits: {
+ const MatchedRule::CapturedOperand *Captured =
+ findCapturedOperand(CapturedOperands, Rendered.Value);
+ if (Captured == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table renders an unknown captured type");
+ const Register *Reg = Captured->Operand.getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table captured type is not a virtual register");
+ auto Type = Function.registerInfo().getType(*Reg);
+ if (!Type)
+ return std::unexpected(Type.error());
+ if (!Type->isCategoryNeutralScalar())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table captured type is not scalar");
+ return MachineOperand::immediate(static_cast<std::int64_t>(Type->scalarBitWidth()));
+ }
+ case MatchedRule::RenderedOperand::Kind::PhysicalRegister: {
+ auto Physical = PhysicalRegister::create(Rendered.Value);
+ if (!Physical || Target.registers().physicalRegister(*Physical) == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table renders an unknown physical register");
+ }
+ auto Flags = registerFlagsFromRaw(Rendered.Flags);
+ if (!Flags)
+ return std::unexpected(Flags.error());
+ return MachineOperand::registerOperand(Register::fromPhysical(*Physical), *Flags);
+ }
+ case MatchedRule::RenderedOperand::Kind::Temporary: {
+ const MaterializedTemporary *Temporary = findTemporary(Temporaries, Rendered.Value);
+ if (Temporary == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table renders an unknown temporary");
+ }
+ auto Flags = registerFlagsFromRaw(Rendered.Flags);
+ if (!Flags)
+ return std::unexpected(Flags.error());
+ return MachineOperand::registerOperand(Temporary->Reg, *Flags);
+ }
+ case MatchedRule::RenderedOperand::Kind::CapturedImmediate: {
+ auto Captured = operandDefConstantValue(Instruction, Function, Rendered.Value);
+ if (!Captured)
+ return std::unexpected(Captured.error());
+ if (!*Captured) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table lost a captured constant");
+ }
+ std::int64_t Value = **Captured;
+ switch (Rendered.Transform) {
+ case GIMatchImmediateTransform::Identity:
+ break;
+ case GIMatchImmediateTransform::Negate:
+ if (Value == std::numeric_limits<std::int64_t>::min()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table immediate negation overflows");
+ }
+ Value = -Value;
+ break;
+ case GIMatchImmediateTransform::PlusOne:
+ if (Value == std::numeric_limits<std::int64_t>::max()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table immediate increment overflows");
+ }
+ ++Value;
+ break;
+ }
+ return MachineOperand::immediate(Value);
+ }
+ case MatchedRule::RenderedOperand::Kind::Immediate:
+ return MachineOperand::immediate(Rendered.Immediate);
+ }
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table has an invalid operand renderer");
+}
+
+[[nodiscard]] bool compatibleReplacement(Register Destination, Register Source,
+ const MachineFunction &Function,
+ const TargetDescription &Target) {
+ if (!Destination.isVirtual() || Source.isNoRegister())
+ return false;
+ auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(Destination);
+ if (!DestinationInfo)
+ return false;
+ const VirtualRegisterInfo &DestinationValue = DestinationInfo->get();
+ if (Source.isPhysical()) {
+ const std::optional<PhysicalRegister> Physical = Source.physicalRegister();
+ if (!Physical)
+ return false;
+ if (DestinationValue.RegClass &&
+ !Target.registers().registerClassContains(*DestinationValue.RegClass, *Physical))
+ return false;
+ return !DestinationValue.Bank ||
+ Target.registers().bankContainsRegister(*DestinationValue.Bank, *Physical);
+ }
+ auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source);
+ if (!SourceInfo)
+ return false;
+ const VirtualRegisterInfo &SourceValue = SourceInfo->get();
+ if (DestinationValue.Type != SourceValue.Type)
+ return false;
+ if (DestinationValue.Bank &&
+ (!SourceValue.Bank ||
+ !Target.registers().registerBanksOverlap(*DestinationValue.Bank, *SourceValue.Bank)))
+ return false;
+ if (DestinationValue.RegClass &&
+ (!SourceValue.RegClass || !Target.registers().registerClassesOverlap(
+ *DestinationValue.RegClass, *SourceValue.RegClass)))
+ return false;
+ return true;
+}
+
+[[nodiscard]] Expected<bool> replaceRootWithCapturedOperand(const MatchedRule &Rule,
+ MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) {
+ if (!Rule.ReplaceRootCapture)
+ return false;
+ const MatchedRule::CapturedOperand *Captured =
+ findCapturedOperand(Rule.CapturedOperands, *Rule.ReplaceRootCapture);
+ const MachineOperand *Definition = Instruction.operandAt(0);
+ const Register *Destination = Definition == nullptr ? nullptr : Definition->getIf<Register>();
+ const Register *Source = Captured == nullptr ? nullptr : Captured->Operand.getIf<Register>();
+ if (Destination == nullptr || Source == nullptr || !Definition->isDef() || Definition->isUse() ||
+ (!Source->isVirtual() && !Source->isPhysical()) || Destination->isNoRegister() ||
+ Source->isNoRegister() || !Destination->isVirtual() ||
+ Function.registerInfo().getUniqueVRegDef(*Destination) != Definition ||
+ !compatibleReplacement(*Destination, *Source, Function, Target)) {
+ return false;
+ }
+ if (Instruction.parent() == nullptr)
+ return makeUnexpected(ErrorCode::InternalError, "generated combiner root is detached");
+ std::vector<MachineOperand *> Uses;
+ for (MachineOperand &Use : Function.registerInfo().useOperands(*Destination, true))
+ Uses.push_back(&Use);
+ for (MachineOperand *Use : Uses) {
+ auto Replaced = Use->setRegister(*Source);
+ if (!Replaced)
+ return std::unexpected(Replaced.error());
+ }
+ auto Erased = eraseInstr(Instruction, Function.registerInfo(), Target);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ return true;
+}
+
+template <typename Finalizer, typename ComplexRenderer>
+[[nodiscard]] Expected<void> applyRule(const MatchedRule &Rule, MachineInstr &Instruction,
+ MachineFunction &Function, const TargetDescription &Target,
+ Finalizer &&Finalize, ComplexRenderer &&RenderComplex) {
+ const InstructionDescriptor *InputDescriptor = Target.instructionDescriptor(Instruction.opcode());
+ if (InputDescriptor == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table input has no instruction descriptor");
+ }
+ if (Rule.ReplaceRootCapture) {
+ auto Replaced = replaceRootWithCapturedOperand(Rule, Instruction, Function, Target);
+ if (!Replaced)
+ return std::unexpected(Replaced.error());
+ if (!*Replaced)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated combiner root replacement is incompatible");
+ for (const Register Reg : Rule.EraseOperandDefinitions) {
+ MachineInstr *Definition = Function.registerInfo().getVRegDef(Reg);
+ if (Definition == nullptr || !isTriviallyDead(*Definition, Function.registerInfo(), Target))
+ continue;
+ auto Erased = eraseInstr(*Definition, Function.registerInfo(), Target);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ }
+ if (Rule.ClearInstructionFlags != 0) {
+ MachineInstrFlags Flags = Instruction.flags();
+ for (std::uint32_t Index = 0; Index < static_cast<std::uint32_t>(MachineInstrFlag::Count);
+ ++Index) {
+ if ((Rule.ClearInstructionFlags & (std::uint32_t{1} << Index)) != 0)
+ Flags.clear(static_cast<MachineInstrFlag>(Index));
+ }
+ Instruction.setFlags(Flags);
+ }
+ return {};
+ }
+ const auto RootPosition =
+ std::ranges::find(Rule.Instructions, true, &MatchedRule::Instruction::ReplacesRoot);
+ if (RootPosition == Rule.Instructions.end())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table selection has no root instruction");
+ const std::size_t RootIndex =
+ static_cast<std::size_t>(std::distance(Rule.Instructions.begin(), RootPosition));
+ const MachineInstrFlags InputFlags = Instruction.flags();
+ const bool InputMayRaiseFPException =
+ InputDescriptor->Flags.has(InstructionFlag::MayRaiseFPException);
+ const std::size_t OriginalExplicitCount = explicitOperandCount(Instruction);
+ const std::size_t OriginalOperandCount = Instruction.operandCount();
+ const Opcode OriginalOpcode = Instruction.opcode();
+
+ std::vector<MachineOperand> OriginalOperands;
+ OriginalOperands.reserve(OriginalOperandCount);
+ for (const MachineOperand &Operand : Instruction.operands())
+ OriginalOperands.push_back(Operand.cloneDetached());
+
+ std::vector<std::pair<std::size_t, std::size_t>> Ties;
+ for (std::size_t Index = 0; Index < OriginalOperandCount; ++Index) {
+ const std::optional<std::size_t> Other = Instruction.operandAt(Index)->tiedOperandIndex();
+ if (Other && Index < *Other)
+ Ties.emplace_back(Index, *Other);
+ }
+
+ std::vector<MaterializedTemporary> Temporaries;
+ Temporaries.reserve(Rule.Temporaries.size());
+ for (const MatchedRule::Temporary &Temporary : Rule.Temporaries) {
+ auto Reg = Function.registerInfo().createVirtualRegister(Temporary.Type);
+ if (!Reg) {
+ auto Cleanup = eraseTemporaries(Function.registerInfo(), Temporaries);
+ if (!Cleanup)
+ return std::unexpected(Cleanup.error());
+ return std::unexpected(Reg.error());
+ }
+ Temporaries.push_back({.ID = Temporary.ID, .Reg = *Reg});
+ }
+
+ struct PendingComplexInstruction final {
+ Opcode ResultOpcode;
+ std::vector<MachineOperand> Operands;
+ };
+
+ std::vector<Register> ComplexRegisters;
+ std::vector<PendingComplexInstruction> PendingComplexInstructions;
+ std::vector<MachineInstr *> AddedInstructions;
+ const auto CleanupGeneratedRegisters = [&]() -> Expected<void> {
+ auto ErasedTemporaries = eraseTemporaries(Function.registerInfo(), Temporaries);
+ if (!ErasedTemporaries)
+ return std::unexpected(ErasedTemporaries.error());
+ return eraseRegisters(Function.registerInfo(), ComplexRegisters);
+ };
+
+ std::vector<std::vector<MachineOperand>> RenderedInstructions;
+ RenderedInstructions.reserve(Rule.Instructions.size());
+ std::vector<std::optional<std::size_t>> SourceToOutput(OriginalOperandCount);
+ for (std::size_t InstructionIndex = 0; InstructionIndex < Rule.Instructions.size();
+ ++InstructionIndex) {
+ const MatchedRule::Instruction &Planned = Rule.Instructions[InstructionIndex];
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Planned.ResultOpcode);
+ if (Descriptor == nullptr ||
+ (!Descriptor->Variadic && Descriptor->Operands.size() != Planned.Operands.size())) {
+ auto Cleanup = CleanupGeneratedRegisters();
+ if (!Cleanup)
+ return std::unexpected(Cleanup.error());
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table result has an incompatible descriptor");
+ }
+ std::vector<MachineOperand> Operands;
+ Operands.reserve(Planned.Operands.size());
+ for (const MatchedRule::RenderedOperand &Rendered : Planned.Operands) {
+ Expected<MachineOperand> Operand = [&]() -> Expected<MachineOperand> {
+ if (Rendered.SourceKind != MatchedRule::RenderedOperand::Kind::ComplexOperand)
+ return renderOperand(Rendered, Instruction, Function, Target, Temporaries,
+ Rule.CapturedOperands, OriginalExplicitCount);
+
+ auto Complex =
+ RenderComplex(Rendered.ComplexRenderer, Instruction, Function, Target, Rendered.Value);
+ if (!Complex)
+ return std::unexpected(Complex.error());
+ if (!*Complex)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table complex operand unexpectedly did not match");
+ if ((*Complex)->Operand && (*Complex)->Instruction)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table complex operand has two renderings");
+ if ((*Complex)->Operand)
+ return std::move(*(*Complex)->Operand);
+ if (!(*Complex)->Instruction)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table complex operand has no rendering");
+
+ GIMatchComplexInstruction &Plan = *(*Complex)->Instruction;
+ if (!Plan.ResultType.isValid())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table complex instruction has no result type");
+ auto Result = Function.registerInfo().createVirtualRegister(Plan.ResultType);
+ if (!Result)
+ return std::unexpected(Result.error());
+ ComplexRegisters.push_back(*Result);
+ auto Definition = MachineOperand::registerOperand(*Result, {RegisterOperandFlag::Define});
+ if (!Definition)
+ return std::unexpected(Definition.error());
+ std::vector<MachineOperand> PlannedOperands;
+ PlannedOperands.reserve(Plan.Operands.size() + 1U);
+ PlannedOperands.push_back(std::move(*Definition));
+ for (MachineOperand &PlanOperand : Plan.Operands)
+ PlannedOperands.push_back(std::move(PlanOperand));
+ PendingComplexInstructions.push_back({
+ .ResultOpcode = Plan.ResultOpcode,
+ .Operands = std::move(PlannedOperands),
+ });
+ return MachineOperand::registerOperand(*Result, {RegisterOperandFlag::Use});
+ }();
+ if (!Operand) {
+ auto Cleanup = CleanupGeneratedRegisters();
+ if (!Cleanup)
+ return std::unexpected(Cleanup.error());
+ return std::unexpected(Operand.error());
+ }
+ if (InstructionIndex == RootIndex &&
+ Rendered.SourceKind == MatchedRule::RenderedOperand::Kind::RootOperand) {
+ if (SourceToOutput[Rendered.Value] &&
+ Instruction.operandAt(Rendered.Value)->tiedOperandIndex()) {
+ auto Cleanup = CleanupGeneratedRegisters();
+ if (!Cleanup)
+ return std::unexpected(Cleanup.error());
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table duplicates a tied operand");
+ }
+ SourceToOutput[Rendered.Value] = Operands.size();
+ }
+ Operands.push_back(std::move(*Operand));
+ }
+ RenderedInstructions.push_back(std::move(Operands));
+ }
+
+ std::vector<RegisterConstraintSnapshot> ConstraintSnapshots;
+ bool RootModified = false;
+ const auto Rollback = [&](Error Failure) -> Expected<void> {
+ for (auto Iterator = AddedInstructions.rbegin(); Iterator != AddedInstructions.rend();
+ ++Iterator) {
+ if ((*Iterator)->parent() == nullptr)
+ continue;
+ auto Erased = eraseInstr(**Iterator, Function.registerInfo(), Target);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ }
+ if (RootModified) {
+ auto Restored = restoreInstruction(Instruction, OriginalOpcode, OriginalOperands, Ties);
+ if (!Restored)
+ return std::unexpected(Restored.error());
+ }
+ auto RestoredConstraints =
+ restoreRegisterConstraints(Function.registerInfo(), ConstraintSnapshots);
+ if (!RestoredConstraints)
+ return std::unexpected(RestoredConstraints.error());
+ auto ErasedGenerated = CleanupGeneratedRegisters();
+ if (!ErasedGenerated)
+ return std::unexpected(ErasedGenerated.error());
+ return std::unexpected(std::move(Failure));
+ };
+
+ for (const PendingComplexInstruction &Pending : PendingComplexInstructions) {
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Pending.ResultOpcode);
+ if (Descriptor == nullptr ||
+ (!Descriptor->Variadic && Descriptor->Operands.size() != Pending.Operands.size())) {
+ return Rollback(Error(ErrorCode::VerificationError,
+ "generated match table complex renderer has an incompatible result"));
+ }
+ auto Constraint = constrainRenderedInstruction(*Descriptor, Pending.Operands, Function, Target,
+ ConstraintSnapshots);
+ if (!Constraint)
+ return Rollback(Constraint.error());
+ }
+
+ for (std::size_t Index = 0; Index < Rule.Instructions.size(); ++Index) {
+ const InstructionDescriptor *Descriptor =
+ Target.instructionDescriptor(Rule.Instructions[Index].ResultOpcode);
+ if (Descriptor == nullptr)
+ return Rollback(Error(ErrorCode::VerificationError,
+ "generated match table selects an unknown instruction opcode"));
+ auto Constraint = constrainRenderedInstruction(*Descriptor, RenderedInstructions[Index],
+ Function, Target, ConstraintSnapshots);
+ if (!Constraint)
+ return Rollback(Constraint.error());
+ }
+
+ MachineIRBuilder Builder(Function, Target);
+ MachineInstr *ComplexAnchor = nullptr;
+ for (PendingComplexInstruction &Pending : PendingComplexInstructions) {
+ auto Point = ComplexAnchor == nullptr ? Builder.setInsertPointBefore(Instruction)
+ : Builder.setInsertPointAfter(*ComplexAnchor);
+ if (!Point)
+ return Rollback(Point.error());
+ auto Built = Builder.buildInstr(Pending.ResultOpcode);
+ if (!Built)
+ return Rollback(Built.error());
+ for (MachineOperand &Operand : Pending.Operands) {
+ auto Added = Built->addOperand(std::move(Operand));
+ if (!Added)
+ return Rollback(Added.error());
+ }
+ auto Complete = Built->build();
+ if (!Complete)
+ return Rollback(Complete.error());
+ AddedInstructions.push_back(*Complete);
+ auto Finalized = Finalize(**Complete, InputFlags, InputMayRaiseFPException);
+ if (!Finalized)
+ return Rollback(Finalized.error());
+ ComplexAnchor = *Complete;
+ }
+
+ const auto BuildAction = [&](std::size_t Index, MachineInstr *&Anchor) -> Expected<void> {
+ auto Point = Anchor == nullptr ? Builder.setInsertPointBefore(Instruction)
+ : Builder.setInsertPointAfter(*Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(Rule.Instructions[Index].ResultOpcode);
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ for (MachineOperand &Operand : RenderedInstructions[Index]) {
+ auto Added = Pending->addOperand(std::move(Operand));
+ if (!Added)
+ return std::unexpected(Added.error());
+ }
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ AddedInstructions.push_back(*Built);
+ auto Finalized = Finalize(**Built, InputFlags, InputMayRaiseFPException);
+ if (!Finalized)
+ return std::unexpected(Finalized.error());
+ Anchor = *Built;
+ return {};
+ };
+
+ MachineInstr *BeforeAnchor = ComplexAnchor;
+ for (std::size_t Index = 0; Index < RootIndex; ++Index) {
+ auto Built = BuildAction(Index, BeforeAnchor);
+ if (!Built)
+ return Rollback(Built.error());
+ }
+
+ const MatchedRule::Instruction &RootAction = Rule.Instructions[RootIndex];
+ RootModified = true;
+ auto SetOpcode = Instruction.setOpcode(RootAction.ResultOpcode);
+ if (!SetOpcode)
+ return Rollback(SetOpcode.error());
+ while (Instruction.operandCount() != 0) {
+ auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U);
+ if (!Erased)
+ return Rollback(Erased.error());
+ }
+ for (MachineOperand &Operand : RenderedInstructions[RootIndex]) {
+ auto Added = Instruction.addOperand(std::move(Operand));
+ if (!Added)
+ return Rollback(Added.error());
+ }
+ for (std::size_t Index = OriginalExplicitCount; Index < OriginalOperandCount; ++Index) {
+ SourceToOutput[Index] = Instruction.operandCount();
+ auto Added = Instruction.addOperand(OriginalOperands[Index].cloneDetached());
+ if (!Added)
+ return Rollback(Added.error());
+ }
+ for (const auto [OldDef, OldUse] : Ties) {
+ const std::optional<std::size_t> &NewDef = SourceToOutput[OldDef];
+ const std::optional<std::size_t> &NewUse = SourceToOutput[OldUse];
+ if (!NewDef || !NewUse)
+ continue;
+ auto Tied = Instruction.tieOperands(*NewDef, *NewUse);
+ if (!Tied)
+ return Rollback(Tied.error());
+ }
+
+ MachineInstr *AfterAnchor = &Instruction;
+ for (std::size_t Index = RootIndex + 1U; Index < Rule.Instructions.size(); ++Index) {
+ auto Built = BuildAction(Index, AfterAnchor);
+ if (!Built)
+ return Rollback(Built.error());
+ }
+
+ for (const Register Reg : Rule.EraseOperandDefinitions) {
+ MachineInstr *Definition = Function.registerInfo().getVRegDef(Reg);
+ if (Definition == nullptr || Definition == &Instruction ||
+ !isTriviallyDead(*Definition, Function.registerInfo(), Target)) {
+ continue;
+ }
+ auto Erased = eraseInstr(*Definition, Function.registerInfo(), Target);
+ if (!Erased)
+ return Rollback(Erased.error());
+ }
+ if (Rule.ClearInstructionFlags != 0) {
+ MachineInstrFlags Flags = Instruction.flags();
+ for (std::uint32_t Index = 0; Index < static_cast<std::uint32_t>(MachineInstrFlag::Count);
+ ++Index) {
+ if ((Rule.ClearInstructionFlags & (std::uint32_t{1} << Index)) != 0)
+ Flags.clear(static_cast<MachineInstrFlag>(Index));
+ }
+ Instruction.setFlags(Flags);
+ }
+ return {};
+}
+
+} // namespace
+
+Expected<void> GIMatchTableExecutor::finalizeGeneratedInstruction(MachineInstr &, MachineFunction &,
+ const TargetDescription &,
+ MachineInstrFlags, bool) const {
+ return {};
+}
+
+Expected<bool> GIMatchTableExecutor::matchesTargetPredicate(std::string_view Name,
+ const MachineInstr &,
+ const MachineFunction &,
+ const TargetDescription &) const {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table uses unknown target predicate '" +
+ std::string(Name) + "'");
+}
+
+Expected<std::optional<GIMatchComplexOperand>>
+GIMatchTableExecutor::renderComplexOperand(std::string_view, const MachineInstr &,
+ const MachineFunction &, const TargetDescription &,
+ std::uint32_t) const {
+ return std::optional<GIMatchComplexOperand>{};
+}
+
+Expected<bool> GIMatchTableExecutor::match(const GIMatchTable &Table,
+ const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &Target) const {
+ for (const GIMatchTableRule &Rule : Table.Rules) {
+ auto Match = matchRule(Table, Rule, Instruction, Function, Target,
+ [this](std::string_view Name, const MachineInstr &Input,
+ const MachineFunction &InputFunction,
+ const TargetDescription &InputTarget) {
+ return matchesTargetPredicate(Name, Input, InputFunction, InputTarget);
+ });
+ if (!Match)
+ return std::unexpected(Match.error());
+ if (!*Match)
+ continue;
+ auto Complex = complexRenderersMatch(
+ **Match, Instruction, Function, Target,
+ [this](std::string_view Name, const MachineInstr &Input,
+ const MachineFunction &InputFunction, const TargetDescription &InputTarget,
+ std::uint32_t OperandIndex) {
+ return renderComplexOperand(Name, Input, InputFunction, InputTarget, OperandIndex);
+ });
+ if (!Complex)
+ return std::unexpected(Complex.error());
+ if (!*Complex)
+ continue;
+ for (const MatchedRule::Instruction &Action : (*Match)->Instructions) {
+ if (Target.instructionDescriptor(Action.ResultOpcode) == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated match table selects an unknown instruction opcode");
+ }
+ }
+ return true;
+ }
+ return false;
+}
+
+Expected<bool> GIMatchTableExecutor::select(const GIMatchTable &Table, MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) const {
+ for (const GIMatchTableRule &Rule : Table.Rules) {
+ auto Match = matchRule(Table, Rule, Instruction, Function, Target,
+ [this](std::string_view Name, const MachineInstr &Input,
+ const MachineFunction &InputFunction,
+ const TargetDescription &InputTarget) {
+ return matchesTargetPredicate(Name, Input, InputFunction, InputTarget);
+ });
+ if (!Match)
+ return std::unexpected(Match.error());
+ if (!*Match)
+ continue;
+ auto Complex = complexRenderersMatch(
+ **Match, Instruction, Function, Target,
+ [this](std::string_view Name, const MachineInstr &Input,
+ const MachineFunction &InputFunction, const TargetDescription &InputTarget,
+ std::uint32_t OperandIndex) {
+ return renderComplexOperand(Name, Input, InputFunction, InputTarget, OperandIndex);
+ });
+ if (!Complex)
+ return std::unexpected(Complex.error());
+ if (!*Complex)
+ continue;
+ auto Applied = applyRule(
+ **Match, Instruction, Function, Target,
+ [this, &Function, &Target](MachineInstr &Output, MachineInstrFlags InputFlags,
+ bool InputMayRaiseFPException) {
+ return finalizeGeneratedInstruction(Output, Function, Target, InputFlags,
+ InputMayRaiseFPException);
+ },
+ [this](std::string_view Name, const MachineInstr &Input,
+ const MachineFunction &InputFunction, const TargetDescription &InputTarget,
+ std::uint32_t OperandIndex) {
+ return renderComplexOperand(Name, Input, InputFunction, InputTarget, OperandIndex);
+ });
+ if (!Applied)
+ return std::unexpected(Applied.error());
+ return true;
+ }
+ return false;
+}
+
+Expected<bool> GIMatchTableExecutor::combine(const GIMatchTable &Table, MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) const {
+ if (Table.Kind != GIMatchTableKind::Combiner)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "GlobalISel combiner received a selection match table");
+ for (const GIMatchTableRule &Rule : Table.Rules) {
+ auto Match = matchRule(Table, Rule, Instruction, Function, Target,
+ [this](std::string_view Name, const MachineInstr &Input,
+ const MachineFunction &InputFunction,
+ const TargetDescription &InputTarget) {
+ return matchesTargetPredicate(Name, Input, InputFunction, InputTarget);
+ });
+ if (!Match)
+ return std::unexpected(Match.error());
+ if (!*Match)
+ continue;
+ if (!(*Match)->ReplaceRootCapture) {
+ for (const MatchedRule::Instruction &Action : (*Match)->Instructions) {
+ if (Target.instructionDescriptor(Action.ResultOpcode) == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "generated combiner selects an unknown instruction opcode");
+ }
+ }
+ auto Applied = applyRule(
+ **Match, Instruction, Function, Target,
+ [this, &Function, &Target](MachineInstr &Output, MachineInstrFlags InputFlags,
+ bool InputMayRaiseFPException) {
+ return finalizeGeneratedInstruction(Output, Function, Target, InputFlags,
+ InputMayRaiseFPException);
+ },
+ [this](std::string_view Name, const MachineInstr &Input,
+ const MachineFunction &InputFunction, const TargetDescription &InputTarget,
+ std::uint32_t OperandIndex) {
+ return renderComplexOperand(Name, Input, InputFunction, InputTarget, OperandIndex);
+ });
+ if (!Applied)
+ return std::unexpected(Applied.error());
+ return true;
+ }
+ return false;
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp b/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp
new file mode 100644
index 0000000..4ce1473
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h"
+
+#include <algorithm>
+
+namespace rcc {
+
+void GISelObserverWrapper::addObserver(GISelChangeObserver &Observer) {
+ Observers.push_back(&Observer);
+}
+
+void GISelObserverWrapper::removeObserver(GISelChangeObserver &Observer) {
+ const auto Position = std::ranges::find(Observers, &Observer);
+ if (Position != Observers.end())
+ Observers.erase(Position);
+}
+
+void GISelObserverWrapper::clearObservers() noexcept {
+ Observers.clear();
+}
+
+void GISelObserverWrapper::erasingInstr(MachineInstr &Instruction) {
+ for (GISelChangeObserver *Observer : Observers)
+ Observer->erasingInstr(Instruction);
+}
+
+void GISelObserverWrapper::createdInstr(MachineInstr &Instruction) {
+ for (GISelChangeObserver *Observer : Observers)
+ Observer->createdInstr(Instruction);
+}
+
+void GISelObserverWrapper::changingInstr(MachineInstr &Instruction) {
+ for (GISelChangeObserver *Observer : Observers)
+ Observer->changingInstr(Instruction);
+}
+
+void GISelObserverWrapper::changedInstr(MachineInstr &Instruction) {
+ for (GISelChangeObserver *Observer : Observers)
+ Observer->changedInstr(Instruction);
+}
+
+void GISelObserverWrapper::handleInsertion(MachineInstr &Instruction) {
+ createdInstr(Instruction);
+}
+
+void GISelObserverWrapper::handleRemoval(MachineInstr &Instruction) {
+ erasingInstr(Instruction);
+}
+
+MachineFunctionDelegateInstaller::MachineFunctionDelegateInstaller(
+ MachineFunction &Function, MachineFunction::Delegate &Delegate)
+ : Function(&Function), Delegate(&Delegate) {
+ Function.setDelegate(Delegate);
+}
+
+MachineFunctionDelegateInstaller::~MachineFunctionDelegateInstaller() {
+ Function->resetDelegate(*Delegate);
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/backend/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
new file mode 100644
index 0000000..85e552c
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -0,0 +1,722 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/GISelValueTracking.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/Utils.h"
+#include "rcc/CodeGen/LowLevelType.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <vector>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] std::optional<Register> registerOperand(const MachineInstr &Instruction,
+ std::size_t Index) noexcept {
+ const MachineOperand *Operand = Instruction.operandAt(Index);
+ return Operand == nullptr ? std::nullopt : Operand->registerValue();
+}
+
+[[nodiscard]] Expected<APInt> zero(std::uint32_t BitWidth) {
+ return APInt::fromUnsigned(BitWidth, 0);
+}
+
+[[nodiscard]] Expected<APInt> resizeMask(const APInt &Value, std::uint32_t BitWidth) {
+ if (Value.bitWidth() == BitWidth)
+ return Value;
+ if (Value.bitWidth() == 0)
+ return zero(BitWidth);
+ if (Value.bitWidth() < BitWidth)
+ return Value.zext(BitWidth);
+ return Value.trunc(BitWidth);
+}
+
+[[nodiscard]] Expected<APInt> highBitsMask(std::uint32_t BitWidth, std::uint32_t LowBitCount) {
+ auto LowBits = APInt::getLowBitsSet(BitWidth, LowBitCount);
+ if (!LowBits)
+ return std::unexpected(LowBits.error());
+ return LowBits->bitwiseNot();
+}
+
+[[nodiscard]] Expected<APInt> mergeMasks(const APInt &Left, const APInt &Right) {
+ return Left.bitwiseOr(Right);
+}
+
+[[nodiscard]] Expected<APInt> intersectMasks(const APInt &Left, const APInt &Right) {
+ return Left.bitwiseAnd(Right);
+}
+
+[[nodiscard]] Expected<APInt> addMasks(const APInt &Left, const APInt &Right) {
+ if (Left.bitWidth() == 0 || Left.bitWidth() != Right.bitWidth())
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "known-bits add operands must have the same nonzero width");
+ std::vector<std::uint64_t> Words(Left.words().size(), 0);
+ unsigned __int128 Carry = 0;
+ for (std::size_t Index = 0; Index != Words.size(); ++Index) {
+ const unsigned __int128 Sum = static_cast<unsigned __int128>(Left.word(Index)) +
+ static_cast<unsigned __int128>(Right.word(Index)) + Carry;
+ Words[Index] = static_cast<std::uint64_t>(Sum);
+ Carry = Sum >> 64U;
+ }
+ return APInt::create(Left.bitWidth(), Words);
+}
+
+struct MaskProduct final {
+ APInt Value;
+ bool Overflow = false;
+};
+
+[[nodiscard]] Expected<MaskProduct> multiplyMasks(const APInt &Left, const APInt &Right) {
+ if (Left.bitWidth() == 0 || Left.bitWidth() != Right.bitWidth())
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "known-bits multiply operands must have the same nonzero width");
+
+ const std::size_t WordCount = Left.words().size();
+ std::vector<std::uint64_t> Product(WordCount * 2U, 0);
+ for (std::size_t LeftIndex = 0; LeftIndex != WordCount; ++LeftIndex) {
+ for (std::size_t RightIndex = 0; RightIndex != WordCount; ++RightIndex) {
+ const std::size_t ProductIndex = LeftIndex + RightIndex;
+ const unsigned __int128 Partial = static_cast<unsigned __int128>(Left.word(LeftIndex)) *
+ static_cast<unsigned __int128>(Right.word(RightIndex));
+ unsigned __int128 Sum = static_cast<unsigned __int128>(Product[ProductIndex]) +
+ static_cast<std::uint64_t>(Partial);
+ Product[ProductIndex] = static_cast<std::uint64_t>(Sum);
+ unsigned __int128 Carry = (Partial >> 64U) + (Sum >> 64U);
+ for (std::size_t Index = ProductIndex + 1U; Carry != 0 && Index != Product.size(); ++Index) {
+ Sum = static_cast<unsigned __int128>(Product[Index]) + static_cast<std::uint64_t>(Carry);
+ Product[Index] = static_cast<std::uint64_t>(Sum);
+ Carry = (Carry >> 64U) + (Sum >> 64U);
+ }
+ }
+ }
+
+ bool Overflow = false;
+ for (std::size_t Index = WordCount; Index != Product.size(); ++Index)
+ Overflow |= Product[Index] != 0;
+ const std::uint32_t HighBits = Left.bitWidth() % 64U;
+ if (HighBits != 0) {
+ const std::uint64_t HighMask = (std::uint64_t{1} << HighBits) - 1U;
+ Overflow |= (Product[WordCount - 1U] & ~HighMask) != 0;
+ Product[WordCount - 1U] &= HighMask;
+ }
+ Product.resize(WordCount);
+ auto Value = APInt::create(Left.bitWidth(), Product);
+ if (!Value)
+ return std::unexpected(Value.error());
+ return MaskProduct{.Value = std::move(*Value), .Overflow = Overflow};
+}
+
+[[nodiscard]] std::uint32_t countTrailingOnes(const APInt &Value) noexcept {
+ std::uint32_t Count = 0;
+ while (Count != Value.bitWidth() && Value.testBit(Count))
+ ++Count;
+ return Count;
+}
+
+[[nodiscard]] Expected<APInt> constantAtWidth(const APInt &Value, std::uint32_t BitWidth) {
+ return resizeMask(Value, BitWidth);
+}
+
+[[nodiscard]] std::optional<std::uint32_t>
+constantShiftAmount(Register Reg, const MachineRegisterInfo &MRI, std::uint32_t BitWidth) {
+ const auto Constant = getIConstantVRegValWithLookThrough(Reg, MRI);
+ if (!Constant || Constant->Value.activeBitWidth() > 32)
+ return std::nullopt;
+ const std::uint64_t Amount = Constant->Value.word(0);
+ if (Amount >= BitWidth)
+ return std::nullopt;
+ return static_cast<std::uint32_t>(Amount);
+}
+
+[[nodiscard]] Expected<APInt> shiftMaskLeft(const APInt &Value, std::uint32_t Amount) {
+ std::vector<std::uint64_t> Words(Value.words().size(), 0);
+ for (std::uint32_t Bit = 0; Bit + Amount < Value.bitWidth(); ++Bit) {
+ if (Value.testBit(Bit)) {
+ const std::uint32_t DestinationBit = Bit + Amount;
+ Words[DestinationBit / 64U] |= std::uint64_t{1} << (DestinationBit % 64U);
+ }
+ }
+ return APInt::create(Value.bitWidth(), Words);
+}
+
+[[nodiscard]] Expected<APInt> shiftMaskRight(const APInt &Value, std::uint32_t Amount) {
+ std::vector<std::uint64_t> Words(Value.words().size(), 0);
+ for (std::uint32_t Bit = Amount; Bit < Value.bitWidth(); ++Bit) {
+ if (Value.testBit(Bit)) {
+ const std::uint32_t DestinationBit = Bit - Amount;
+ Words[DestinationBit / 64U] |= std::uint64_t{1} << (DestinationBit % 64U);
+ }
+ }
+ return APInt::create(Value.bitWidth(), Words);
+}
+
+[[nodiscard]] std::uint32_t constantNumSignBits(const APInt &Value) noexcept {
+ if (Value.bitWidth() == 0)
+ return 1;
+ if (Value.isNegative())
+ return static_cast<std::uint32_t>(Value.countlOne());
+ auto Inverted = Value.bitwiseNot();
+ return Inverted ? static_cast<std::uint32_t>(Inverted->countlOne()) : 1U;
+}
+
+} // namespace
+
+GISelValueTracking::GISelValueTracking(const MachineFunction &Function,
+ const TargetDescription &Target, unsigned MaxDepth) noexcept
+ : Function(Function), MRI(Function.registerInfo()), Target(Target), MaxDepth(MaxDepth) {}
+
+Expected<GISelValueTracking::KnownBits>
+GISelValueTracking::makeUnknown(std::uint32_t BitWidth) const {
+ auto Zero = zero(BitWidth);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto One = zero(BitWidth);
+ if (!One)
+ return std::unexpected(One.error());
+ return KnownBits{.Zero = std::move(*Zero), .One = std::move(*One)};
+}
+
+Expected<GISelValueTracking::KnownBits> GISelValueTracking::getKnownBits(Register Reg,
+ unsigned Depth) const {
+ auto Type = MRI.getType(Reg);
+ if (!Type)
+ return std::unexpected(Type.error());
+ const std::uint32_t BitWidth = Type->scalarBitWidth();
+ auto Unknown = makeUnknown(BitWidth);
+ if (!Unknown)
+ return std::unexpected(Unknown.error());
+ if (Depth >= MaxDepth)
+ return Unknown;
+
+ const MachineInstr *Definition = MRI.getVRegDef(Reg);
+ if (Definition == nullptr)
+ return Unknown;
+
+ const auto Opcode = static_cast<GenericOpcode>(Definition->opcode().value());
+ if (Opcode == GenericOpcode::G_CONSTANT) {
+ const MachineOperand *ValueOperand = Definition->operandAt(1);
+ const auto *Immediate =
+ ValueOperand == nullptr ? nullptr : ValueOperand->getIf<IntegerImmediateOperand>();
+ if (Immediate == nullptr)
+ return Unknown;
+ auto Value = constantAtWidth(Immediate->Value, BitWidth);
+ if (!Value)
+ return std::unexpected(Value.error());
+ auto KnownZero = Value->bitwiseNot();
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*Value)};
+ }
+
+ if (Opcode == GenericOpcode::COPY) {
+ const auto Source = registerOperand(*Definition, 1);
+ if (!Source || !Source->isVirtual())
+ return Unknown;
+ auto SourceBits = getKnownBits(*Source, Depth);
+ if (!SourceBits)
+ return std::unexpected(SourceBits.error());
+ auto KnownZero = resizeMask(SourceBits->Zero, BitWidth);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto KnownOne = resizeMask(SourceBits->One, BitWidth);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)};
+ }
+
+ if (Opcode == GenericOpcode::G_ICMP || Opcode == GenericOpcode::G_FCMP) {
+ if (!Type->isVector() && BitWidth > 1 &&
+ Target.getBooleanContents(false, Opcode == GenericOpcode::G_FCMP) ==
+ BooleanContent::ZeroOrOne) {
+ auto KnownZero = highBitsMask(BitWidth, 1);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ Unknown->Zero = std::move(*KnownZero);
+ }
+ return Unknown;
+ }
+
+ switch (Opcode) {
+ case GenericOpcode::G_UADDO:
+ case GenericOpcode::G_UADDE:
+ case GenericOpcode::G_USUBO:
+ case GenericOpcode::G_USUBE:
+ case GenericOpcode::G_SADDO:
+ case GenericOpcode::G_SADDE:
+ case GenericOpcode::G_SSUBO:
+ case GenericOpcode::G_SSUBE:
+ case GenericOpcode::G_UMULO:
+ case GenericOpcode::G_SMULO: {
+ const auto BooleanResult = registerOperand(*Definition, 1);
+ if (BooleanResult && *BooleanResult == Reg && !Type->isVector() && BitWidth > 1 &&
+ Target.getBooleanContents(false, false) == BooleanContent::ZeroOrOne) {
+ auto KnownZero = highBitsMask(BitWidth, 1);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ Unknown->Zero = std::move(*KnownZero);
+ }
+ return Unknown;
+ }
+ default:
+ break;
+ }
+
+ const auto Source = registerOperand(*Definition, 1);
+ if (!Source)
+ return Unknown;
+
+ if (Opcode == GenericOpcode::G_TRUNC || Opcode == GenericOpcode::G_ANYEXT ||
+ Opcode == GenericOpcode::G_ZEXT || Opcode == GenericOpcode::G_SEXT ||
+ Opcode == GenericOpcode::G_ASSERT_ZEXT || Opcode == GenericOpcode::G_ASSERT_SEXT ||
+ Opcode == GenericOpcode::G_SEXT_INREG) {
+ auto SourceBits = getKnownBits(*Source, Depth + 1U);
+ if (!SourceBits)
+ return std::unexpected(SourceBits.error());
+ auto SourceType = MRI.getType(*Source);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ const std::uint32_t SourceWidth = SourceType->scalarBitWidth();
+ auto KnownZero = resizeMask(SourceBits->Zero, BitWidth);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto KnownOne = resizeMask(SourceBits->One, BitWidth);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+
+ if (Opcode == GenericOpcode::G_ZEXT && SourceWidth < BitWidth) {
+ auto HighZero = highBitsMask(BitWidth, SourceWidth);
+ if (!HighZero)
+ return std::unexpected(HighZero.error());
+ KnownZero = mergeMasks(*KnownZero, *HighZero);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ }
+
+ std::uint32_t ExtendedFrom = SourceWidth;
+ if (Opcode == GenericOpcode::G_ASSERT_ZEXT || Opcode == GenericOpcode::G_ASSERT_SEXT ||
+ Opcode == GenericOpcode::G_SEXT_INREG) {
+ const MachineOperand *WidthOperand = Definition->operandAt(2);
+ const auto *Width = WidthOperand == nullptr ? nullptr : WidthOperand->getIf<std::int64_t>();
+ if (Width == nullptr || *Width <= 0 || static_cast<std::uint64_t>(*Width) > BitWidth)
+ return Unknown;
+ ExtendedFrom = static_cast<std::uint32_t>(*Width);
+ }
+
+ if (Opcode == GenericOpcode::G_ASSERT_ZEXT) {
+ auto HighZero = highBitsMask(BitWidth, ExtendedFrom);
+ if (!HighZero)
+ return std::unexpected(HighZero.error());
+ KnownZero = mergeMasks(*KnownZero, *HighZero);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ }
+
+ if ((Opcode == GenericOpcode::G_SEXT || Opcode == GenericOpcode::G_ASSERT_SEXT ||
+ Opcode == GenericOpcode::G_SEXT_INREG) &&
+ ExtendedFrom < BitWidth) {
+ auto HighBits = highBitsMask(BitWidth, ExtendedFrom);
+ if (!HighBits)
+ return std::unexpected(HighBits.error());
+ if (KnownZero->testBit(ExtendedFrom - 1U)) {
+ KnownZero = mergeMasks(*KnownZero, *HighBits);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ } else if (KnownOne->testBit(ExtendedFrom - 1U)) {
+ KnownOne = mergeMasks(*KnownOne, *HighBits);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ }
+ }
+
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)};
+ }
+
+ if (Opcode == GenericOpcode::G_AND || Opcode == GenericOpcode::G_OR ||
+ Opcode == GenericOpcode::G_XOR) {
+ const auto Right = registerOperand(*Definition, 2);
+ if (!Right)
+ return Unknown;
+ auto LeftBits = getKnownBits(*Source, Depth + 1U);
+ if (!LeftBits)
+ return std::unexpected(LeftBits.error());
+ auto RightBits = getKnownBits(*Right, Depth + 1U);
+ if (!RightBits)
+ return std::unexpected(RightBits.error());
+
+ if (Opcode == GenericOpcode::G_AND) {
+ auto KnownZero = mergeMasks(LeftBits->Zero, RightBits->Zero);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto KnownOne = intersectMasks(LeftBits->One, RightBits->One);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)};
+ }
+ if (Opcode == GenericOpcode::G_OR) {
+ auto KnownZero = intersectMasks(LeftBits->Zero, RightBits->Zero);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto KnownOne = mergeMasks(LeftBits->One, RightBits->One);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)};
+ }
+
+ auto BothZero = intersectMasks(LeftBits->Zero, RightBits->Zero);
+ if (!BothZero)
+ return std::unexpected(BothZero.error());
+ auto BothOne = intersectMasks(LeftBits->One, RightBits->One);
+ if (!BothOne)
+ return std::unexpected(BothOne.error());
+ auto KnownZero = mergeMasks(*BothZero, *BothOne);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto LeftZeroRightOne = intersectMasks(LeftBits->Zero, RightBits->One);
+ if (!LeftZeroRightOne)
+ return std::unexpected(LeftZeroRightOne.error());
+ auto LeftOneRightZero = intersectMasks(LeftBits->One, RightBits->Zero);
+ if (!LeftOneRightZero)
+ return std::unexpected(LeftOneRightZero.error());
+ auto KnownOne = mergeMasks(*LeftZeroRightOne, *LeftOneRightZero);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)};
+ }
+
+ if (Opcode == GenericOpcode::G_ADD) {
+ const auto Right = registerOperand(*Definition, 2);
+ if (!Right)
+ return Unknown;
+ auto LeftBits = getKnownBits(*Source, Depth + 1U);
+ if (!LeftBits)
+ return std::unexpected(LeftBits.error());
+ auto RightBits = getKnownBits(*Right, Depth + 1U);
+ if (!RightBits)
+ return std::unexpected(RightBits.error());
+
+ auto LeftMaximum = LeftBits->Zero.bitwiseNot();
+ if (!LeftMaximum)
+ return std::unexpected(LeftMaximum.error());
+ auto RightMaximum = RightBits->Zero.bitwiseNot();
+ if (!RightMaximum)
+ return std::unexpected(RightMaximum.error());
+ auto PossibleSumZero = addMasks(*LeftMaximum, *RightMaximum);
+ if (!PossibleSumZero)
+ return std::unexpected(PossibleSumZero.error());
+ auto PossibleSumOne = addMasks(LeftBits->One, RightBits->One);
+ if (!PossibleSumOne)
+ return std::unexpected(PossibleSumOne.error());
+
+ auto CarryKnownZero = PossibleSumZero->bitwiseXor(LeftBits->Zero);
+ if (!CarryKnownZero)
+ return std::unexpected(CarryKnownZero.error());
+ CarryKnownZero = CarryKnownZero->bitwiseXor(RightBits->Zero);
+ if (!CarryKnownZero)
+ return std::unexpected(CarryKnownZero.error());
+ CarryKnownZero = CarryKnownZero->bitwiseNot();
+ if (!CarryKnownZero)
+ return std::unexpected(CarryKnownZero.error());
+ auto CarryKnownOne = PossibleSumOne->bitwiseXor(LeftBits->One);
+ if (!CarryKnownOne)
+ return std::unexpected(CarryKnownOne.error());
+ CarryKnownOne = CarryKnownOne->bitwiseXor(RightBits->One);
+ if (!CarryKnownOne)
+ return std::unexpected(CarryKnownOne.error());
+
+ auto LeftKnown = mergeMasks(LeftBits->Zero, LeftBits->One);
+ if (!LeftKnown)
+ return std::unexpected(LeftKnown.error());
+ auto RightKnown = mergeMasks(RightBits->Zero, RightBits->One);
+ if (!RightKnown)
+ return std::unexpected(RightKnown.error());
+ auto CarryKnown = mergeMasks(*CarryKnownZero, *CarryKnownOne);
+ if (!CarryKnown)
+ return std::unexpected(CarryKnown.error());
+ auto Known = intersectMasks(*LeftKnown, *RightKnown);
+ if (!Known)
+ return std::unexpected(Known.error());
+ Known = intersectMasks(*Known, *CarryKnown);
+ if (!Known)
+ return std::unexpected(Known.error());
+
+ auto NotPossibleSumZero = PossibleSumZero->bitwiseNot();
+ if (!NotPossibleSumZero)
+ return std::unexpected(NotPossibleSumZero.error());
+ auto KnownZero = intersectMasks(*NotPossibleSumZero, *Known);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto KnownOne = intersectMasks(*PossibleSumOne, *Known);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)};
+ }
+
+ if (Opcode == GenericOpcode::G_MUL) {
+ const auto Right = registerOperand(*Definition, 2);
+ if (!Right)
+ return Unknown;
+ auto LeftBits = getKnownBits(*Source, Depth + 1U);
+ if (!LeftBits)
+ return std::unexpected(LeftBits.error());
+ auto RightBits = getKnownBits(*Right, Depth + 1U);
+ if (!RightBits)
+ return std::unexpected(RightBits.error());
+
+ auto LeftMaximum = LeftBits->Zero.bitwiseNot();
+ if (!LeftMaximum)
+ return std::unexpected(LeftMaximum.error());
+ auto RightMaximum = RightBits->Zero.bitwiseNot();
+ if (!RightMaximum)
+ return std::unexpected(RightMaximum.error());
+ auto MaximumProduct = multiplyMasks(*LeftMaximum, *RightMaximum);
+ if (!MaximumProduct)
+ return std::unexpected(MaximumProduct.error());
+ const std::uint32_t LeadingZeroes =
+ MaximumProduct->Overflow
+ ? 0
+ : BitWidth - static_cast<std::uint32_t>(MaximumProduct->Value.activeBitWidth());
+
+ auto LeftKnown = mergeMasks(LeftBits->Zero, LeftBits->One);
+ if (!LeftKnown)
+ return std::unexpected(LeftKnown.error());
+ auto RightKnown = mergeMasks(RightBits->Zero, RightBits->One);
+ if (!RightKnown)
+ return std::unexpected(RightKnown.error());
+ const std::uint32_t LeftTrailingKnown = countTrailingOnes(*LeftKnown);
+ const std::uint32_t RightTrailingKnown = countTrailingOnes(*RightKnown);
+ const std::uint32_t LeftTrailingZeroes = countTrailingOnes(LeftBits->Zero);
+ const std::uint32_t RightTrailingZeroes = countTrailingOnes(RightBits->Zero);
+ const std::uint32_t TrailingZeroes = LeftTrailingZeroes + RightTrailingZeroes;
+ const std::uint32_t SmallestOperand =
+ std::min(LeftTrailingKnown - LeftTrailingZeroes, RightTrailingKnown - RightTrailingZeroes);
+ const std::uint32_t ResultBitsKnown = std::min(SmallestOperand + TrailingZeroes, BitWidth);
+
+ auto BottomKnown = multiplyMasks(LeftBits->One, RightBits->One);
+ if (!BottomKnown)
+ return std::unexpected(BottomKnown.error());
+ auto LowMask = APInt::getLowBitsSet(BitWidth, ResultBitsKnown);
+ if (!LowMask)
+ return std::unexpected(LowMask.error());
+ auto NotBottomKnown = BottomKnown->Value.bitwiseNot();
+ if (!NotBottomKnown)
+ return std::unexpected(NotBottomKnown.error());
+ auto KnownZero = intersectMasks(*NotBottomKnown, *LowMask);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto HighZero = highBitsMask(BitWidth, BitWidth - LeadingZeroes);
+ if (!HighZero)
+ return std::unexpected(HighZero.error());
+ KnownZero = mergeMasks(*KnownZero, *HighZero);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto KnownOne = intersectMasks(BottomKnown->Value, *LowMask);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)};
+ }
+
+ if (Opcode == GenericOpcode::G_SHL || Opcode == GenericOpcode::G_LSHR ||
+ Opcode == GenericOpcode::G_ASHR) {
+ const auto AmountReg = registerOperand(*Definition, 2);
+ if (!AmountReg)
+ return Unknown;
+ const auto Amount = constantShiftAmount(*AmountReg, MRI, BitWidth);
+ if (!Amount)
+ return Unknown;
+ auto SourceBits = getKnownBits(*Source, Depth + 1U);
+ if (!SourceBits)
+ return std::unexpected(SourceBits.error());
+
+ if (Opcode == GenericOpcode::G_SHL) {
+ auto KnownZero = shiftMaskLeft(SourceBits->Zero, *Amount);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto LowZero = APInt::getLowBitsSet(BitWidth, *Amount);
+ if (!LowZero)
+ return std::unexpected(LowZero.error());
+ KnownZero = mergeMasks(*KnownZero, *LowZero);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto KnownOne = shiftMaskLeft(SourceBits->One, *Amount);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)};
+ }
+
+ auto KnownZero = shiftMaskRight(SourceBits->Zero, *Amount);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ auto KnownOne = shiftMaskRight(SourceBits->One, *Amount);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ auto HighBits = highBitsMask(BitWidth, BitWidth - *Amount);
+ if (!HighBits)
+ return std::unexpected(HighBits.error());
+ if (Opcode == GenericOpcode::G_LSHR || SourceBits->Zero.testBit(BitWidth - 1U)) {
+ KnownZero = mergeMasks(*KnownZero, *HighBits);
+ if (!KnownZero)
+ return std::unexpected(KnownZero.error());
+ } else if (SourceBits->One.testBit(BitWidth - 1U)) {
+ KnownOne = mergeMasks(*KnownOne, *HighBits);
+ if (!KnownOne)
+ return std::unexpected(KnownOne.error());
+ }
+ return KnownBits{.Zero = std::move(*KnownZero), .One = std::move(*KnownOne)};
+ }
+
+ return Unknown;
+}
+
+Expected<APInt> GISelValueTracking::getKnownZeroes(Register Reg) const {
+ auto Known = getKnownBits(Reg);
+ if (!Known)
+ return std::unexpected(Known.error());
+ return std::move(Known->Zero);
+}
+
+Expected<std::uint32_t> GISelValueTracking::computeNumSignBits(Register Reg, unsigned Depth) const {
+ auto Type = MRI.getType(Reg);
+ if (!Type)
+ return std::unexpected(Type.error());
+ const std::uint32_t BitWidth = Type->scalarBitWidth();
+ if (Depth >= MaxDepth)
+ return 1U;
+
+ const MachineInstr *Definition = MRI.getVRegDef(Reg);
+ if (Definition == nullptr)
+ return 1U;
+ const auto Opcode = static_cast<GenericOpcode>(Definition->opcode().value());
+ std::uint32_t FirstAnswer = 1U;
+
+ if (Opcode == GenericOpcode::G_CONSTANT) {
+ const MachineOperand *ValueOperand = Definition->operandAt(1);
+ const auto *Immediate =
+ ValueOperand == nullptr ? nullptr : ValueOperand->getIf<IntegerImmediateOperand>();
+ if (Immediate == nullptr)
+ return 1U;
+ auto Value = constantAtWidth(Immediate->Value, BitWidth);
+ if (!Value)
+ return std::unexpected(Value.error());
+ return constantNumSignBits(*Value);
+ }
+
+ const auto Source = registerOperand(*Definition, 1);
+ if (Opcode == GenericOpcode::COPY && Source && Source->isVirtual())
+ return computeNumSignBits(*Source, Depth);
+
+ if (Opcode == GenericOpcode::G_SEXT && Source) {
+ auto SourceType = MRI.getType(*Source);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ auto SourceSignBits = computeNumSignBits(*Source, Depth + 1U);
+ if (!SourceSignBits)
+ return std::unexpected(SourceSignBits.error());
+ return std::min(BitWidth, *SourceSignBits + BitWidth - SourceType->scalarBitWidth());
+ }
+
+ if ((Opcode == GenericOpcode::G_ASSERT_SEXT || Opcode == GenericOpcode::G_SEXT_INREG) && Source) {
+ const MachineOperand *WidthOperand = Definition->operandAt(2);
+ const auto *Width = WidthOperand == nullptr ? nullptr : WidthOperand->getIf<std::int64_t>();
+ if (Width != nullptr && *Width > 0 && static_cast<std::uint64_t>(*Width) <= BitWidth) {
+ auto SourceSignBits = computeNumSignBits(*Source, Depth + 1U);
+ if (!SourceSignBits)
+ return std::unexpected(SourceSignBits.error());
+ return std::max(*SourceSignBits, BitWidth - static_cast<std::uint32_t>(*Width) + 1U);
+ }
+ }
+
+ if ((Opcode == GenericOpcode::G_AND || Opcode == GenericOpcode::G_OR ||
+ Opcode == GenericOpcode::G_XOR) &&
+ Source) {
+ const auto Right = registerOperand(*Definition, 2);
+ if (Right) {
+ auto LeftSignBits = computeNumSignBits(*Source, Depth + 1U);
+ if (!LeftSignBits)
+ return std::unexpected(LeftSignBits.error());
+ if (*LeftSignBits != 1) {
+ auto RightSignBits = computeNumSignBits(*Right, Depth + 1U);
+ if (!RightSignBits)
+ return std::unexpected(RightSignBits.error());
+ if (*RightSignBits != 1)
+ FirstAnswer = std::min(*LeftSignBits, *RightSignBits);
+ }
+ }
+ }
+
+ if (Opcode == GenericOpcode::G_ASHR && Source) {
+ const auto Right = registerOperand(*Definition, 2);
+ if (Right) {
+ auto SourceSignBits = computeNumSignBits(*Source, Depth + 1U);
+ if (!SourceSignBits)
+ return std::unexpected(SourceSignBits.error());
+ FirstAnswer = *SourceSignBits;
+ if (const auto Amount = constantShiftAmount(*Right, MRI, BitWidth))
+ FirstAnswer = std::min(BitWidth, FirstAnswer + *Amount);
+ }
+ }
+
+ if (Opcode == GenericOpcode::G_TRUNC && Source) {
+ auto SourceType = MRI.getType(*Source);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ const std::uint32_t SourceWidth = SourceType->scalarBitWidth();
+ auto SourceSignBits = computeNumSignBits(*Source, Depth + 1U);
+ if (!SourceSignBits)
+ return std::unexpected(SourceSignBits.error());
+ if (SourceWidth > BitWidth && *SourceSignBits > SourceWidth - BitWidth)
+ return *SourceSignBits - (SourceWidth - BitWidth);
+ }
+
+ if (Opcode == GenericOpcode::G_ICMP || Opcode == GenericOpcode::G_FCMP) {
+ if (BitWidth > 1) {
+ const BooleanContent Contents =
+ Target.getBooleanContents(Type->isVector(), Opcode == GenericOpcode::G_FCMP);
+ if (Contents == BooleanContent::ZeroOrNegativeOne)
+ return BitWidth;
+ if (Contents == BooleanContent::ZeroOrOne)
+ return BitWidth - 1U;
+ }
+ }
+
+ switch (Opcode) {
+ case GenericOpcode::G_UADDO:
+ case GenericOpcode::G_UADDE:
+ case GenericOpcode::G_USUBO:
+ case GenericOpcode::G_USUBE:
+ case GenericOpcode::G_SADDO:
+ case GenericOpcode::G_SADDE:
+ case GenericOpcode::G_SSUBO:
+ case GenericOpcode::G_SSUBE:
+ case GenericOpcode::G_UMULO:
+ case GenericOpcode::G_SMULO: {
+ const auto BooleanResult = registerOperand(*Definition, 1);
+ if (BooleanResult && *BooleanResult == Reg &&
+ Target.getBooleanContents(Type->isVector(), false) == BooleanContent::ZeroOrNegativeOne)
+ return BitWidth;
+ break;
+ }
+ default:
+ break;
+ }
+
+ auto Known = getKnownBits(Reg, Depth);
+ if (!Known)
+ return std::unexpected(Known.error());
+ std::uint32_t KnownSignBits = 1;
+ if (Known->Zero.testBit(BitWidth - 1U))
+ KnownSignBits = static_cast<std::uint32_t>(Known->Zero.countlOne());
+ else if (Known->One.testBit(BitWidth - 1U))
+ KnownSignBits = static_cast<std::uint32_t>(Known->One.countlOne());
+ return std::max(FirstAnswer, std::max(1U, KnownSignBits));
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp
new file mode 100644
index 0000000..0814641
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -0,0 +1,398 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/InstructionSelect.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/Utils.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <string>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] const MachinePassContract &instructionSelectContract() {
+ static const MachinePassContract Contract{
+ .Name = "instruction-select",
+ .Requires = {MachineFunctionProperty::Legalized, MachineFunctionProperty::RegBankSelected},
+ .Sets = {MachineFunctionProperty::Selected},
+ .Clears = {},
+ .RequiredAnalyses = {},
+ .VerificationStage = PipelineStage::Selected,
+ };
+ return Contract;
+}
+
+struct OpcodeState final {
+ MachineInstrStableID Instruction;
+ Opcode OpcodeValue;
+ std::uint64_t OperandFingerprint = 0;
+
+ friend bool operator==(const OpcodeState &Left, const OpcodeState &Right) noexcept = default;
+};
+
+struct OpcodeStateHash final {
+ [[nodiscard]] std::size_t operator()(const OpcodeState &State) const noexcept {
+ return std::hash<std::uint64_t>{}(State.Instruction.value()) ^
+ (std::hash<std::uint32_t>{}(State.OpcodeValue.value()) << 1U) ^
+ (std::hash<std::uint64_t>{}(State.OperandFingerprint) << 2U);
+ }
+};
+
+void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept {
+ State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U);
+}
+
+[[nodiscard]] std::uint64_t operandFingerprint(const MachineInstr &Instruction) noexcept {
+ std::uint64_t Result = 0xcbf29ce484222325ULL;
+ combineFingerprint(Result, Instruction.flags().rawValue());
+ combineFingerprint(Result, Instruction.operandCount());
+ for (const MachineOperand &Operand : Instruction.operands()) {
+ combineFingerprint(Result, static_cast<std::uint64_t>(Operand.kind()));
+ combineFingerprint(Result, Operand.targetFlags());
+ combineFingerprint(Result, Operand.registerFlags().rawValue());
+ combineFingerprint(Result, Operand.subReg().value());
+ if (const Register *Reg = Operand.getIf<Register>())
+ combineFingerprint(Result, Reg->opaqueValue());
+ else if (const std::int64_t *Immediate = Operand.getIf<std::int64_t>())
+ combineFingerprint(Result, static_cast<std::uint64_t>(*Immediate));
+ else if (const auto *Integer = Operand.getIf<IntegerImmediateOperand>())
+ combineFingerprint(Result, Integer->Value.bitWidth());
+ else if (const auto *Floating = Operand.getIf<FloatingImmediateOperand>())
+ combineFingerprint(Result, static_cast<std::uint64_t>(Floating->Bits.semantics()));
+ else if (const auto *Block = Operand.getIf<BasicBlockOperand>())
+ combineFingerprint(Result, Block->Block == nullptr ? 0 : Block->Block->stableID().value());
+ }
+ return Result;
+}
+
+[[nodiscard]] OpcodeState stateOf(const MachineInstr &Instruction) noexcept {
+ return {Instruction.stableID(), Instruction.opcode(), operandFingerprint(Instruction)};
+}
+
+[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) {
+ std::vector<MachineInstr *> Result;
+ for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next())
+ for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next())
+ Result.push_back(Instruction);
+ return Result;
+}
+
+[[nodiscard]] std::vector<MachineBasicBlock *> stableCFGPostOrder(MachineFunction &Function) {
+ std::vector<MachineBasicBlock *> Roots;
+ for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next())
+ Roots.push_back(Block);
+ std::ranges::sort(Roots, {},
+ [](const MachineBasicBlock *Block) { return Block->stableID().value(); });
+ if (MachineBasicBlock *Entry = Function.firstBlock(); Entry != nullptr) {
+ const auto Position = std::ranges::find(Roots, Entry);
+ if (Position != Roots.end())
+ std::rotate(Roots.begin(), Position, std::next(Position));
+ }
+
+ struct TraversalFrame final {
+ MachineBasicBlock *Block = nullptr;
+ std::vector<MachineBasicBlock *> Successors;
+ std::size_t NextSuccessor = 0;
+ };
+ const auto MakeFrame = [](MachineBasicBlock &Block) {
+ TraversalFrame Frame;
+ Frame.Block = &Block;
+ for (const MachineSuccessor &Successor : Block.successors()) {
+ if (Successor.Block != nullptr)
+ Frame.Successors.push_back(Successor.Block);
+ }
+ std::ranges::sort(Frame.Successors, {}, [](const MachineBasicBlock *Successor) {
+ return Successor->stableID().value();
+ });
+ return Frame;
+ };
+
+ std::vector<MachineBasicBlock *> Result;
+ std::unordered_set<MachineBasicBlock *> Visited;
+ std::vector<TraversalFrame> Stack;
+ for (MachineBasicBlock *Root : Roots) {
+ if (!Visited.insert(Root).second)
+ continue;
+ Stack.push_back(MakeFrame(*Root));
+ while (!Stack.empty()) {
+ TraversalFrame &Frame = Stack.back();
+ if (Frame.NextSuccessor < Frame.Successors.size()) {
+ MachineBasicBlock *Successor = Frame.Successors[Frame.NextSuccessor++];
+ if (Visited.insert(Successor).second)
+ Stack.push_back(MakeFrame(*Successor));
+ continue;
+ }
+ Result.push_back(Frame.Block);
+ Stack.pop_back();
+ }
+ }
+ return Result;
+}
+
+[[nodiscard]] std::vector<MachineInstr *> bottomUpInstructions(MachineFunction &Function) {
+ std::vector<MachineInstr *> Result;
+ for (MachineBasicBlock *Block : stableCFGPostOrder(Function)) {
+ for (MachineInstr *Instruction = Block->lastInstruction(); Instruction != nullptr;
+ Instruction = Instruction->previous())
+ Result.push_back(Instruction);
+ }
+ return Result;
+}
+
+[[nodiscard]] Error passFailure(std::string Message,
+ ErrorCode Code = ErrorCode::VerificationError) {
+ return Error(Code, std::move(Message));
+}
+
+} // namespace
+
+Expected<void> InstructionSelector::propagateSelectionFlags(MachineInstr &Output,
+ MachineInstrFlags InputFlags,
+ bool InputMayRaiseFPException,
+ const TargetDescription &Target) const {
+ const InstructionDescriptor *OutputDescriptor = Target.instructionDescriptor(Output.opcode());
+ if (OutputDescriptor == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "instruction selector produced an unknown instruction descriptor");
+ }
+ if (!InputMayRaiseFPException &&
+ OutputDescriptor->Flags.has(InstructionFlag::MayRaiseFPException)) {
+ InputFlags.set(MachineInstrFlag::NoFPExcept);
+ }
+ Output.setFlags(InputFlags);
+ return {};
+}
+
+Expected<void> InstructionSelector::propagateSelectionFlags(MachineInstr &Output,
+ const MachineInstr &Input,
+ const TargetDescription &Target) const {
+ const InstructionDescriptor *InputDescriptor = Target.instructionDescriptor(Input.opcode());
+ if (InputDescriptor == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "instruction selector input has an unknown instruction descriptor");
+ }
+ return propagateSelectionFlags(Output, Input.flags(),
+ InputDescriptor->Flags.has(InstructionFlag::MayRaiseFPException),
+ Target);
+}
+
+Expected<void> InstructionSelector::finalizeGeneratedInstruction(
+ MachineInstr &Output, MachineFunction &Function, const TargetDescription &Target,
+ MachineInstrFlags InputFlags, bool InputMayRaiseFPException) const {
+ auto Flags = propagateSelectionFlags(Output, InputFlags, InputMayRaiseFPException, Target);
+ if (!Flags)
+ return std::unexpected(Flags.error());
+ return constrainSelectedInstruction(Output, Function, Target);
+}
+
+const MachinePassContract &InstructionSelect::contract() const noexcept {
+ return instructionSelectContract();
+}
+
+Expected<MachinePassResult> InstructionSelect::run(MachineFunction &Function,
+ const TargetDescription &Target,
+ MachineFunctionAnalysisManager &Analyses) {
+ if (Function.properties().has(MachineFunctionProperty::FailedISel))
+ return MachinePassResult{
+ .Changed = false,
+ .Changes = {},
+ .Preserved = PreservedAnalyses::all(),
+ };
+ if (Info == nullptr)
+ return std::unexpected(passFailure("instruction select has no target behavior interface",
+ ErrorCode::InvalidArgument));
+ return runImplementation(Function, Target, Analyses);
+}
+
+Expected<MachinePassResult>
+InstructionSelect::runImplementation(MachineFunction &Function, const TargetDescription &Target,
+ MachineFunctionAnalysisManager &Analyses) {
+
+ MachinePassResult Result;
+ std::vector<MachineInstr *> Worklist;
+ std::unordered_set<MachineInstrStableID> DiscoveredInstructions;
+ std::unordered_set<OpcodeState, OpcodeStateHash> SeenStates;
+ for (MachineInstr *Instruction : bottomUpInstructions(Function)) {
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode());
+ if (Descriptor == nullptr)
+ return std::unexpected(
+ passFailure("instruction selector encountered an unknown instruction descriptor"));
+ auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo());
+ if (!Validation)
+ return std::unexpected(
+ withErrorContext(Validation.error(), "instruction selector input is invalid: "));
+ Worklist.push_back(Instruction);
+ DiscoveredInstructions.insert(Instruction->stableID());
+ }
+ const auto DiscoverNewInstructions = [&]() -> Expected<void> {
+ for (MachineInstr *Candidate : bottomUpInstructions(Function)) {
+ const InstructionDescriptor *CandidateDescriptor =
+ Target.instructionDescriptor(Candidate->opcode());
+ if (CandidateDescriptor == nullptr) {
+ return std::unexpected(
+ passFailure("instruction selector produced an unknown instruction descriptor"));
+ }
+ if (DiscoveredInstructions.insert(Candidate->stableID()).second)
+ Worklist.push_back(Candidate);
+ }
+ return {};
+ };
+
+ for (std::size_t Index = 0; Index < Worklist.size(); ++Index) {
+ MachineInstr *Instruction = Worklist[Index];
+ if (Instruction == nullptr || Instruction->parent() == nullptr)
+ continue;
+ if (isTriviallyDead(*Instruction, Function.registerInfo(), Target)) {
+ auto Removed = eraseInstr(*Instruction, Function.registerInfo(), Target);
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ continue;
+ }
+ const InstructionDescriptor *CurrentDescriptor =
+ Target.instructionDescriptor(Instruction->opcode());
+ if (CurrentDescriptor == nullptr)
+ return std::unexpected(
+ passFailure("instruction selector encountered an unknown instruction descriptor"));
+ const bool IsGeneric = CurrentDescriptor->Flags.has(InstructionFlag::Generic);
+ const bool IsCopy = Instruction->opcode() == opcode(GenericOpcode::COPY);
+ if (!IsGeneric && !IsCopy)
+ continue;
+ const OpcodeState Before = stateOf(*Instruction);
+ if (!SeenStates.insert(Before).second) {
+ return std::unexpected(passFailure("instruction selector entered a repeated state for '" +
+ CurrentDescriptor->Name + "'",
+ ErrorCode::InternalError));
+ }
+ std::string Rule = "select.";
+ Rule.append(CurrentDescriptor->Name);
+ auto Accepted = Analyses.requestAction(Rule, Function, Instruction->stableID());
+ if (!Accepted)
+ return std::unexpected(Accepted.error());
+ if (!*Accepted) {
+ return std::unexpected(passFailure("required selection action was rejected for '" +
+ CurrentDescriptor->Name + "'",
+ ErrorCode::VerificationError));
+ }
+ const MachineInstrFlags InputFlags = Instruction->flags();
+ const bool InputMayRaiseFPException =
+ CurrentDescriptor->Flags.has(InstructionFlag::MayRaiseFPException);
+ auto Selection = Info->select(*Instruction, Function, Target);
+ if (!Selection)
+ return std::unexpected(Selection.error());
+ if (Instruction->parent() == nullptr) {
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ auto Discovered = DiscoverNewInstructions();
+ if (!Discovered)
+ return std::unexpected(Discovered.error());
+ continue;
+ }
+ const InstructionDescriptor *Selected = Target.instructionDescriptor(Instruction->opcode());
+ if (Selected == nullptr)
+ return std::unexpected(
+ passFailure("instruction selector produced an unknown instruction descriptor"));
+ if (IsGeneric) {
+ auto FlagPropagation =
+ Info->propagateSelectionFlags(*Instruction, InputFlags, InputMayRaiseFPException, Target);
+ if (!FlagPropagation)
+ return std::unexpected(FlagPropagation.error());
+ auto Constraint = Info->constrainSelectedInstruction(*Instruction, Function, Target);
+ if (!Constraint) {
+ return std::unexpected(withErrorContext(
+ Constraint.error(), "instruction selector could not constrain selected registers: "));
+ }
+ }
+ auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo());
+ if (!Validation)
+ return std::unexpected(withErrorContext(Validation.error(),
+ "instruction selector produced invalid MachineIR: "));
+ const OpcodeState After = stateOf(*Instruction);
+ if (IsGeneric && After == Before)
+ return std::unexpected(
+ passFailure("instruction selector reported success without observable progress",
+ ErrorCode::InternalError));
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ if (Selected->Flags.has(InstructionFlag::Generic))
+ Worklist.push_back(Instruction);
+ auto Discovered = DiscoverNewInstructions();
+ if (!Discovered)
+ return std::unexpected(Discovered.error());
+ }
+ for (MachineInstr *Instruction : layoutInstructions(Function)) {
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode());
+ if (Descriptor == nullptr)
+ return std::unexpected(
+ passFailure("instruction selector produced an unknown instruction descriptor"));
+ auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo());
+ if (!Validation)
+ return std::unexpected(withErrorContext(Validation.error(),
+ "instruction selector produced invalid MachineIR: "));
+ if (Descriptor->Flags.has(InstructionFlag::Generic))
+ return std::unexpected(
+ passFailure("instruction selector left a generic opcode in the function"));
+ }
+ for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) {
+ for (MachineInstr *Instruction = Block->lastInstruction(); Instruction != nullptr;) {
+ MachineInstr *Previous = Instruction->previous();
+ if (Instruction->opcode() != opcode(GenericOpcode::COPY)) {
+ Instruction = Previous;
+ continue;
+ }
+ const MachineOperand *SourceOperand = Instruction->operandAt(1);
+ const MachineOperand *DestinationOperand = Instruction->operandAt(0);
+ const Register *Source =
+ SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Source == nullptr || Destination == nullptr || !Source->isVirtual() ||
+ !Destination->isVirtual() || SourceOperand->subReg().value() != 0) {
+ Instruction = Previous;
+ continue;
+ }
+ auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source);
+ if (!SourceInfo)
+ return std::unexpected(SourceInfo.error());
+ auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination);
+ if (!DestinationInfo)
+ return std::unexpected(DestinationInfo.error());
+ if (!SourceInfo->get().RegClass ||
+ SourceInfo->get().RegClass != DestinationInfo->get().RegClass) {
+ Instruction = Previous;
+ continue;
+ }
+ auto Replaced = Function.registerInfo().replaceRegWith(*Destination, *Source);
+ if (!Replaced)
+ return std::unexpected(Replaced.error());
+ auto Removed = Instruction->eraseFromParent();
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ Instruction = Previous;
+ }
+ }
+ auto RegisterValidation = Target.validateRegisterInfo(Function.registerInfo());
+ if (!RegisterValidation)
+ return std::unexpected(withErrorContext(RegisterValidation.error(),
+ "instruction selection produced invalid registers: "));
+ Function.registerInfo().clearAllVirtualRegisterTypes();
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Registers);
+ Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
+ return Result;
+}
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
new file mode 100644
index 0000000..0601040
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
@@ -0,0 +1,493 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/Legalizer.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/CSEInfo.h"
+#include "rcc/CodeGen/GlobalISel/CSEMIRBuilder.h"
+#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "rcc/CodeGen/GlobalISel/GISelValueTracking.h"
+#include "rcc/CodeGen/GlobalISel/GISelWorkList.h"
+#include "rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h"
+#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h"
+#include "rcc/CodeGen/GlobalISel/Utils.h"
+#include "rcc/CodeGen/LiveVariables.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <memory>
+#include <ranges>
+#include <string>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] const MachinePassContract &legalizerContract() {
+ static const MachinePassContract Contract{
+ .Name = "legalizer",
+ // LLVM's GlobalISel legalizer operates on SSA form and does not require
+ // the optional register-liveness tracking property.
+ .Requires = {MachineFunctionProperty::IsSSA},
+ .Sets = {MachineFunctionProperty::Legalized},
+ .Clears = {},
+ .RequiredAnalyses = {},
+ .VerificationStage = PipelineStage::Legalized,
+ };
+ return Contract;
+}
+
+[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) {
+ std::vector<MachineInstr *> Result;
+ for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next())
+ for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next())
+ Result.push_back(Instruction);
+ return Result;
+}
+
+[[nodiscard]] bool isArtifact(const MachineInstr &Instruction) noexcept {
+ switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+ default:
+ return false;
+ case GenericOpcode::G_TRUNC:
+ case GenericOpcode::G_ZEXT:
+ case GenericOpcode::G_ANYEXT:
+ case GenericOpcode::G_SEXT:
+ case GenericOpcode::G_MERGE_VALUES:
+ case GenericOpcode::G_UNMERGE_VALUES:
+ case GenericOpcode::G_CONCAT_VECTORS:
+ case GenericOpcode::G_BUILD_VECTOR:
+ case GenericOpcode::G_EXTRACT:
+ case GenericOpcode::G_INSERT:
+ return true;
+ }
+}
+
+[[nodiscard]] bool isIntrinsicOpcode(Opcode InstructionOpcode) noexcept {
+ switch (static_cast<GenericOpcode>(InstructionOpcode.value())) {
+ default:
+ return false;
+ case GenericOpcode::G_INTRINSIC:
+ case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
+ case GenericOpcode::G_INTRINSIC_CONVERGENT:
+ case GenericOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
+ return true;
+ }
+}
+
+using InstList = GISelWorkList<256>;
+using ArtifactList = GISelWorkList<128>;
+
+class LegalizerWorkListManager final : public GISelChangeObserver {
+public:
+ LegalizerWorkListManager(InstList &Instructions, ArtifactList &Artifacts)
+ : Instructions(Instructions), Artifacts(Artifacts) {}
+
+ void erasingInstr(MachineInstr &Instruction) override {
+ Instructions.remove(&Instruction);
+ Artifacts.remove(&Instruction);
+ }
+
+ void createdInstr(MachineInstr &Instruction) override {
+ createdOrChangedInstr(Instruction);
+ }
+
+ void changingInstr(MachineInstr &) override {}
+
+ void changedInstr(MachineInstr &Instruction) override {
+ createdOrChangedInstr(Instruction);
+ }
+
+private:
+ void createdOrChangedInstr(MachineInstr &Instruction) {
+ if (!isGenericOpcode(Instruction.opcode()))
+ return;
+ if (isArtifact(Instruction))
+ Artifacts.insert(&Instruction);
+ else
+ Instructions.insert(&Instruction);
+ }
+
+ InstList &Instructions;
+ ArtifactList &Artifacts;
+};
+
+[[nodiscard]] std::vector<MachineBasicBlock *> reversePostOrderBlocks(MachineFunction &Function) {
+ struct TraversalFrame final {
+ MachineBasicBlock *Block = nullptr;
+ std::vector<MachineBasicBlock *> Successors;
+ std::size_t NextSuccessor = 0;
+ };
+
+ const auto MakeFrame = [](MachineBasicBlock &Block) {
+ TraversalFrame Frame;
+ Frame.Block = &Block;
+ for (const MachineSuccessor &Successor : Block.successors()) {
+ if (Successor.Block != nullptr)
+ Frame.Successors.push_back(Successor.Block);
+ }
+ std::ranges::sort(Frame.Successors, {}, [](const MachineBasicBlock *Successor) {
+ return Successor->stableID().value();
+ });
+ return Frame;
+ };
+
+ std::vector<MachineBasicBlock *> Roots;
+ for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next())
+ Roots.push_back(Block);
+ std::ranges::sort(Roots, {},
+ [](const MachineBasicBlock *Block) { return Block->stableID().value(); });
+ if (MachineBasicBlock *Entry = Function.firstBlock(); Entry != nullptr) {
+ const auto Position = std::ranges::find(Roots, Entry);
+ if (Position != Roots.end())
+ std::rotate(Roots.begin(), Position, std::next(Position));
+ }
+
+ std::vector<MachineBasicBlock *> PostOrder;
+ std::unordered_set<MachineBasicBlock *> Visited;
+ std::vector<TraversalFrame> Stack;
+ for (MachineBasicBlock *Root : Roots) {
+ if (!Visited.insert(Root).second)
+ continue;
+ Stack.push_back(MakeFrame(*Root));
+ while (!Stack.empty()) {
+ TraversalFrame &Frame = Stack.back();
+ if (Frame.NextSuccessor < Frame.Successors.size()) {
+ MachineBasicBlock *Successor = Frame.Successors[Frame.NextSuccessor++];
+ if (Visited.insert(Successor).second)
+ Stack.push_back(MakeFrame(*Successor));
+ continue;
+ }
+ PostOrder.push_back(Frame.Block);
+ Stack.pop_back();
+ }
+ }
+ std::ranges::reverse(PostOrder);
+ return PostOrder;
+}
+
+[[nodiscard]] Error passFailure(std::string Message,
+ ErrorCode Code = ErrorCode::VerificationError) {
+ return Error(Code, std::move(Message));
+}
+
+[[nodiscard]] std::string_view legalizationActionName(LegalizationAction Action) noexcept {
+ switch (Action) {
+ case LegalizationAction::Legal:
+ return "legal";
+ case LegalizationAction::Unsupported:
+ return "unsupported";
+ case LegalizationAction::Custom:
+ return "custom";
+ case LegalizationAction::Lower:
+ return "lower";
+ case LegalizationAction::Libcall:
+ return "libcall";
+ case LegalizationAction::WidenScalar:
+ return "widen-scalar";
+ case LegalizationAction::NarrowScalar:
+ return "narrow-scalar";
+ case LegalizationAction::MoreElements:
+ return "more-elements";
+ case LegalizationAction::FewerElements:
+ return "fewer-elements";
+ case LegalizationAction::Bitcast:
+ return "bitcast";
+ case LegalizationAction::ChangeElementType:
+ return "change-element-type";
+ case LegalizationAction::ChangeAddressSpace:
+ return "change-address-space";
+ }
+ return "unknown";
+}
+
+[[nodiscard]] Expected<void> clearTrackedLivenessFlags(MachineFunction &Function,
+ MachinePassResult &Result) {
+ if (!Result.Changed || !Function.properties().has(MachineFunctionProperty::TracksLiveness))
+ return {};
+ bool Changed = false;
+ for (MachineInstr *Instruction : layoutInstructions(Function)) {
+ for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) {
+ MachineOperand &Operand = *Instruction->operandAt(Index);
+ if (!Operand.isRegister())
+ continue;
+ RegisterOperandFlags Flags = Operand.registerFlags();
+ if (!Flags.has(RegisterOperandFlag::Kill) && !Flags.has(RegisterOperandFlag::Dead))
+ continue;
+ Flags.clear(RegisterOperandFlag::Kill);
+ Flags.clear(RegisterOperandFlag::Dead);
+ auto Updated = Operand.setRegisterFlags(Flags);
+ if (!Updated)
+ return std::unexpected(Updated.error());
+ Changed = true;
+ }
+ }
+ if (Changed)
+ Result.Changes.add(MachinePassChange::Operands);
+ return {};
+}
+
+} // namespace
+
+const MachinePassContract &Legalizer::contract() const noexcept {
+ return legalizerContract();
+}
+
+Expected<MachinePassResult> Legalizer::run(MachineFunction &Function,
+ const TargetDescription &Target,
+ MachineFunctionAnalysisManager &Analyses) {
+ if (Function.properties().has(MachineFunctionProperty::FailedISel))
+ return MachinePassResult{
+ .Changed = false,
+ .Changes = {},
+ .Preserved = PreservedAnalyses::all(),
+ };
+ if (Info == nullptr)
+ return std::unexpected(
+ passFailure("legalizer has no target behavior interface", ErrorCode::InvalidArgument));
+ if (AbortMode != GlobalISelAbortMode::Enable) {
+ if (AbortMode != GlobalISelAbortMode::Disable &&
+ AbortMode != GlobalISelAbortMode::DisableWithDiagnostic) {
+ return std::unexpected(passFailure("legalizer has an unknown GlobalISel abort mode",
+ ErrorCode::InvalidArgument));
+ }
+ auto Result = runImplementation(Function, Target, Analyses);
+ if (Result || Result.error().code() != ErrorCode::Unsupported)
+ return Result;
+ Function.properties().set(MachineFunctionProperty::FailedISel);
+ return MachinePassResult{
+ .Changed = true,
+ .Changes = {MachinePassChange::Instructions},
+ .Preserved = PreservedAnalyses::none(),
+ };
+ }
+
+ return runImplementation(Function, Target, Analyses);
+}
+
+Expected<MachinePassResult> Legalizer::runImplementation(MachineFunction &Function,
+ const TargetDescription &Target,
+ MachineFunctionAnalysisManager &Analyses) {
+ MachinePassResult Result;
+ InstList InstructionWorklist;
+ ArtifactList ArtifactWorklist;
+ std::vector<MachineInstr *> RetryList;
+
+ for (MachineBasicBlock *Block : reversePostOrderBlocks(Function)) {
+ for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ if (!isGenericOpcode(Instruction->opcode()))
+ continue;
+ if (isArtifact(*Instruction))
+ ArtifactWorklist.deferredInsert(Instruction);
+ else
+ InstructionWorklist.deferredInsert(Instruction);
+ }
+ }
+ ArtifactWorklist.finalize();
+ InstructionWorklist.finalize();
+
+ LegalizerWorkListManager WorkListObserver(InstructionWorklist, ArtifactWorklist);
+ GISelCSEInfo CSEInfo;
+ CSEInfo.setCSEConfig(getStandardCSEConfigForOpt(Optimization));
+ auto CSEAnalysis = CSEInfo.analyze(Function);
+ if (!CSEAnalysis)
+ return std::unexpected(
+ withErrorContext(CSEAnalysis.error(), "GlobalISel CSE analysis failed: "));
+ std::array<GISelChangeObserver *, 2> AuxiliaryObservers{&WorkListObserver, &CSEInfo};
+ GISelObserverWrapper Observer{std::span<GISelChangeObserver *const>(AuxiliaryObservers)};
+ MachineFunctionDelegateInstaller ObserverInstaller(Function, Observer);
+ CSEMIRBuilder Builder(Function, Target);
+ Builder.setCSEInfo(&CSEInfo);
+ const LibcallLoweringInfo *ActiveLibcalls =
+ Libcalls.CallLowering == nullptr ? nullptr : &Libcalls;
+ LegalizerHelper Helper(Function, *Info, Observer, Builder, ActiveLibcalls);
+ GISelValueTracking ValueTracking(Function, Target);
+ LegalizationArtifactCombiner ArtifactCombiner(Builder, Function.registerInfo(), *Info,
+ &ValueTracking);
+
+ do {
+ while (!InstructionWorklist.empty()) {
+ MachineInstr *Instruction = InstructionWorklist.popBack();
+ if (Instruction == nullptr || Instruction->parent() == nullptr)
+ continue;
+
+ if (isTriviallyDead(*Instruction, Function.registerInfo(), Target)) {
+ auto Removed = eraseInstr(*Instruction, Function.registerInfo(), Target);
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ continue;
+ }
+
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode());
+ if (Descriptor == nullptr)
+ return std::unexpected(
+ passFailure("legalizer encountered an unknown instruction descriptor"));
+ if (!Descriptor->Flags.has(InstructionFlag::Generic))
+ continue;
+
+ auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo());
+ if (!Validation) {
+ return std::unexpected(
+ withErrorContext(Validation.error(), "legalizer input is invalid: "));
+ }
+
+ if (isIntrinsicOpcode(Instruction->opcode())) {
+ auto Accepted =
+ Analyses.requestAction("legalize.intrinsic", Function, Instruction->stableID());
+ if (!Accepted)
+ return std::unexpected(Accepted.error());
+ if (!*Accepted)
+ return std::unexpected(passFailure("required intrinsic legalization action was rejected",
+ ErrorCode::VerificationError));
+ auto IntrinsicResult = Info->legalizeIntrinsic(Helper, *Instruction);
+ if (!IntrinsicResult)
+ return std::unexpected(IntrinsicResult.error());
+ if (!*IntrinsicResult) {
+ const std::string_view Name = Descriptor->Name;
+ return std::unexpected(passFailure("target legalizer rejected intrinsic instruction '" +
+ std::string(Name) + "'",
+ ErrorCode::Unsupported));
+ }
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ continue;
+ }
+
+ auto Decision = Info->getDecision(*Instruction, Function, Target);
+ if (!Decision)
+ return std::unexpected(Decision.error());
+ if (Decision->Action == LegalizationAction::Legal)
+ continue;
+ if (Decision->Action == LegalizationAction::Unsupported) {
+ if (isArtifact(*Instruction)) {
+ RetryList.push_back(Instruction);
+ continue;
+ }
+ return std::unexpected(
+ passFailure("target legalizer rejected generic instruction '" + Descriptor->Name + "'",
+ ErrorCode::Unsupported));
+ }
+
+ auto LoweringValidation = Info->validateLowering(*Instruction, Function, Target, *Decision);
+ if (!LoweringValidation) {
+ return std::unexpected(withErrorContext(LoweringValidation.error(),
+ "legalizer lowering precondition failed: "));
+ }
+ std::string Rule = "legalize.";
+ Rule.append(legalizationActionName(Decision->Action));
+ Rule.push_back('.');
+ Rule.append(Descriptor->Name);
+ auto Accepted = Analyses.requestAction(Rule, Function, Instruction->stableID());
+ if (!Accepted)
+ return std::unexpected(Accepted.error());
+ if (!*Accepted) {
+ return std::unexpected(
+ passFailure("required legalizer action was rejected for '" + Descriptor->Name + "'",
+ ErrorCode::VerificationError));
+ }
+ Expected<void> Lowered = Helper.legalize(*Instruction, *Decision);
+ if (!Lowered) {
+ if (Lowered.error().code() == ErrorCode::Unsupported && Instruction->parent() != nullptr &&
+ isArtifact(*Instruction)) {
+ RetryList.push_back(Instruction);
+ continue;
+ }
+ return std::unexpected(Lowered.error());
+ }
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ }
+
+ if (!RetryList.empty()) {
+ if (!ArtifactWorklist.empty()) {
+ for (MachineInstr *Instruction : RetryList)
+ ArtifactWorklist.insert(Instruction);
+ RetryList.clear();
+ } else {
+ MachineInstr *Instruction = RetryList.front();
+ const InstructionDescriptor *Descriptor =
+ Instruction == nullptr ? nullptr : Target.instructionDescriptor(Instruction->opcode());
+ return std::unexpected(passFailure(
+ "target legalizer rejected generic artifact '" +
+ std::string(Descriptor == nullptr ? "<unknown>" : Descriptor->Name) + "'",
+ ErrorCode::Unsupported));
+ }
+ }
+
+ while (!ArtifactWorklist.empty()) {
+ MachineInstr *Instruction = ArtifactWorklist.popBack();
+ if (Instruction == nullptr || Instruction->parent() == nullptr)
+ continue;
+ if (isTriviallyDead(*Instruction, Function.registerInfo(), Target)) {
+ auto Removed = eraseInstr(*Instruction, Function.registerInfo(), Target);
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ continue;
+ }
+ std::vector<MachineInstr *> DeadInstructions;
+ auto Combined =
+ ArtifactCombiner.tryCombineInstruction(*Instruction, DeadInstructions, Observer);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ if (*Combined) {
+ auto Removed = eraseInstrs(DeadInstructions, Function.registerInfo(), Target);
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ continue;
+ }
+ InstructionWorklist.insert(Instruction);
+ }
+ } while (!InstructionWorklist.empty());
+
+ auto CSEValidation = CSEInfo.verify();
+ if (!CSEValidation) {
+ return std::unexpected(
+ withErrorContext(CSEValidation.error(), "GlobalISel CSE verification failed: "));
+ }
+
+ for (MachineInstr *Instruction : layoutInstructions(Function)) {
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode());
+ if (Descriptor == nullptr)
+ return std::unexpected(passFailure("legalizer produced an unknown instruction descriptor"));
+ if (!Descriptor->Flags.has(InstructionFlag::Generic))
+ continue;
+ auto Decision = Info->getDecision(*Instruction, Function, Target);
+ if (!Decision)
+ return std::unexpected(Decision.error());
+ if (Decision->Action != LegalizationAction::Legal &&
+ Decision->Action != LegalizationAction::Custom)
+ return std::unexpected(passFailure(
+ "legalizer reached a fixed point with an illegal generic instruction '" +
+ Descriptor->Name + "'",
+ Decision->Action == LegalizationAction::Unsupported ? ErrorCode::Unsupported
+ : ErrorCode::InternalError));
+ auto FinalValidation = Target.validateInstruction(*Instruction, Function.registerInfo());
+ if (!FinalValidation)
+ return std::unexpected(
+ withErrorContext(FinalValidation.error(), "legalizer produced invalid MachineIR: "));
+ }
+ auto Liveness = clearTrackedLivenessFlags(Function, Result);
+ if (!Liveness)
+ return std::unexpected(
+ withErrorContext(Liveness.error(), "legalizer liveness-flag invalidation failed: "));
+ Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
+ return Result;
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
new file mode 100644
index 0000000..8bf6e80
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -0,0 +1,7120 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h"
+
+#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "rcc/CodeGen/GlobalISel/Utils.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetCallLowering.h"
+
+#include <algorithm>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <initializer_list>
+#include <limits>
+#include <numeric>
+#include <optional>
+#include <span>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace rcc {
+namespace {
+
+struct ScalarLayout final {
+ std::uint64_t StoreSize = 0;
+ std::uint64_t AllocationSize = 0;
+ std::uint64_t ABIAlignment = 0;
+};
+
+[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) {
+ if (const std::int64_t *Value = Operand.getIf<std::int64_t>())
+ return *Value;
+ const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+ if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "legalizer requires an immediate no wider than 64 bits");
+ }
+ std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front();
+ const std::uint32_t Width = Integer->Value.bitWidth();
+ if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0)
+ Raw |= ~std::uint64_t{0} << Width;
+ return std::bit_cast<std::int64_t>(Raw);
+}
+
+[[nodiscard]] std::optional<ScalarLayout> scalarLayout(LowLevelType Type) noexcept {
+ if (!Type.isScalarValue() || Type.scalarBitWidth() == 0)
+ return std::nullopt;
+ const std::uint64_t StoreSize = (Type.scalarBitWidth() + 7U) / 8U;
+ const std::uint64_t AllocationSize = std::bit_ceil(StoreSize);
+ return ScalarLayout{StoreSize, AllocationSize, std::min<std::uint64_t>(AllocationSize, 8)};
+}
+
+[[nodiscard]] Expected<const MachineMemOperand *> createMemoryAccess(MachineFunction &Function,
+ MemoryAccessFlag Access,
+ LowLevelType Type,
+ const ScalarLayout &Layout) {
+ auto Size = MemorySize::fixed(Layout.StoreSize);
+ auto Align = Alignment::create(Layout.ABIAlignment);
+ if (!Size || !Align)
+ return makeUnexpected(ErrorCode::InternalError,
+ "legalizer cannot represent scalar memory layout");
+ MachineMemOperandDesc Description;
+ Description.Accesses = {Access};
+ Description.Size = *Size;
+ Description.MemoryType = Type;
+ Description.Align = *Align;
+ return Function.createMemOperand(std::move(Description));
+}
+
+[[nodiscard]] Expected<Register> resultRegister(MachineInstr &Instruction) {
+ const MachineOperand *Definition = Instruction.operandAt(0);
+ const Register *Result = Definition == nullptr ? nullptr : Definition->getIf<Register>();
+ if (Result == nullptr || !Definition->isDef()) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "machine IR builder returned an instruction without a register result");
+ }
+ return *Result;
+}
+
+[[nodiscard]] Expected<void> setInsertPointAtFirstTerminator(MachineIRBuilder &Builder,
+ MachineBasicBlock &Block) {
+ for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ const InstructionDescriptor *Descriptor =
+ Builder.target().instructionDescriptor(Instruction->opcode());
+ if (Descriptor == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "PHI predecessor contains an unknown instruction");
+ if (Descriptor->Flags.has(InstructionFlag::Terminator))
+ return Builder.setInsertPointBefore(*Instruction);
+ }
+ return Builder.setInsertPoint(Block);
+}
+
+[[nodiscard]] Expected<void> setInsertPointAfterPHIs(MachineIRBuilder &Builder,
+ MachineBasicBlock &Block) {
+ for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ const InstructionDescriptor *Descriptor =
+ Builder.target().instructionDescriptor(Instruction->opcode());
+ if (Descriptor == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "PHI block contains an unknown instruction");
+ if (!Descriptor->Flags.has(InstructionFlag::Phi) &&
+ !Descriptor->Flags.has(InstructionFlag::Meta)) {
+ return Builder.setInsertPointBefore(*Instruction);
+ }
+ }
+ return Builder.setInsertPoint(Block);
+}
+
+[[nodiscard]] Expected<Register> buildConstantRegister(MachineIRBuilder &Builder, LowLevelType Type,
+ APInt Value) {
+ auto Built = Builder.buildConstant(Type, std::move(Value));
+ if (!Built)
+ return std::unexpected(Built.error());
+ return resultRegister(**Built);
+}
+
+[[nodiscard]] Expected<Register> buildConstantRegister(MachineIRBuilder &Builder, LowLevelType Type,
+ std::int64_t Value) {
+ auto Immediate = APInt::fromSigned(Type.scalarBitWidth(), Value);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ return buildConstantRegister(Builder, Type, std::move(*Immediate));
+}
+
+[[nodiscard]] Expected<Register> buildResultInstruction(MachineIRBuilder &Builder,
+ GenericOpcode Generic, LowLevelType Type,
+ std::initializer_list<Register> Sources) {
+ auto Built = Builder.buildInstr(opcode(Generic), {Type}, Sources);
+ if (!Built)
+ return std::unexpected(Built.error());
+ return resultRegister(**Built);
+}
+
+[[nodiscard]] Expected<Register>
+buildFlaggedBinaryInstruction(MachineIRBuilder &Builder, GenericOpcode Generic, LowLevelType Type,
+ Register Left, Register Right, MachineInstrFlags Flags) {
+ auto Destination = Builder.function().registerInfo().createVirtualRegister(Type);
+ if (!Destination)
+ return std::unexpected(Destination.error());
+ auto Pending = Builder.buildInstr(opcode(Generic));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ if (auto Added = Pending->addDef(*Destination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Pending->addUse(Left); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Pending->addUse(Right); !Added)
+ return std::unexpected(Added.error());
+ if (auto Set = Pending->setFlags(Flags); !Set)
+ return std::unexpected(Set.error());
+ if (auto Built = Pending->build(); !Built)
+ return std::unexpected(Built.error());
+ return *Destination;
+}
+
+[[nodiscard]] Expected<std::int64_t> addMemoryOffset(std::int64_t Offset,
+ std::uint64_t ByteOffset) {
+ if (ByteOffset > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory split offset exceeds the representable machine memory range");
+ }
+ const auto SignedByteOffset = static_cast<std::int64_t>(ByteOffset);
+ if (Offset > std::numeric_limits<std::int64_t>::max() - SignedByteOffset) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory split offset exceeds the representable machine memory range");
+ }
+ return Offset + SignedByteOffset;
+}
+
+[[nodiscard]] Expected<const MachineMemOperand *>
+deriveMemoryOperand(MachineFunction &Function, const MachineMemOperand &Original,
+ LowLevelType MemoryType, std::uint64_t ByteOffset) {
+ if (!MemoryType.isScalarValue() || MemoryType.minimumBitWidth() == 0 ||
+ MemoryType.minimumBitWidth() % 8U != 0) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory splitting requires a byte-sized scalar memory type");
+ }
+
+ const std::uint64_t ByteSize = MemoryType.minimumBitWidth() / 8U;
+ auto Size = MemorySize::fixed(ByteSize);
+ if (!Size)
+ return std::unexpected(Size.error());
+
+ MachineMemOperandDesc Description = Original.description();
+ auto Offset = addMemoryOffset(Description.Offset, ByteOffset);
+ if (!Offset)
+ return std::unexpected(Offset.error());
+ Description.Offset = *Offset;
+ if (Description.OffsetRange) {
+ auto Lower = addMemoryOffset(Description.OffsetRange->Lower, ByteOffset);
+ auto Upper = addMemoryOffset(Description.OffsetRange->Upper, ByteOffset);
+ if (!Lower)
+ return std::unexpected(Lower.error());
+ if (!Upper)
+ return std::unexpected(Upper.error());
+ Description.OffsetRange = MemoryOffsetRange{*Lower, *Upper};
+ }
+
+ const std::optional<Alignment> BaseAlignment = Description.BaseAlign;
+ std::uint64_t EffectiveAlignment = BaseAlignment.value_or(Description.Align).bytes();
+ if (Description.Offset != 0) {
+ const auto OffsetBits = static_cast<std::uint64_t>(Description.Offset);
+ const std::uint64_t OffsetAlignment = std::uint64_t{1} << std::countr_zero(OffsetBits);
+ EffectiveAlignment = std::min(EffectiveAlignment, OffsetAlignment);
+ }
+ auto AlignmentValue = Alignment::create(EffectiveAlignment);
+ if (!AlignmentValue)
+ return std::unexpected(AlignmentValue.error());
+
+ Description.Size = *Size;
+ Description.MemoryType = MemoryType;
+ Description.Align = *AlignmentValue;
+ Description.BaseAlign = BaseAlignment;
+ return Function.createMemOperand(std::move(Description));
+}
+
+[[nodiscard]] Expected<Register>
+buildObjectPointerOffset(MachineIRBuilder &Builder, Register Pointer, std::uint64_t ByteOffset) {
+ if (ByteOffset == 0)
+ return Pointer;
+
+ auto PointerType = Builder.function().registerInfo().getType(Pointer);
+ if (!PointerType)
+ return std::unexpected(PointerType.error());
+ if (!PointerType->isPointer()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "memory splitting requires a pointer address operand");
+ }
+ const std::uint32_t PointerBits = PointerType->scalarBitWidth();
+ if (PointerBits < 64U && (ByteOffset >> PointerBits) != 0) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory split offset does not fit the pointer index type");
+ }
+ auto OffsetType = LowLevelType::makeCategoryNeutralScalar(PointerBits);
+ if (!OffsetType)
+ return std::unexpected(OffsetType.error());
+ auto OffsetValue = APInt::fromUnsigned(PointerBits, ByteOffset);
+ if (!OffsetValue)
+ return std::unexpected(OffsetValue.error());
+ auto Offset = buildConstantRegister(Builder, *OffsetType, std::move(*OffsetValue));
+ if (!Offset)
+ return std::unexpected(Offset.error());
+ return buildFlaggedBinaryInstruction(
+ Builder, GenericOpcode::G_PTR_ADD, *PointerType, Pointer, *Offset,
+ MachineInstrFlags{MachineInstrFlag::NoUnsignedWrap, MachineInstrFlag::InBounds});
+}
+
+[[nodiscard]] Expected<APInt> repeatedByte(std::uint32_t BitWidth, std::uint8_t Byte) {
+ if (BitWidth == 0 || BitWidth > APInt::MaxBitWidth || BitWidth % 8U != 0)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "repeated-byte integer has an invalid bit width");
+ constexpr std::uint64_t ByteSplat = 0x0101010101010101ULL;
+ std::vector<std::uint64_t> Words((static_cast<std::size_t>(BitWidth) + 63U) / 64U,
+ ByteSplat * Byte);
+ const std::uint32_t HighBits = BitWidth % 64U;
+ if (HighBits != 0)
+ Words.back() &= (std::uint64_t{1} << HighBits) - 1U;
+ return APInt::create(BitWidth, Words);
+}
+
+[[nodiscard]] Expected<APInt> oneBitSet(std::uint32_t BitWidth, std::uint32_t Bit) {
+ if (BitWidth == 0 || BitWidth > APInt::MaxBitWidth || Bit >= BitWidth)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "single-bit integer has an invalid bit position");
+ std::vector<std::uint64_t> Words((static_cast<std::size_t>(BitWidth) + 63U) / 64U, 0);
+ Words[Bit / 64U] = std::uint64_t{1} << (Bit % 64U);
+ return APInt::create(BitWidth, Words);
+}
+
+[[nodiscard]] Expected<Register> buildICmpRegister(MachineIRBuilder &Builder,
+ IntegerPredicate Predicate,
+ LowLevelType ResultType, Register Left,
+ Register Right) {
+ auto Built = Builder.buildICmp(Predicate, ResultType, Left, Right);
+ if (!Built)
+ return std::unexpected(Built.error());
+ return resultRegister(**Built);
+}
+
+[[nodiscard]] IntegerPredicate minMaxToCompare(GenericOpcode Generic) {
+ switch (Generic) {
+ case GenericOpcode::G_SMIN:
+ return IntegerPredicate::SignedLessThan;
+ case GenericOpcode::G_SMAX:
+ return IntegerPredicate::SignedGreaterThan;
+ case GenericOpcode::G_UMIN:
+ return IntegerPredicate::UnsignedLessThan;
+ case GenericOpcode::G_UMAX:
+ return IntegerPredicate::UnsignedGreaterThan;
+ default:
+ std::unreachable();
+ }
+}
+
+[[nodiscard]] bool isEqualityPredicate(IntegerPredicate Predicate) noexcept {
+ return Predicate == IntegerPredicate::Equal || Predicate == IntegerPredicate::NotEqual;
+}
+
+[[nodiscard]] Expected<IntegerPredicate> unsignedPredicate(IntegerPredicate Predicate) {
+ switch (Predicate) {
+ case IntegerPredicate::Equal:
+ case IntegerPredicate::NotEqual:
+ case IntegerPredicate::UnsignedGreaterThan:
+ case IntegerPredicate::UnsignedGreaterEqual:
+ case IntegerPredicate::UnsignedLessThan:
+ case IntegerPredicate::UnsignedLessEqual:
+ return Predicate;
+ case IntegerPredicate::SignedGreaterThan:
+ return IntegerPredicate::UnsignedGreaterThan;
+ case IntegerPredicate::SignedGreaterEqual:
+ return IntegerPredicate::UnsignedGreaterEqual;
+ case IntegerPredicate::SignedLessThan:
+ return IntegerPredicate::UnsignedLessThan;
+ case IntegerPredicate::SignedLessEqual:
+ return IntegerPredicate::UnsignedLessEqual;
+ case IntegerPredicate::Count:
+ return makeUnexpected(ErrorCode::VerificationError,
+ "integer comparison has an invalid predicate");
+ }
+ return makeUnexpected(ErrorCode::VerificationError,
+ "integer comparison has an invalid predicate");
+}
+
+[[nodiscard]] Expected<LowLevelType> booleanTypeFor(LowLevelType Type) {
+ auto BooleanElementType = LowLevelType::makeCategoryNeutralScalar(1);
+ if (!BooleanElementType)
+ return std::unexpected(BooleanElementType.error());
+ if (!Type.isVector())
+ return *BooleanElementType;
+ if (Type.isScalableVector())
+ return LowLevelType::makeScalableVector(Type.minimumElementCount(), *BooleanElementType);
+ return LowLevelType::makeFixedVector(Type.minimumElementCount(), *BooleanElementType);
+}
+
+[[nodiscard]] Expected<Register> buildConstantBefore(MachineIRBuilder &Builder,
+ MachineInstr &Anchor, LowLevelType Type,
+ std::uint64_t Value) {
+ auto Immediate = APInt::fromUnsigned(Type.scalarBitWidth(), Value);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ if (auto Point = Builder.setInsertPointBefore(Anchor); !Point)
+ return std::unexpected(Point.error());
+ return buildConstantRegister(Builder, Type, std::move(*Immediate));
+}
+
+[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
+ GenericOpcode Generic, LowLevelType Type,
+ Register Left, Register Right) {
+ auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
+ if (!Result)
+ return std::unexpected(Result.error());
+ if (auto Point = Builder.setInsertPointBefore(Anchor); !Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(opcode(Generic));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ if (auto Added = Pending->addDef(*Result); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Pending->addUse(Left); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Pending->addUse(Right); !Added)
+ return std::unexpected(Added.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return *Result;
+}
+
+[[nodiscard]] Expected<std::string> integerLibcallName(GenericOpcode Generic,
+ std::uint32_t BitWidth) {
+ std::string_view Prefix;
+ switch (Generic) {
+ case GenericOpcode::G_MUL:
+ Prefix = "__mul";
+ break;
+ case GenericOpcode::G_SDIV:
+ Prefix = "__div";
+ break;
+ case GenericOpcode::G_UDIV:
+ Prefix = "__udiv";
+ break;
+ case GenericOpcode::G_SREM:
+ Prefix = "__mod";
+ break;
+ case GenericOpcode::G_UREM:
+ Prefix = "__umod";
+ break;
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "integer libcall has no runtime function mapping");
+ }
+
+ std::string_view Suffix;
+ switch (BitWidth) {
+ case 32:
+ Suffix = "si3";
+ break;
+ case 64:
+ Suffix = "di3";
+ break;
+ case 128:
+ Suffix = "ti3";
+ break;
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "integer libcall requires a 32, 64, or 128-bit scalar");
+ }
+ return std::string(Prefix) + std::string(Suffix);
+}
+
+[[nodiscard]] ABIExtensionKind integerLibcallExtension(GenericOpcode Generic,
+ std::uint32_t BitWidth) noexcept {
+ if (BitWidth >= 64)
+ return ABIExtensionKind::None;
+ if (Generic == GenericOpcode::G_SDIV || Generic == GenericOpcode::G_SREM)
+ return ABIExtensionKind::SignExtend;
+ if (Generic == GenericOpcode::G_UDIV || Generic == GenericOpcode::G_UREM)
+ return ABIExtensionKind::ZeroExtend;
+ return ABIExtensionKind::AnyExtend;
+}
+
+[[nodiscard]] Expected<std::string_view> integerLibcallTypeCode(std::uint32_t BitWidth) {
+ switch (BitWidth) {
+ case 32:
+ return "si";
+ case 64:
+ return "di";
+ case 128:
+ return "ti";
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "conversion libcall requires a 32, 64, or 128-bit integer");
+ }
+}
+
+[[nodiscard]] Expected<std::string_view> floatingLibcallTypeCode(std::uint32_t BitWidth) {
+ switch (BitWidth) {
+ case 32:
+ return "sf";
+ case 64:
+ return "df";
+ case 128:
+ return "tf";
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "conversion libcall requires a 32, 64, or 128-bit float");
+ }
+}
+
+[[nodiscard]] Expected<std::string> floatingBinaryLibcallName(GenericOpcode Generic,
+ std::uint32_t BitWidth) {
+ if (Generic != GenericOpcode::G_FREM) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "floating libcall has no runtime function mapping");
+ }
+ switch (BitWidth) {
+ case 32:
+ return "fmodf";
+ case 64:
+ return "fmod";
+ case 128:
+ return "fmodl";
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "floating libcall requires a 32, 64, or 128-bit scalar");
+ }
+}
+
+[[nodiscard]] Expected<std::string> conversionLibcallName(GenericOpcode Generic,
+ std::uint32_t DestinationWidth,
+ std::uint32_t SourceWidth) {
+ const bool DestinationIsFloat =
+ Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP ||
+ Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC;
+ const bool SourceIsFloat =
+ Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI ||
+ Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC;
+ auto DestinationCode = DestinationIsFloat ? floatingLibcallTypeCode(DestinationWidth)
+ : integerLibcallTypeCode(DestinationWidth);
+ auto SourceCode =
+ SourceIsFloat ? floatingLibcallTypeCode(SourceWidth) : integerLibcallTypeCode(SourceWidth);
+ if (!DestinationCode)
+ return std::unexpected(DestinationCode.error());
+ if (!SourceCode)
+ return std::unexpected(SourceCode.error());
+
+ std::string Name;
+ switch (Generic) {
+ case GenericOpcode::G_FPEXT:
+ Name = "__extend";
+ break;
+ case GenericOpcode::G_FPTRUNC:
+ Name = "__trunc";
+ break;
+ case GenericOpcode::G_FPTOSI:
+ Name = "__fix";
+ break;
+ case GenericOpcode::G_FPTOUI:
+ Name = "__fixuns";
+ break;
+ case GenericOpcode::G_SITOFP:
+ Name = "__float";
+ break;
+ case GenericOpcode::G_UITOFP:
+ Name = "__floatun";
+ break;
+ default:
+ return makeUnexpected(ErrorCode::Unsupported, "opcode has no conversion libcall mapping");
+ }
+ Name += *SourceCode;
+ Name += *DestinationCode;
+ if (Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC)
+ Name += '2';
+ return Name;
+}
+
+[[nodiscard]] Expected<ABIValue>
+libcallABIValue(Register Value, LowLevelType Type, ABIValueCategory Category,
+ std::uint32_t OriginalValue, std::uint16_t FragmentIndex,
+ std::uint16_t FragmentCount, std::uint32_t FragmentOffsetBytes,
+ Alignment OriginalAlignment, ABIExtensionKind Extension) {
+ auto OneByte = Alignment::create(1);
+ if (!OneByte)
+ return std::unexpected(OneByte.error());
+ return ABIValue{
+ .Value = Value,
+ .Type = Type,
+ .Category = Category,
+ .OriginalValue = OriginalValue,
+ .FragmentIndex = FragmentIndex,
+ .FragmentCount = FragmentCount,
+ .FragmentOffsetBytes = FragmentOffsetBytes,
+ .OriginalAlignment = OriginalAlignment,
+ .Extension = Extension,
+ .Flags = {},
+ .ByValSizeBytes = 0,
+ .ByValAlignment = *OneByte,
+ };
+}
+
+} // namespace
+
+LegalizerHelper::LegalizerHelper(MachineFunction &Function, const LegalizerInfo &Info,
+ GISelChangeObserver &Observer, MachineIRBuilder &MIRBuilder,
+ const LibcallLoweringInfo *Libcalls)
+ : MIRBuilder(MIRBuilder), Observer(Observer), Function(Function), MRI(Function.registerInfo()),
+ Info(Info), Libcalls(Libcalls) {}
+
+Expected<void> LegalizerHelper::widenScalarMergeValues(MachineInstr &Instruction,
+ std::size_t TypeIndex,
+ LowLevelType WideType) {
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_MERGE_VALUES widening requires type index one");
+ if (Instruction.operandCount() < 3)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_MERGE_VALUES requires a destination and sources");
+
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ const Register *FirstSource = Instruction.operandAt(1)->getIf<Register>();
+ if (Destination == nullptr || FirstSource == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_MERGE_VALUES requires register operands");
+ auto DestinationType = MRI.getType(*Destination);
+ auto SourceType = MRI.getType(*FirstSource);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ if (DestinationType->isVector())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "vector G_MERGE_VALUES widening is not supported");
+
+ const std::uint64_t DestinationSize = DestinationType->minimumBitWidth();
+ const std::uint64_t SourceSize = SourceType->minimumBitWidth();
+ const std::uint64_t WideSize = WideType.minimumBitWidth();
+ const std::uint64_t MergeCount = (DestinationSize + WideSize - 1U) / WideSize;
+ const std::size_t SourceCount = Instruction.operandCount() - 1U;
+ const std::uint64_t PartSize = DestinationSize / SourceCount;
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ if (WideSize >= DestinationSize) {
+ auto BuiltResult = MIRBuilder.buildZExt(WideType, *FirstSource);
+ if (!BuiltResult)
+ return std::unexpected(BuiltResult.error());
+ auto Result = resultRegister(**BuiltResult);
+ if (!Result)
+ return std::unexpected(Result.error());
+ Register ResultRegister = *Result;
+
+ for (std::size_t Index = 2; Index != Instruction.operandCount(); ++Index) {
+ const Register *Source = Instruction.operandAt(Index)->getIf<Register>();
+ if (Source == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_MERGE_VALUES requires register sources");
+ auto ActualSourceType = MRI.getType(*Source);
+ if (!ActualSourceType)
+ return std::unexpected(ActualSourceType.error());
+ if (*ActualSourceType != *SourceType || ActualSourceType->minimumBitWidth() != PartSize)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_MERGE_VALUES source types are inconsistent");
+
+ auto BuiltExtendedInput = MIRBuilder.buildZExt(WideType, *Source);
+ if (!BuiltExtendedInput)
+ return std::unexpected(BuiltExtendedInput.error());
+ auto ExtendedInput = resultRegister(**BuiltExtendedInput);
+ if (!ExtendedInput)
+ return std::unexpected(ExtendedInput.error());
+ auto ShiftAmount = buildConstantRegister(MIRBuilder, WideType,
+ static_cast<std::int64_t>((Index - 1U) * PartSize));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto BuiltShiftedInput = MIRBuilder.buildShl(WideType, *ExtendedInput, *ShiftAmount);
+ if (!BuiltShiftedInput)
+ return std::unexpected(BuiltShiftedInput.error());
+ auto ShiftedInput = resultRegister(**BuiltShiftedInput);
+ if (!ShiftedInput)
+ return std::unexpected(ShiftedInput.error());
+
+ Register NextResult = *Destination;
+ if (Index + 1U != Instruction.operandCount() || WideType != *DestinationType) {
+ auto NewResult = MRI.createVirtualRegister(WideType);
+ if (!NewResult)
+ return std::unexpected(NewResult.error());
+ NextResult = *NewResult;
+ }
+ if (auto Built = MIRBuilder.buildOr(NextResult, ResultRegister, *ShiftedInput); !Built)
+ return std::unexpected(Built.error());
+ ResultRegister = NextResult;
+ }
+
+ if (WideSize > DestinationSize) {
+ if (auto Built = MIRBuilder.buildTrunc(*Destination, ResultRegister); !Built)
+ return std::unexpected(Built.error());
+ } else if (DestinationType->isPointer()) {
+ if (auto Built = MIRBuilder.buildIntToPtr(*Destination, ResultRegister); !Built)
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+ }
+
+ auto GCDType = LowLevelType::makeCategoryNeutralScalar(std::gcd(SourceSize, WideSize));
+ auto WideDestinationType = LowLevelType::makeCategoryNeutralScalar(MergeCount * WideSize);
+ if (!GCDType)
+ return std::unexpected(GCDType.error());
+ if (!WideDestinationType)
+ return std::unexpected(WideDestinationType.error());
+
+ std::vector<Register> UnmergedRegisters;
+ for (std::size_t Index = 1; Index != Instruction.operandCount(); ++Index) {
+ const Register *Source = Instruction.operandAt(Index)->getIf<Register>();
+ if (Source == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_MERGE_VALUES requires register sources");
+ if (GCDType->minimumBitWidth() == SourceSize) {
+ UnmergedRegisters.push_back(*Source);
+ continue;
+ }
+ auto Extracted = extractGCDType(UnmergedRegisters, *GCDType, *Source);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ }
+
+ const std::uint64_t RequiredPartCount = MergeCount * WideSize;
+ if (UnmergedRegisters.size() != RequiredPartCount) {
+ auto BuiltUndef = MIRBuilder.buildUndef(*GCDType);
+ if (!BuiltUndef)
+ return std::unexpected(BuiltUndef.error());
+ auto UndefRegister = resultRegister(**BuiltUndef);
+ if (!UndefRegister)
+ return std::unexpected(UndefRegister.error());
+ while (UnmergedRegisters.size() != RequiredPartCount)
+ UnmergedRegisters.push_back(*UndefRegister);
+ }
+
+ const std::size_t PartsPerGCD = static_cast<std::size_t>(WideSize / GCDType->minimumBitWidth());
+ std::vector<Register> NewMergeRegisters;
+ NewMergeRegisters.reserve(static_cast<std::size_t>(MergeCount));
+ for (std::size_t Index = 0; Index != MergeCount; ++Index) {
+ std::vector<Register> Parts(
+ UnmergedRegisters.begin() + static_cast<std::ptrdiff_t>(Index * PartsPerGCD),
+ UnmergedRegisters.begin() + static_cast<std::ptrdiff_t>((Index + 1U) * PartsPerGCD));
+ auto BuiltMerge = MIRBuilder.buildMergeLikeInstr(WideType, Parts);
+ if (!BuiltMerge)
+ return std::unexpected(BuiltMerge.error());
+ auto MergeRegister = resultRegister(**BuiltMerge);
+ if (!MergeRegister)
+ return std::unexpected(MergeRegister.error());
+ NewMergeRegisters.push_back(*MergeRegister);
+ }
+
+ if (DestinationSize == WideDestinationType->minimumBitWidth()) {
+ if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, NewMergeRegisters); !Built)
+ return std::unexpected(Built.error());
+ } else {
+ auto BuiltFinalMerge = MIRBuilder.buildMergeLikeInstr(*WideDestinationType, NewMergeRegisters);
+ if (!BuiltFinalMerge)
+ return std::unexpected(BuiltFinalMerge.error());
+ auto FinalMerge = resultRegister(**BuiltFinalMerge);
+ if (!FinalMerge)
+ return std::unexpected(FinalMerge.error());
+ if (auto Built = MIRBuilder.buildTrunc(*Destination, *FinalMerge); !Built)
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::widenScalarUnmergeValues(MachineInstr &Instruction,
+ std::size_t TypeIndex,
+ LowLevelType WideType) {
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_UNMERGE_VALUES widening requires type index zero");
+ if (Instruction.operandCount() < 2)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_UNMERGE_VALUES requires destinations and a source");
+
+ const std::size_t DestinationCount = Instruction.operandCount() - 1U;
+ const Register *Source = Instruction.operandAt(DestinationCount)->getIf<Register>();
+ const Register *FirstDestination = Instruction.operandAt(0)->getIf<Register>();
+ if (Source == nullptr || FirstDestination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_UNMERGE_VALUES requires register operands");
+ auto SourceType = MRI.getType(*Source);
+ auto DestinationType = MRI.getType(*FirstDestination);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (SourceType->isVector() || !DestinationType->isCategoryNeutralScalar())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_UNMERGE_VALUES widening requires scalar types");
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ Register SourceRegister = *Source;
+ if (WideType.minimumBitWidth() >= SourceType->minimumBitWidth()) {
+ if (SourceType->isPointer()) {
+ auto IntegerSourceType =
+ LowLevelType::makeCategoryNeutralScalar(SourceType->minimumBitWidth());
+ if (!IntegerSourceType)
+ return std::unexpected(IntegerSourceType.error());
+ auto IntegerSource = MRI.createVirtualRegister(*IntegerSourceType);
+ if (!IntegerSource)
+ return std::unexpected(IntegerSource.error());
+ if (auto Built = MIRBuilder.buildPtrToInt(*IntegerSource, SourceRegister); !Built)
+ return std::unexpected(Built.error());
+ *SourceType = *IntegerSourceType;
+ SourceRegister = *IntegerSource;
+ }
+
+ if (WideType.minimumBitWidth() > SourceType->minimumBitWidth()) {
+ auto WideSource = MRI.createVirtualRegister(WideType);
+ if (!WideSource)
+ return std::unexpected(WideSource.error());
+ if (auto Built = MIRBuilder.buildAnyExt(*WideSource, SourceRegister); !Built)
+ return std::unexpected(Built.error());
+ *SourceType = WideType;
+ SourceRegister = *WideSource;
+ }
+
+ const std::uint64_t DestinationSize = DestinationType->minimumBitWidth();
+ if (auto Built = MIRBuilder.buildTrunc(*FirstDestination, SourceRegister); !Built)
+ return std::unexpected(Built.error());
+ for (std::size_t Index = 1; Index != DestinationCount; ++Index) {
+ const Register *Destination = Instruction.operandAt(Index)->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_UNMERGE_VALUES requires register destinations");
+ auto ShiftedSource = MRI.createVirtualRegister(*SourceType);
+ if (!ShiftedSource)
+ return std::unexpected(ShiftedSource.error());
+ auto ShiftAmount = buildConstantRegister(MIRBuilder, *SourceType,
+ static_cast<std::int64_t>(DestinationSize * Index));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ if (auto Built = MIRBuilder.buildLShr(*ShiftedSource, SourceRegister, *ShiftAmount); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildTrunc(*Destination, *ShiftedSource); !Built)
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+ }
+
+ auto LCMType = getLCMType(*SourceType, WideType);
+ if (!LCMType)
+ return std::unexpected(LCMType.error());
+ Register WideSource = SourceRegister;
+ if (LCMType->minimumBitWidth() != SourceType->minimumBitWidth()) {
+ if (SourceType->isPointer())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "widening a pointer G_UNMERGE_VALUES source is not implemented");
+ auto ExtendedSource = MRI.createVirtualRegister(*LCMType);
+ if (!ExtendedSource)
+ return std::unexpected(ExtendedSource.error());
+ if (auto Built = MIRBuilder.buildAnyExt(*ExtendedSource, WideSource); !Built)
+ return std::unexpected(Built.error());
+ WideSource = *ExtendedSource;
+ }
+
+ const std::size_t UnmergeCount =
+ static_cast<std::size_t>(LCMType->minimumBitWidth() / WideType.minimumBitWidth());
+ std::vector<Register> UnmergeRegisters;
+ UnmergeRegisters.reserve(UnmergeCount);
+ for (std::size_t Index = 0; Index != UnmergeCount; ++Index) {
+ auto Part = MRI.createVirtualRegister(WideType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ UnmergeRegisters.push_back(*Part);
+ }
+ if (auto Built = MIRBuilder.buildUnmergeValues(UnmergeRegisters, WideSource); !Built)
+ return std::unexpected(Built.error());
+
+ auto GCDType = getGCDType(WideType, *DestinationType);
+ if (!GCDType)
+ return std::unexpected(GCDType.error());
+ const std::size_t PartsPerRemerge =
+ static_cast<std::size_t>(DestinationType->minimumBitWidth() / GCDType->minimumBitWidth());
+
+ if (PartsPerRemerge == 1) {
+ const std::size_t PartsPerUnmerge =
+ static_cast<std::size_t>(WideType.minimumBitWidth() / DestinationType->minimumBitWidth());
+ for (std::size_t Index = 0; Index != UnmergeRegisters.size(); ++Index) {
+ std::vector<Register> Destinations;
+ Destinations.reserve(PartsPerUnmerge);
+ for (std::size_t PartIndex = 0; PartIndex != PartsPerUnmerge; ++PartIndex) {
+ const std::size_t DestinationIndex = Index * PartsPerUnmerge + PartIndex;
+ if (DestinationIndex < DestinationCount) {
+ const Register *Destination = Instruction.operandAt(DestinationIndex)->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_UNMERGE_VALUES requires register destinations");
+ Destinations.push_back(*Destination);
+ } else {
+ auto DeadDestination = MRI.createVirtualRegister(*DestinationType);
+ if (!DeadDestination)
+ return std::unexpected(DeadDestination.error());
+ Destinations.push_back(*DeadDestination);
+ }
+ }
+ if (auto Built = MIRBuilder.buildUnmergeValues(Destinations, UnmergeRegisters[Index]); !Built)
+ return std::unexpected(Built.error());
+ }
+ } else {
+ std::vector<Register> Parts;
+ for (Register UnmergeRegister : UnmergeRegisters) {
+ auto Extracted = extractGCDType(Parts, *GCDType, UnmergeRegister);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ }
+ for (std::size_t Index = 0; Index != DestinationCount; ++Index) {
+ const Register *Destination = Instruction.operandAt(Index)->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_UNMERGE_VALUES requires register destinations");
+ std::vector<Register> RemergeParts(
+ Parts.begin() + static_cast<std::ptrdiff_t>(Index * PartsPerRemerge),
+ Parts.begin() + static_cast<std::ptrdiff_t>((Index + 1U) * PartsPerRemerge));
+ if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, RemergeParts); !Built)
+ return std::unexpected(Built.error());
+ }
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::insertParts(Register Destination, LowLevelType ResultType,
+ LowLevelType PartType,
+ const std::vector<Register> &PartRegisters,
+ LowLevelType LeftoverType,
+ const std::vector<Register> &LeftoverRegisters) {
+ if (!LeftoverType.isValid()) {
+ if (!LeftoverRegisters.empty())
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "leftover registers require a leftover type");
+ return MIRBuilder.buildMergeLikeInstr(Destination, PartRegisters).transform([](MachineInstr *) {
+ });
+ }
+
+ if (ResultType.isVector()) {
+ if (LeftoverRegisters.size() != 1)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "mixed subvector merge requires one leftover register");
+ std::vector<Register> AllRegisters = PartRegisters;
+ AllRegisters.insert(AllRegisters.end(), LeftoverRegisters.begin(), LeftoverRegisters.end());
+ return MIRBuilder.buildMergeLikeInstr(Destination, AllRegisters).transform([](MachineInstr *) {
+ });
+ }
+
+ std::vector<Register> GCDRegisters;
+ auto FirstGCD = getGCDType(ResultType, LeftoverType);
+ if (!FirstGCD)
+ return std::unexpected(FirstGCD.error());
+ auto GCDType = getGCDType(*FirstGCD, PartType);
+ if (!GCDType)
+ return std::unexpected(GCDType.error());
+ for (Register Part : PartRegisters) {
+ auto Extracted = extractGCDType(GCDRegisters, *GCDType, Part);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ }
+ for (Register Part : LeftoverRegisters) {
+ auto Extracted = extractGCDType(GCDRegisters, *GCDType, Part);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ }
+ auto LCMType = buildLCMMergePieces(ResultType, LeftoverType, *GCDType, GCDRegisters);
+ if (!LCMType)
+ return std::unexpected(LCMType.error());
+ return buildWidenedRemergeToDst(Destination, *LCMType, GCDRegisters);
+}
+
+Expected<void> LegalizerHelper::extractGCDType(std::vector<Register> &Parts, LowLevelType GCDType,
+ Register Source) {
+ auto SourceType = MRI.getType(Source);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ if (*SourceType == GCDType) {
+ Parts.push_back(Source);
+ return {};
+ }
+ const std::uint64_t SourceSize = SourceType->minimumBitWidth();
+ const std::uint64_t GCDSize = GCDType.minimumBitWidth();
+ if (GCDSize == 0 || SourceSize % GCDSize != 0)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "GCD type does not evenly divide the source type");
+ return extractParts(Source, GCDType, static_cast<int>(SourceSize / GCDSize), Parts, MIRBuilder,
+ MRI);
+}
+
+Expected<LowLevelType> LegalizerHelper::extractGCDType(std::vector<Register> &Parts,
+ LowLevelType DestinationType,
+ LowLevelType NarrowType, Register Source) {
+ auto SourceType = MRI.getType(Source);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ auto FirstGCD = getGCDType(*SourceType, NarrowType);
+ if (!FirstGCD)
+ return std::unexpected(FirstGCD.error());
+ auto GCDType = getGCDType(*FirstGCD, DestinationType);
+ if (!GCDType)
+ return std::unexpected(GCDType.error());
+ auto Extracted = extractGCDType(Parts, *GCDType, Source);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ return *GCDType;
+}
+
+Expected<LowLevelType> LegalizerHelper::buildLCMMergePieces(LowLevelType DestinationType,
+ LowLevelType NarrowType,
+ LowLevelType GCDType,
+ std::vector<Register> &Registers,
+ GenericOpcode PadStrategy) {
+ auto LCMType = getLCMType(DestinationType, NarrowType);
+ if (!LCMType)
+ return std::unexpected(LCMType.error());
+ const int PartCount = static_cast<int>(LCMType->minimumBitWidth() / NarrowType.minimumBitWidth());
+ const int SubPartCount =
+ static_cast<int>(NarrowType.minimumBitWidth() / GCDType.minimumBitWidth());
+ const int OriginalSourceCount = static_cast<int>(Registers.size());
+
+ Register PadRegister;
+ if (OriginalSourceCount < PartCount * SubPartCount) {
+ if (PadStrategy == GenericOpcode::G_ZEXT) {
+ auto Pad = buildConstantRegister(MIRBuilder, GCDType, 0);
+ if (!Pad)
+ return std::unexpected(Pad.error());
+ PadRegister = *Pad;
+ } else if (PadStrategy == GenericOpcode::G_ANYEXT) {
+ auto Built = MIRBuilder.buildUndef(GCDType);
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto Pad = resultRegister(**Built);
+ if (!Pad)
+ return std::unexpected(Pad.error());
+ PadRegister = *Pad;
+ } else if (PadStrategy == GenericOpcode::G_SEXT) {
+ if (Registers.empty())
+ return makeUnexpected(ErrorCode::InternalError,
+ "sign-extension padding requires a source part");
+ auto ShiftType = LowLevelType::makeCategoryNeutralScalar(64);
+ if (!ShiftType)
+ return std::unexpected(ShiftType.error());
+ auto ShiftAmount = buildConstantRegister(
+ MIRBuilder, *ShiftType, static_cast<std::int64_t>(GCDType.minimumBitWidth() - 1U));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto Pad = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, GCDType,
+ {Registers.back(), *ShiftAmount});
+ if (!Pad)
+ return std::unexpected(Pad.error());
+ PadRegister = *Pad;
+ } else {
+ return makeUnexpected(ErrorCode::InternalError,
+ "LCM merge received an invalid extension strategy");
+ }
+ }
+
+ std::vector<Register> Remerge(static_cast<std::size_t>(PartCount));
+ std::vector<Register> SubMerge(static_cast<std::size_t>(SubPartCount));
+ Register AllPadRegister;
+ for (int PartIndex = 0; PartIndex != PartCount; ++PartIndex) {
+ bool AllMergePartsArePadding = true;
+ for (int SubPartIndex = 0; SubPartIndex != SubPartCount; ++SubPartIndex) {
+ const int Index = PartIndex * SubPartCount + SubPartIndex;
+ if (Index >= OriginalSourceCount) {
+ SubMerge[static_cast<std::size_t>(SubPartIndex)] = PadRegister;
+ continue;
+ }
+ SubMerge[static_cast<std::size_t>(SubPartIndex)] = Registers[static_cast<std::size_t>(Index)];
+ AllMergePartsArePadding = false;
+ }
+
+ if (AllMergePartsArePadding && AllPadRegister.isNoRegister()) {
+ if (PadStrategy == GenericOpcode::G_ZEXT) {
+ auto Pad = buildConstantRegister(MIRBuilder, NarrowType, 0);
+ if (!Pad)
+ return std::unexpected(Pad.error());
+ AllPadRegister = *Pad;
+ } else if (PadStrategy == GenericOpcode::G_ANYEXT) {
+ auto Built = MIRBuilder.buildUndef(NarrowType);
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto Pad = resultRegister(**Built);
+ if (!Pad)
+ return std::unexpected(Pad.error());
+ AllPadRegister = *Pad;
+ }
+ }
+ if (!AllPadRegister.isNoRegister()) {
+ Remerge[static_cast<std::size_t>(PartIndex)] = AllPadRegister;
+ continue;
+ }
+ if (SubPartCount == 1) {
+ Remerge[static_cast<std::size_t>(PartIndex)] = SubMerge.front();
+ } else {
+ auto Built = MIRBuilder.buildMergeLikeInstr(NarrowType, SubMerge);
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto RemergeRegister = resultRegister(**Built);
+ if (!RemergeRegister)
+ return std::unexpected(RemergeRegister.error());
+ Remerge[static_cast<std::size_t>(PartIndex)] = *RemergeRegister;
+ }
+ if (AllMergePartsArePadding && AllPadRegister.isNoRegister())
+ AllPadRegister = Remerge[static_cast<std::size_t>(PartIndex)];
+ }
+ Registers = std::move(Remerge);
+ return *LCMType;
+}
+
+Expected<void>
+LegalizerHelper::buildWidenedRemergeToDst(Register Destination, LowLevelType LCMType,
+ const std::vector<Register> &RemergeRegisters) {
+ auto DestinationType = MRI.getType(Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (*DestinationType == LCMType)
+ return MIRBuilder.buildMergeLikeInstr(Destination, RemergeRegisters)
+ .transform([](MachineInstr *) {});
+
+ auto Remerge = MRI.createVirtualRegister(LCMType);
+ if (!Remerge)
+ return std::unexpected(Remerge.error());
+ auto BuiltMerge = MIRBuilder.buildMergeLikeInstr(*Remerge, RemergeRegisters);
+ if (!BuiltMerge)
+ return std::unexpected(BuiltMerge.error());
+ if (!DestinationType->isVector() && !LCMType.isVector())
+ return MIRBuilder.buildTrunc(Destination, *Remerge).transform([](MachineInstr *) {});
+
+ if (LCMType.isVector()) {
+ const std::size_t DefinitionCount =
+ static_cast<std::size_t>(LCMType.minimumBitWidth() / DestinationType->minimumBitWidth());
+ std::vector<Register> Definitions;
+ Definitions.reserve(DefinitionCount);
+ Definitions.push_back(Destination);
+ for (std::size_t Index = 1; Index != DefinitionCount; ++Index) {
+ auto Definition = MRI.createVirtualRegister(*DestinationType);
+ if (!Definition)
+ return std::unexpected(Definition.error());
+ Definitions.push_back(*Definition);
+ }
+ return MIRBuilder.buildUnmergeValues(Definitions, *Remerge).transform([](MachineInstr *) {});
+ }
+ return makeUnexpected(ErrorCode::Unsupported,
+ "widened remerge has an unsupported type combination");
+}
+
+Expected<void> LegalizerHelper::multiplyRegisters(std::vector<Register> &Destinations,
+ const std::vector<Register> &FirstSources,
+ const std::vector<Register> &SecondSources,
+ LowLevelType NarrowType) {
+ if (FirstSources.empty() || FirstSources.size() != SecondSources.size() ||
+ (Destinations.size() != FirstSources.size() &&
+ Destinations.size() != FirstSources.size() * 2U)) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "multi-precision multiplication has incompatible register parts");
+ }
+ auto BooleanType = LowLevelType::makeCategoryNeutralScalar(1);
+ if (!BooleanType)
+ return std::unexpected(BooleanType.error());
+
+ const auto BuildUAddo = [&](Register Left,
+ Register Right) -> Expected<std::pair<Register, Register>> {
+ auto Sum = MRI.createVirtualRegister(NarrowType);
+ auto Carry = MRI.createVirtualRegister(*BooleanType);
+ if (!Sum)
+ return std::unexpected(Sum.error());
+ if (!Carry)
+ return std::unexpected(Carry.error());
+ auto Built =
+ MIRBuilder.buildInstr(opcode(GenericOpcode::G_UADDO), {*Sum, *Carry}, {Left, Right});
+ if (!Built)
+ return std::unexpected(Built.error());
+ return std::pair{*Sum, *Carry};
+ };
+
+ const std::size_t SourcePartCount = FirstSources.size();
+ const std::size_t DestinationPartCount = Destinations.size();
+ auto Product = buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, NarrowType,
+ {FirstSources.front(), SecondSources.front()});
+ if (!Product)
+ return std::unexpected(Product.error());
+ Destinations.front() = *Product;
+
+ Register PreviousCarrySum;
+ std::vector<Register> Factors;
+ for (std::size_t DestinationIndex = 1; DestinationIndex != DestinationPartCount;
+ ++DestinationIndex) {
+ // Collect high halves from the preceding product diagonal.
+ const std::size_t HighStart =
+ DestinationIndex < SourcePartCount ? 0 : DestinationIndex - SourcePartCount;
+ const std::size_t HighEnd = std::min(DestinationIndex - 1U, SourcePartCount - 1U);
+ for (std::size_t Index = HighStart; Index <= HighEnd; ++Index) {
+ auto HighProduct = buildResultInstruction(
+ MIRBuilder, GenericOpcode::G_UMULH, NarrowType,
+ {FirstSources[DestinationIndex - 1U - Index], SecondSources[Index]});
+ if (!HighProduct)
+ return std::unexpected(HighProduct.error());
+ Factors.push_back(*HighProduct);
+ }
+
+ // Visit the current low-half diagonal from its low first-source part.
+ const std::size_t LowStart =
+ DestinationIndex + 1U < SourcePartCount ? 0 : DestinationIndex - SourcePartCount + 1U;
+ const std::size_t LowEnd = std::min(DestinationIndex, SourcePartCount - 1U);
+ for (std::size_t ReverseIndex = LowEnd + 1U; ReverseIndex != LowStart; --ReverseIndex) {
+ const std::size_t Index = ReverseIndex - 1U;
+ auto LowProduct =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, NarrowType,
+ {FirstSources[DestinationIndex - Index], SecondSources[Index]});
+ if (!LowProduct)
+ return std::unexpected(LowProduct.error());
+ Factors.push_back(*LowProduct);
+ }
+ if (DestinationIndex != 1U)
+ Factors.push_back(PreviousCarrySum);
+ if (Factors.size() < 2)
+ return makeUnexpected(ErrorCode::InternalError,
+ "multi-precision multiplication has too few factors");
+
+ Register FactorSum;
+ Register CarrySum;
+ if (DestinationIndex + 1U != DestinationPartCount) {
+ auto SumAndCarry = BuildUAddo(Factors[0], Factors[1]);
+ if (!SumAndCarry)
+ return std::unexpected(SumAndCarry.error());
+ FactorSum = SumAndCarry->first;
+ auto ExtendedCarry = MRI.createVirtualRegister(NarrowType);
+ if (!ExtendedCarry)
+ return std::unexpected(ExtendedCarry.error());
+ if (auto Built = MIRBuilder.buildZExt(*ExtendedCarry, SumAndCarry->second); !Built)
+ return std::unexpected(Built.error());
+ CarrySum = *ExtendedCarry;
+
+ for (std::size_t Index = 2; Index != Factors.size(); ++Index) {
+ SumAndCarry = BuildUAddo(FactorSum, Factors[Index]);
+ if (!SumAndCarry)
+ return std::unexpected(SumAndCarry.error());
+ FactorSum = SumAndCarry->first;
+ auto ExtendedCarryPart = MRI.createVirtualRegister(NarrowType);
+ if (!ExtendedCarryPart)
+ return std::unexpected(ExtendedCarryPart.error());
+ if (auto Built = MIRBuilder.buildZExt(*ExtendedCarryPart, SumAndCarry->second); !Built)
+ return std::unexpected(Built.error());
+ auto NextCarrySum = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, NarrowType,
+ {CarrySum, *ExtendedCarryPart});
+ if (!NextCarrySum)
+ return std::unexpected(NextCarrySum.error());
+ CarrySum = *NextCarrySum;
+ }
+ } else {
+ auto Sum = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, NarrowType,
+ {Factors[0], Factors[1]});
+ if (!Sum)
+ return std::unexpected(Sum.error());
+ FactorSum = *Sum;
+ for (std::size_t Index = 2; Index != Factors.size(); ++Index) {
+ Sum = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, NarrowType,
+ {FactorSum, Factors[Index]});
+ if (!Sum)
+ return std::unexpected(Sum.error());
+ FactorSum = *Sum;
+ }
+ }
+
+ PreviousCarrySum = CarrySum;
+ Destinations[DestinationIndex] = FactorSum;
+ Factors.clear();
+ }
+ return {};
+}
+
+Expected<void> LegalizerHelper::widenScalarSrc(MachineInstr &Instruction, LowLevelType WideType,
+ std::size_t OperandIndex,
+ GenericOpcode ExtensionOpcode) {
+ MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Source == nullptr || !Source->isVirtual() || !Operand->isUse())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "scalar widening source is not a virtual-register use");
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto Extension = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {WideType}, {*Source});
+ if (!Extension)
+ return std::unexpected(Extension.error());
+ auto WideSource = resultRegister(**Extension);
+ if (!WideSource)
+ return std::unexpected(WideSource.error());
+ return Operand->setRegister(*WideSource);
+}
+
+Expected<void> LegalizerHelper::narrowScalarSrc(MachineInstr &Instruction, LowLevelType NarrowType,
+ std::size_t OperandIndex) {
+ MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Source == nullptr || !Source->isVirtual() || !Operand->isUse())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "scalar narrowing source is not a virtual-register use");
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto Truncation = MIRBuilder.buildTrunc(NarrowType, *Source);
+ if (!Truncation)
+ return std::unexpected(Truncation.error());
+ auto NarrowSource = resultRegister(**Truncation);
+ if (!NarrowSource)
+ return std::unexpected(NarrowSource.error());
+ return Operand->setRegister(*NarrowSource);
+}
+
+Expected<void> LegalizerHelper::narrowScalarDst(MachineInstr &Instruction, LowLevelType NarrowType,
+ std::size_t OperandIndex,
+ GenericOpcode ExtensionOpcode) {
+ MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const Register *Destination = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Destination == nullptr || !Destination->isVirtual() || !Operand->isDef())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "scalar narrowing destination is not a virtual-register definition");
+
+ const Register OriginalDestination = *Destination;
+ auto NarrowDestination = MRI.createVirtualRegister(NarrowType);
+ if (!NarrowDestination)
+ return std::unexpected(NarrowDestination.error());
+ if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Extension = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {OriginalDestination},
+ {*NarrowDestination});
+ !Extension) {
+ return std::unexpected(Extension.error());
+ }
+ return Operand->setRegister(*NarrowDestination);
+}
+
+Expected<void> LegalizerHelper::widenScalarDst(MachineInstr &Instruction, LowLevelType WideType,
+ std::size_t OperandIndex,
+ GenericOpcode TruncOpcode) {
+ MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const Register *Destination = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Destination == nullptr || !Destination->isVirtual() || !Operand->isDef())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "scalar widening destination is not a virtual-register definition");
+
+ auto WideDestination = MRI.createVirtualRegister(WideType);
+ if (!WideDestination)
+ return std::unexpected(WideDestination.error());
+ if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto Truncation = MIRBuilder.buildInstr(opcode(TruncOpcode));
+ if (!Truncation)
+ return std::unexpected(Truncation.error());
+ if (auto Added = Truncation->addDef(*Destination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Truncation->addUse(*WideDestination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Truncation->build(); !Built)
+ return std::unexpected(Built.error());
+ return Operand->setRegister(*WideDestination);
+}
+
+Expected<void> LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &Instruction,
+ std::size_t TypeIndex,
+ LowLevelType WideType) {
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ GenericOpcode ArithmeticOpcode;
+ GenericOpcode ExtensionOpcode;
+ bool HasCarryIn = false;
+ switch (Generic) {
+ case GenericOpcode::G_SADDO:
+ ArithmeticOpcode = GenericOpcode::G_ADD;
+ ExtensionOpcode = GenericOpcode::G_SEXT;
+ break;
+ case GenericOpcode::G_SSUBO:
+ ArithmeticOpcode = GenericOpcode::G_SUB;
+ ExtensionOpcode = GenericOpcode::G_SEXT;
+ break;
+ case GenericOpcode::G_UADDO:
+ ArithmeticOpcode = GenericOpcode::G_ADD;
+ ExtensionOpcode = GenericOpcode::G_ZEXT;
+ break;
+ case GenericOpcode::G_USUBO:
+ ArithmeticOpcode = GenericOpcode::G_SUB;
+ ExtensionOpcode = GenericOpcode::G_ZEXT;
+ break;
+ case GenericOpcode::G_SADDE:
+ ArithmeticOpcode = GenericOpcode::G_UADDE;
+ ExtensionOpcode = GenericOpcode::G_SEXT;
+ HasCarryIn = true;
+ break;
+ case GenericOpcode::G_SSUBE:
+ ArithmeticOpcode = GenericOpcode::G_USUBE;
+ ExtensionOpcode = GenericOpcode::G_SEXT;
+ HasCarryIn = true;
+ break;
+ case GenericOpcode::G_UADDE:
+ ArithmeticOpcode = GenericOpcode::G_UADDE;
+ ExtensionOpcode = GenericOpcode::G_ZEXT;
+ HasCarryIn = true;
+ break;
+ case GenericOpcode::G_USUBE:
+ ArithmeticOpcode = GenericOpcode::G_USUBE;
+ ExtensionOpcode = GenericOpcode::G_ZEXT;
+ HasCarryIn = true;
+ break;
+ default:
+ return makeUnexpected(ErrorCode::InternalError,
+ "unexpected opcode during add/sub overflow widening");
+ }
+
+ if (TypeIndex == 1) {
+ if (HasCarryIn) {
+ if (auto Result = widenScalarSrc(Instruction, WideType, 4,
+ MIRBuilder.getBoolExtOp(WideType.isVector(), false));
+ !Result) {
+ return Result;
+ }
+ }
+ return widenScalarDst(Instruction, WideType, 1);
+ }
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "add/sub overflow widening requires type index zero or one");
+ if (Instruction.operandCount() != (HasCarryIn ? 5U : 4U))
+ return makeUnexpected(ErrorCode::VerificationError,
+ "add/sub overflow widening has an invalid operand count");
+
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *OverflowOperand = Instruction.operandAt(1);
+ const MachineOperand *LeftOperand = Instruction.operandAt(2);
+ const MachineOperand *RightOperand = Instruction.operandAt(3);
+ const MachineOperand *CarryInOperand = HasCarryIn ? Instruction.operandAt(4) : nullptr;
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Overflow =
+ OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ const Register *CarryIn = CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf<Register>();
+ if (Destination == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr ||
+ (HasCarryIn && CarryIn == nullptr)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "add/sub overflow widening requires register operands");
+ }
+
+ auto OriginalType = MRI.getType(*Destination);
+ auto OverflowType = MRI.getType(*Overflow);
+ if (!OriginalType)
+ return std::unexpected(OriginalType.error());
+ if (!OverflowType)
+ return std::unexpected(OverflowType.error());
+
+ auto WideLeft = MRI.createVirtualRegister(WideType);
+ auto WideRight = MRI.createVirtualRegister(WideType);
+ auto WideResult = MRI.createVirtualRegister(WideType);
+ auto TruncatedResult = MRI.createVirtualRegister(*OriginalType);
+ auto ExtendedResult = MRI.createVirtualRegister(WideType);
+ if (!WideLeft)
+ return std::unexpected(WideLeft.error());
+ if (!WideRight)
+ return std::unexpected(WideRight.error());
+ if (!WideResult)
+ return std::unexpected(WideResult.error());
+ if (!TruncatedResult)
+ return std::unexpected(TruncatedResult.error());
+ if (!ExtendedResult)
+ return std::unexpected(ExtendedResult.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*WideLeft}, {*Left}); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*WideRight}, {*Right}); !Built)
+ return std::unexpected(Built.error());
+ if (HasCarryIn) {
+ if (auto Built = MIRBuilder.buildInstr(opcode(ArithmeticOpcode), {*WideResult, *OverflowType},
+ {*WideLeft, *WideRight, *CarryIn});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ } else if (auto Built = MIRBuilder.buildInstr(opcode(ArithmeticOpcode), {*WideResult},
+ {*WideLeft, *WideRight});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildTrunc(*TruncatedResult, *WideResult); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built =
+ MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*ExtendedResult}, {*TruncatedResult});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::NotEqual, *Overflow, *WideResult, *ExtendedResult);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildTrunc(*Destination, *WideResult); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::widenScalarMulo(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType WideType) {
+ if (TypeIndex == 1)
+ return widenScalarDst(Instruction, WideType, 1);
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiply-overflow widening requires type index zero or one");
+ if (Instruction.operandCount() != 4)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiply-overflow widening requires four register operands");
+
+ const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SMULO);
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ const Register *OriginalOverflow = Instruction.operandAt(1)->getIf<Register>();
+ const Register *Left = Instruction.operandAt(2)->getIf<Register>();
+ const Register *Right = Instruction.operandAt(3)->getIf<Register>();
+ if (Destination == nullptr || OriginalOverflow == nullptr || Left == nullptr || Right == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiply-overflow widening requires register operands");
+
+ auto SourceType = MRI.getType(*Left);
+ auto OverflowType = MRI.getType(*OriginalOverflow);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ if (!OverflowType)
+ return std::unexpected(OverflowType.error());
+ const std::uint32_t SourceBitWidth = SourceType->scalarBitWidth();
+ if (SourceBitWidth == 0 || !WideType.isCategoryNeutralScalar())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiply-overflow widening requires scalar types");
+
+ const GenericOpcode ExtensionOpcode = IsSigned ? GenericOpcode::G_SEXT : GenericOpcode::G_ZEXT;
+ auto WideLeft = buildResultInstruction(MIRBuilder, ExtensionOpcode, WideType, {*Left});
+ auto WideRight = buildResultInstruction(MIRBuilder, ExtensionOpcode, WideType, {*Right});
+ if (!WideLeft)
+ return std::unexpected(WideLeft.error());
+ if (!WideRight)
+ return std::unexpected(WideRight.error());
+
+ const bool WideMulCanOverflow = static_cast<std::uint64_t>(WideType.scalarBitWidth()) <
+ 2U * static_cast<std::uint64_t>(SourceBitWidth);
+ Register Mul;
+ Register WideOverflow;
+ if (WideMulCanOverflow) {
+ auto Built =
+ MIRBuilder.buildInstr(opcode(IsSigned ? GenericOpcode::G_SMULO : GenericOpcode::G_UMULO),
+ {WideType, *OverflowType}, {*WideLeft, *WideRight});
+ if (!Built)
+ return std::unexpected(Built.error());
+ const MachineOperand *ResultOperand = (*Built)->operandAt(0);
+ const MachineOperand *OverflowOperand = (*Built)->operandAt(1);
+ const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
+ const Register *Overflow =
+ OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf<Register>();
+ if (Result == nullptr || Overflow == nullptr)
+ return makeUnexpected(ErrorCode::InternalError,
+ "multiply-overflow widening produced invalid results");
+ Mul = *Result;
+ WideOverflow = *Overflow;
+ } else {
+ auto Built =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, WideType, {*WideLeft, *WideRight});
+ if (!Built)
+ return std::unexpected(Built.error());
+ Mul = *Built;
+ }
+
+ if (auto Built = MIRBuilder.buildTrunc(*Destination, Mul); !Built)
+ return std::unexpected(Built.error());
+
+ Register ExtendedResult;
+ if (IsSigned) {
+ auto Built = MIRBuilder.buildSExtInReg(WideType, Mul, SourceBitWidth);
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto Result = resultRegister(**Built);
+ if (!Result)
+ return std::unexpected(Result.error());
+ ExtendedResult = *Result;
+ } else {
+ auto Mask = APInt::getLowBitsSet(WideType.scalarBitWidth(), SourceBitWidth);
+ if (!Mask)
+ return std::unexpected(Mask.error());
+ auto MaskRegister = buildConstantRegister(MIRBuilder, WideType, std::move(*Mask));
+ if (!MaskRegister)
+ return std::unexpected(MaskRegister.error());
+ auto Built =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, WideType, {Mul, *MaskRegister});
+ if (!Built)
+ return std::unexpected(Built.error());
+ ExtendedResult = *Built;
+ }
+
+ if (WideMulCanOverflow) {
+ auto Overflow = MRI.createVirtualRegister(*OverflowType);
+ if (!Overflow)
+ return std::unexpected(Overflow.error());
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::NotEqual, *Overflow, Mul, ExtendedResult);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildOr(*OriginalOverflow, WideOverflow, *Overflow); !Built)
+ return std::unexpected(Built.error());
+ } else if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::NotEqual, *OriginalOverflow, Mul,
+ ExtendedResult);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::widenScalar(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType WideType) {
+ auto Query = LegalizerInfo::buildQuery(Instruction, Function, MIRBuilder.target());
+ if (!Query)
+ return std::unexpected(Query.error());
+ if (!WideType.isCategoryNeutralScalar() || TypeIndex >= Query->Types.size() ||
+ !Query->Types[TypeIndex].isCategoryNeutralScalar() ||
+ WideType.scalarBitWidth() <= Query->Types[TypeIndex].scalarBitWidth()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "scalar widening has an invalid type index or target type");
+ }
+
+ const auto WidenSource = [&](std::size_t OperandIndex, GenericOpcode ExtensionOpcode) {
+ return widenScalarSrc(Instruction, WideType, OperandIndex, ExtensionOpcode);
+ };
+ const auto WidenDestination = [&](std::size_t OperandIndex = 0,
+ GenericOpcode TruncOpcode = GenericOpcode::G_TRUNC) {
+ return widenScalarDst(Instruction, WideType, OperandIndex, TruncOpcode);
+ };
+ const auto Apply = [](Expected<void> Result) -> Expected<void> {
+ if (!Result)
+ return std::unexpected(Result.error());
+ return {};
+ };
+
+ Observer.changingInstr(Instruction);
+ switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+ case GenericOpcode::G_MERGE_VALUES:
+ return widenScalarMergeValues(Instruction, TypeIndex, WideType);
+ case GenericOpcode::G_UNMERGE_VALUES:
+ return widenScalarUnmergeValues(Instruction, TypeIndex, WideType);
+ case GenericOpcode::G_ATOMICRMW_XCHG:
+ case GenericOpcode::G_ATOMICRMW_ADD:
+ case GenericOpcode::G_ATOMICRMW_SUB:
+ case GenericOpcode::G_ATOMICRMW_AND:
+ case GenericOpcode::G_ATOMICRMW_OR:
+ case GenericOpcode::G_ATOMICRMW_XOR:
+ case GenericOpcode::G_ATOMICRMW_MIN:
+ case GenericOpcode::G_ATOMICRMW_MAX:
+ case GenericOpcode::G_ATOMICRMW_UMIN:
+ case GenericOpcode::G_ATOMICRMW_UMAX:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "atomic RMW value widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_ATOMIC_CMPXCHG:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "compare-exchange widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenSource(3, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
+ if (TypeIndex == 0) {
+ if (auto Result = Apply(WidenSource(3, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenSource(4, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+ }
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "compare-exchange success widening has an invalid type index");
+ if (auto Result = Apply(WidenDestination(1)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_CONSTANT: {
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "constant widening has an invalid type index");
+ MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const auto *Immediate =
+ SourceOperand == nullptr ? nullptr : SourceOperand->getIf<IntegerImmediateOperand>();
+ if (Immediate == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_CONSTANT widening requires an integer immediate");
+ const GenericOpcode ExtensionOpcode =
+ Info.getExtOpcodeForWideningConstant(Query->Types[TypeIndex]);
+ if (ExtensionOpcode != GenericOpcode::G_SEXT && ExtensionOpcode != GenericOpcode::G_ZEXT &&
+ ExtensionOpcode != GenericOpcode::G_ANYEXT) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "constant widening selected an invalid extension opcode");
+ }
+ auto Extended = ExtensionOpcode == GenericOpcode::G_SEXT
+ ? Immediate->Value.sext(WideType.scalarBitWidth())
+ : Immediate->Value.zext(WideType.scalarBitWidth());
+ if (!Extended)
+ return std::unexpected(Extended.error());
+ if (auto Replaced = SourceOperand->setIntegerImmediate(std::move(*Extended)); !Replaced)
+ return std::unexpected(Replaced.error());
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+ }
+
+ case GenericOpcode::G_IMPLICIT_DEF:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "implicit definition widening has an invalid type index");
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_BRCOND:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_BRCOND widening requires type index zero");
+ if (auto Result = Apply(WidenSource(0, MIRBuilder.getBoolExtOp(false, false))); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_CTTZ:
+ case GenericOpcode::G_CTTZ_ZERO_POISON:
+ case GenericOpcode::G_CTLZ:
+ case GenericOpcode::G_CTLZ_ZERO_POISON:
+ case GenericOpcode::G_CTLS:
+ case GenericOpcode::G_CTPOP: {
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (TypeIndex == 0) {
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+ }
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "bit-count widening has an invalid type index");
+
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "bit-count widening requires register operands");
+ auto SourceType = MRI.getType(*Source);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+
+ GenericOpcode ExtensionOpcode = GenericOpcode::G_ZEXT;
+ switch (Generic) {
+ case GenericOpcode::G_CTTZ:
+ case GenericOpcode::G_CTTZ_ZERO_POISON:
+ case GenericOpcode::G_CTLZ_ZERO_POISON:
+ ExtensionOpcode = GenericOpcode::G_ANYEXT;
+ break;
+ case GenericOpcode::G_CTLS:
+ ExtensionOpcode = GenericOpcode::G_SEXT;
+ break;
+ default:
+ break;
+ }
+
+ auto WideSource = buildResultInstruction(MIRBuilder, ExtensionOpcode, WideType, {*Source});
+ if (!WideSource)
+ return std::unexpected(WideSource.error());
+ GenericOpcode WideOpcode = Generic;
+ if (Generic == GenericOpcode::G_CTTZ) {
+ auto TopBitValue = oneBitSet(WideType.scalarBitWidth(), SourceType->scalarBitWidth());
+ if (!TopBitValue)
+ return std::unexpected(TopBitValue.error());
+ auto TopBit = buildConstantRegister(MIRBuilder, WideType, std::move(*TopBitValue));
+ if (!TopBit)
+ return std::unexpected(TopBit.error());
+ auto NonZeroSource =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, WideType, {*WideSource, *TopBit});
+ if (!NonZeroSource)
+ return std::unexpected(NonZeroSource.error());
+ WideSource = *NonZeroSource;
+ WideOpcode = GenericOpcode::G_CTTZ_ZERO_POISON;
+ }
+
+ const std::uint32_t SizeDifference = WideType.scalarBitWidth() - SourceType->scalarBitWidth();
+ if (Generic == GenericOpcode::G_CTLZ_ZERO_POISON) {
+ auto ShiftAmount =
+ buildConstantRegister(MIRBuilder, WideType, static_cast<std::int64_t>(SizeDifference));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, WideType,
+ {*WideSource, *ShiftAmount});
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ WideSource = *Shifted;
+ }
+
+ auto Result = buildResultInstruction(MIRBuilder, WideOpcode, WideType, {*WideSource});
+ if (!Result)
+ return std::unexpected(Result.error());
+ if (Generic == GenericOpcode::G_CTLZ || Generic == GenericOpcode::G_CTLS) {
+ auto Difference =
+ buildConstantRegister(MIRBuilder, WideType, static_cast<std::int64_t>(SizeDifference));
+ if (!Difference)
+ return std::unexpected(Difference.error());
+ if (Generic == GenericOpcode::G_CTLZ) {
+ Result = buildFlaggedBinaryInstruction(MIRBuilder, GenericOpcode::G_SUB, WideType, *Result,
+ *Difference,
+ MachineInstrFlags{MachineInstrFlag::NoUnsignedWrap});
+ } else {
+ Result = buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, WideType,
+ {*Result, *Difference});
+ }
+ if (!Result)
+ return std::unexpected(Result.error());
+ }
+
+ if (auto Built = MIRBuilder.buildZExtOrTrunc(*Destination, *Result); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+ }
+
+ case GenericOpcode::G_BSWAP: {
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "byte-swap widening has an invalid type index");
+ MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "byte-swap widening requires a register destination");
+ const Register OriginalDestination = *Destination;
+ auto OriginalType = MRI.getType(OriginalDestination);
+ if (!OriginalType)
+ return std::unexpected(OriginalType.error());
+ auto WideDestination = MRI.createVirtualRegister(WideType);
+ auto Shifted = MRI.createVirtualRegister(WideType);
+ if (!WideDestination)
+ return std::unexpected(WideDestination.error());
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Replaced = DestinationOperand->setRegister(*WideDestination); !Replaced)
+ return std::unexpected(Replaced.error());
+
+ if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point)
+ return std::unexpected(Point.error());
+ const std::uint32_t Difference = WideType.scalarBitWidth() - OriginalType->scalarBitWidth();
+ auto ShiftAmount = buildConstantRegister(MIRBuilder, WideType, Difference);
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ if (auto Built = MIRBuilder.buildLShr(*Shifted, *WideDestination, *ShiftAmount); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildTrunc(OriginalDestination, *Shifted); !Built)
+ return std::unexpected(Built.error());
+ break;
+ }
+
+ case GenericOpcode::G_LOAD:
+ case GenericOpcode::G_SEXTLOAD:
+ case GenericOpcode::G_ZEXTLOAD:
+ case GenericOpcode::G_FPEXTLOAD:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "load widening has an invalid type index");
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_STORE: {
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "store widening has an invalid type index");
+ const Register *Source = Instruction.operandAt(0)->getIf<Register>();
+ if (Source == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "store widening requires a register source");
+ auto SourceType = MRI.getType(*Source);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ if (!SourceType->isCategoryNeutralScalar())
+ return makeUnexpected(ErrorCode::Unsupported, "non-scalar store widening is not implemented");
+ const GenericOpcode ExtensionOpcode =
+ SourceType->scalarBitWidth() == 1 ? GenericOpcode::G_ZEXT : GenericOpcode::G_ANYEXT;
+ if (auto Result = Apply(WidenSource(0, ExtensionOpcode)); !Result)
+ return Result;
+ break;
+ }
+
+ case GenericOpcode::G_FPTRUNCSTORE:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "floating truncating store widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(0, GenericOpcode::G_FPEXT)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_SADDO:
+ case GenericOpcode::G_SSUBO:
+ case GenericOpcode::G_UADDO:
+ case GenericOpcode::G_USUBO:
+ case GenericOpcode::G_SADDE:
+ case GenericOpcode::G_SSUBE:
+ case GenericOpcode::G_UADDE:
+ case GenericOpcode::G_USUBE:
+ return widenScalarAddSubOverflow(Instruction, TypeIndex, WideType);
+
+ case GenericOpcode::G_SMULO:
+ case GenericOpcode::G_UMULO:
+ if (TypeIndex != 1)
+ return widenScalarMulo(Instruction, TypeIndex, WideType);
+ if (auto Result = widenScalarMulo(Instruction, TypeIndex, WideType); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_ADD:
+ case GenericOpcode::G_SUB:
+ case GenericOpcode::G_MUL:
+ case GenericOpcode::G_AND:
+ case GenericOpcode::G_OR:
+ case GenericOpcode::G_XOR:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "binary operation widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_SDIV:
+ case GenericOpcode::G_SREM:
+ case GenericOpcode::G_SMIN:
+ case GenericOpcode::G_SMAX:
+ case GenericOpcode::G_ABDS:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "signed operation widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_SEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenSource(2, GenericOpcode::G_SEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_UDIV:
+ case GenericOpcode::G_UREM:
+ case GenericOpcode::G_UMIN:
+ case GenericOpcode::G_UMAX:
+ case GenericOpcode::G_ABDU:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "unsigned operation widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ZEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ZEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_SHL:
+ case GenericOpcode::G_LSHR:
+ case GenericOpcode::G_ASHR:
+ if (TypeIndex == 0) {
+ GenericOpcode ExtensionOpcode = GenericOpcode::G_ANYEXT;
+ if (Instruction.opcode() == opcode(GenericOpcode::G_LSHR))
+ ExtensionOpcode = GenericOpcode::G_ZEXT;
+ else if (Instruction.opcode() == opcode(GenericOpcode::G_ASHR))
+ ExtensionOpcode = GenericOpcode::G_SEXT;
+ if (auto Result = Apply(WidenSource(1, ExtensionOpcode)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+ }
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "shift widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ZEXT)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_ROTL:
+ case GenericOpcode::G_ROTR:
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::Unsupported, "rotate value widening is not defined");
+ if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ZEXT)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_FREEZE:
+ case GenericOpcode::G_CONSTANT_FOLD_BARRIER:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "unary operation widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_ABS:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "absolute-value widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_SEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_ICMP:
+ if (TypeIndex == 0) {
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+ }
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "integer comparison widening has an invalid type index");
+ {
+ const MachineOperand *PredicateMachineOperand = Instruction.operandAt(1);
+ const auto *Predicate = PredicateMachineOperand == nullptr
+ ? nullptr
+ : PredicateMachineOperand->getIf<PredicateOperand>();
+ const IntegerPredicate *Integer =
+ Predicate == nullptr ? nullptr : Predicate->integerPredicate();
+ if (Integer == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_ICMP widening requires an integer predicate");
+
+ bool IsSigned = false;
+ switch (*Integer) {
+ case IntegerPredicate::SignedGreaterThan:
+ case IntegerPredicate::SignedGreaterEqual:
+ case IntegerPredicate::SignedLessThan:
+ case IntegerPredicate::SignedLessEqual:
+ IsSigned = true;
+ break;
+ case IntegerPredicate::Equal:
+ case IntegerPredicate::NotEqual:
+ case IntegerPredicate::UnsignedGreaterThan:
+ case IntegerPredicate::UnsignedGreaterEqual:
+ case IntegerPredicate::UnsignedLessThan:
+ case IntegerPredicate::UnsignedLessEqual:
+ break;
+ case IntegerPredicate::Count:
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_ICMP widening received an invalid integer predicate");
+ }
+ const GenericOpcode ExtensionOpcode =
+ IsSigned || Info.isSExtCheaperThanZExt(Query->Types[TypeIndex], WideType)
+ ? GenericOpcode::G_SEXT
+ : GenericOpcode::G_ZEXT;
+ if (auto Result = Apply(WidenSource(2, ExtensionOpcode)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenSource(3, ExtensionOpcode)); !Result)
+ return Result;
+ }
+ break;
+
+ case GenericOpcode::G_FCMP:
+ if (TypeIndex == 0) {
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+ }
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "floating comparison widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(2, GenericOpcode::G_FPEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenSource(3, GenericOpcode::G_FPEXT)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_SELECT:
+ if (TypeIndex == 0) {
+ if (auto Result = Apply(WidenSource(2, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenSource(3, GenericOpcode::G_ANYEXT)); !Result)
+ return Result;
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+ }
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "select widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ZEXT)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_PHI: {
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_PHI widening requires type index zero");
+ if (Instruction.operandCount() < 3 || Instruction.operandCount() % 2U == 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_PHI widening requires incoming value/block pairs");
+
+ for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
+ MachineOperand *ValueOperand = Instruction.operandAt(Index);
+ const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U);
+ const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+ const auto *Incoming =
+ BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>();
+ if (Value == nullptr || Incoming == nullptr || Incoming->Block == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_PHI widening has an invalid incoming pair");
+ if (auto Point = setInsertPointAtFirstTerminator(MIRBuilder, *Incoming->Block); !Point)
+ return std::unexpected(Point.error());
+ auto Extension = MIRBuilder.buildInstr(opcode(GenericOpcode::G_ANYEXT), {WideType}, {*Value});
+ if (!Extension)
+ return std::unexpected(Extension.error());
+ auto WideValue = resultRegister(**Extension);
+ if (!WideValue)
+ return std::unexpected(WideValue.error());
+ if (auto Changed = ValueOperand->setRegister(*WideValue); !Changed)
+ return std::unexpected(Changed.error());
+ }
+
+ MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ MachineBasicBlock *Block = Instruction.parent();
+ if (Destination == nullptr || Block == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_PHI widening has an invalid destination");
+ const Register OriginalDestination = *Destination;
+ auto WideDestination = MRI.createVirtualRegister(WideType);
+ if (!WideDestination)
+ return std::unexpected(WideDestination.error());
+ if (auto Changed = DestinationOperand->setRegister(*WideDestination); !Changed)
+ return std::unexpected(Changed.error());
+ if (auto Point = setInsertPointAfterPHIs(MIRBuilder, *Block); !Point)
+ return std::unexpected(Point.error());
+ if (auto Truncation = MIRBuilder.buildTrunc(OriginalDestination, *WideDestination);
+ !Truncation) {
+ return std::unexpected(Truncation.error());
+ }
+ break;
+ }
+
+ case GenericOpcode::G_FADD:
+ case GenericOpcode::G_FMUL:
+ case GenericOpcode::G_FSUB:
+ case GenericOpcode::G_FMA:
+ case GenericOpcode::G_FMAD:
+ case GenericOpcode::G_FNEG:
+ case GenericOpcode::G_FABS:
+ case GenericOpcode::G_FCANONICALIZE:
+ case GenericOpcode::G_FMINNUM:
+ case GenericOpcode::G_FMAXNUM:
+ case GenericOpcode::G_FMINNUM_IEEE:
+ case GenericOpcode::G_FMAXNUM_IEEE:
+ case GenericOpcode::G_FMINIMUM:
+ case GenericOpcode::G_FMAXIMUM:
+ case GenericOpcode::G_FMINIMUMNUM:
+ case GenericOpcode::G_FMAXIMUMNUM:
+ case GenericOpcode::G_FDIV:
+ case GenericOpcode::G_FREM:
+ case GenericOpcode::G_FCEIL:
+ case GenericOpcode::G_FFLOOR:
+ case GenericOpcode::G_FCOS:
+ case GenericOpcode::G_FSIN:
+ case GenericOpcode::G_FTAN:
+ case GenericOpcode::G_FACOS:
+ case GenericOpcode::G_FASIN:
+ case GenericOpcode::G_FATAN:
+ case GenericOpcode::G_FATAN2:
+ case GenericOpcode::G_FCOSH:
+ case GenericOpcode::G_FSINH:
+ case GenericOpcode::G_FTANH:
+ case GenericOpcode::G_FLOG10:
+ case GenericOpcode::G_FLOG:
+ case GenericOpcode::G_FLOG2:
+ case GenericOpcode::G_FRINT:
+ case GenericOpcode::G_FNEARBYINT:
+ case GenericOpcode::G_FSQRT:
+ case GenericOpcode::G_FEXP:
+ case GenericOpcode::G_FEXP2:
+ case GenericOpcode::G_FEXP10:
+ case GenericOpcode::G_FPOW:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "floating operation widening has an invalid type index");
+ for (std::size_t OperandIndex = 1; OperandIndex < Instruction.operandCount(); ++OperandIndex) {
+ if (auto Result = Apply(WidenSource(OperandIndex, GenericOpcode::G_FPEXT)); !Result)
+ return Result;
+ }
+ if (auto Result = Apply(WidenDestination(0, GenericOpcode::G_FPTRUNC)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_FPEXT:
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "floating-point extension destination widening is not defined");
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_FPEXT)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_FPTOSI:
+ case GenericOpcode::G_FPTOUI:
+ case GenericOpcode::G_INTRINSIC_LRINT:
+ case GenericOpcode::G_INTRINSIC_LLRINT:
+ case GenericOpcode::G_IS_FPCLASS:
+ if (TypeIndex == 0) {
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+ }
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "floating-to-integer widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_FPEXT)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_SITOFP:
+ case GenericOpcode::G_UITOFP:
+ if (TypeIndex == 0) {
+ if (auto Result = Apply(WidenDestination(0, GenericOpcode::G_FPTRUNC)); !Result)
+ return Result;
+ break;
+ }
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "integer-to-floating widening has an invalid type index");
+ if (auto Result = Apply(WidenSource(1, Instruction.opcode() == opcode(GenericOpcode::G_SITOFP)
+ ? GenericOpcode::G_SEXT
+ : GenericOpcode::G_ZEXT));
+ !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_INTTOPTR:
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_INTTOPTR destination widening is not defined");
+ if (auto Result = Apply(WidenSource(1, GenericOpcode::G_ZEXT)); !Result)
+ return Result;
+ break;
+
+ case GenericOpcode::G_PTRTOINT:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::Unsupported, "G_PTRTOINT source widening is not defined");
+ if (auto Result = Apply(WidenDestination()); !Result)
+ return Result;
+ break;
+
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "scalar widening is not implemented for this opcode");
+ }
+
+ Observer.changedInstr(Instruction);
+ return {};
+}
+
+Expected<void> LegalizerHelper::narrowScalarAddSub(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::Unsupported, "add/sub narrowing requires type index zero");
+
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "add/sub narrowing requires a register destination");
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (DestinationType->isVector())
+ return makeUnexpected(ErrorCode::Unsupported, "vector add/sub narrowing is not implemented");
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ GenericOpcode FirstOpcode;
+ GenericOpcode MiddleOpcode;
+ GenericOpcode FinalOpcode;
+ switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+ case GenericOpcode::G_SADDO:
+ case GenericOpcode::G_SADDE:
+ case GenericOpcode::G_UADDO:
+ case GenericOpcode::G_UADDE:
+ case GenericOpcode::G_ADD:
+ FirstOpcode = GenericOpcode::G_UADDO;
+ MiddleOpcode = GenericOpcode::G_UADDE;
+ FinalOpcode = Instruction.opcode() == opcode(GenericOpcode::G_SADDO) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_SADDE)
+ ? GenericOpcode::G_SADDE
+ : GenericOpcode::G_UADDE;
+ break;
+ case GenericOpcode::G_SSUBO:
+ case GenericOpcode::G_SSUBE:
+ case GenericOpcode::G_USUBO:
+ case GenericOpcode::G_USUBE:
+ case GenericOpcode::G_SUB:
+ FirstOpcode = GenericOpcode::G_USUBO;
+ MiddleOpcode = GenericOpcode::G_USUBE;
+ FinalOpcode = Instruction.opcode() == opcode(GenericOpcode::G_SSUBO) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_SSUBE)
+ ? GenericOpcode::G_SSUBE
+ : GenericOpcode::G_USUBE;
+ break;
+ default:
+ return makeUnexpected(ErrorCode::InternalError,
+ "unexpected add/sub opcode during scalar narrowing");
+ }
+
+ const InstructionDescriptor *Descriptor =
+ MIRBuilder.target().instructionDescriptor(Instruction.opcode());
+ if (Descriptor == nullptr)
+ return makeUnexpected(ErrorCode::InternalError,
+ "add/sub narrowing has no instruction descriptor");
+ const std::size_t DefinitionCount = Descriptor->NumExplicitDefs;
+ const MachineOperand *FirstSourceOperand = Instruction.operandAt(DefinitionCount);
+ const MachineOperand *SecondSourceOperand = Instruction.operandAt(DefinitionCount + 1);
+ const Register *FirstSource =
+ FirstSourceOperand == nullptr ? nullptr : FirstSourceOperand->getIf<Register>();
+ const Register *SecondSource =
+ SecondSourceOperand == nullptr ? nullptr : SecondSourceOperand->getIf<Register>();
+ if (FirstSource == nullptr || SecondSource == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "add/sub narrowing requires two register sources");
+
+ Register CarryDestination;
+ Register CarryIn;
+ if (DefinitionCount == 2) {
+ const MachineOperand *CarryOperand = Instruction.operandAt(1);
+ const Register *Carry = CarryOperand == nullptr ? nullptr : CarryOperand->getIf<Register>();
+ if (Carry == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "add/sub narrowing has an invalid carry destination");
+ CarryDestination = *Carry;
+ }
+ if (Instruction.operandCount() == DefinitionCount + 3) {
+ const MachineOperand *CarryOperand = Instruction.operandAt(DefinitionCount + 2);
+ const Register *Carry = CarryOperand == nullptr ? nullptr : CarryOperand->getIf<Register>();
+ if (Carry == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "add/sub narrowing has an invalid carry input");
+ CarryIn = *Carry;
+ }
+
+ LowLevelType LeftoverType;
+ LowLevelType DummyType;
+ std::vector<Register> FirstParts;
+ std::vector<Register> SecondParts;
+ std::vector<Register> FirstLeftovers;
+ std::vector<Register> SecondLeftovers;
+ std::vector<Register> DestinationParts;
+ auto FirstExtracted = extractParts(*FirstSource, *DestinationType, NarrowType, LeftoverType,
+ FirstParts, FirstLeftovers, MIRBuilder, MRI);
+ if (!FirstExtracted)
+ return std::unexpected(FirstExtracted.error());
+ auto SecondExtracted = extractParts(*SecondSource, *DestinationType, NarrowType, DummyType,
+ SecondParts, SecondLeftovers, MIRBuilder, MRI);
+ if (!SecondExtracted)
+ return std::unexpected(SecondExtracted.error());
+
+ const std::size_t NarrowPartCount = FirstParts.size();
+ FirstParts.insert(FirstParts.end(), FirstLeftovers.begin(), FirstLeftovers.end());
+ SecondParts.insert(SecondParts.end(), SecondLeftovers.begin(), SecondLeftovers.end());
+ if (FirstParts.size() != SecondParts.size())
+ return makeUnexpected(ErrorCode::InternalError,
+ "add/sub source splits have different part counts");
+ DestinationParts.reserve(FirstParts.size());
+
+ auto BooleanType = LowLevelType::makeCategoryNeutralScalar(1);
+ if (!BooleanType)
+ return std::unexpected(BooleanType.error());
+ for (std::size_t Index = 0; Index != FirstParts.size(); ++Index) {
+ auto PartType = MRI.getType(FirstParts[Index]);
+ if (!PartType)
+ return std::unexpected(PartType.error());
+ auto PartDestination = MRI.createVirtualRegister(*PartType);
+ if (!PartDestination)
+ return std::unexpected(PartDestination.error());
+
+ Register CarryOut;
+ if (Index + 1 == FirstParts.size() && !CarryDestination.isNoRegister()) {
+ CarryOut = CarryDestination;
+ } else {
+ auto NewCarry = MRI.createVirtualRegister(*BooleanType);
+ if (!NewCarry)
+ return std::unexpected(NewCarry.error());
+ CarryOut = *NewCarry;
+ }
+
+ Expected<MachineInstr *> Built;
+ if (CarryIn.isNoRegister()) {
+ Built = MIRBuilder.buildInstr(opcode(FirstOpcode), {*PartDestination, CarryOut},
+ {FirstParts[Index], SecondParts[Index]});
+ } else if (Index + 1 == FirstParts.size()) {
+ Built = MIRBuilder.buildInstr(opcode(FinalOpcode), {*PartDestination, CarryOut},
+ {FirstParts[Index], SecondParts[Index], CarryIn});
+ } else {
+ Built = MIRBuilder.buildInstr(opcode(MiddleOpcode), {*PartDestination, CarryOut},
+ {FirstParts[Index], SecondParts[Index], CarryIn});
+ }
+ if (!Built)
+ return std::unexpected(Built.error());
+ DestinationParts.push_back(*PartDestination);
+ CarryIn = CarryOut;
+ }
+
+ std::vector<Register> MainDestinations(DestinationParts.begin(),
+ DestinationParts.begin() +
+ static_cast<std::ptrdiff_t>(NarrowPartCount));
+ std::vector<Register> LeftoverDestinations(DestinationParts.begin() +
+ static_cast<std::ptrdiff_t>(NarrowPartCount),
+ DestinationParts.end());
+ auto Inserted = insertParts(*Destination, *DestinationType, NarrowType, MainDestinations,
+ LeftoverType, LeftoverDestinations);
+ if (!Inserted)
+ return std::unexpected(Inserted.error());
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::narrowScalarMul(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ if (TypeIndex != 0 || Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "multiply narrowing requires type index zero and two sources");
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *FirstSourceOperand = Instruction.operandAt(1);
+ const MachineOperand *SecondSourceOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *FirstSource =
+ FirstSourceOperand == nullptr ? nullptr : FirstSourceOperand->getIf<Register>();
+ const Register *SecondSource =
+ SecondSourceOperand == nullptr ? nullptr : SecondSourceOperand->getIf<Register>();
+ if (Destination == nullptr || FirstSource == nullptr || SecondSource == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiply narrowing requires register operands");
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (DestinationType->isVector() || !NarrowType.isCategoryNeutralScalar())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "multiply narrowing requires scalar integer types");
+
+ const std::uint64_t DestinationSize = DestinationType->minimumBitWidth();
+ const std::uint64_t NarrowSize = NarrowType.minimumBitWidth();
+ if (NarrowSize == 0 || NarrowSize >= DestinationSize || DestinationSize % NarrowSize != 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "multiply narrowing requires an even smaller scalar type");
+ const std::size_t PartCount = static_cast<std::size_t>(DestinationSize / NarrowSize);
+ const bool IsMultiplyHigh = Instruction.opcode() == opcode(GenericOpcode::G_UMULH);
+ const std::size_t TemporaryPartCount = PartCount * (IsMultiplyHigh ? 2U : 1U);
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ std::vector<Register> FirstParts;
+ std::vector<Register> SecondParts;
+ std::vector<Register> TemporaryDestinations(TemporaryPartCount);
+ if (auto Extracted = extractParts(*FirstSource, NarrowType, static_cast<int>(PartCount),
+ FirstParts, MIRBuilder, MRI);
+ !Extracted) {
+ return std::unexpected(Extracted.error());
+ }
+ if (auto Extracted = extractParts(*SecondSource, NarrowType, static_cast<int>(PartCount),
+ SecondParts, MIRBuilder, MRI);
+ !Extracted) {
+ return std::unexpected(Extracted.error());
+ }
+ if (auto Multiplied =
+ multiplyRegisters(TemporaryDestinations, FirstParts, SecondParts, NarrowType);
+ !Multiplied) {
+ return std::unexpected(Multiplied.error());
+ }
+
+ std::vector<Register> ResultParts(TemporaryDestinations.end() -
+ static_cast<std::ptrdiff_t>(PartCount),
+ TemporaryDestinations.end());
+ if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, ResultParts); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::narrowScalarExt(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::Unsupported, "extension narrowing requires type index zero");
+
+ const auto Destination = Instruction.operandAt(0)->registerValue();
+ const auto Source = Instruction.operandAt(1)->registerValue();
+ if (!Destination || !Source)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "extension narrowing requires register operands");
+
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (DestinationType->isVector())
+ return makeUnexpected(ErrorCode::Unsupported, "vector extension narrowing is not implemented");
+
+ std::vector<Register> Parts;
+ auto GCDType = extractGCDType(Parts, *DestinationType, NarrowType, *Source);
+ if (!GCDType)
+ return std::unexpected(GCDType.error());
+ auto LCMType = buildLCMMergePieces(*DestinationType, NarrowType, *GCDType, Parts,
+ static_cast<GenericOpcode>(Instruction.opcode().value()));
+ if (!LCMType)
+ return std::unexpected(LCMType.error());
+ auto Remerged = buildWidenedRemergeToDst(*Destination, *LCMType, Parts);
+ if (!Remerged)
+ return std::unexpected(Remerged.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::narrowScalarBasic(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ if (TypeIndex != 0 || Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "basic scalar narrowing requires type index zero and two sources");
+
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ const Register *FirstSource = Instruction.operandAt(1)->getIf<Register>();
+ const Register *SecondSource = Instruction.operandAt(2)->getIf<Register>();
+ if (Destination == nullptr || FirstSource == nullptr || SecondSource == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "basic scalar narrowing requires register operands");
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+
+ LowLevelType LeftoverType;
+ LowLevelType UnusedType;
+ std::vector<Register> FirstParts;
+ std::vector<Register> FirstLeftovers;
+ std::vector<Register> SecondParts;
+ std::vector<Register> SecondLeftovers;
+ auto FirstExtracted = extractParts(*FirstSource, *DestinationType, NarrowType, LeftoverType,
+ FirstParts, FirstLeftovers, MIRBuilder, MRI);
+ if (!FirstExtracted)
+ return std::unexpected(FirstExtracted.error());
+ auto SecondExtracted = extractParts(*SecondSource, *DestinationType, NarrowType, UnusedType,
+ SecondParts, SecondLeftovers, MIRBuilder, MRI);
+ if (!SecondExtracted)
+ return std::unexpected(SecondExtracted.error());
+ if (!*FirstExtracted || !*SecondExtracted || FirstParts.size() != SecondParts.size() ||
+ FirstLeftovers.size() != SecondLeftovers.size()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "basic scalar operands cannot be split consistently");
+ }
+
+ std::vector<Register> DestinationParts;
+ DestinationParts.reserve(FirstParts.size());
+ for (std::size_t Index = 0; Index != FirstParts.size(); ++Index) {
+ auto Built = MIRBuilder.buildInstr(Instruction.opcode(), {NarrowType},
+ {FirstParts[Index], SecondParts[Index]});
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto Result = resultRegister(**Built);
+ if (!Result)
+ return std::unexpected(Result.error());
+ DestinationParts.push_back(*Result);
+ }
+
+ std::vector<Register> DestinationLeftovers;
+ DestinationLeftovers.reserve(FirstLeftovers.size());
+ for (std::size_t Index = 0; Index != FirstLeftovers.size(); ++Index) {
+ auto Built = MIRBuilder.buildInstr(Instruction.opcode(), {LeftoverType},
+ {FirstLeftovers[Index], SecondLeftovers[Index]});
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto Result = resultRegister(**Built);
+ if (!Result)
+ return std::unexpected(Result.error());
+ DestinationLeftovers.push_back(*Result);
+ }
+
+ if (auto Inserted = insertParts(*Destination, *DestinationType, NarrowType, DestinationParts,
+ LeftoverType, DestinationLeftovers);
+ !Inserted) {
+ return std::unexpected(Inserted.error());
+ }
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::narrowScalarSelect(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ if (TypeIndex != 0 || Instruction.operandCount() != 4)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "select narrowing requires type index zero and three sources");
+
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ const Register *Condition = Instruction.operandAt(1)->getIf<Register>();
+ const Register *TrueSource = Instruction.operandAt(2)->getIf<Register>();
+ const Register *FalseSource = Instruction.operandAt(3)->getIf<Register>();
+ if (Destination == nullptr || Condition == nullptr || TrueSource == nullptr ||
+ FalseSource == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "select narrowing requires register operands");
+ }
+ auto DestinationType = MRI.getType(*Destination);
+ auto ConditionType = MRI.getType(*Condition);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (!ConditionType)
+ return std::unexpected(ConditionType.error());
+ if (ConditionType->isVector())
+ return makeUnexpected(ErrorCode::Unsupported, "vector select narrowing is not implemented");
+
+ LowLevelType LeftoverType;
+ LowLevelType UnusedType;
+ std::vector<Register> TrueParts;
+ std::vector<Register> TrueLeftovers;
+ std::vector<Register> FalseParts;
+ std::vector<Register> FalseLeftovers;
+ auto TrueExtracted = extractParts(*TrueSource, *DestinationType, NarrowType, LeftoverType,
+ TrueParts, TrueLeftovers, MIRBuilder, MRI);
+ if (!TrueExtracted)
+ return std::unexpected(TrueExtracted.error());
+ auto FalseExtracted = extractParts(*FalseSource, *DestinationType, NarrowType, UnusedType,
+ FalseParts, FalseLeftovers, MIRBuilder, MRI);
+ if (!FalseExtracted)
+ return std::unexpected(FalseExtracted.error());
+ if (!*TrueExtracted || !*FalseExtracted || TrueParts.size() != FalseParts.size() ||
+ TrueLeftovers.size() != FalseLeftovers.size()) {
+ return makeUnexpected(ErrorCode::Unsupported, "select operands cannot be split consistently");
+ }
+
+ std::vector<Register> DestinationParts;
+ DestinationParts.reserve(TrueParts.size());
+ for (std::size_t Index = 0; Index != TrueParts.size(); ++Index) {
+ auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {NarrowType},
+ {*Condition, TrueParts[Index], FalseParts[Index]});
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto Result = resultRegister(**Built);
+ if (!Result)
+ return std::unexpected(Result.error());
+ DestinationParts.push_back(*Result);
+ }
+
+ std::vector<Register> DestinationLeftovers;
+ DestinationLeftovers.reserve(TrueLeftovers.size());
+ for (std::size_t Index = 0; Index != TrueLeftovers.size(); ++Index) {
+ auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {LeftoverType},
+ {*Condition, TrueLeftovers[Index], FalseLeftovers[Index]});
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto Result = resultRegister(**Built);
+ if (!Result)
+ return std::unexpected(Result.error());
+ DestinationLeftovers.push_back(*Result);
+ }
+
+ if (auto Inserted = insertParts(*Destination, *DestinationType, NarrowType, DestinationParts,
+ LeftoverType, DestinationLeftovers);
+ !Inserted) {
+ return std::unexpected(Inserted.error());
+ }
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &Instruction,
+ const APInt &Amount,
+ LowLevelType HalfType,
+ LowLevelType AmountType) {
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ const Register *Source = Instruction.operandAt(1)->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "constant shift narrowing requires register operands");
+
+ auto LowPart = MRI.createVirtualRegister(HalfType);
+ auto HighPart = MRI.createVirtualRegister(HalfType);
+ if (!LowPart)
+ return std::unexpected(LowPart.error());
+ if (!HighPart)
+ return std::unexpected(HighPart.error());
+ if (auto Built = MIRBuilder.buildUnmergeValues({*LowPart, *HighPart}, *Source); !Built)
+ return std::unexpected(Built.error());
+
+ if (Amount.isZero()) {
+ if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, {*LowPart, *HighPart}); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+ }
+
+ const std::uint64_t HalfBits = HalfType.minimumBitWidth();
+ const std::uint64_t TotalBits = HalfBits * 2U;
+ const bool AmountExceeds64Bits = Amount.activeBitWidth() > 64;
+ const std::uint64_t AmountValue = Amount.word(0);
+ const bool ExceedsTotal = AmountExceeds64Bits || AmountValue > TotalBits;
+ const bool ExceedsHalf = AmountExceeds64Bits || AmountValue > HalfBits;
+
+ const auto BuildConstant = [&](LowLevelType Type, std::uint64_t Value) {
+ return buildConstantRegister(MIRBuilder, Type, static_cast<std::int64_t>(Value));
+ };
+ const auto BuildBinary = [&](GenericOpcode Generic, Register Left, Register Right) {
+ return buildResultInstruction(MIRBuilder, Generic, HalfType, {Left, Right});
+ };
+
+ Register LowResult;
+ Register HighResult;
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (Generic == GenericOpcode::G_SHL) {
+ if (ExceedsTotal) {
+ auto Zero = BuildConstant(HalfType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ LowResult = *Zero;
+ HighResult = *Zero;
+ } else if (ExceedsHalf) {
+ auto Zero = BuildConstant(HalfType, 0);
+ auto Shift = BuildConstant(AmountType, AmountValue - HalfBits);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ auto Shifted = BuildBinary(GenericOpcode::G_SHL, *LowPart, *Shift);
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ LowResult = *Zero;
+ HighResult = *Shifted;
+ } else if (AmountValue == HalfBits) {
+ auto Zero = BuildConstant(HalfType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ LowResult = *Zero;
+ HighResult = *LowPart;
+ } else {
+ auto Shift = BuildConstant(AmountType, AmountValue);
+ auto InverseShift = BuildConstant(AmountType, HalfBits - AmountValue);
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ if (!InverseShift)
+ return std::unexpected(InverseShift.error());
+ auto LowShift = BuildBinary(GenericOpcode::G_SHL, *LowPart, *Shift);
+ auto HighShift = BuildBinary(GenericOpcode::G_SHL, *HighPart, *Shift);
+ auto Carry = BuildBinary(GenericOpcode::G_LSHR, *LowPart, *InverseShift);
+ if (!LowShift)
+ return std::unexpected(LowShift.error());
+ if (!HighShift)
+ return std::unexpected(HighShift.error());
+ if (!Carry)
+ return std::unexpected(Carry.error());
+ auto Combined = BuildBinary(GenericOpcode::G_OR, *HighShift, *Carry);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ LowResult = *LowShift;
+ HighResult = *Combined;
+ }
+ } else if (Generic == GenericOpcode::G_LSHR) {
+ if (ExceedsTotal) {
+ auto Zero = BuildConstant(HalfType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ LowResult = *Zero;
+ HighResult = *Zero;
+ } else if (ExceedsHalf) {
+ auto Shift = BuildConstant(AmountType, AmountValue - HalfBits);
+ auto Zero = BuildConstant(HalfType, 0);
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Shifted = BuildBinary(GenericOpcode::G_LSHR, *HighPart, *Shift);
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ LowResult = *Shifted;
+ HighResult = *Zero;
+ } else if (AmountValue == HalfBits) {
+ auto Zero = BuildConstant(HalfType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ LowResult = *HighPart;
+ HighResult = *Zero;
+ } else {
+ auto Shift = BuildConstant(AmountType, AmountValue);
+ auto InverseShift = BuildConstant(AmountType, HalfBits - AmountValue);
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ if (!InverseShift)
+ return std::unexpected(InverseShift.error());
+ auto LowShift = BuildBinary(GenericOpcode::G_LSHR, *LowPart, *Shift);
+ auto Carry = BuildBinary(GenericOpcode::G_SHL, *HighPart, *InverseShift);
+ auto HighShift = BuildBinary(GenericOpcode::G_LSHR, *HighPart, *Shift);
+ if (!LowShift)
+ return std::unexpected(LowShift.error());
+ if (!Carry)
+ return std::unexpected(Carry.error());
+ if (!HighShift)
+ return std::unexpected(HighShift.error());
+ auto Combined = BuildBinary(GenericOpcode::G_OR, *LowShift, *Carry);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ LowResult = *Combined;
+ HighResult = *HighShift;
+ }
+ } else if (Generic == GenericOpcode::G_ASHR) {
+ auto SignShift = BuildConstant(AmountType, HalfBits - 1U);
+ if (!SignShift)
+ return std::unexpected(SignShift.error());
+ auto Sign = BuildBinary(GenericOpcode::G_ASHR, *HighPart, *SignShift);
+ if (!Sign)
+ return std::unexpected(Sign.error());
+ if (ExceedsTotal) {
+ LowResult = *Sign;
+ HighResult = *Sign;
+ } else if (ExceedsHalf) {
+ auto Shift = BuildConstant(AmountType, AmountValue - HalfBits);
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ auto Shifted = BuildBinary(GenericOpcode::G_ASHR, *HighPart, *Shift);
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ LowResult = *Shifted;
+ HighResult = *Sign;
+ } else if (AmountValue == HalfBits) {
+ LowResult = *HighPart;
+ HighResult = *Sign;
+ } else {
+ auto Shift = BuildConstant(AmountType, AmountValue);
+ auto InverseShift = BuildConstant(AmountType, HalfBits - AmountValue);
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ if (!InverseShift)
+ return std::unexpected(InverseShift.error());
+ auto LowShift = BuildBinary(GenericOpcode::G_LSHR, *LowPart, *Shift);
+ auto Carry = BuildBinary(GenericOpcode::G_SHL, *HighPart, *InverseShift);
+ auto HighShift = BuildBinary(GenericOpcode::G_ASHR, *HighPart, *Shift);
+ if (!LowShift)
+ return std::unexpected(LowShift.error());
+ if (!Carry)
+ return std::unexpected(Carry.error());
+ if (!HighShift)
+ return std::unexpected(HighShift.error());
+ auto Combined = BuildBinary(GenericOpcode::G_OR, *LowShift, *Carry);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ LowResult = *Combined;
+ HighResult = *HighShift;
+ }
+ } else {
+ return makeUnexpected(ErrorCode::InternalError,
+ "constant shift narrowing received a non-shift opcode");
+ }
+
+ if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, {LowResult, HighResult}); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::narrowScalarShift(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ if (Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "shift narrowing requires one destination and two sources");
+ if (TypeIndex == 1) {
+ Observer.changingInstr(Instruction);
+ auto Narrowed = narrowScalarSrc(Instruction, NarrowType, 2);
+ if (!Narrowed)
+ return std::unexpected(Narrowed.error());
+ Observer.changedInstr(Instruction);
+ return {};
+ }
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::Unsupported, "shift narrowing has an invalid type index");
+
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ const Register *Source = Instruction.operandAt(1)->getIf<Register>();
+ const Register *Amount = Instruction.operandAt(2)->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr || Amount == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "shift narrowing requires register operands");
+ auto DestinationType = MRI.getType(*Destination);
+ auto AmountType = MRI.getType(*Amount);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (!AmountType)
+ return std::unexpected(AmountType.error());
+ if (!DestinationType->isCategoryNeutralScalar())
+ return makeUnexpected(ErrorCode::Unsupported, "vector shift narrowing is not implemented");
+
+ const std::uint64_t DestinationBits = DestinationType->scalarBitWidth();
+ if (DestinationBits == 0 || DestinationBits % 2U != 0)
+ return makeUnexpected(ErrorCode::Unsupported, "shift narrowing requires an even scalar width");
+
+ if (NarrowType.isCategoryNeutralScalar() && NarrowType.scalarBitWidth() != 0 &&
+ DestinationBits % NarrowType.scalarBitWidth() == 0 &&
+ DestinationBits / NarrowType.scalarBitWidth() >= 8U) {
+ return narrowScalarShiftMultiway(Instruction, NarrowType);
+ }
+
+ auto HalfType = LowLevelType::makeCategoryNeutralScalar(DestinationBits / 2U);
+ if (!HalfType)
+ return std::unexpected(HalfType.error());
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Constant = getIConstantVRegValWithLookThrough(*Amount, MRI))
+ return narrowScalarShiftByConstant(Instruction, Constant->Value, *HalfType, *AmountType);
+
+ auto HalfBits = buildConstantRegister(MIRBuilder, *AmountType,
+ static_cast<std::int64_t>(DestinationBits / 2U));
+ if (!HalfBits)
+ return std::unexpected(HalfBits.error());
+ auto LowPart = MRI.createVirtualRegister(*HalfType);
+ auto HighPart = MRI.createVirtualRegister(*HalfType);
+ if (!LowPart)
+ return std::unexpected(LowPart.error());
+ if (!HighPart)
+ return std::unexpected(HighPart.error());
+ if (auto Built = MIRBuilder.buildUnmergeValues({*LowPart, *HighPart}, *Source); !Built)
+ return std::unexpected(Built.error());
+
+ auto AmountExcess =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, {*Amount, *HalfBits});
+ auto AmountLack =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, {*HalfBits, *Amount});
+ auto ZeroAmount = buildConstantRegister(MIRBuilder, *AmountType, 0);
+ auto ConditionType = LowLevelType::makeCategoryNeutralScalar(1);
+ if (!AmountExcess)
+ return std::unexpected(AmountExcess.error());
+ if (!AmountLack)
+ return std::unexpected(AmountLack.error());
+ if (!ZeroAmount)
+ return std::unexpected(ZeroAmount.error());
+ if (!ConditionType)
+ return std::unexpected(ConditionType.error());
+ auto IsShort = buildICmpRegister(MIRBuilder, IntegerPredicate::UnsignedLessThan, *ConditionType,
+ *Amount, *HalfBits);
+ auto IsZero =
+ buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *ConditionType, *Amount, *ZeroAmount);
+ if (!IsShort)
+ return std::unexpected(IsShort.error());
+ if (!IsZero)
+ return std::unexpected(IsZero.error());
+
+ const auto Build = [&](GenericOpcode Generic, Register Left, Register Right) {
+ return buildResultInstruction(MIRBuilder, Generic, *HalfType, {Left, Right});
+ };
+ const auto Select = [&](Register Condition, Register TrueValue, Register FalseValue) {
+ return buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, *HalfType,
+ {Condition, TrueValue, FalseValue});
+ };
+
+ Register LowResult;
+ Register HighResult;
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (Generic == GenericOpcode::G_SHL) {
+ auto LowShort = Build(GenericOpcode::G_SHL, *LowPart, *Amount);
+ auto LowCarry = Build(GenericOpcode::G_LSHR, *LowPart, *AmountLack);
+ auto HighShift = Build(GenericOpcode::G_SHL, *HighPart, *Amount);
+ if (!LowShort)
+ return std::unexpected(LowShort.error());
+ if (!LowCarry)
+ return std::unexpected(LowCarry.error());
+ if (!HighShift)
+ return std::unexpected(HighShift.error());
+ auto HighShort = Build(GenericOpcode::G_OR, *LowCarry, *HighShift);
+ auto LowLong = buildConstantRegister(MIRBuilder, *HalfType, 0);
+ auto HighLong = Build(GenericOpcode::G_SHL, *LowPart, *AmountExcess);
+ if (!HighShort)
+ return std::unexpected(HighShort.error());
+ if (!LowLong)
+ return std::unexpected(LowLong.error());
+ if (!HighLong)
+ return std::unexpected(HighLong.error());
+ auto Low = Select(*IsShort, *LowShort, *LowLong);
+ auto HighNonZero = Select(*IsShort, *HighShort, *HighLong);
+ if (!Low)
+ return std::unexpected(Low.error());
+ if (!HighNonZero)
+ return std::unexpected(HighNonZero.error());
+ auto High = Select(*IsZero, *HighPart, *HighNonZero);
+ if (!High)
+ return std::unexpected(High.error());
+ LowResult = *Low;
+ HighResult = *High;
+ } else if (Generic == GenericOpcode::G_LSHR || Generic == GenericOpcode::G_ASHR) {
+ auto HighShort = Build(Generic, *HighPart, *Amount);
+ auto LowShift = Build(GenericOpcode::G_LSHR, *LowPart, *Amount);
+ auto HighCarry = Build(GenericOpcode::G_SHL, *HighPart, *AmountLack);
+ if (!HighShort)
+ return std::unexpected(HighShort.error());
+ if (!LowShift)
+ return std::unexpected(LowShift.error());
+ if (!HighCarry)
+ return std::unexpected(HighCarry.error());
+ auto LowShort = Build(GenericOpcode::G_OR, *LowShift, *HighCarry);
+ if (!LowShort)
+ return std::unexpected(LowShort.error());
+
+ Expected<Register> HighLong =
+ Generic == GenericOpcode::G_LSHR
+ ? buildConstantRegister(MIRBuilder, *HalfType, 0)
+ : buildConstantRegister(MIRBuilder, *AmountType,
+ static_cast<std::int64_t>(DestinationBits / 2U - 1U))
+ .and_then([&](Register Shift) {
+ return Build(GenericOpcode::G_ASHR, *HighPart, Shift);
+ });
+ auto LowLong = Build(Generic, *HighPart, *AmountExcess);
+ if (!HighLong)
+ return std::unexpected(HighLong.error());
+ if (!LowLong)
+ return std::unexpected(LowLong.error());
+ auto LowNonZero = Select(*IsShort, *LowShort, *LowLong);
+ if (!LowNonZero)
+ return std::unexpected(LowNonZero.error());
+ auto Low = Select(*IsZero, *LowPart, *LowNonZero);
+ auto High = Select(*IsShort, *HighShort, *HighLong);
+ if (!Low)
+ return std::unexpected(Low.error());
+ if (!High)
+ return std::unexpected(High.error());
+ LowResult = *Low;
+ HighResult = *High;
+ } else {
+ return makeUnexpected(ErrorCode::InternalError, "shift narrowing received a non-shift opcode");
+ }
+
+ if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, {LowResult, HighResult}); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+}
+
+Expected<Register> LegalizerHelper::buildConstantShiftPart(GenericOpcode Generic,
+ std::size_t PartIndex,
+ std::size_t PartCount,
+ const std::vector<Register> &SourceParts,
+ const ShiftParameters &Parameters,
+ LowLevelType TargetType) {
+ const auto WordShift = getIConstantVRegValWithLookThrough(Parameters.WordShift, MRI, false);
+ const auto BitShift = getIConstantVRegValWithLookThrough(Parameters.BitShift, MRI, false);
+ if (!WordShift || !BitShift || WordShift->Value.activeBitWidth() > 64 ||
+ BitShift->Value.activeBitWidth() > 64) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "multiway shift parameters are not representable constants");
+ }
+ const std::uint64_t ShiftWords = WordShift->Value.word(0);
+ const std::uint64_t ShiftBits = BitShift->Value.word(0);
+ const bool NeedsCarry = ShiftBits != 0;
+ const auto Build = [&](GenericOpcode Opcode, Register Left, Register Right) {
+ return buildResultInstruction(MIRBuilder, Opcode, TargetType, {Left, Right});
+ };
+
+ if (Generic == GenericOpcode::G_SHL) {
+ if (PartIndex < ShiftWords)
+ return Parameters.Zero;
+ const std::size_t SourceIndex = PartIndex - static_cast<std::size_t>(ShiftWords);
+ if (!NeedsCarry)
+ return SourceParts[SourceIndex];
+ auto High = Build(GenericOpcode::G_SHL, SourceParts[SourceIndex], Parameters.BitShift);
+ if (!High)
+ return std::unexpected(High.error());
+ if (SourceIndex == 0)
+ return *High;
+ auto Low =
+ Build(GenericOpcode::G_LSHR, SourceParts[SourceIndex - 1U], Parameters.InverseBitShift);
+ if (!Low)
+ return std::unexpected(Low.error());
+ return Build(GenericOpcode::G_OR, *High, *Low);
+ }
+
+ if (Generic == GenericOpcode::G_LSHR) {
+ const std::uint64_t SourceIndex = PartIndex + ShiftWords;
+ if (SourceIndex >= PartCount)
+ return Parameters.Zero;
+ if (!NeedsCarry)
+ return SourceParts[static_cast<std::size_t>(SourceIndex)];
+ auto Low = Build(GenericOpcode::G_LSHR, SourceParts[static_cast<std::size_t>(SourceIndex)],
+ Parameters.BitShift);
+ if (!Low)
+ return std::unexpected(Low.error());
+ if (SourceIndex + 1U >= PartCount)
+ return *Low;
+ auto High = Build(GenericOpcode::G_SHL, SourceParts[static_cast<std::size_t>(SourceIndex + 1U)],
+ Parameters.InverseBitShift);
+ if (!High)
+ return std::unexpected(High.error());
+ return Build(GenericOpcode::G_OR, *Low, *High);
+ }
+
+ if (Generic == GenericOpcode::G_ASHR) {
+ const std::uint64_t SourceIndex = PartIndex + ShiftWords;
+ if (SourceIndex >= PartCount)
+ return Parameters.SignBit;
+ if (!NeedsCarry)
+ return SourceParts[static_cast<std::size_t>(SourceIndex)];
+ const GenericOpcode MainOpcode =
+ SourceIndex + 1U == PartCount ? GenericOpcode::G_ASHR : GenericOpcode::G_LSHR;
+ auto Low =
+ Build(MainOpcode, SourceParts[static_cast<std::size_t>(SourceIndex)], Parameters.BitShift);
+ if (!Low)
+ return std::unexpected(Low.error());
+ const Register HighSource = SourceIndex + 1U < PartCount
+ ? SourceParts[static_cast<std::size_t>(SourceIndex + 1U)]
+ : Parameters.SignBit;
+ auto High = Build(GenericOpcode::G_SHL, HighSource, Parameters.InverseBitShift);
+ if (!High)
+ return std::unexpected(High.error());
+ return Build(GenericOpcode::G_OR, *Low, *High);
+ }
+
+ return makeUnexpected(ErrorCode::InternalError,
+ "constant shift part received a non-shift opcode");
+}
+
+Expected<Register> LegalizerHelper::buildVariableShiftPart(GenericOpcode Generic,
+ Register MainOperand,
+ Register ShiftAmount,
+ LowLevelType TargetType,
+ Register CarryOperand) {
+ const GenericOpcode MainOpcode =
+ Generic == GenericOpcode::G_ASHR ? GenericOpcode::G_LSHR : Generic;
+ auto MainShifted =
+ buildResultInstruction(MIRBuilder, MainOpcode, TargetType, {MainOperand, ShiftAmount});
+ if (!MainShifted)
+ return std::unexpected(MainShifted.error());
+ if (CarryOperand.isNoRegister())
+ return *MainShifted;
+
+ auto AmountType = MRI.getType(ShiftAmount);
+ if (!AmountType)
+ return std::unexpected(AmountType.error());
+ auto ZeroAmount = buildConstantRegister(MIRBuilder, *AmountType, 0);
+ auto BoolType = LowLevelType::makeCategoryNeutralScalar(1);
+ if (!ZeroAmount)
+ return std::unexpected(ZeroAmount.error());
+ if (!BoolType)
+ return std::unexpected(BoolType.error());
+ auto IsZero =
+ buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *BoolType, ShiftAmount, *ZeroAmount);
+ auto TargetBits = buildConstantRegister(MIRBuilder, *AmountType,
+ static_cast<std::int64_t>(TargetType.scalarBitWidth()));
+ if (!IsZero)
+ return std::unexpected(IsZero.error());
+ if (!TargetBits)
+ return std::unexpected(TargetBits.error());
+ auto InverseAmount = buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType,
+ {*TargetBits, ShiftAmount});
+ if (!InverseAmount)
+ return std::unexpected(InverseAmount.error());
+
+ const GenericOpcode CarryOpcode =
+ Generic == GenericOpcode::G_SHL ? GenericOpcode::G_LSHR : GenericOpcode::G_SHL;
+ auto CarryBits =
+ buildResultInstruction(MIRBuilder, CarryOpcode, TargetType, {CarryOperand, *InverseAmount});
+ auto Zero = buildConstantRegister(MIRBuilder, TargetType, 0);
+ if (!CarryBits)
+ return std::unexpected(CarryBits.error());
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto SafeCarry = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, TargetType,
+ {*IsZero, *Zero, *CarryBits});
+ if (!SafeCarry)
+ return std::unexpected(SafeCarry.error());
+ return buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, TargetType,
+ {*MainShifted, *SafeCarry});
+}
+
+Expected<void> LegalizerHelper::narrowScalarShiftByConstantMultiway(MachineInstr &Instruction,
+ const APInt &Amount,
+ LowLevelType TargetType,
+ LowLevelType AmountType) {
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ const Register *Source = Instruction.operandAt(1)->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr || Amount.activeBitWidth() > 64)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "constant multiway shift requires representable register operands");
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ const std::uint64_t DestinationBits = DestinationType->scalarBitWidth();
+ const std::uint64_t TargetBits = TargetType.scalarBitWidth();
+ if (TargetBits == 0 || DestinationBits % TargetBits != 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiway shift target type does not divide the destination");
+ const std::size_t PartCount = static_cast<std::size_t>(DestinationBits / TargetBits);
+
+ std::vector<Register> SourceParts;
+ auto Extracted =
+ extractParts(*Source, TargetType, static_cast<int>(PartCount), SourceParts, MIRBuilder, MRI);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ if (Amount.isZero()) {
+ if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, SourceParts); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+ }
+
+ const std::uint64_t AmountValue = Amount.word(0);
+ const std::uint64_t ShiftWords = AmountValue / TargetBits;
+ const std::uint64_t ShiftBits = AmountValue % TargetBits;
+ ShiftParameters Parameters;
+ auto WordShift =
+ buildConstantRegister(MIRBuilder, AmountType, static_cast<std::int64_t>(ShiftWords));
+ auto BitShift =
+ buildConstantRegister(MIRBuilder, AmountType, static_cast<std::int64_t>(ShiftBits));
+ auto InverseBitShift = buildConstantRegister(MIRBuilder, AmountType,
+ static_cast<std::int64_t>(TargetBits - ShiftBits));
+ auto Zero = buildConstantRegister(MIRBuilder, TargetType, 0);
+ if (!WordShift)
+ return std::unexpected(WordShift.error());
+ if (!BitShift)
+ return std::unexpected(BitShift.error());
+ if (!InverseBitShift)
+ return std::unexpected(InverseBitShift.error());
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ Parameters.WordShift = *WordShift;
+ Parameters.BitShift = *BitShift;
+ Parameters.InverseBitShift = *InverseBitShift;
+ Parameters.Zero = *Zero;
+
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (Generic == GenericOpcode::G_ASHR) {
+ auto SignAmount =
+ buildConstantRegister(MIRBuilder, AmountType, static_cast<std::int64_t>(TargetBits - 1U));
+ if (!SignAmount)
+ return std::unexpected(SignAmount.error());
+ auto SignBit = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, TargetType,
+ {SourceParts.back(), *SignAmount});
+ if (!SignBit)
+ return std::unexpected(SignBit.error());
+ Parameters.SignBit = *SignBit;
+ }
+
+ std::vector<Register> DestinationParts;
+ DestinationParts.reserve(PartCount);
+ for (std::size_t Index = 0; Index != PartCount; ++Index) {
+ auto Part =
+ buildConstantShiftPart(Generic, Index, PartCount, SourceParts, Parameters, TargetType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ DestinationParts.push_back(*Part);
+ }
+ if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, DestinationParts); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::narrowScalarShiftMultiway(MachineInstr &Instruction,
+ LowLevelType TargetType) {
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ const Register *Source = Instruction.operandAt(1)->getIf<Register>();
+ const Register *Amount = Instruction.operandAt(2)->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr || Amount == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiway shift narrowing requires register operands");
+ auto DestinationType = MRI.getType(*Destination);
+ auto AmountType = MRI.getType(*Amount);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (!AmountType)
+ return std::unexpected(AmountType.error());
+ const std::uint64_t DestinationBits = DestinationType->scalarBitWidth();
+ const std::uint64_t TargetBits = TargetType.scalarBitWidth();
+ if (TargetBits == 0 || DestinationBits % TargetBits != 0 || !std::has_single_bit(TargetBits)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiway shift requires a power-of-two divisor type");
+ }
+ const std::size_t PartCount = static_cast<std::size_t>(DestinationBits / TargetBits);
+ if (auto Constant = getIConstantVRegValWithLookThrough(*Amount, MRI)) {
+ return narrowScalarShiftByConstantMultiway(Instruction, Constant->Value, TargetType,
+ *AmountType);
+ }
+
+ std::vector<Register> SourceParts;
+ auto Extracted =
+ extractParts(*Source, TargetType, static_cast<int>(PartCount), SourceParts, MIRBuilder, MRI);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+
+ auto ZeroAmount = buildConstantRegister(MIRBuilder, *AmountType, 0);
+ auto BoolType = LowLevelType::makeCategoryNeutralScalar(1);
+ if (!ZeroAmount)
+ return std::unexpected(ZeroAmount.error());
+ if (!BoolType)
+ return std::unexpected(BoolType.error());
+ auto IsZero =
+ buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *BoolType, *Amount, *ZeroAmount);
+ if (!IsZero)
+ return std::unexpected(IsZero.error());
+
+ const auto TargetLog2 = static_cast<std::int64_t>(std::countr_zero(TargetBits));
+ auto Log2 = buildConstantRegister(MIRBuilder, *AmountType, TargetLog2);
+ auto Mask =
+ buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(TargetBits - 1U));
+ if (!Log2)
+ return std::unexpected(Log2.error());
+ if (!Mask)
+ return std::unexpected(Mask.error());
+ auto WordShift =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *AmountType, {*Amount, *Log2});
+ auto BitShift =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType, {*Amount, *Mask});
+ auto Zero = buildConstantRegister(MIRBuilder, TargetType, 0);
+ if (!WordShift)
+ return std::unexpected(WordShift.error());
+ if (!BitShift)
+ return std::unexpected(BitShift.error());
+ if (!Zero)
+ return std::unexpected(Zero.error());
+
+ Register FillValue = *Zero;
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (Generic == GenericOpcode::G_ASHR) {
+ auto SignAmount =
+ buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(TargetBits - 1U));
+ if (!SignAmount)
+ return std::unexpected(SignAmount.error());
+ auto Sign = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, TargetType,
+ {SourceParts.back(), *SignAmount});
+ if (!Sign)
+ return std::unexpected(Sign.error());
+ FillValue = *Sign;
+ }
+
+ std::vector<Register> DestinationParts(PartCount);
+ for (std::size_t PartIndex = 0; PartIndex != PartCount; ++PartIndex) {
+ Register InBoundsResult = FillValue;
+ for (std::size_t WordIndex = 0; WordIndex != PartCount; ++WordIndex) {
+ auto WordConstant =
+ buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(WordIndex));
+ if (!WordConstant)
+ return std::unexpected(WordConstant.error());
+ auto IsWord = buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *BoolType, *WordShift,
+ *WordConstant);
+ if (!IsWord)
+ return std::unexpected(IsWord.error());
+
+ std::ptrdiff_t MainIndex;
+ std::ptrdiff_t CarryIndex;
+ if (Generic == GenericOpcode::G_SHL) {
+ MainIndex = static_cast<std::ptrdiff_t>(PartIndex) - static_cast<std::ptrdiff_t>(WordIndex);
+ CarryIndex = MainIndex - 1;
+ } else if (Generic == GenericOpcode::G_LSHR || Generic == GenericOpcode::G_ASHR) {
+ MainIndex = static_cast<std::ptrdiff_t>(PartIndex) + static_cast<std::ptrdiff_t>(WordIndex);
+ CarryIndex = MainIndex + 1;
+ } else {
+ return makeUnexpected(ErrorCode::InternalError,
+ "multiway shift received a non-shift opcode");
+ }
+
+ Register Result = FillValue;
+ if (MainIndex >= 0 && MainIndex < static_cast<std::ptrdiff_t>(PartCount)) {
+ Register Carry;
+ if (CarryIndex >= 0 && CarryIndex < static_cast<std::ptrdiff_t>(PartCount))
+ Carry = SourceParts[static_cast<std::size_t>(CarryIndex)];
+ else if (Generic == GenericOpcode::G_ASHR &&
+ CarryIndex >= static_cast<std::ptrdiff_t>(PartCount))
+ Carry = FillValue;
+ auto Shifted =
+ buildVariableShiftPart(Generic, SourceParts[static_cast<std::size_t>(MainIndex)],
+ *BitShift, TargetType, Carry);
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ Result = *Shifted;
+ }
+
+ auto Selected = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, TargetType,
+ {*IsWord, Result, InBoundsResult});
+ if (!Selected)
+ return std::unexpected(Selected.error());
+ InBoundsResult = *Selected;
+ }
+ auto Selected = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, TargetType,
+ {*IsZero, SourceParts[PartIndex], InBoundsResult});
+ if (!Selected)
+ return std::unexpected(Selected.error());
+ DestinationParts[PartIndex] = *Selected;
+ }
+
+ if (auto Built = MIRBuilder.buildMergeLikeInstr(*Destination, DestinationParts); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::narrowScalarLoadStore(MachineInstr &Instruction,
+ std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (Generic != GenericOpcode::G_LOAD && Generic != GenericOpcode::G_STORE) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "memory scalar narrowing received an unsupported opcode");
+ }
+ if (TypeIndex != 0 || !NarrowType.isCategoryNeutralScalar() ||
+ NarrowType.minimumBitWidth() == 0 || NarrowType.minimumBitWidth() % 8U != 0) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "memory scalar narrowing requires a byte-sized scalar value type");
+ }
+ if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 ||
+ Instruction.memOperands().front() == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "memory scalar narrowing requires one address and one memory operand");
+ }
+
+ const MachineOperand *ValueOperand = Instruction.operandAt(0);
+ const MachineOperand *AddressOperand = Instruction.operandAt(1);
+ const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+ const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>();
+ if (Value == nullptr || Address == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "memory scalar narrowing requires register value and address operands");
+ }
+ auto ValueType = MRI.getType(*Value);
+ if (!ValueType)
+ return std::unexpected(ValueType.error());
+ if (!ValueType->isCategoryNeutralScalar() ||
+ NarrowType.minimumBitWidth() >= ValueType->minimumBitWidth()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory scalar narrowing requires a smaller integer scalar type");
+ }
+
+ const MachineMemOperand &Memory = *Instruction.memOperands().front();
+ const MachineMemOperandDesc &Description = Memory.description();
+ if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory scalar narrowing requires a fixed typed memory operand");
+ }
+ const std::uint64_t MemorySizeInBits = Description.Size.minimumBytes() * 8U;
+ if (MemorySizeInBits != ValueType->minimumBitWidth()) {
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ if (Generic == GenericOpcode::G_LOAD) {
+ auto Temporary = MRI.createVirtualRegister(NarrowType);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ auto Load = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD));
+ if (!Load)
+ return std::unexpected(Load.error());
+ if (auto Added = Load->addDef(*Temporary); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Load->addUse(*Address); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Load->addMemOperand(Memory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Load->build(); !Built)
+ return std::unexpected(Built.error());
+ if (auto Extended = MIRBuilder.buildAnyExt(*Value, *Temporary); !Extended)
+ return std::unexpected(Extended.error());
+ } else {
+ auto Temporary = MRI.createVirtualRegister(NarrowType);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ if (auto Truncated = MIRBuilder.buildTrunc(*Temporary, *Value); !Truncated)
+ return std::unexpected(Truncated.error());
+ auto Store = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE));
+ if (!Store)
+ return std::unexpected(Store.error());
+ if (auto Added = Store->addUse(*Temporary); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Store->addUse(*Address); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Store->addMemOperand(Memory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Store->build(); !Built)
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+ }
+
+ return reduceLoadStoreWidth(Instruction, TypeIndex, NarrowType);
+}
+
+Expected<void> LegalizerHelper::reduceLoadStoreWidth(MachineInstr &Instruction,
+ std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (Generic != GenericOpcode::G_LOAD && Generic != GenericOpcode::G_STORE) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "memory width reduction received an unsupported opcode");
+ }
+ if (TypeIndex != 0 || !NarrowType.isCategoryNeutralScalar() ||
+ NarrowType.minimumBitWidth() == 0 || NarrowType.minimumBitWidth() % 8U != 0) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory width reduction requires a byte-sized scalar target type");
+ }
+ if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 ||
+ Instruction.memOperands().front() == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "memory width reduction requires one address and one memory operand");
+ }
+
+ const bool IsLoad = Generic == GenericOpcode::G_LOAD;
+ const MachineOperand *ValueOperand = Instruction.operandAt(0);
+ const MachineOperand *AddressOperand = Instruction.operandAt(1);
+ const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+ const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>();
+ if (Value == nullptr || Address == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "memory width reduction requires register value and address operands");
+ }
+ auto ValueType = MRI.getType(*Value);
+ if (!ValueType)
+ return std::unexpected(ValueType.error());
+ if (!ValueType->isCategoryNeutralScalar()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory width reduction does not support pointer or vector values");
+ }
+
+ const MachineMemOperand &Memory = *Instruction.memOperands().front();
+ const MachineMemOperandDesc &Description = Memory.description();
+ if (Memory.isAtomic()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory width reduction does not support atomic operations");
+ }
+ if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType ||
+ Description.MemoryType->isVector()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory width reduction requires a fixed scalar memory operand");
+ }
+ const std::uint64_t ValueBits = ValueType->minimumBitWidth();
+ if (ValueBits != Description.Size.minimumBytes() * 8U ||
+ NarrowType.minimumBitWidth() >= ValueBits) {
+ return makeUnexpected(
+ ErrorCode::Unsupported,
+ "memory width reduction cannot split the requested value and memory widths");
+ }
+
+ const std::size_t PartCount = static_cast<std::size_t>(ValueBits / NarrowType.minimumBitWidth());
+ const std::uint64_t LeftoverBits = ValueBits - PartCount * NarrowType.minimumBitWidth();
+ LowLevelType ExpectedLeftoverType;
+ if (LeftoverBits != 0) {
+ auto Type = LowLevelType::makeCategoryNeutralScalar(LeftoverBits);
+ if (!Type)
+ return std::unexpected(Type.error());
+ if (LeftoverBits % 8U != 0) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory width reduction leaves a non-byte-sized scalar fragment");
+ }
+ ExpectedLeftoverType = *Type;
+ }
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ std::vector<Register> Parts;
+ std::vector<Register> Leftovers;
+ LowLevelType LeftoverType;
+ if (IsLoad) {
+ Parts.reserve(PartCount);
+ for (std::size_t Index = 0; Index != PartCount; ++Index) {
+ auto Part = MRI.createVirtualRegister(NarrowType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ Parts.push_back(*Part);
+ }
+ LeftoverType = ExpectedLeftoverType;
+ if (ExpectedLeftoverType.isValid()) {
+ auto Leftover = MRI.createVirtualRegister(ExpectedLeftoverType);
+ if (!Leftover)
+ return std::unexpected(Leftover.error());
+ Leftovers.push_back(*Leftover);
+ }
+ } else {
+ auto Extracted = extractParts(*Value, *ValueType, NarrowType, LeftoverType, Parts, Leftovers,
+ MIRBuilder, MRI);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ if (!*Extracted || Parts.size() != PartCount || LeftoverType != ExpectedLeftoverType ||
+ (ExpectedLeftoverType.isValid() && Leftovers.size() != 1U)) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "memory width reduction cannot split the store value consistently");
+ }
+ }
+
+ const auto SplitPieces = [&](LowLevelType PartType, const std::vector<Register> &Values,
+ std::uint64_t &Offset) -> Expected<void> {
+ const std::uint64_t PartBytes = PartType.minimumBitWidth() / 8U;
+ for (Register Part : Values) {
+ auto AddressPart = buildObjectPointerOffset(MIRBuilder, *Address, Offset);
+ if (!AddressPart)
+ return std::unexpected(AddressPart.error());
+ auto MemoryPart = deriveMemoryOperand(Function, Memory, PartType, Offset);
+ if (!MemoryPart)
+ return std::unexpected(MemoryPart.error());
+
+ auto Access =
+ MIRBuilder.buildInstr(opcode(IsLoad ? GenericOpcode::G_LOAD : GenericOpcode::G_STORE));
+ if (!Access)
+ return std::unexpected(Access.error());
+ if (IsLoad) {
+ if (auto Added = Access->addDef(Part); !Added)
+ return std::unexpected(Added.error());
+ } else if (auto Added = Access->addUse(Part); !Added) {
+ return std::unexpected(Added.error());
+ }
+ if (auto Added = Access->addUse(*AddressPart); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Access->addMemOperand(**MemoryPart); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Access->build(); !Built)
+ return std::unexpected(Built.error());
+ Offset += PartBytes;
+ }
+ return {};
+ };
+
+ std::uint64_t Offset = 0;
+ if (auto Split = SplitPieces(NarrowType, Parts, Offset); !Split)
+ return std::unexpected(Split.error());
+ if (LeftoverType.isValid()) {
+ if (auto Split = SplitPieces(LeftoverType, Leftovers, Offset); !Split)
+ return std::unexpected(Split.error());
+ }
+
+ if (IsLoad) {
+ if (auto Inserted = insertParts(*Value, *ValueType, NarrowType, Parts, LeftoverType, Leftovers);
+ !Inserted) {
+ return std::unexpected(Inserted.error());
+ }
+ }
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::narrowScalar(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+ case GenericOpcode::G_IMPLICIT_DEF: {
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_IMPLICIT_DEF narrowing requires a register destination");
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+
+ const std::uint64_t DestinationSize = DestinationType->minimumBitWidth();
+ const std::uint64_t NarrowSize = NarrowType.minimumBitWidth();
+ if (NarrowSize == 0 || NarrowSize >= DestinationSize)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_IMPLICIT_DEF narrowing requires a smaller nonzero type");
+
+ if (DestinationSize % NarrowSize != 0) {
+ LowLevelType ImplicitType = NarrowType;
+ if (DestinationType->isVector()) {
+ auto VectorType =
+ DestinationType->isScalableVector()
+ ? LowLevelType::makeScalableVector(DestinationType->minimumElementCount(),
+ NarrowType)
+ : LowLevelType::makeFixedVector(DestinationType->minimumElementCount(), NarrowType);
+ if (!VectorType)
+ return std::unexpected(VectorType.error());
+ ImplicitType = *VectorType;
+ }
+ auto Implicit = MIRBuilder.buildUndef(ImplicitType);
+ if (!Implicit)
+ return std::unexpected(Implicit.error());
+ auto ImplicitReg = resultRegister(**Implicit);
+ if (!ImplicitReg)
+ return std::unexpected(ImplicitReg.error());
+ if (auto Extended = MIRBuilder.buildAnyExt(*Destination, *ImplicitReg); !Extended)
+ return std::unexpected(Extended.error());
+ } else {
+ const std::size_t PartCount = static_cast<std::size_t>(DestinationSize / NarrowSize);
+ std::vector<Register> Parts;
+ Parts.reserve(PartCount);
+ for (std::size_t Index = 0; Index != PartCount; ++Index) {
+ auto Part = MIRBuilder.buildUndef(NarrowType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ auto PartReg = resultRegister(**Part);
+ if (!PartReg)
+ return std::unexpected(PartReg.error());
+ Parts.push_back(*PartReg);
+ }
+ if (auto Merged = MIRBuilder.buildMergeLikeInstr(*Destination, Parts); !Merged)
+ return std::unexpected(Merged.error());
+ }
+
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_CONSTANT: {
+ const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+ const auto *Immediate = Instruction.operandAt(1)->getIf<IntegerImmediateOperand>();
+ if (Destination == nullptr || Immediate == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_CONSTANT narrowing requires register and integer operands");
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+
+ const std::uint64_t TotalSize = DestinationType->minimumBitWidth();
+ const std::uint64_t NarrowSize = NarrowType.minimumBitWidth();
+ if (NarrowSize == 0 || NarrowSize >= TotalSize || NarrowSize > APInt::MaxBitWidth ||
+ TotalSize > APInt::MaxBitWidth) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_CONSTANT narrowing requires a smaller representable type");
+ }
+
+ const std::size_t PartCount = static_cast<std::size_t>(TotalSize / NarrowSize);
+ std::vector<Register> Parts;
+ Parts.reserve(PartCount);
+ for (std::size_t Index = 0; Index != PartCount; ++Index) {
+ auto Shifted = Immediate->Value.logicalShiftRight(Index * NarrowSize);
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ APInt PartValue = std::move(*Shifted);
+ if (PartValue.bitWidth() > NarrowSize) {
+ auto Truncated = PartValue.trunc(static_cast<std::uint32_t>(NarrowSize));
+ if (!Truncated)
+ return std::unexpected(Truncated.error());
+ PartValue = std::move(*Truncated);
+ }
+ auto Part = buildConstantRegister(MIRBuilder, NarrowType, std::move(PartValue));
+ if (!Part)
+ return std::unexpected(Part.error());
+ Parts.push_back(*Part);
+ }
+
+ LowLevelType LeftoverType;
+ std::vector<Register> Leftovers;
+ const std::uint64_t LeftoverBits = TotalSize - PartCount * NarrowSize;
+ if (LeftoverBits != 0) {
+ auto Type = LowLevelType::makeCategoryNeutralScalar(LeftoverBits);
+ if (!Type)
+ return std::unexpected(Type.error());
+ LeftoverType = *Type;
+ auto Shifted = Immediate->Value.logicalShiftRight(PartCount * NarrowSize);
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ APInt LeftoverValue = std::move(*Shifted);
+ if (LeftoverValue.bitWidth() > LeftoverBits) {
+ auto Truncated = LeftoverValue.trunc(static_cast<std::uint32_t>(LeftoverBits));
+ if (!Truncated)
+ return std::unexpected(Truncated.error());
+ LeftoverValue = std::move(*Truncated);
+ }
+ auto Leftover = buildConstantRegister(MIRBuilder, LeftoverType, std::move(LeftoverValue));
+ if (!Leftover)
+ return std::unexpected(Leftover.error());
+ Leftovers.push_back(*Leftover);
+ }
+
+ if (auto Inserted =
+ insertParts(*Destination, *DestinationType, NarrowType, Parts, LeftoverType, Leftovers);
+ !Inserted) {
+ return std::unexpected(Inserted.error());
+ }
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_ADD:
+ case GenericOpcode::G_SUB:
+ case GenericOpcode::G_SADDO:
+ case GenericOpcode::G_SSUBO:
+ case GenericOpcode::G_SADDE:
+ case GenericOpcode::G_SSUBE:
+ case GenericOpcode::G_UADDO:
+ case GenericOpcode::G_USUBO:
+ case GenericOpcode::G_UADDE:
+ case GenericOpcode::G_USUBE:
+ return narrowScalarAddSub(Instruction, TypeIndex, NarrowType);
+ case GenericOpcode::G_MUL:
+ case GenericOpcode::G_UMULH:
+ return narrowScalarMul(Instruction, TypeIndex, NarrowType);
+ case GenericOpcode::G_LOAD:
+ case GenericOpcode::G_STORE:
+ return narrowScalarLoadStore(Instruction, TypeIndex, NarrowType);
+ case GenericOpcode::G_SEXT:
+ case GenericOpcode::G_ZEXT:
+ case GenericOpcode::G_ANYEXT:
+ return narrowScalarExt(Instruction, TypeIndex, NarrowType);
+ case GenericOpcode::G_INTTOPTR:
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_INTTOPTR destination narrowing is not defined");
+ Observer.changingInstr(Instruction);
+ if (auto Narrowed = narrowScalarSrc(Instruction, NarrowType, 1); !Narrowed)
+ return std::unexpected(Narrowed.error());
+ Observer.changedInstr(Instruction);
+ return {};
+ case GenericOpcode::G_PTRTOINT:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::Unsupported, "G_PTRTOINT source narrowing is not defined");
+ Observer.changingInstr(Instruction);
+ if (auto Narrowed = narrowScalarDst(Instruction, NarrowType, 0, GenericOpcode::G_ZEXT);
+ !Narrowed) {
+ return std::unexpected(Narrowed.error());
+ }
+ Observer.changedInstr(Instruction);
+ return {};
+ case GenericOpcode::G_PHI: {
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::Unsupported, "G_PHI narrowing requires type index zero");
+ if (Instruction.operandCount() < 3 || Instruction.operandCount() % 2U == 0)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_PHI narrowing requires incoming value/block pairs");
+ MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_PHI narrowing requires a register destination");
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ const std::uint64_t DestinationSize = DestinationType->minimumBitWidth();
+ const std::uint64_t NarrowSize = NarrowType.minimumBitWidth();
+ if (NarrowSize == 0 || NarrowSize >= DestinationSize || DestinationSize % NarrowSize != 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_PHI narrowing requires an exact scalar split");
+
+ const std::size_t PartCount = static_cast<std::size_t>(DestinationSize / NarrowSize);
+ const std::size_t IncomingCount = (Instruction.operandCount() - 1U) / 2U;
+ std::vector<std::vector<Register>> IncomingParts(IncomingCount);
+ std::vector<MachineBasicBlock *> IncomingBlocks(IncomingCount);
+ Observer.changingInstr(Instruction);
+ for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
+ const MachineOperand *ValueOperand = Instruction.operandAt(Index);
+ const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U);
+ const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+ const auto *Incoming =
+ BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>();
+ if (Value == nullptr || Incoming == nullptr || Incoming->Block == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_PHI narrowing has an invalid incoming pair");
+ if (auto Point = setInsertPointAtFirstTerminator(MIRBuilder, *Incoming->Block); !Point)
+ return std::unexpected(Point.error());
+ const std::size_t IncomingIndex = Index / 2U;
+ IncomingBlocks[IncomingIndex] = Incoming->Block;
+ auto &Parts = IncomingParts[IncomingIndex];
+ Parts.reserve(PartCount);
+ for (std::size_t PartIndex = 0; PartIndex != PartCount; ++PartIndex) {
+ auto Part = MRI.createVirtualRegister(NarrowType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ Parts.push_back(*Part);
+ }
+ if (auto Unmerged = MIRBuilder.buildUnmergeValues(Parts, *Value); !Unmerged)
+ return std::unexpected(Unmerged.error());
+ }
+
+ MachineBasicBlock &Block = *Instruction.parent();
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ std::vector<Register> Destinations;
+ Destinations.reserve(PartCount);
+ for (std::size_t PartIndex = 0; PartIndex != PartCount; ++PartIndex) {
+ auto DestinationPart = MRI.createVirtualRegister(NarrowType);
+ if (!DestinationPart)
+ return std::unexpected(DestinationPart.error());
+ auto Phi = MIRBuilder.buildInstr(opcode(GenericOpcode::G_PHI));
+ if (!Phi)
+ return std::unexpected(Phi.error());
+ if (auto Added = Phi->addDef(*DestinationPart); !Added)
+ return std::unexpected(Added.error());
+ for (std::size_t IncomingIndex = 0; IncomingIndex != IncomingCount; ++IncomingIndex) {
+ if (auto Added = Phi->addUse(IncomingParts[IncomingIndex][PartIndex]); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Phi->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added)
+ return std::unexpected(Added.error());
+ }
+ if (auto Built = Phi->build(); !Built)
+ return std::unexpected(Built.error());
+ Destinations.push_back(*DestinationPart);
+ }
+
+ if (auto Point = setInsertPointAfterPHIs(MIRBuilder, Block); !Point)
+ return std::unexpected(Point.error());
+ if (auto Merged = MIRBuilder.buildMergeLikeInstr(*Destination, Destinations); !Merged)
+ return std::unexpected(Merged.error());
+ Observer.changedInstr(Instruction);
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_FCMP:
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "floating comparison source narrowing is not defined");
+ Observer.changingInstr(Instruction);
+ if (auto Narrowed = narrowScalarDst(Instruction, NarrowType, 0, GenericOpcode::G_ZEXT);
+ !Narrowed) {
+ return std::unexpected(Narrowed.error());
+ }
+ Observer.changedInstr(Instruction);
+ return {};
+ case GenericOpcode::G_SEXT_INREG: {
+ if (TypeIndex != 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_SEXT_INREG narrowing requires type index zero");
+
+ MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ MachineOperand *SourceOperand = Instruction.operandAt(1);
+ MachineOperand *WidthOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr || WidthOperand == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_SEXT_INREG narrowing requires register and width operands");
+ auto SizeInBits = signedImmediate(*WidthOperand);
+ if (!SizeInBits)
+ return std::unexpected(SizeInBits.error());
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+
+ const std::uint64_t TotalSize = DestinationType->minimumBitWidth();
+ const std::uint64_t NarrowSize = NarrowType.minimumBitWidth();
+ if (*SizeInBits <= 0 || static_cast<std::uint64_t>(*SizeInBits) > TotalSize ||
+ NarrowSize == 0 || NarrowSize >= TotalSize) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_SEXT_INREG narrowing has invalid scalar widths");
+ }
+
+ if (NarrowSize > static_cast<std::uint64_t>(*SizeInBits)) {
+ const Register OriginalDestination = *Destination;
+ Observer.changingInstr(Instruction);
+ auto NarrowedSource = narrowScalarSrc(Instruction, NarrowType, 1);
+ if (!NarrowedSource)
+ return std::unexpected(NarrowedSource.error());
+ auto NarrowDestination = MRI.createVirtualRegister(NarrowType);
+ if (!NarrowDestination)
+ return std::unexpected(NarrowDestination.error());
+ if (auto Changed = DestinationOperand->setRegister(*NarrowDestination); !Changed)
+ return std::unexpected(Changed.error());
+ if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Extended = MIRBuilder.buildSExt(OriginalDestination, *NarrowDestination); !Extended)
+ return std::unexpected(Extended.error());
+ Observer.changedInstr(Instruction);
+ return {};
+ }
+
+ if (TotalSize % NarrowSize != 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_SEXT_INREG cannot be split evenly to the requested type");
+ const std::size_t PartCount = static_cast<std::size_t>(TotalSize / NarrowSize);
+ std::vector<Register> SourceParts;
+ SourceParts.reserve(PartCount);
+ for (std::size_t Index = 0; Index != PartCount; ++Index) {
+ auto Part = MRI.createVirtualRegister(NarrowType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ SourceParts.push_back(*Part);
+ }
+ if (auto Unmerged = MIRBuilder.buildUnmergeValues(SourceParts, *Source); !Unmerged)
+ return std::unexpected(Unmerged.error());
+
+ auto ShiftAmount =
+ buildConstantRegister(MIRBuilder, NarrowType, static_cast<std::int64_t>(NarrowSize - 1U));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+
+ std::vector<Register> DestinationParts;
+ DestinationParts.reserve(PartCount);
+ Register FullExtension;
+ Register PartialExtension;
+ const std::uint64_t ExtensionBits = static_cast<std::uint64_t>(*SizeInBits);
+ for (std::size_t Index = 0; Index != PartCount; ++Index) {
+ if ((Index + 1U) * NarrowSize <= ExtensionBits) {
+ DestinationParts.push_back(SourceParts[Index]);
+ PartialExtension = SourceParts[Index];
+ continue;
+ }
+ if (Index * NarrowSize >= ExtensionBits) {
+ if (PartialExtension.isNoRegister())
+ return makeUnexpected(ErrorCode::InternalError,
+ "G_SEXT_INREG split has no sign-bearing part");
+ if (FullExtension.isNoRegister()) {
+ auto Extended = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, NarrowType,
+ {PartialExtension, *ShiftAmount});
+ if (!Extended)
+ return std::unexpected(Extended.error());
+ FullExtension = *Extended;
+ }
+ DestinationParts.push_back(FullExtension);
+ continue;
+ }
+
+ auto Extended = MIRBuilder.buildSExtInReg(
+ NarrowType, SourceParts[Index], static_cast<std::uint32_t>(ExtensionBits % NarrowSize));
+ if (!Extended)
+ return std::unexpected(Extended.error());
+ auto ExtendedRegister = resultRegister(**Extended);
+ if (!ExtendedRegister)
+ return std::unexpected(ExtendedRegister.error());
+ DestinationParts.push_back(*ExtendedRegister);
+ PartialExtension = *ExtendedRegister;
+ }
+
+ if (auto Merged = MIRBuilder.buildMergeLikeInstr(*Destination, DestinationParts); !Merged)
+ return std::unexpected(Merged.error());
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_AND:
+ case GenericOpcode::G_OR:
+ case GenericOpcode::G_XOR:
+ return narrowScalarBasic(Instruction, TypeIndex, NarrowType);
+ case GenericOpcode::G_SELECT:
+ return narrowScalarSelect(Instruction, TypeIndex, NarrowType);
+ case GenericOpcode::G_SHL:
+ case GenericOpcode::G_LSHR:
+ case GenericOpcode::G_ASHR:
+ return narrowScalarShift(Instruction, TypeIndex, NarrowType);
+ case GenericOpcode::G_ICMP: {
+ if (TypeIndex != 1)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_ICMP narrowing requires the source type index");
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *PredicateMachineOperand = Instruction.operandAt(1);
+ const MachineOperand *LeftOperand = Instruction.operandAt(2);
+ const MachineOperand *RightOperand = Instruction.operandAt(3);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const auto *PredicateValue = PredicateMachineOperand == nullptr
+ ? nullptr
+ : PredicateMachineOperand->getIf<PredicateOperand>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ const IntegerPredicate *Predicate =
+ PredicateValue == nullptr ? nullptr : PredicateValue->integerPredicate();
+ if (Destination == nullptr || Left == nullptr || Right == nullptr || Predicate == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_ICMP narrowing requires an integer predicate and register operands");
+ }
+ auto SourceType = MRI.getType(*Left);
+ auto ResultType = MRI.getType(*Destination);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ if (!ResultType)
+ return std::unexpected(ResultType.error());
+
+ LowLevelType LeftoverType;
+ LowLevelType UnusedType;
+ std::vector<Register> LeftParts;
+ std::vector<Register> LeftLeftovers;
+ std::vector<Register> RightParts;
+ std::vector<Register> RightLeftovers;
+ auto LeftExtracted = extractParts(*Left, *SourceType, NarrowType, LeftoverType, LeftParts,
+ LeftLeftovers, MIRBuilder, MRI);
+ if (!LeftExtracted)
+ return std::unexpected(LeftExtracted.error());
+ auto RightExtracted = extractParts(*Right, *SourceType, NarrowType, UnusedType, RightParts,
+ RightLeftovers, MIRBuilder, MRI);
+ if (!RightExtracted)
+ return std::unexpected(RightExtracted.error());
+ if (!*LeftExtracted || !*RightExtracted || LeftParts.size() != RightParts.size() ||
+ LeftLeftovers.size() != RightLeftovers.size()) {
+ return makeUnexpected(ErrorCode::Unsupported, "G_ICMP operands cannot be split consistently");
+ }
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (isEqualityPredicate(*Predicate)) {
+ auto Zero = buildConstantRegister(MIRBuilder, NarrowType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+
+ std::vector<Register> Xors;
+ Xors.reserve(LeftParts.size() + LeftLeftovers.size());
+ for (std::size_t Index = 0; Index != LeftParts.size(); ++Index) {
+ auto Xor = buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, NarrowType,
+ {LeftParts[Index], RightParts[Index]});
+ if (!Xor)
+ return std::unexpected(Xor.error());
+ Xors.push_back(*Xor);
+ }
+
+ std::vector<Register> WidenedXors;
+ for (std::size_t Index = 0; Index != LeftLeftovers.size(); ++Index) {
+ auto Xor = buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, LeftoverType,
+ {LeftLeftovers[Index], RightLeftovers[Index]});
+ if (!Xor)
+ return std::unexpected(Xor.error());
+ auto GCDType = extractGCDType(WidenedXors, NarrowType, LeftoverType, *Xor);
+ if (!GCDType)
+ return std::unexpected(GCDType.error());
+ auto LCMType = buildLCMMergePieces(LeftoverType, NarrowType, *GCDType, WidenedXors,
+ GenericOpcode::G_ZEXT);
+ if (!LCMType)
+ return std::unexpected(LCMType.error());
+ Xors.insert(Xors.end(), WidenedXors.begin(), WidenedXors.end());
+ }
+
+ if (Xors.size() < 2) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "G_ICMP narrowing produced fewer than two comparison parts");
+ }
+ auto Combined =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, NarrowType, {Xors[0], Xors[1]});
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ for (std::size_t Index = 2; Index != Xors.size(); ++Index) {
+ auto Next = buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, NarrowType,
+ {*Combined, Xors[Index]});
+ if (!Next)
+ return std::unexpected(Next.error());
+ Combined = std::move(Next);
+ }
+ if (auto Built = MIRBuilder.buildICmp(*Predicate, *Destination, *Combined, *Zero); !Built)
+ return std::unexpected(Built.error());
+ } else {
+ Register ComparisonInput;
+ const auto buildComparisonPart = [&](Register LeftPart, Register RightPart,
+ bool IsFinal) -> Expected<Register> {
+ IntegerPredicate PartPredicate = *Predicate;
+ if (!IsFinal) {
+ auto Unsigned = unsignedPredicate(*Predicate);
+ if (!Unsigned)
+ return std::unexpected(Unsigned.error());
+ PartPredicate = *Unsigned;
+ }
+ Register ComparisonOutput = *Destination;
+ if (!IsFinal) {
+ auto Temporary = MRI.createVirtualRegister(*ResultType);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ ComparisonOutput = *Temporary;
+ }
+ if (ComparisonInput.isNoRegister()) {
+ auto Built = MIRBuilder.buildICmp(PartPredicate, ComparisonOutput, LeftPart, RightPart);
+ if (!Built)
+ return std::unexpected(Built.error());
+ } else {
+ auto Comparison = MRI.createVirtualRegister(*ResultType);
+ auto Equal = MRI.createVirtualRegister(*ResultType);
+ if (!Comparison)
+ return std::unexpected(Comparison.error());
+ if (!Equal)
+ return std::unexpected(Equal.error());
+ if (auto Built = MIRBuilder.buildICmp(PartPredicate, *Comparison, LeftPart, RightPart);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::Equal, *Equal, LeftPart, RightPart);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built =
+ MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {ComparisonOutput},
+ {*Equal, ComparisonInput, *Comparison});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ }
+ return ComparisonOutput;
+ };
+
+ for (std::size_t Index = 0; Index != LeftParts.size(); ++Index) {
+ const bool IsFinal = Index + 1U == LeftParts.size() && LeftLeftovers.empty();
+ auto Comparison = buildComparisonPart(LeftParts[Index], RightParts[Index], IsFinal);
+ if (!Comparison)
+ return std::unexpected(Comparison.error());
+ ComparisonInput = *Comparison;
+ }
+ for (std::size_t Index = 0; Index != LeftLeftovers.size(); ++Index) {
+ const bool IsFinal = Index + 1U == LeftLeftovers.size();
+ auto Comparison = buildComparisonPart(LeftLeftovers[Index], RightLeftovers[Index], IsFinal);
+ if (!Comparison)
+ return std::unexpected(Comparison.error());
+ ComparisonInput = *Comparison;
+ }
+ }
+
+ return Instruction.eraseFromParent();
+ }
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "scalar narrowing is not implemented for this opcode");
+ }
+}
+
+Expected<void> LegalizerHelper::fewerElements(MachineInstr &Instruction, std::size_t TypeIndex,
+ LowLevelType NarrowType) {
+ auto Query = LegalizerInfo::buildQuery(Instruction, Function, MIRBuilder.target());
+ if (!Query)
+ return std::unexpected(Query.error());
+ if (TypeIndex >= Query->Types.size() || TypeIndex >= Query->TypeOperands.size()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "vector legalization has an invalid type index");
+ }
+
+ const LowLevelType VectorType = Query->Types[TypeIndex];
+ const std::optional<LowLevelType> ElementType = VectorType.scalarElementType();
+ if (!VectorType.isVector() || VectorType.isScalableVector() || !ElementType ||
+ *ElementType != NarrowType || !NarrowType.isCategoryNeutralScalar()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "vector legalization requires fixed vectors split into scalar elements");
+ }
+
+ std::size_t ExplicitOperandCount = 0;
+ while (ExplicitOperandCount < Instruction.operandCount() &&
+ !Instruction.operandAt(ExplicitOperandCount)->isImplicit()) {
+ ++ExplicitOperandCount;
+ }
+ const std::vector<std::size_t> &TypedOperands = Query->TypeOperands[TypeIndex];
+ if (TypedOperands.size() != ExplicitOperandCount || TypedOperands.empty()) {
+ return makeUnexpected(
+ ErrorCode::Unsupported,
+ "vector legalization requires all explicit operands to share one vector type");
+ }
+
+ std::vector<std::size_t> DefinitionIndices;
+ std::vector<std::size_t> SourceIndices;
+ DefinitionIndices.reserve(TypedOperands.size());
+ SourceIndices.reserve(TypedOperands.size());
+ for (const std::size_t OperandIndex : TypedOperands) {
+ const MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+ const Register *RegisterOperand = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (RegisterOperand == nullptr || !RegisterOperand->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "vector legalization requires virtual-register operands");
+ }
+ auto OperandType = MRI.getType(*RegisterOperand);
+ if (!OperandType)
+ return std::unexpected(OperandType.error());
+ if (*OperandType != VectorType) {
+ return makeUnexpected(
+ ErrorCode::VerificationError,
+ "vector legalization type operands do not match the requested vector type");
+ }
+ if (Operand->isDef() && !Operand->isUse()) {
+ DefinitionIndices.push_back(OperandIndex);
+ } else if (Operand->isUse() && !Operand->isDef()) {
+ SourceIndices.push_back(OperandIndex);
+ } else {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "vector legalization does not support tied or non-register operands");
+ }
+ }
+ if (DefinitionIndices.empty() || SourceIndices.empty()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "vector legalization requires register definitions and sources");
+ }
+
+ const std::size_t ElementCount = VectorType.minimumElementCount();
+ std::vector<std::vector<Register>> SourceParts;
+ SourceParts.reserve(SourceIndices.size());
+ for (const std::size_t SourceIndex : SourceIndices) {
+ const Register Source = *Instruction.operandAt(SourceIndex)->getIf<Register>();
+ std::vector<Register> Parts;
+ auto Extracted =
+ extractParts(Source, NarrowType, static_cast<int>(ElementCount), Parts, MIRBuilder, MRI);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ SourceParts.push_back(std::move(Parts));
+ }
+
+ std::vector<std::vector<Register>> DestinationParts(DefinitionIndices.size());
+ for (std::vector<Register> &Parts : DestinationParts)
+ Parts.reserve(ElementCount);
+ for (std::size_t ElementIndex = 0; ElementIndex < ElementCount; ++ElementIndex) {
+ auto ScalarInstruction = MIRBuilder.buildInstr(Instruction.opcode());
+ if (!ScalarInstruction)
+ return std::unexpected(ScalarInstruction.error());
+ if (auto SetFlags = ScalarInstruction->setFlags(Instruction.flags()); !SetFlags)
+ return std::unexpected(SetFlags.error());
+ for (std::size_t DefinitionNumber = 0; DefinitionNumber < DefinitionIndices.size();
+ ++DefinitionNumber) {
+ auto Destination = MRI.createVirtualRegister(NarrowType);
+ if (!Destination)
+ return std::unexpected(Destination.error());
+ if (auto Added = ScalarInstruction->addDef(*Destination); !Added)
+ return std::unexpected(Added.error());
+ DestinationParts[DefinitionNumber].push_back(*Destination);
+ }
+ for (const std::vector<Register> &Parts : SourceParts) {
+ if (auto Added = ScalarInstruction->addUse(Parts[ElementIndex]); !Added)
+ return std::unexpected(Added.error());
+ }
+ if (auto Built = ScalarInstruction->build(); !Built)
+ return std::unexpected(Built.error());
+ }
+
+ for (std::size_t DefinitionNumber = 0; DefinitionNumber < DefinitionIndices.size();
+ ++DefinitionNumber) {
+ const Register Destination =
+ *Instruction.operandAt(DefinitionIndices[DefinitionNumber])->getIf<Register>();
+ if (auto Merged =
+ MIRBuilder.buildMergeLikeInstr(Destination, DestinationParts[DefinitionNumber]);
+ !Merged) {
+ return std::unexpected(Merged.error());
+ }
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerVAArg(MachineInstr &Instruction) {
+ if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 ||
+ !Instruction.memOperands().empty()) {
+ return makeUnexpected(ErrorCode::VerificationError, "G_VAARG has invalid operands");
+ }
+
+ MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ MachineOperand *ListOperand = Instruction.operandAt(1);
+ MachineOperand *AlignmentOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>();
+ if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() ||
+ !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() ||
+ AlignmentOperand == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_VAARG requires virtual-register operands");
+ }
+
+ auto DestinationInfo = MRI.virtualRegisterInfo(*Destination);
+ auto PointerInfo = MRI.virtualRegisterInfo(*ListPointer);
+ auto RequestedAlignment = signedImmediate(*AlignmentOperand);
+ if (!DestinationInfo || !PointerInfo)
+ return makeUnexpected(ErrorCode::VerificationError, "G_VAARG has invalid register information");
+ const std::optional<LowLevelType> &DestinationTypeInfo = DestinationInfo->get().Type;
+ const std::optional<LowLevelType> &PointerTypeInfo = PointerInfo->get().Type;
+ if (!DestinationTypeInfo.has_value() || !PointerTypeInfo.has_value() || !RequestedAlignment ||
+ *RequestedAlignment <= 0 ||
+ !std::has_single_bit(static_cast<std::uint64_t>(*RequestedAlignment))) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_VAARG has invalid type or alignment information");
+ }
+
+ const LowLevelType DestinationType = DestinationTypeInfo.value();
+ const LowLevelType PointerType = PointerTypeInfo.value();
+ const auto DestinationLayout = scalarLayout(DestinationType);
+ const auto PointerLayout = scalarLayout(PointerType);
+ if (!DestinationLayout || !PointerLayout || !PointerType.isPointer()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_VAARG requires scalar result and pointer list types");
+ }
+
+ auto ListLoadMemory =
+ createMemoryAccess(Function, MemoryAccessFlag::Load, PointerType, *PointerLayout);
+ auto ListStoreMemory =
+ createMemoryAccess(Function, MemoryAccessFlag::Store, PointerType, *PointerLayout);
+ if (!ListLoadMemory)
+ return std::unexpected(ListLoadMemory.error());
+ if (!ListStoreMemory)
+ return std::unexpected(ListStoreMemory.error());
+
+ auto CurrentList = MRI.createVirtualRegister(PointerType);
+ if (!CurrentList)
+ return std::unexpected(CurrentList.error());
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto ListLoad = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD));
+ if (!ListLoad)
+ return std::unexpected(ListLoad.error());
+ if (auto Added = ListLoad->addDef(*CurrentList); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = ListLoad->addUse(*ListPointer); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = ListLoad->build(); !Built)
+ return std::unexpected(Built.error());
+
+ Register ValueAddress = *CurrentList;
+ const std::uint64_t AlignmentValue = static_cast<std::uint64_t>(*RequestedAlignment);
+ auto PointerScalarType = LowLevelType::makeCategoryNeutralScalar(PointerType.scalarBitWidth());
+ if (!PointerScalarType)
+ return std::unexpected(PointerScalarType.error());
+ if (AlignmentValue > 1) {
+ auto AlignmentOffset =
+ buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, AlignmentValue - 1U);
+ if (!AlignmentOffset)
+ return std::unexpected(AlignmentOffset.error());
+ auto BiasedAddress = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTR_ADD,
+ PointerType, ValueAddress, *AlignmentOffset);
+ if (!BiasedAddress)
+ return std::unexpected(BiasedAddress.error());
+ auto AlignmentMask =
+ buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, ~(AlignmentValue - 1U));
+ if (!AlignmentMask)
+ return std::unexpected(AlignmentMask.error());
+ auto AlignedAddress = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTRMASK,
+ PointerType, *BiasedAddress, *AlignmentMask);
+ if (!AlignedAddress)
+ return std::unexpected(AlignedAddress.error());
+ ValueAddress = *AlignedAddress;
+ }
+
+ auto Increment = buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType,
+ DestinationLayout->AllocationSize);
+ if (!Increment)
+ return std::unexpected(Increment.error());
+ auto NextList = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTR_ADD, PointerType,
+ ValueAddress, *Increment);
+ if (!NextList)
+ return std::unexpected(NextList.error());
+
+ auto ListStore = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE));
+ if (!ListStore)
+ return std::unexpected(ListStore.error());
+ if (auto Added = ListStore->addUse(*NextList); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = ListStore->addUse(*ListPointer); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = ListStore->build(); !Built)
+ return std::unexpected(Built.error());
+
+ if (!MRI.useEmpty(*Destination)) {
+ auto ValueLoadMemory =
+ createMemoryAccess(Function, MemoryAccessFlag::Load, DestinationType, *DestinationLayout);
+ if (!ValueLoadMemory)
+ return std::unexpected(ValueLoadMemory.error());
+ auto ValueLoad = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD));
+ if (!ValueLoad)
+ return std::unexpected(ValueLoad.error());
+ if (auto Added = ValueLoad->addDef(*Destination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = ValueLoad->addUse(ValueAddress); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = ValueLoad->build(); !Built)
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerLoad(MachineInstr &Instruction) {
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (Generic != GenericOpcode::G_LOAD && Generic != GenericOpcode::G_SEXTLOAD &&
+ Generic != GenericOpcode::G_ZEXTLOAD) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "load lowering received an unsupported opcode");
+ }
+ if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 ||
+ Instruction.memOperands().front() == nullptr) {
+ return makeUnexpected(
+ ErrorCode::VerificationError,
+ "load lowering requires one destination, one address, and one memory operand");
+ }
+
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *AddressOperand = Instruction.operandAt(1);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>();
+ if (Destination == nullptr || Address == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "load lowering requires register destination and address operands");
+ }
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (!DestinationType->isScalarValue()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "load lowering does not support vector result types");
+ }
+
+ const MachineMemOperand &Memory = *Instruction.memOperands().front();
+ const MachineMemOperandDesc &Description = Memory.description();
+ if (Memory.isAtomic())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "load lowering does not support atomic operations");
+ if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType ||
+ Description.MemoryType->isVector()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "load lowering requires a fixed scalar memory operand");
+ }
+ const LowLevelType MemoryType = *Description.MemoryType;
+ const std::uint64_t MemoryBits = MemoryType.minimumBitWidth();
+ const std::uint64_t MemoryStoreBits = Description.Size.minimumBytes() * 8U;
+ if (MemoryBits == 0 || MemoryBits > MemoryStoreBits) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "load lowering has an inconsistent memory type and size");
+ }
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ const auto BuildLoad = [&](GenericOpcode LoadOpcode, Register Result, Register Pointer,
+ const MachineMemOperand &LoadMemory) -> Expected<void> {
+ auto Load = MIRBuilder.buildInstr(opcode(LoadOpcode));
+ if (!Load)
+ return std::unexpected(Load.error());
+ if (auto Added = Load->addDef(Result); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Load->addUse(Pointer); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Load->addMemOperand(LoadMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Load->build(); !Built)
+ return std::unexpected(Built.error());
+ return {};
+ };
+
+ if (MemoryBits != MemoryStoreBits) {
+ if (!MemoryType.isCategoryNeutralScalar()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "non-byte-sized pointer loads are not supported");
+ }
+ auto WideMemoryType = LowLevelType::makeCategoryNeutralScalar(MemoryStoreBits);
+ if (!WideMemoryType)
+ return std::unexpected(WideMemoryType.error());
+ auto WideMemory = deriveMemoryOperand(Function, Memory, *WideMemoryType, 0);
+ if (!WideMemory)
+ return std::unexpected(WideMemory.error());
+
+ LowLevelType LoadType = *DestinationType;
+ Register LoadDestination = *Destination;
+ if (MemoryStoreBits > DestinationType->minimumBitWidth()) {
+ LoadType = *WideMemoryType;
+ auto Temporary = MRI.createVirtualRegister(LoadType);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ LoadDestination = *Temporary;
+ }
+
+ if (Generic == GenericOpcode::G_SEXTLOAD) {
+ auto Loaded = MRI.createVirtualRegister(LoadType);
+ if (!Loaded)
+ return std::unexpected(Loaded.error());
+ if (auto Built = BuildLoad(GenericOpcode::G_LOAD, *Loaded, *Address, **WideMemory); !Built)
+ return std::unexpected(Built.error());
+ if (auto Extended = MIRBuilder.buildSExtInReg(LoadDestination, *Loaded,
+ static_cast<std::uint32_t>(MemoryBits));
+ !Extended) {
+ return std::unexpected(Extended.error());
+ }
+ } else if (Generic == GenericOpcode::G_ZEXTLOAD || *WideMemoryType == LoadType) {
+ auto Loaded = MRI.createVirtualRegister(LoadType);
+ if (!Loaded)
+ return std::unexpected(Loaded.error());
+ if (auto Built = BuildLoad(GenericOpcode::G_LOAD, *Loaded, *Address, **WideMemory); !Built)
+ return std::unexpected(Built.error());
+ auto Assert = MIRBuilder.buildInstr(opcode(GenericOpcode::G_ASSERT_ZEXT));
+ if (!Assert)
+ return std::unexpected(Assert.error());
+ if (auto Added = Assert->addDef(LoadDestination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Assert->addUse(*Loaded); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Assert->addImmediate(static_cast<std::int64_t>(MemoryBits)); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Assert->build(); !Built)
+ return std::unexpected(Built.error());
+ } else if (auto Built =
+ BuildLoad(GenericOpcode::G_LOAD, LoadDestination, *Address, **WideMemory);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+
+ if (*DestinationType != LoadType) {
+ if (auto Truncated = MIRBuilder.buildTrunc(*Destination, LoadDestination); !Truncated)
+ return std::unexpected(Truncated.error());
+ }
+ return Instruction.eraseFromParent();
+ }
+
+ std::uint64_t LargeSplitBits = 0;
+ std::uint64_t SmallSplitBits = 0;
+ if (!std::has_single_bit(MemoryBits)) {
+ LargeSplitBits = std::bit_floor(MemoryBits);
+ SmallSplitBits = MemoryBits - LargeSplitBits;
+ } else {
+ if (MemoryBits < 16U) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "load lowering cannot split a byte-sized memory operation");
+ }
+ LargeSplitBits = MemoryBits / 2U;
+ SmallSplitBits = LargeSplitBits;
+ }
+ if (LargeSplitBits == 0 || SmallSplitBits == 0 || LargeSplitBits % 8U != 0 ||
+ SmallSplitBits % 8U != 0) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "load lowering cannot split the requested memory width into bytes");
+ }
+
+ const std::uint64_t AnyExtBits = std::bit_ceil(DestinationType->minimumBitWidth());
+ if (AnyExtBits == 0 || AnyExtBits > LowLevelType::MaxBitWidth) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "load lowering result width cannot be represented by the machine IR");
+ }
+ auto AnyExtType = LowLevelType::makeCategoryNeutralScalar(AnyExtBits);
+ auto LargeMemoryType = LowLevelType::makeCategoryNeutralScalar(LargeSplitBits);
+ auto SmallMemoryType = LowLevelType::makeCategoryNeutralScalar(SmallSplitBits);
+ if (!AnyExtType)
+ return std::unexpected(AnyExtType.error());
+ if (!LargeMemoryType)
+ return std::unexpected(LargeMemoryType.error());
+ if (!SmallMemoryType)
+ return std::unexpected(SmallMemoryType.error());
+ auto LargeMemory = deriveMemoryOperand(Function, Memory, *LargeMemoryType, 0);
+ auto SmallMemory = deriveMemoryOperand(Function, Memory, *SmallMemoryType, LargeSplitBits / 8U);
+ if (!LargeMemory)
+ return std::unexpected(LargeMemory.error());
+ if (!SmallMemory)
+ return std::unexpected(SmallMemory.error());
+
+ auto LargeLoad = MRI.createVirtualRegister(*AnyExtType);
+ if (!LargeLoad)
+ return std::unexpected(LargeLoad.error());
+ if (auto Built = BuildLoad(GenericOpcode::G_ZEXTLOAD, *LargeLoad, *Address, **LargeMemory);
+ !Built)
+ return std::unexpected(Built.error());
+ auto SmallAddress = buildObjectPointerOffset(MIRBuilder, *Address, LargeSplitBits / 8U);
+ if (!SmallAddress)
+ return std::unexpected(SmallAddress.error());
+ auto SmallLoad = MRI.createVirtualRegister(*AnyExtType);
+ if (!SmallLoad)
+ return std::unexpected(SmallLoad.error());
+ if (auto Built = BuildLoad(Generic, *SmallLoad, *SmallAddress, **SmallMemory); !Built)
+ return std::unexpected(Built.error());
+ auto ShiftAmount =
+ buildConstantRegister(MIRBuilder, *AnyExtType, static_cast<std::int64_t>(LargeSplitBits));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *AnyExtType,
+ {*SmallLoad, *ShiftAmount});
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+
+ if (*AnyExtType == *DestinationType) {
+ if (auto Combined = MIRBuilder.buildOr(*Destination, *Shifted, *LargeLoad); !Combined)
+ return std::unexpected(Combined.error());
+ } else {
+ auto Combined = buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *AnyExtType,
+ {*Shifted, *LargeLoad});
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ if (AnyExtBits != DestinationType->minimumBitWidth()) {
+ if (auto Truncated = MIRBuilder.buildTrunc(*Destination, *Combined); !Truncated)
+ return std::unexpected(Truncated.error());
+ } else if (DestinationType->isPointer()) {
+ if (auto Converted = MIRBuilder.buildIntToPtr(*Destination, *Combined); !Converted)
+ return std::unexpected(Converted.error());
+ } else {
+ return makeUnexpected(ErrorCode::InternalError,
+ "load lowering produced an incompatible scalar result type");
+ }
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerStore(MachineInstr &Instruction) {
+ if (Instruction.opcode() != opcode(GenericOpcode::G_STORE)) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "store lowering received an unsupported opcode");
+ }
+ if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 ||
+ Instruction.memOperands().front() == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "store lowering requires one value, one address, and one memory operand");
+ }
+
+ const MachineOperand *ValueOperand = Instruction.operandAt(0);
+ const MachineOperand *AddressOperand = Instruction.operandAt(1);
+ const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+ const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>();
+ if (Value == nullptr || Address == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "store lowering requires register value and address operands");
+ }
+ auto ValueType = MRI.getType(*Value);
+ if (!ValueType)
+ return std::unexpected(ValueType.error());
+ if (!ValueType->isScalarValue()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "store lowering does not support vector value types");
+ }
+
+ const MachineMemOperand &Memory = *Instruction.memOperands().front();
+ const MachineMemOperandDesc &Description = Memory.description();
+ if (Memory.isAtomic())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "store lowering does not support atomic operations");
+ if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType ||
+ Description.MemoryType->isVector()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "store lowering requires a fixed scalar memory operand");
+ }
+ const LowLevelType MemoryType = *Description.MemoryType;
+ const std::uint64_t StoreBits = MemoryType.minimumBitWidth();
+ const std::uint64_t StoreSizeBits = Description.Size.minimumBytes() * 8U;
+ if (StoreBits == 0 || StoreBits > StoreSizeBits) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "store lowering has an inconsistent memory type and size");
+ }
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ const auto BuildStore = [&](Register StoredValue, Register Pointer,
+ const MachineMemOperand &StoreMemory) -> Expected<void> {
+ auto Store = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE));
+ if (!Store)
+ return std::unexpected(Store.error());
+ if (auto Added = Store->addUse(StoredValue); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Store->addUse(Pointer); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Store->addMemOperand(StoreMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Store->build(); !Built)
+ return std::unexpected(Built.error());
+ return {};
+ };
+
+ Register Source = *Value;
+ LowLevelType SourceType = *ValueType;
+ if (StoreBits != StoreSizeBits) {
+ if (!MemoryType.isCategoryNeutralScalar() || !SourceType.isCategoryNeutralScalar()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "non-byte-sized pointer stores are not supported");
+ }
+ auto WideMemoryType = LowLevelType::makeCategoryNeutralScalar(StoreSizeBits);
+ if (!WideMemoryType)
+ return std::unexpected(WideMemoryType.error());
+ if (StoreSizeBits > SourceType.minimumBitWidth()) {
+ auto Extended =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_ANYEXT, *WideMemoryType, {Source});
+ if (!Extended)
+ return std::unexpected(Extended.error());
+ Source = *Extended;
+ SourceType = *WideMemoryType;
+ }
+ auto MaskValue =
+ APInt::getLowBitsSet(SourceType.scalarBitWidth(), static_cast<std::uint32_t>(StoreBits));
+ if (!MaskValue)
+ return std::unexpected(MaskValue.error());
+ auto Mask = buildConstantRegister(MIRBuilder, SourceType, std::move(*MaskValue));
+ if (!Mask)
+ return std::unexpected(Mask.error());
+ auto ZeroExtended =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, SourceType, {Source, *Mask});
+ if (!ZeroExtended)
+ return std::unexpected(ZeroExtended.error());
+ auto WideMemory = deriveMemoryOperand(Function, Memory, *WideMemoryType, 0);
+ if (!WideMemory)
+ return std::unexpected(WideMemory.error());
+ if (auto Built = BuildStore(*ZeroExtended, *Address, **WideMemory); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+ }
+
+ std::uint64_t LargeSplitBits = 0;
+ std::uint64_t SmallSplitBits = 0;
+ if (!std::has_single_bit(StoreBits)) {
+ LargeSplitBits = std::bit_floor(StoreBits);
+ SmallSplitBits = StoreBits - LargeSplitBits;
+ } else {
+ if (StoreBits < 16U) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "store lowering cannot split a byte-sized memory operation");
+ }
+ LargeSplitBits = StoreBits / 2U;
+ SmallSplitBits = LargeSplitBits;
+ }
+ if (LargeSplitBits == 0 || SmallSplitBits == 0 || LargeSplitBits % 8U != 0 ||
+ SmallSplitBits % 8U != 0) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "store lowering cannot split the requested memory width into bytes");
+ }
+
+ const std::uint64_t AnyExtBits = std::bit_ceil(StoreBits);
+ if (AnyExtBits == 0 || AnyExtBits > LowLevelType::MaxBitWidth) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "store lowering value width cannot be represented by the machine IR");
+ }
+ auto NewSourceType = LowLevelType::makeCategoryNeutralScalar(AnyExtBits);
+ auto LargeMemoryType = LowLevelType::makeCategoryNeutralScalar(LargeSplitBits);
+ auto SmallMemoryType = LowLevelType::makeCategoryNeutralScalar(SmallSplitBits);
+ if (!NewSourceType)
+ return std::unexpected(NewSourceType.error());
+ if (!LargeMemoryType)
+ return std::unexpected(LargeMemoryType.error());
+ if (!SmallMemoryType)
+ return std::unexpected(SmallMemoryType.error());
+
+ if (SourceType.isPointer()) {
+ auto IntegerPointerType = LowLevelType::makeCategoryNeutralScalar(SourceType.scalarBitWidth());
+ if (!IntegerPointerType)
+ return std::unexpected(IntegerPointerType.error());
+ auto Converted = buildResultInstruction(MIRBuilder, GenericOpcode::G_PTRTOINT,
+ *IntegerPointerType, {Source});
+ if (!Converted)
+ return std::unexpected(Converted.error());
+ Source = *Converted;
+ SourceType = *IntegerPointerType;
+ }
+ auto Extended = MIRBuilder.buildAnyExtOrTrunc(*NewSourceType, Source);
+ if (!Extended)
+ return std::unexpected(Extended.error());
+ auto ExtendedValue = resultRegister(**Extended);
+ if (!ExtendedValue)
+ return std::unexpected(ExtendedValue.error());
+ auto ShiftAmount =
+ buildConstantRegister(MIRBuilder, *NewSourceType, static_cast<std::int64_t>(LargeSplitBits));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto SmallValue = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *NewSourceType,
+ {*ExtendedValue, *ShiftAmount});
+ if (!SmallValue)
+ return std::unexpected(SmallValue.error());
+ auto SmallAddress = buildObjectPointerOffset(MIRBuilder, *Address, LargeSplitBits / 8U);
+ if (!SmallAddress)
+ return std::unexpected(SmallAddress.error());
+ auto LargeMemory = deriveMemoryOperand(Function, Memory, *LargeMemoryType, 0);
+ auto SmallMemory = deriveMemoryOperand(Function, Memory, *SmallMemoryType, LargeSplitBits / 8U);
+ if (!LargeMemory)
+ return std::unexpected(LargeMemory.error());
+ if (!SmallMemory)
+ return std::unexpected(SmallMemory.error());
+ if (auto Built = BuildStore(*ExtendedValue, *Address, **LargeMemory); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = BuildStore(*SmallValue, *SmallAddress, **SmallMemory); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lower(MachineInstr &Instruction,
+ const LegalizationDecision &Decision) {
+ if (Decision.Action != LegalizationAction::Lower)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "lower received a non-lower legalization action");
+ switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+ case GenericOpcode::G_ABS:
+ return lowerAbsToAddXor(Instruction);
+ case GenericOpcode::G_LOAD:
+ case GenericOpcode::G_SEXTLOAD:
+ case GenericOpcode::G_ZEXTLOAD:
+ return lowerLoad(Instruction);
+ case GenericOpcode::G_STORE:
+ return lowerStore(Instruction);
+ case GenericOpcode::G_SMIN:
+ case GenericOpcode::G_SMAX:
+ case GenericOpcode::G_UMIN:
+ case GenericOpcode::G_UMAX:
+ return lowerMinMax(Instruction);
+ case GenericOpcode::G_SCMP:
+ case GenericOpcode::G_UCMP:
+ return lowerThreewayCompare(Instruction);
+ case GenericOpcode::G_SDIVREM:
+ case GenericOpcode::G_UDIVREM:
+ return lowerDivRem(Instruction);
+ case GenericOpcode::G_SMULH:
+ case GenericOpcode::G_UMULH:
+ return lowerHighMultiply(Instruction);
+ case GenericOpcode::G_SMULO:
+ case GenericOpcode::G_UMULO:
+ return lowerMulo(Instruction);
+ case GenericOpcode::G_CTLZ_ZERO_POISON:
+ case GenericOpcode::G_CTTZ_ZERO_POISON:
+ case GenericOpcode::G_CTLZ:
+ case GenericOpcode::G_CTTZ:
+ case GenericOpcode::G_CTPOP:
+ case GenericOpcode::G_CTLS:
+ return lowerBitCount(Instruction);
+ case GenericOpcode::G_BSWAP:
+ return lowerBswap(Instruction);
+ case GenericOpcode::G_FSHL:
+ case GenericOpcode::G_FSHR:
+ return lowerFunnelShift(Instruction);
+ case GenericOpcode::G_ROTL:
+ case GenericOpcode::G_ROTR:
+ return lowerRotate(Instruction);
+ case GenericOpcode::G_DYN_STACKALLOC:
+ return lowerDynStackAlloc(Instruction);
+ case GenericOpcode::G_STACKSAVE:
+ return lowerStackSave(Instruction);
+ case GenericOpcode::G_STACKRESTORE:
+ return lowerStackRestore(Instruction);
+ case GenericOpcode::G_MEMCPY_INLINE:
+ return lowerMemcpyInline(Instruction);
+ case GenericOpcode::G_ABDS:
+ case GenericOpcode::G_ABDU: {
+ const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_ABDS);
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ auto Type = MRI.getType(Destination);
+ if (!Type)
+ return std::unexpected(Type.error());
+
+ const GenericOpcode MinOpcode = IsSigned ? GenericOpcode::G_SMIN : GenericOpcode::G_UMIN;
+ const GenericOpcode MaxOpcode = IsSigned ? GenericOpcode::G_SMAX : GenericOpcode::G_UMAX;
+ auto MinLegal = Info.isLegal(LegalizationQuery(opcode(MinOpcode), {*Type}));
+ if (!MinLegal)
+ return std::unexpected(MinLegal.error());
+ auto MaxLegal = Info.isLegal(LegalizationQuery(opcode(MaxOpcode), {*Type}));
+ if (!MaxLegal)
+ return std::unexpected(MaxLegal.error());
+ if (*MinLegal && *MaxLegal)
+ return lowerAbsDiffToMinMax(Instruction);
+ return lowerAbsDiffToSelect(Instruction);
+ }
+ case GenericOpcode::G_SADDO:
+ case GenericOpcode::G_SSUBO:
+ case GenericOpcode::G_SADDE:
+ case GenericOpcode::G_SSUBE:
+ return lowerSignedAddSubOverflow(Instruction);
+ case GenericOpcode::G_UADDSAT:
+ case GenericOpcode::G_SADDSAT:
+ case GenericOpcode::G_USUBSAT:
+ case GenericOpcode::G_SSUBSAT: {
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ const bool IsSigned =
+ Generic == GenericOpcode::G_SADDSAT || Generic == GenericOpcode::G_SSUBSAT;
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "saturating add/sub lowering requires a register destination");
+ auto Type = MRI.getType(*Destination);
+ if (!Type)
+ return std::unexpected(Type.error());
+
+ const GenericOpcode MinOpcode = IsSigned ? GenericOpcode::G_SMIN : GenericOpcode::G_UMIN;
+ auto MinLegal = Info.isLegal(LegalizationQuery(opcode(MinOpcode), {*Type}));
+ if (!MinLegal)
+ return std::unexpected(MinLegal.error());
+ if (!IsSigned && *MinLegal)
+ return lowerSaturatingAddSubToMinMax(Instruction);
+ if (IsSigned) {
+ auto MaxLegal = Info.isLegal(LegalizationQuery(opcode(GenericOpcode::G_SMAX), {*Type}));
+ if (!MaxLegal)
+ return std::unexpected(MaxLegal.error());
+ if (*MinLegal && *MaxLegal)
+ return lowerSaturatingAddSubToMinMax(Instruction);
+ }
+ return lowerSaturatingAddSubToOverflow(Instruction);
+ }
+ case GenericOpcode::G_SSHLSAT:
+ case GenericOpcode::G_USHLSAT:
+ return lowerSaturatingShiftLeft(Instruction);
+ case GenericOpcode::G_SEXT_INREG: {
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Source = *Instruction.operandAt(1)->getIf<Register>();
+ auto SizeInBits = signedImmediate(*Instruction.operandAt(2));
+ if (!SizeInBits)
+ return std::unexpected(SizeInBits.error());
+ auto Type = MRI.getType(Destination);
+ if (!Type)
+ return std::unexpected(Type.error());
+
+ auto Temporary = MRI.createVirtualRegister(*Type);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ const std::int64_t Shift = static_cast<std::int64_t>(Type->scalarBitWidth()) - *SizeInBits;
+ auto ShiftAmount = buildConstantRegister(MIRBuilder, *Type, Shift);
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ if (auto Built = MIRBuilder.buildShl(*Temporary, Source, *ShiftAmount); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildAShr(Destination, *Temporary, *ShiftAmount); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_VAARG:
+ return lowerVAArg(Instruction);
+ case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
+ const Register OldValue = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Success = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Address = *Instruction.operandAt(2)->getIf<Register>();
+ const Register Compare = *Instruction.operandAt(3)->getIf<Register>();
+ const Register NewValue = *Instruction.operandAt(4)->getIf<Register>();
+ auto NewOldValue = MRI.cloneVirtualRegister(OldValue);
+ if (!NewOldValue)
+ return std::unexpected(NewOldValue.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto Atomic = MIRBuilder.buildInstr(opcode(GenericOpcode::G_ATOMIC_CMPXCHG));
+ if (!Atomic)
+ return std::unexpected(Atomic.error());
+ if (auto Added = Atomic->addDef(*NewOldValue); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Atomic->addUse(Address); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Atomic->addUse(Compare); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Atomic->addUse(NewValue); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Atomic->addMemOperand(*Instruction.memOperands().front()); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Atomic->build(); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::Equal, Success, *NewOldValue, Compare);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildCopy(OldValue, *NewOldValue); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_ATOMICRMW_SUB: {
+ const Register Result = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Address = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Value = *Instruction.operandAt(2)->getIf<Register>();
+ auto ValueType = MRI.getType(Value);
+ if (!ValueType)
+ return std::unexpected(ValueType.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto Zero = buildConstantRegister(MIRBuilder, *ValueType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Negated =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *ValueType, {*Zero, Value});
+ if (!Negated)
+ return std::unexpected(Negated.error());
+
+ auto AtomicAdd = MIRBuilder.buildInstr(opcode(GenericOpcode::G_ATOMICRMW_ADD));
+ if (!AtomicAdd)
+ return std::unexpected(AtomicAdd.error());
+ if (auto Added = AtomicAdd->addDef(Result); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = AtomicAdd->addUse(Address); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = AtomicAdd->addUse(*Negated); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = AtomicAdd->addMemOperand(*Instruction.memOperands().front()); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = AtomicAdd->build(); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_UADDO: {
+ const Register Result = *Instruction.operandAt(0)->getIf<Register>();
+ const Register CarryOut = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(2)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(3)->getIf<Register>();
+ auto NewResult = MRI.cloneVirtualRegister(Result);
+ if (!NewResult)
+ return std::unexpected(NewResult.error());
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildAdd(*NewResult, Left, Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::UnsignedLessThan, CarryOut, *NewResult, Right);
+ !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildCopy(Result, *NewResult); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_UADDE: {
+ const Register Result = *Instruction.operandAt(0)->getIf<Register>();
+ const Register CarryOut = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(2)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(3)->getIf<Register>();
+ const Register CarryIn = *Instruction.operandAt(4)->getIf<Register>();
+ auto ResultType = MRI.getType(Result);
+ auto ConditionType = MRI.getType(CarryOut);
+ if (!ResultType)
+ return std::unexpected(ResultType.error());
+ if (!ConditionType)
+ return std::unexpected(ConditionType.error());
+ auto NewResult = MRI.cloneVirtualRegister(Result);
+ auto TemporaryResult = MRI.createVirtualRegister(*ResultType);
+ auto Carry = MRI.createVirtualRegister(*ConditionType);
+ auto ExtendedCarryIn = MRI.createVirtualRegister(*ResultType);
+ auto ResultIsZero = MRI.createVirtualRegister(*ConditionType);
+ auto SecondCarry = MRI.createVirtualRegister(*ConditionType);
+ if (!NewResult)
+ return std::unexpected(NewResult.error());
+ if (!TemporaryResult)
+ return std::unexpected(TemporaryResult.error());
+ if (!Carry)
+ return std::unexpected(Carry.error());
+ if (!ExtendedCarryIn)
+ return std::unexpected(ExtendedCarryIn.error());
+ if (!ResultIsZero)
+ return std::unexpected(ResultIsZero.error());
+ if (!SecondCarry)
+ return std::unexpected(SecondCarry.error());
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildAdd(*TemporaryResult, Left, Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::UnsignedLessThan, *Carry,
+ *TemporaryResult, Left);
+ !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildZExt(*ExtendedCarryIn, CarryIn); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildAdd(*NewResult, *TemporaryResult, *ExtendedCarryIn); !Built)
+ return std::unexpected(Built.error());
+ auto Zero = buildConstantRegister(MIRBuilder, *ResultType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::Equal, *ResultIsZero, *NewResult, *Zero);
+ !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildAnd(*SecondCarry, *ResultIsZero, CarryIn); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildOr(CarryOut, *Carry, *SecondCarry); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildCopy(Result, *NewResult); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_USUBO: {
+ const Register Result = *Instruction.operandAt(0)->getIf<Register>();
+ const Register BorrowOut = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(2)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(3)->getIf<Register>();
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildSub(Result, Left, Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::UnsignedLessThan, BorrowOut, Left, Right);
+ !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+ }
+ case GenericOpcode::G_USUBE: {
+ const Register Result = *Instruction.operandAt(0)->getIf<Register>();
+ const Register BorrowOut = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(2)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(3)->getIf<Register>();
+ const Register BorrowIn = *Instruction.operandAt(4)->getIf<Register>();
+ auto ResultType = MRI.getType(Result);
+ auto ConditionType = MRI.getType(BorrowOut);
+ if (!ResultType)
+ return std::unexpected(ResultType.error());
+ if (!ConditionType)
+ return std::unexpected(ConditionType.error());
+ auto TemporaryResult = MRI.createVirtualRegister(*ResultType);
+ auto Borrow = MRI.createVirtualRegister(*ConditionType);
+ auto ExtendedBorrowIn = MRI.createVirtualRegister(*ResultType);
+ auto TemporaryResultIsZero = MRI.createVirtualRegister(*ConditionType);
+ auto SecondBorrow = MRI.createVirtualRegister(*ConditionType);
+ if (!TemporaryResult)
+ return std::unexpected(TemporaryResult.error());
+ if (!Borrow)
+ return std::unexpected(Borrow.error());
+ if (!ExtendedBorrowIn)
+ return std::unexpected(ExtendedBorrowIn.error());
+ if (!TemporaryResultIsZero)
+ return std::unexpected(TemporaryResultIsZero.error());
+ if (!SecondBorrow)
+ return std::unexpected(SecondBorrow.error());
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildSub(*TemporaryResult, Left, Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::UnsignedGreaterThan, *Borrow,
+ *TemporaryResult, Left);
+ !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildZExt(*ExtendedBorrowIn, BorrowIn); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildSub(Result, *TemporaryResult, *ExtendedBorrowIn); !Built)
+ return std::unexpected(Built.error());
+ auto Zero = buildConstantRegister(MIRBuilder, *ResultType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::Equal, *TemporaryResultIsZero,
+ *TemporaryResult, *Zero);
+ !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildAnd(*SecondBorrow, *TemporaryResultIsZero, BorrowIn); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildOr(BorrowOut, *Borrow, *SecondBorrow); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+ }
+ default: {
+ const InstructionDescriptor *Descriptor =
+ MIRBuilder.target().instructionDescriptor(Instruction.opcode());
+ const std::string_view Name =
+ Descriptor == nullptr ? std::string_view{"<unknown>"} : Descriptor->Name;
+ return makeUnexpected(ErrorCode::Unsupported,
+ "generic lowering is not implemented for opcode '" + std::string(Name) +
+ "'");
+ }
+ }
+}
+
+Expected<void> LegalizerHelper::lowerDynStackAlloc(MachineInstr &Instruction) {
+ if (Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_DYN_STACKALLOC requires destination, size, and alignment operands");
+ if (MIRBuilder.target().stackGrowthDirection() != StackGrowthDirection::Down)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "dynamic stack allocation requires a downward-growing stack");
+
+ const PhysicalRegister StackPointer = MIRBuilder.target().stackPointerRegisterToSaveRestore();
+ if (!StackPointer.isValid())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "target does not define a stack pointer for dynamic allocation");
+
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *SizeOperand = Instruction.operandAt(1);
+ const MachineOperand *AlignmentOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *AllocationSize =
+ SizeOperand == nullptr ? nullptr : SizeOperand->getIf<Register>();
+ if (Destination == nullptr || AllocationSize == nullptr || AlignmentOperand == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError, "G_DYN_STACKALLOC has invalid operands");
+
+ auto PointerType = MRI.getType(*Destination);
+ auto AllocationSizeType = MRI.getType(*AllocationSize);
+ auto AlignmentValue = signedImmediate(*AlignmentOperand);
+ if (!PointerType)
+ return std::unexpected(PointerType.error());
+ if (!AllocationSizeType)
+ return std::unexpected(AllocationSizeType.error());
+ if (!AlignmentValue)
+ return std::unexpected(AlignmentValue.error());
+ if (!PointerType->isPointer() || !AllocationSizeType->isCategoryNeutralScalar() ||
+ PointerType->scalarBitWidth() != AllocationSizeType->scalarBitWidth()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_DYN_STACKALLOC requires pointer-width integer allocation size");
+ }
+ if (*AlignmentValue <= 0 || !std::has_single_bit(static_cast<std::uint64_t>(*AlignmentValue))) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_DYN_STACKALLOC alignment must be a positive power of two");
+ }
+
+ auto StackPointerCopy = MRI.createVirtualRegister(*PointerType);
+ auto IntegerStackPointer = MRI.createVirtualRegister(*AllocationSizeType);
+ auto Allocation = MRI.createVirtualRegister(*AllocationSizeType);
+ if (!StackPointerCopy)
+ return std::unexpected(StackPointerCopy.error());
+ if (!IntegerStackPointer)
+ return std::unexpected(IntegerStackPointer.error());
+ if (!Allocation)
+ return std::unexpected(Allocation.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildCopy(*StackPointerCopy, Register::fromPhysical(StackPointer));
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildPtrToInt(*IntegerStackPointer, *StackPointerCopy); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildSub(*Allocation, *IntegerStackPointer, *AllocationSize); !Built)
+ return std::unexpected(Built.error());
+
+ Register AlignedAllocation = *Allocation;
+ if (*AlignmentValue > 1) {
+ auto LowBits = APInt::fromUnsigned(AllocationSizeType->scalarBitWidth(),
+ static_cast<std::uint64_t>(*AlignmentValue) - 1U);
+ if (!LowBits)
+ return std::unexpected(LowBits.error());
+ auto AlignmentMask = LowBits->bitwiseNot();
+ if (!AlignmentMask)
+ return std::unexpected(AlignmentMask.error());
+ auto Mask = buildConstantRegister(MIRBuilder, *AllocationSizeType, std::move(*AlignmentMask));
+ if (!Mask)
+ return std::unexpected(Mask.error());
+ auto MaskedAllocation = MRI.createVirtualRegister(*AllocationSizeType);
+ if (!MaskedAllocation)
+ return std::unexpected(MaskedAllocation.error());
+ if (auto Built = MIRBuilder.buildAnd(*MaskedAllocation, *Allocation, *Mask); !Built)
+ return std::unexpected(Built.error());
+ AlignedAllocation = *MaskedAllocation;
+ }
+
+ auto TargetPointer = MRI.createVirtualRegister(*PointerType);
+ if (!TargetPointer)
+ return std::unexpected(TargetPointer.error());
+ if (auto Built = MIRBuilder.buildIntToPtr(*TargetPointer, AlignedAllocation); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildCopy(Register::fromPhysical(StackPointer), *TargetPointer);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildCopy(*Destination, *TargetPointer); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerStackSave(MachineInstr &Instruction) {
+ const PhysicalRegister StackPointer = MIRBuilder.target().stackPointerRegisterToSaveRestore();
+ if (!StackPointer.isValid())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "target does not define a stack pointer for stack save");
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Instruction.operandCount() != 1 || Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_STACKSAVE requires one register destination");
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildCopy(*Destination, Register::fromPhysical(StackPointer));
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerStackRestore(MachineInstr &Instruction) {
+ const PhysicalRegister StackPointer = MIRBuilder.target().stackPointerRegisterToSaveRestore();
+ if (!StackPointer.isValid())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "target does not define a stack pointer for stack restore");
+ const MachineOperand *SourceOperand = Instruction.operandAt(0);
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Instruction.operandCount() != 1 || Source == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_STACKRESTORE requires one register source");
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildCopy(Register::fromPhysical(StackPointer), *Source); !Built)
+ return std::unexpected(Built.error());
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerMemcpyInline(MachineInstr &Instruction) {
+ if (Instruction.operandCount() != 3 || Instruction.memOperands().size() != 2 ||
+ Instruction.memOperands()[0] == nullptr || Instruction.memOperands()[1] == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_MEMCPY_INLINE requires destination, source, length, and two MMOs");
+ }
+
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const MachineOperand *LengthOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ const Register *Length = LengthOperand == nullptr ? nullptr : LengthOperand->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr || Length == nullptr ||
+ !Destination->isVirtual() || !Source->isVirtual() || !Length->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_MEMCPY_INLINE requires virtual-register operands");
+ }
+
+ const MachineMemOperand &StoreMemory = *Instruction.memOperands()[0];
+ const MachineMemOperand &LoadMemory = *Instruction.memOperands()[1];
+ if (!StoreMemory.isStore() || StoreMemory.isLoad() || !LoadMemory.isLoad() ||
+ LoadMemory.isStore() || StoreMemory.isAtomic() || LoadMemory.isAtomic()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "G_MEMCPY_INLINE requires non-atomic store and load memory operands");
+ }
+
+ const std::optional<ValueAndVReg> ConstantLength =
+ getIConstantVRegValWithLookThrough(*Length, MRI);
+ if (!ConstantLength || ConstantLength->Value.bitWidth() == 0 ||
+ ConstantLength->Value.bitWidth() > 64) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "G_MEMCPY_INLINE requires a constant length no wider than 64 bits");
+ }
+ const std::uint64_t KnownLength = ConstantLength->Value.words().front();
+ if (KnownLength == 0) {
+ return Instruction.eraseFromParent();
+ }
+
+ auto ByteType = LowLevelType::makeCategoryNeutralScalar(8);
+ if (!ByteType)
+ return std::unexpected(ByteType.error());
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ MachineFunction &Owner = MIRBuilder.function();
+ for (std::uint64_t Offset = 0; Offset != KnownLength; ++Offset) {
+ auto LoadAddress = buildObjectPointerOffset(MIRBuilder, *Source, Offset);
+ if (!LoadAddress)
+ return std::unexpected(LoadAddress.error());
+ auto ByteLoadMemory = deriveMemoryOperand(Owner, LoadMemory, *ByteType, Offset);
+ if (!ByteLoadMemory)
+ return std::unexpected(ByteLoadMemory.error());
+ auto LoadedByte = MRI.createVirtualRegister(*ByteType);
+ if (!LoadedByte)
+ return std::unexpected(LoadedByte.error());
+ auto Load = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD));
+ if (!Load)
+ return std::unexpected(Load.error());
+ if (auto Added = Load->addDef(*LoadedByte); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Load->addUse(*LoadAddress); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Load->addMemOperand(**ByteLoadMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Load->build(); !Built)
+ return std::unexpected(Built.error());
+
+ auto StoreAddress = buildObjectPointerOffset(MIRBuilder, *Destination, Offset);
+ if (!StoreAddress)
+ return std::unexpected(StoreAddress.error());
+ auto ByteStoreMemory = deriveMemoryOperand(Owner, StoreMemory, *ByteType, Offset);
+ if (!ByteStoreMemory)
+ return std::unexpected(ByteStoreMemory.error());
+ auto Store = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE));
+ if (!Store)
+ return std::unexpected(Store.error());
+ if (auto Added = Store->addUse(*LoadedByte); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Store->addUse(*StoreAddress); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Store->addMemOperand(**ByteStoreMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = Store->build(); !Built)
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerAbsToAddXor(MachineInstr &Instruction) {
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Source = *Instruction.operandAt(1)->getIf<Register>();
+ auto Type = MRI.getType(Destination);
+ if (!Type)
+ return std::unexpected(Type.error());
+
+ auto Shift = MRI.createVirtualRegister(*Type);
+ auto Add = MRI.createVirtualRegister(*Type);
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ if (!Add)
+ return std::unexpected(Add.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto ShiftAmount = buildConstantRegister(MIRBuilder, *Type,
+ static_cast<std::int64_t>(Type->scalarBitWidth() - 1U));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ if (auto Built = MIRBuilder.buildAShr(*Shift, Source, *ShiftAmount); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildAdd(*Add, Source, *Shift); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildXor(Destination, *Add, *Shift); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerAbsDiffToSelect(MachineInstr &Instruction) {
+ const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_ABDS);
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(2)->getIf<Register>();
+ auto Type = MRI.getType(Left);
+ if (!Type)
+ return std::unexpected(Type.error());
+ auto ConditionType = LowLevelType::makeCategoryNeutralScalar(1);
+ if (!ConditionType)
+ return std::unexpected(ConditionType.error());
+
+ auto LeftSub = MRI.createVirtualRegister(*Type);
+ auto RightSub = MRI.createVirtualRegister(*Type);
+ auto Condition = MRI.createVirtualRegister(*ConditionType);
+ if (!LeftSub)
+ return std::unexpected(LeftSub.error());
+ if (!RightSub)
+ return std::unexpected(RightSub.error());
+ if (!Condition)
+ return std::unexpected(Condition.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildSub(*LeftSub, Left, Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildSub(*RightSub, Right, Left); !Built)
+ return std::unexpected(Built.error());
+ const IntegerPredicate Predicate =
+ IsSigned ? IntegerPredicate::SignedGreaterThan : IntegerPredicate::UnsignedGreaterThan;
+ if (auto Built = MIRBuilder.buildICmp(Predicate, *Condition, Left, Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {Destination},
+ {*Condition, *LeftSub, *RightSub});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerAbsDiffToMinMax(MachineInstr &Instruction) {
+ const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_ABDS);
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(2)->getIf<Register>();
+ auto Type = MRI.getType(Left);
+ if (!Type)
+ return std::unexpected(Type.error());
+
+ auto Maximum = MRI.createVirtualRegister(*Type);
+ auto Minimum = MRI.createVirtualRegister(*Type);
+ if (!Maximum)
+ return std::unexpected(Maximum.error());
+ if (!Minimum)
+ return std::unexpected(Minimum.error());
+
+ const GenericOpcode MaxOpcode = IsSigned ? GenericOpcode::G_SMAX : GenericOpcode::G_UMAX;
+ const GenericOpcode MinOpcode = IsSigned ? GenericOpcode::G_SMIN : GenericOpcode::G_UMIN;
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(MaxOpcode), {*Maximum}, {Left, Right}); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(MinOpcode), {*Minimum}, {Left, Right}); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildSub(Destination, *Maximum, *Minimum); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerMinMax(MachineInstr &Instruction) {
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(2)->getIf<Register>();
+
+ auto DestinationType = MRI.getType(Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ auto ConditionType = booleanTypeFor(*DestinationType);
+ if (!ConditionType)
+ return std::unexpected(ConditionType.error());
+
+ auto Condition = MRI.createVirtualRegister(*ConditionType);
+ if (!Condition)
+ return std::unexpected(Condition.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ const auto Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (auto Built = MIRBuilder.buildICmp(minMaxToCompare(Generic), *Condition, Left, Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {Destination},
+ {*Condition, Left, Right});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerThreewayCompare(MachineInstr &Instruction) {
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(2)->getIf<Register>();
+ auto DestinationType = MRI.getType(Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ auto ConditionType = booleanTypeFor(*DestinationType);
+ if (!ConditionType)
+ return std::unexpected(ConditionType.error());
+
+ auto IsGreater = MRI.createVirtualRegister(*ConditionType);
+ auto IsLess = MRI.createVirtualRegister(*ConditionType);
+ if (!IsGreater)
+ return std::unexpected(IsGreater.error());
+ if (!IsLess)
+ return std::unexpected(IsLess.error());
+
+ const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SCMP);
+ const IntegerPredicate GreaterPredicate =
+ IsSigned ? IntegerPredicate::SignedGreaterThan : IntegerPredicate::UnsignedGreaterThan;
+ const IntegerPredicate LessPredicate =
+ IsSigned ? IntegerPredicate::SignedLessThan : IntegerPredicate::UnsignedLessThan;
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = MIRBuilder.buildICmp(GreaterPredicate, *IsGreater, Left, Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildICmp(LessPredicate, *IsLess, Left, Right); !Built)
+ return std::unexpected(Built.error());
+
+ const BooleanContent Contents =
+ MIRBuilder.target().getBooleanContents(DestinationType->isVector(), false);
+ if (Contents == BooleanContent::Undefined) {
+ auto Zero = buildConstantRegister(MIRBuilder, *DestinationType, 0);
+ auto One = buildConstantRegister(MIRBuilder, *DestinationType, 1);
+ auto MinusOne = buildConstantRegister(MIRBuilder, *DestinationType, -1);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (!One)
+ return std::unexpected(One.error());
+ if (!MinusOne)
+ return std::unexpected(MinusOne.error());
+ auto ZeroOrOne = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, *DestinationType,
+ {*IsGreater, *One, *Zero});
+ if (!ZeroOrOne)
+ return std::unexpected(ZeroOrOne.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {Destination},
+ {*IsLess, *MinusOne, *ZeroOrOne});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ } else {
+ if (Contents == BooleanContent::ZeroOrNegativeOne)
+ std::swap(IsGreater, IsLess);
+ const GenericOpcode ExtensionOpcode = Contents == BooleanContent::ZeroOrNegativeOne
+ ? GenericOpcode::G_SEXT
+ : GenericOpcode::G_ZEXT;
+ auto ExtendedGreater = MRI.createVirtualRegister(*DestinationType);
+ auto ExtendedLess = MRI.createVirtualRegister(*DestinationType);
+ if (!ExtendedGreater)
+ return std::unexpected(ExtendedGreater.error());
+ if (!ExtendedLess)
+ return std::unexpected(ExtendedLess.error());
+ if (auto Built =
+ MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*ExtendedGreater}, {*IsGreater});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildInstr(opcode(ExtensionOpcode), {*ExtendedLess}, {*IsLess});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildSub(Destination, *ExtendedGreater, *ExtendedLess); !Built)
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerDivRem(MachineInstr &Instruction) {
+ const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SDIVREM);
+ const Register Quotient = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Remainder = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Dividend = *Instruction.operandAt(2)->getIf<Register>();
+ const Register Divisor = *Instruction.operandAt(3)->getIf<Register>();
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ const GenericOpcode DivideOpcode = IsSigned ? GenericOpcode::G_SDIV : GenericOpcode::G_UDIV;
+ const GenericOpcode RemainderOpcode = IsSigned ? GenericOpcode::G_SREM : GenericOpcode::G_UREM;
+ if (auto Built = MIRBuilder.buildInstr(opcode(DivideOpcode), {Quotient}, {Dividend, Divisor});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildInstr(opcode(RemainderOpcode), {Remainder}, {Dividend, Divisor});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerHighMultiply(MachineInstr &Instruction) {
+ if (Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "high multiply lowering requires a destination and two sources");
+ const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SMULH);
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *FirstSourceOperand = Instruction.operandAt(1);
+ const MachineOperand *SecondSourceOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *FirstSource =
+ FirstSourceOperand == nullptr ? nullptr : FirstSourceOperand->getIf<Register>();
+ const Register *SecondSource =
+ SecondSourceOperand == nullptr ? nullptr : SecondSourceOperand->getIf<Register>();
+ if (Destination == nullptr || FirstSource == nullptr || SecondSource == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "high multiply lowering requires register operands");
+ auto OriginalType = MRI.getType(*Destination);
+ if (!OriginalType)
+ return std::unexpected(OriginalType.error());
+ if (!OriginalType->isCategoryNeutralScalar() || OriginalType->scalarBitWidth() == 0 ||
+ OriginalType->scalarBitWidth() > LowLevelType::MaxBitWidth / 2U) {
+ return makeUnexpected(
+ ErrorCode::Unsupported,
+ "high multiply lowering requires a scalar type with a representable double width");
+ }
+ const std::uint32_t SizeInBits = OriginalType->scalarBitWidth();
+ auto WideType = LowLevelType::makeCategoryNeutralScalar(SizeInBits * 2U);
+ if (!WideType)
+ return std::unexpected(WideType.error());
+
+ auto WideFirstSource = MRI.createVirtualRegister(*WideType);
+ auto WideSecondSource = MRI.createVirtualRegister(*WideType);
+ if (!WideFirstSource)
+ return std::unexpected(WideFirstSource.error());
+ if (!WideSecondSource)
+ return std::unexpected(WideSecondSource.error());
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = IsSigned ? MIRBuilder.buildSExt(*WideFirstSource, *FirstSource)
+ : MIRBuilder.buildZExt(*WideFirstSource, *FirstSource);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = IsSigned ? MIRBuilder.buildSExt(*WideSecondSource, *SecondSource)
+ : MIRBuilder.buildZExt(*WideSecondSource, *SecondSource);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ auto Product = buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, *WideType,
+ {*WideFirstSource, *WideSecondSource});
+ if (!Product)
+ return std::unexpected(Product.error());
+ auto ShiftAmount = buildConstantRegister(MIRBuilder, *WideType, SizeInBits);
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto Shifted =
+ buildResultInstruction(MIRBuilder, IsSigned ? GenericOpcode::G_ASHR : GenericOpcode::G_LSHR,
+ *WideType, {*Product, *ShiftAmount});
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ if (auto Built = MIRBuilder.buildTrunc(*Destination, *Shifted); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerMulo(MachineInstr &Instruction) {
+ if (Instruction.operandCount() != 4)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiply-overflow lowering requires four register operands");
+
+ const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SMULO);
+ const Register *ResultOperand = Instruction.operandAt(0)->getIf<Register>();
+ const Register *OverflowOperand = Instruction.operandAt(1)->getIf<Register>();
+ const Register *LeftOperand = Instruction.operandAt(2)->getIf<Register>();
+ const Register *RightOperand = Instruction.operandAt(3)->getIf<Register>();
+ if (ResultOperand == nullptr || OverflowOperand == nullptr || LeftOperand == nullptr ||
+ RightOperand == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "multiply-overflow lowering requires register operands");
+ }
+ const Register Result = *ResultOperand;
+ const Register Overflow = *OverflowOperand;
+ const Register Left = *LeftOperand;
+ const Register Right = *RightOperand;
+ auto ResultType = MRI.getType(Result);
+ if (!ResultType)
+ return std::unexpected(ResultType.error());
+ if (!ResultType->isCategoryNeutralScalar() || ResultType->scalarBitWidth() == 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "multiply-overflow lowering requires a scalar result type");
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ Observer.changingInstr(Instruction);
+ if (auto Changed = Instruction.setOpcode(opcode(GenericOpcode::G_MUL)); !Changed)
+ return std::unexpected(Changed.error());
+ if (auto Changed = Instruction.eraseOperand(1); !Changed)
+ return std::unexpected(Changed.error());
+ Observer.changedInstr(Instruction);
+
+ auto HighPart =
+ buildResultInstruction(MIRBuilder, IsSigned ? GenericOpcode::G_SMULH : GenericOpcode::G_UMULH,
+ *ResultType, {Left, Right});
+ if (!HighPart)
+ return std::unexpected(HighPart.error());
+ auto Zero = buildConstantRegister(MIRBuilder, *ResultType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+
+ if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (IsSigned) {
+ auto ShiftAmount =
+ buildConstantRegister(MIRBuilder, *ResultType, ResultType->scalarBitWidth() - 1U);
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, *ResultType,
+ {Result, *ShiftAmount});
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::NotEqual, Overflow, *HighPart, *Shifted);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ } else if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::NotEqual, Overflow, *HighPart, *Zero);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ return {};
+}
+
+Expected<void> LegalizerHelper::lowerBitCount(MachineInstr &Instruction) {
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "bit-count lowering requires register operands");
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ auto SourceType = MRI.getType(*Source);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ if (!DestinationType->isCategoryNeutralScalar() || !SourceType->isCategoryNeutralScalar())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "non-scalar bit-count lowering is not implemented");
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ const auto IsSupported = [this](GenericOpcode Opcode, LowLevelType ResultType,
+ LowLevelType OperandType) -> Expected<bool> {
+ auto Decision = Info.getDecision(LegalizationQuery(opcode(Opcode), {ResultType, OperandType}));
+ if (!Decision)
+ return std::unexpected(Decision.error());
+ return Decision->Action == LegalizationAction::Legal ||
+ Decision->Action == LegalizationAction::Libcall ||
+ Decision->Action == LegalizationAction::Custom;
+ };
+ const auto EraseOriginal = [&Instruction]() -> Expected<void> {
+ return Instruction.eraseFromParent();
+ };
+
+ switch (Generic) {
+ case GenericOpcode::G_CTLZ_ZERO_POISON:
+ case GenericOpcode::G_CTTZ_ZERO_POISON: {
+ const GenericOpcode Replacement = Generic == GenericOpcode::G_CTLZ_ZERO_POISON
+ ? GenericOpcode::G_CTLZ
+ : GenericOpcode::G_CTTZ;
+ Observer.changingInstr(Instruction);
+ if (auto Changed = Instruction.setOpcode(opcode(Replacement)); !Changed)
+ return std::unexpected(Changed.error());
+ Observer.changedInstr(Instruction);
+ return {};
+ }
+
+ case GenericOpcode::G_CTLZ: {
+ const std::uint32_t Size = SourceType->scalarBitWidth();
+ auto PoisonSupported =
+ IsSupported(GenericOpcode::G_CTLZ_ZERO_POISON, *DestinationType, *SourceType);
+ if (!PoisonSupported)
+ return std::unexpected(PoisonSupported.error());
+ if (*PoisonSupported) {
+ auto PoisonCount = buildResultInstruction(MIRBuilder, GenericOpcode::G_CTLZ_ZERO_POISON,
+ *DestinationType, {*Source});
+ if (!PoisonCount)
+ return std::unexpected(PoisonCount.error());
+ auto Zero = buildConstantRegister(MIRBuilder, *SourceType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto ConditionType = booleanTypeFor(*SourceType);
+ if (!ConditionType)
+ return std::unexpected(ConditionType.error());
+ auto IsZero =
+ buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *ConditionType, *Source, *Zero);
+ if (!IsZero)
+ return std::unexpected(IsZero.error());
+ auto SizeConstant =
+ buildConstantRegister(MIRBuilder, *DestinationType, static_cast<std::int64_t>(Size));
+ if (!SizeConstant)
+ return std::unexpected(SizeConstant.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {*Destination},
+ {*IsZero, *SizeConstant, *PoisonCount});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ return EraseOriginal();
+ }
+
+ Register Value = *Source;
+ const std::uint32_t ExpandedSize = std::bit_ceil(Size);
+ for (std::uint32_t Shift = 1; Shift <= ExpandedSize / 2U; Shift <<= 1U) {
+ auto ShiftAmount =
+ buildConstantRegister(MIRBuilder, *SourceType, static_cast<std::int64_t>(Shift));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType,
+ {Value, *ShiftAmount});
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ auto Propagated =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *SourceType, {Value, *Shifted});
+ if (!Propagated)
+ return std::unexpected(Propagated.error());
+ Value = *Propagated;
+ }
+ auto Population =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_CTPOP, *DestinationType, {Value});
+ if (!Population)
+ return std::unexpected(Population.error());
+ auto SizeConstant =
+ buildConstantRegister(MIRBuilder, *DestinationType, static_cast<std::int64_t>(Size));
+ if (!SizeConstant)
+ return std::unexpected(SizeConstant.error());
+ if (auto Built = MIRBuilder.buildSub(*Destination, *SizeConstant, *Population); !Built)
+ return std::unexpected(Built.error());
+ return EraseOriginal();
+ }
+
+ case GenericOpcode::G_CTTZ: {
+ const std::uint32_t Size = SourceType->scalarBitWidth();
+ auto PoisonSupported =
+ IsSupported(GenericOpcode::G_CTTZ_ZERO_POISON, *DestinationType, *SourceType);
+ if (!PoisonSupported)
+ return std::unexpected(PoisonSupported.error());
+ if (*PoisonSupported) {
+ auto PoisonCount = buildResultInstruction(MIRBuilder, GenericOpcode::G_CTTZ_ZERO_POISON,
+ *DestinationType, {*Source});
+ if (!PoisonCount)
+ return std::unexpected(PoisonCount.error());
+ auto Zero = buildConstantRegister(MIRBuilder, *SourceType, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto ConditionType = booleanTypeFor(*DestinationType);
+ if (!ConditionType)
+ return std::unexpected(ConditionType.error());
+ auto IsZero =
+ buildICmpRegister(MIRBuilder, IntegerPredicate::Equal, *ConditionType, *Source, *Zero);
+ if (!IsZero)
+ return std::unexpected(IsZero.error());
+ auto SizeConstant =
+ buildConstantRegister(MIRBuilder, *DestinationType, static_cast<std::int64_t>(Size));
+ if (!SizeConstant)
+ return std::unexpected(SizeConstant.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {*Destination},
+ {*IsZero, *SizeConstant, *PoisonCount});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ return EraseOriginal();
+ }
+
+ auto MinusOne = buildConstantRegister(MIRBuilder, *SourceType, -1);
+ if (!MinusOne)
+ return std::unexpected(MinusOne.error());
+ auto Inverted =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, *SourceType, {*Source, *MinusOne});
+ if (!Inverted)
+ return std::unexpected(Inverted.error());
+ auto Decremented =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType, {*Source, *MinusOne});
+ if (!Decremented)
+ return std::unexpected(Decremented.error());
+ auto Masked = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType,
+ {*Inverted, *Decremented});
+ if (!Masked)
+ return std::unexpected(Masked.error());
+
+ auto PopulationSupported = IsSupported(GenericOpcode::G_CTPOP, *SourceType, *SourceType);
+ if (!PopulationSupported)
+ return std::unexpected(PopulationSupported.error());
+ auto LeadingZeroSupported = IsSupported(GenericOpcode::G_CTLZ, *SourceType, *SourceType);
+ if (!LeadingZeroSupported)
+ return std::unexpected(LeadingZeroSupported.error());
+ if (!*PopulationSupported && *LeadingZeroSupported) {
+ auto SizeConstant =
+ buildConstantRegister(MIRBuilder, *SourceType, static_cast<std::int64_t>(Size));
+ if (!SizeConstant)
+ return std::unexpected(SizeConstant.error());
+ auto LeadingZeros =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_CTLZ, *SourceType, {*Masked});
+ if (!LeadingZeros)
+ return std::unexpected(LeadingZeros.error());
+ if (auto Built = MIRBuilder.buildSub(*Destination, *SizeConstant, *LeadingZeros); !Built)
+ return std::unexpected(Built.error());
+ return EraseOriginal();
+ }
+
+ Observer.changingInstr(Instruction);
+ if (auto Changed = Instruction.setOpcode(opcode(GenericOpcode::G_CTPOP)); !Changed)
+ return std::unexpected(Changed.error());
+ if (auto Changed = SourceOperand->setRegister(*Masked); !Changed)
+ return std::unexpected(Changed.error());
+ Observer.changedInstr(Instruction);
+ return {};
+ }
+
+ case GenericOpcode::G_CTPOP: {
+ const std::uint32_t Size = SourceType->scalarBitWidth();
+ if (Size > 128 || Size % 8U != 0)
+ return makeUnexpected(
+ ErrorCode::Unsupported,
+ "population-count lowering requires a byte-sized scalar up to 128 bits");
+
+ auto One = buildConstantRegister(MIRBuilder, *SourceType, 1);
+ if (!One)
+ return std::unexpected(One.error());
+ auto ShiftedByOne =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType, {*Source, *One});
+ if (!ShiftedByOne)
+ return std::unexpected(ShiftedByOne.error());
+ auto PairMaskValue = repeatedByte(Size, 0x55);
+ if (!PairMaskValue)
+ return std::unexpected(PairMaskValue.error());
+ auto PairMask = buildConstantRegister(MIRBuilder, *SourceType, std::move(*PairMaskValue));
+ if (!PairMask)
+ return std::unexpected(PairMask.error());
+ auto HighPairBits = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType,
+ {*ShiftedByOne, *PairMask});
+ if (!HighPairBits)
+ return std::unexpected(HighPairBits.error());
+ auto PairCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *SourceType,
+ {*Source, *HighPairBits});
+ if (!PairCounts)
+ return std::unexpected(PairCounts.error());
+
+ auto Two = buildConstantRegister(MIRBuilder, *SourceType, 2);
+ if (!Two)
+ return std::unexpected(Two.error());
+ auto ShiftedPairs =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType, {*PairCounts, *Two});
+ if (!ShiftedPairs)
+ return std::unexpected(ShiftedPairs.error());
+ auto NibbleMaskValue = repeatedByte(Size, 0x33);
+ if (!NibbleMaskValue)
+ return std::unexpected(NibbleMaskValue.error());
+ auto NibbleMask = buildConstantRegister(MIRBuilder, *SourceType, std::move(*NibbleMaskValue));
+ if (!NibbleMask)
+ return std::unexpected(NibbleMask.error());
+ auto HighNibbleCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType,
+ {*ShiftedPairs, *NibbleMask});
+ if (!HighNibbleCounts)
+ return std::unexpected(HighNibbleCounts.error());
+ auto LowNibbleCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType,
+ {*PairCounts, *NibbleMask});
+ if (!LowNibbleCounts)
+ return std::unexpected(LowNibbleCounts.error());
+ auto NibbleCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType,
+ {*HighNibbleCounts, *LowNibbleCounts});
+ if (!NibbleCounts)
+ return std::unexpected(NibbleCounts.error());
+
+ auto Four = buildConstantRegister(MIRBuilder, *SourceType, 4);
+ if (!Four)
+ return std::unexpected(Four.error());
+ auto ShiftedNibbles = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType,
+ {*NibbleCounts, *Four});
+ if (!ShiftedNibbles)
+ return std::unexpected(ShiftedNibbles.error());
+ auto DirtyByteCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType,
+ {*ShiftedNibbles, *NibbleCounts});
+ if (!DirtyByteCounts)
+ return std::unexpected(DirtyByteCounts.error());
+ auto ByteMaskValue = repeatedByte(Size, 0x0f);
+ if (!ByteMaskValue)
+ return std::unexpected(ByteMaskValue.error());
+ auto ByteMask = buildConstantRegister(MIRBuilder, *SourceType, std::move(*ByteMaskValue));
+ if (!ByteMask)
+ return std::unexpected(ByteMask.error());
+ auto ByteCounts = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *SourceType,
+ {*DirtyByteCounts, *ByteMask});
+ if (!ByteCounts)
+ return std::unexpected(ByteCounts.error());
+
+ if (Size == 16) {
+ auto Eight = buildConstantRegister(MIRBuilder, *SourceType, 8);
+ if (!Eight)
+ return std::unexpected(Eight.error());
+ auto HighByteCount = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *SourceType,
+ {*ByteCounts, *Eight});
+ if (!HighByteCount)
+ return std::unexpected(HighByteCount.error());
+ auto Sum = buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType,
+ {*ByteCounts, *HighByteCount});
+ if (!Sum)
+ return std::unexpected(Sum.error());
+ auto ResultMaskValue = APInt::fromUnsigned(Size, 0xff);
+ if (!ResultMaskValue)
+ return std::unexpected(ResultMaskValue.error());
+ auto ResultMask = buildConstantRegister(MIRBuilder, *SourceType, std::move(*ResultMaskValue));
+ if (!ResultMask)
+ return std::unexpected(ResultMask.error());
+ if (auto Built = MIRBuilder.buildAnd(*Destination, *Sum, *ResultMask); !Built)
+ return std::unexpected(Built.error());
+ return EraseOriginal();
+ }
+
+ auto MultiplyMaskValue = repeatedByte(Size, 0x01);
+ if (!MultiplyMaskValue)
+ return std::unexpected(MultiplyMaskValue.error());
+ auto MultiplyMask =
+ buildConstantRegister(MIRBuilder, *SourceType, std::move(*MultiplyMaskValue));
+ if (!MultiplyMask)
+ return std::unexpected(MultiplyMask.error());
+ auto FinalShift =
+ buildConstantRegister(MIRBuilder, *SourceType, static_cast<std::int64_t>(Size - 8U));
+ if (!FinalShift)
+ return std::unexpected(FinalShift.error());
+
+ auto MultiplyDecision =
+ Info.getDecision(LegalizationQuery(opcode(GenericOpcode::G_MUL), {*SourceType}));
+ if (!MultiplyDecision)
+ return std::unexpected(MultiplyDecision.error());
+ const bool IsMultiplySupported = MultiplyDecision->Action == LegalizationAction::Legal ||
+ MultiplyDecision->Action == LegalizationAction::WidenScalar ||
+ MultiplyDecision->Action == LegalizationAction::Custom;
+ if (IsMultiplySupported) {
+ auto Product = buildResultInstruction(MIRBuilder, GenericOpcode::G_MUL, *SourceType,
+ {*ByteCounts, *MultiplyMask});
+ if (!Product)
+ return std::unexpected(Product.error());
+ if (auto Built = MIRBuilder.buildLShr(*Destination, *Product, *FinalShift); !Built)
+ return std::unexpected(Built.error());
+ } else {
+ Register Sum = *ByteCounts;
+ for (std::uint32_t Shift = 8; Shift < Size; Shift *= 2U) {
+ auto ShiftAmount =
+ buildConstantRegister(MIRBuilder, *SourceType, static_cast<std::int64_t>(Shift));
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *SourceType,
+ {Sum, *ShiftAmount});
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ auto NewSum =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *SourceType, {Sum, *Shifted});
+ if (!NewSum)
+ return std::unexpected(NewSum.error());
+ Sum = *NewSum;
+ }
+ if (auto Built = MIRBuilder.buildLShr(*Destination, Sum, *FinalShift); !Built)
+ return std::unexpected(Built.error());
+ }
+ return EraseOriginal();
+ }
+
+ case GenericOpcode::G_CTLS: {
+ auto SignIndex = buildConstantRegister(
+ MIRBuilder, *SourceType, static_cast<std::int64_t>(SourceType->scalarBitWidth() - 1U));
+ if (!SignIndex)
+ return std::unexpected(SignIndex.error());
+ auto One = buildConstantRegister(MIRBuilder, *DestinationType, 1);
+ if (!One)
+ return std::unexpected(One.error());
+ auto Sign = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, *SourceType,
+ {*Source, *SignIndex});
+ if (!Sign)
+ return std::unexpected(Sign.error());
+ auto DifferentBits =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, *SourceType, {*Source, *Sign});
+ if (!DifferentBits)
+ return std::unexpected(DifferentBits.error());
+ auto LeadingZeros = buildResultInstruction(MIRBuilder, GenericOpcode::G_CTLZ, *DestinationType,
+ {*DifferentBits});
+ if (!LeadingZeros)
+ return std::unexpected(LeadingZeros.error());
+ if (auto Built = MIRBuilder.buildSub(*Destination, *LeadingZeros, *One); !Built)
+ return std::unexpected(Built.error());
+ return EraseOriginal();
+ }
+
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "bit-count lowering received an unexpected opcode");
+ }
+}
+
+Expected<void> LegalizerHelper::lowerBswap(MachineInstr &Instruction) {
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Source = *Instruction.operandAt(1)->getIf<Register>();
+ auto Type = MRI.getType(Source);
+ if (!Type)
+ return std::unexpected(Type.error());
+
+ const std::uint32_t SizeInBytes = (Type->scalarBitWidth() + 7U) / 8U;
+ const std::uint32_t BaseShiftAmount = (SizeInBytes - 1U) * 8U;
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto ShiftAmount = buildConstantRegister(MIRBuilder, *Type, BaseShiftAmount);
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto LowByteShiftedLeft =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *Type, {Source, *ShiftAmount});
+ auto HighByteShiftedRight =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *Type, {Source, *ShiftAmount});
+ if (!LowByteShiftedLeft)
+ return std::unexpected(LowByteShiftedLeft.error());
+ if (!HighByteShiftedRight)
+ return std::unexpected(HighByteShiftedRight.error());
+
+ Register Result;
+ if (SizeInBytes == 2U) {
+ if (auto Built = MIRBuilder.buildOr(Destination, *HighByteShiftedRight, *LowByteShiftedLeft);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ Result = Destination;
+ } else {
+ auto OuterBytes = buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *Type,
+ {*HighByteShiftedRight, *LowByteShiftedLeft});
+ if (!OuterBytes)
+ return std::unexpected(OuterBytes.error());
+ Result = *OuterBytes;
+ }
+
+ for (std::uint32_t Index = 1; Index < SizeInBytes / 2U; ++Index) {
+ const std::uint32_t MaskBits = std::uint32_t{0xff} << (Index * 8U);
+ const std::int64_t MaskValue = static_cast<std::int32_t>(MaskBits);
+ auto Mask = buildConstantRegister(MIRBuilder, *Type, MaskValue);
+ auto InnerShiftAmount = buildConstantRegister(MIRBuilder, *Type, BaseShiftAmount - 16U * Index);
+ if (!Mask)
+ return std::unexpected(Mask.error());
+ if (!InnerShiftAmount)
+ return std::unexpected(InnerShiftAmount.error());
+
+ auto LowByte = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *Type, {Source, *Mask});
+ if (!LowByte)
+ return std::unexpected(LowByte.error());
+ auto LowByteShifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *Type,
+ {*LowByte, *InnerShiftAmount});
+ if (!LowByteShifted)
+ return std::unexpected(LowByteShifted.error());
+ auto WithLowByte =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *Type, {Result, *LowByteShifted});
+ if (!WithLowByte)
+ return std::unexpected(WithLowByte.error());
+
+ auto SourceShifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *Type,
+ {Source, *InnerShiftAmount});
+ if (!SourceShifted)
+ return std::unexpected(SourceShifted.error());
+ auto HighByte =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *Type, {*SourceShifted, *Mask});
+ if (!HighByte)
+ return std::unexpected(HighByte.error());
+
+ if (Index + 1U == SizeInBytes / 2U) {
+ if (auto Built = MIRBuilder.buildOr(Destination, *WithLowByte, *HighByte); !Built)
+ return std::unexpected(Built.error());
+ Result = Destination;
+ } else {
+ auto Combined =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *Type, {*WithLowByte, *HighByte});
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ Result = *Combined;
+ }
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerFunnelShift(MachineInstr &Instruction) {
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(2)->getIf<Register>();
+ const Register Amount = *Instruction.operandAt(3)->getIf<Register>();
+ auto DestinationType = MRI.getType(Destination);
+ auto AmountType = MRI.getType(Amount);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (!AmountType)
+ return std::unexpected(AmountType.error());
+
+ const std::uint32_t ElementWidth = DestinationType->scalarBitWidth();
+ const bool IsLeft = Instruction.opcode() == opcode(GenericOpcode::G_FSHL);
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ auto Mask =
+ buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(ElementWidth - 1U));
+ if (!Mask)
+ return std::unexpected(Mask.error());
+
+ Register ShiftAmount;
+ Register InverseShiftAmount;
+ if (std::has_single_bit(ElementWidth)) {
+ auto Shift =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType, {Amount, *Mask});
+ auto AllOnes = buildConstantRegister(MIRBuilder, *AmountType, -1);
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ if (!AllOnes)
+ return std::unexpected(AllOnes.error());
+ auto Inverted =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, *AmountType, {Amount, *AllOnes});
+ if (!Inverted)
+ return std::unexpected(Inverted.error());
+ auto Inverse =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType, {*Inverted, *Mask});
+ if (!Inverse)
+ return std::unexpected(Inverse.error());
+ ShiftAmount = *Shift;
+ InverseShiftAmount = *Inverse;
+ } else {
+ auto Width =
+ buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(ElementWidth));
+ if (!Width)
+ return std::unexpected(Width.error());
+ auto Shift =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_UREM, *AmountType, {Amount, *Width});
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ auto Inverse =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, {*Mask, *Shift});
+ if (!Inverse)
+ return std::unexpected(Inverse.error());
+ ShiftAmount = *Shift;
+ InverseShiftAmount = *Inverse;
+ }
+
+ auto One = buildConstantRegister(MIRBuilder, *AmountType, 1);
+ if (!One)
+ return std::unexpected(One.error());
+
+ Register ShiftedLeft;
+ Register ShiftedRight;
+ if (IsLeft) {
+ auto ShiftLeft = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *DestinationType,
+ {Left, ShiftAmount});
+ if (!ShiftLeft)
+ return std::unexpected(ShiftLeft.error());
+ auto FirstShiftRight =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *DestinationType, {Right, *One});
+ if (!FirstShiftRight)
+ return std::unexpected(FirstShiftRight.error());
+ auto ShiftRight = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *DestinationType,
+ {*FirstShiftRight, InverseShiftAmount});
+ if (!ShiftRight)
+ return std::unexpected(ShiftRight.error());
+ ShiftedLeft = *ShiftLeft;
+ ShiftedRight = *ShiftRight;
+ } else {
+ auto FirstShiftLeft =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *DestinationType, {Left, *One});
+ if (!FirstShiftLeft)
+ return std::unexpected(FirstShiftLeft.error());
+ auto ShiftLeft = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *DestinationType,
+ {*FirstShiftLeft, InverseShiftAmount});
+ if (!ShiftLeft)
+ return std::unexpected(ShiftLeft.error());
+ auto ShiftRight = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *DestinationType,
+ {Right, ShiftAmount});
+ if (!ShiftRight)
+ return std::unexpected(ShiftRight.error());
+ ShiftedLeft = *ShiftLeft;
+ ShiftedRight = *ShiftRight;
+ }
+
+ auto Result = MIRBuilder.buildInstr(opcode(GenericOpcode::G_OR));
+ if (!Result)
+ return std::unexpected(Result.error());
+ if (auto Added = Result->addDef(Destination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Result->addUse(ShiftedLeft); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Result->addUse(ShiftedRight); !Added)
+ return std::unexpected(Added.error());
+ if (auto Flags = Result->setFlags({MachineInstrFlag::Disjoint}); !Flags)
+ return std::unexpected(Flags.error());
+ if (auto Built = Result->build(); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerRotate(MachineInstr &Instruction) {
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Source = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Amount = *Instruction.operandAt(2)->getIf<Register>();
+ auto DestinationType = MRI.getType(Destination);
+ auto AmountType = MRI.getType(Amount);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (!AmountType)
+ return std::unexpected(AmountType.error());
+
+ const std::uint32_t ElementWidth = DestinationType->scalarBitWidth();
+ const bool IsLeft = Instruction.opcode() == opcode(GenericOpcode::G_ROTL);
+ const GenericOpcode ShiftOpcode = IsLeft ? GenericOpcode::G_SHL : GenericOpcode::G_LSHR;
+ const GenericOpcode ReverseShiftOpcode = IsLeft ? GenericOpcode::G_LSHR : GenericOpcode::G_SHL;
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto Zero = buildConstantRegister(MIRBuilder, *AmountType, 0);
+ auto WidthMinusOne =
+ buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(ElementWidth - 1U));
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (!WidthMinusOne)
+ return std::unexpected(WidthMinusOne.error());
+
+ Register Shifted;
+ Register ReverseShifted;
+ if (std::has_single_bit(ElementWidth)) {
+ auto NegativeAmount =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType, {*Zero, Amount});
+ auto ShiftAmount = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType,
+ {Amount, *WidthMinusOne});
+ if (!NegativeAmount)
+ return std::unexpected(NegativeAmount.error());
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto Shift =
+ buildResultInstruction(MIRBuilder, ShiftOpcode, *DestinationType, {Source, *ShiftAmount});
+ auto ReverseAmount = buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, *AmountType,
+ {*NegativeAmount, *WidthMinusOne});
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ if (!ReverseAmount)
+ return std::unexpected(ReverseAmount.error());
+ auto ReverseShift = buildResultInstruction(MIRBuilder, ReverseShiftOpcode, *DestinationType,
+ {Source, *ReverseAmount});
+ if (!ReverseShift)
+ return std::unexpected(ReverseShift.error());
+ Shifted = *Shift;
+ ReverseShifted = *ReverseShift;
+ } else {
+ auto Width =
+ buildConstantRegister(MIRBuilder, *AmountType, static_cast<std::int64_t>(ElementWidth));
+ if (!Width)
+ return std::unexpected(Width.error());
+ auto ShiftAmount =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_UREM, *AmountType, {Amount, *Width});
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto Shift =
+ buildResultInstruction(MIRBuilder, ShiftOpcode, *DestinationType, {Source, *ShiftAmount});
+ auto ReverseAmount = buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *AmountType,
+ {*WidthMinusOne, *ShiftAmount});
+ auto One = buildConstantRegister(MIRBuilder, *AmountType, 1);
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ if (!ReverseAmount)
+ return std::unexpected(ReverseAmount.error());
+ if (!One)
+ return std::unexpected(One.error());
+ auto Inner =
+ buildResultInstruction(MIRBuilder, ReverseShiftOpcode, *DestinationType, {Source, *One});
+ if (!Inner)
+ return std::unexpected(Inner.error());
+ auto ReverseShift = buildResultInstruction(MIRBuilder, ReverseShiftOpcode, *DestinationType,
+ {*Inner, *ReverseAmount});
+ if (!ReverseShift)
+ return std::unexpected(ReverseShift.error());
+ Shifted = *Shift;
+ ReverseShifted = *ReverseShift;
+ }
+
+ auto Result = MIRBuilder.buildInstr(opcode(GenericOpcode::G_OR));
+ if (!Result)
+ return std::unexpected(Result.error());
+ if (auto Added = Result->addDef(Destination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Result->addUse(Shifted); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = Result->addUse(ReverseShifted); !Added)
+ return std::unexpected(Added.error());
+ if (auto Flags = Result->setFlags({MachineInstrFlag::Disjoint}); !Flags)
+ return std::unexpected(Flags.error());
+ if (auto Built = Result->build(); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::libcall(MachineInstr &Instruction) {
+ if (Libcalls == nullptr || Libcalls->CallLowering == nullptr) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "libcall legalization requires target call lowering information");
+ }
+
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ auto *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr || !DestinationOperand->isDef()) {
+ return makeUnexpected(ErrorCode::VerificationError, "libcall requires a register destination");
+ }
+ auto DestinationType = MRI.getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (!DestinationType->isCategoryNeutralScalar()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "libcall requires a category-neutral scalar destination");
+ }
+
+ const bool IsConversion =
+ Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC ||
+ Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI ||
+ Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP;
+ const bool IsFloatingBinary = Generic == GenericOpcode::G_FREM;
+ if (Instruction.operandCount() < 2 || (IsConversion && Instruction.operandCount() != 2)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "libcall has an invalid number of operands");
+ }
+
+ const MachineOperand *FirstSourceOperand = Instruction.operandAt(1);
+ const Register *FirstSource =
+ FirstSourceOperand == nullptr ? nullptr : FirstSourceOperand->getIf<Register>();
+ if (FirstSource == nullptr || !FirstSourceOperand->isUse()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "libcall requires register source operands");
+ }
+ auto SourceType = MRI.getType(*FirstSource);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ if (!SourceType->isCategoryNeutralScalar()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "libcall requires category-neutral scalar sources");
+ }
+
+ Expected<std::string> Name =
+ IsConversion ? conversionLibcallName(Generic, DestinationType->scalarBitWidth(),
+ SourceType->scalarBitWidth())
+ : IsFloatingBinary ? floatingBinaryLibcallName(Generic, DestinationType->scalarBitWidth())
+ : integerLibcallName(Generic, DestinationType->scalarBitWidth());
+ if (!Name)
+ return std::unexpected(Name.error());
+
+ const bool DestinationIsFloat =
+ Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC ||
+ Generic == GenericOpcode::G_SITOFP || Generic == GenericOpcode::G_UITOFP || IsFloatingBinary;
+ const bool SourceIsFloat =
+ Generic == GenericOpcode::G_FPEXT || Generic == GenericOpcode::G_FPTRUNC ||
+ Generic == GenericOpcode::G_FPTOSI || Generic == GenericOpcode::G_FPTOUI || IsFloatingBinary;
+ const ABIValueCategory DestinationCategory =
+ DestinationIsFloat ? ABIValueCategory::FloatingPoint : ABIValueCategory::Integer;
+ const ABIValueCategory SourceCategory =
+ SourceIsFloat ? ABIValueCategory::FloatingPoint : ABIValueCategory::Integer;
+ auto ExtensionFor = [&](ABIValueCategory Category, bool IsResult,
+ std::uint32_t BitWidth) noexcept {
+ if (Category != ABIValueCategory::Integer || BitWidth >= 64)
+ return ABIExtensionKind::None;
+ if (!IsConversion)
+ return integerLibcallExtension(Generic, BitWidth);
+ if ((IsResult && Generic == GenericOpcode::G_FPTOSI) ||
+ (!IsResult && Generic == GenericOpcode::G_SITOFP)) {
+ return ABIExtensionKind::SignExtend;
+ }
+ return ABIExtensionKind::ZeroExtend;
+ };
+
+ auto FragmentType = LowLevelType::makeCategoryNeutralScalar(64);
+ if (!FragmentType)
+ return std::unexpected(FragmentType.error());
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ auto AppendValue = [&](Register Value, LowLevelType Type, ABIValueCategory Category,
+ ABIExtensionKind Extension, std::uint32_t OriginalValue, bool IsResult,
+ std::vector<ABIValue> &Values,
+ std::vector<Register> *Fragments) -> Expected<void> {
+ const std::uint32_t BitWidth = Type.scalarBitWidth();
+ auto OriginalAlignment = Alignment::create((BitWidth + 7U) / 8U);
+ if (!OriginalAlignment)
+ return std::unexpected(OriginalAlignment.error());
+ if (BitWidth != 128) {
+ auto ABI = libcallABIValue(Value, Type, Category, OriginalValue, 0, 1, 0, *OriginalAlignment,
+ Extension);
+ if (!ABI)
+ return std::unexpected(ABI.error());
+ Values.push_back(*ABI);
+ if (Fragments != nullptr)
+ Fragments->push_back(Value);
+ return {};
+ }
+
+ std::vector<Register> PartRegisters;
+ PartRegisters.reserve(2);
+ for (std::size_t Index = 0; Index != 2; ++Index) {
+ auto Part = MRI.createVirtualRegister(*FragmentType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ PartRegisters.push_back(*Part);
+ }
+ if (!IsResult) {
+ auto Unmerged = MIRBuilder.buildUnmergeValues(PartRegisters, Value);
+ if (!Unmerged)
+ return std::unexpected(Unmerged.error());
+ }
+ for (std::uint16_t Index = 0; Index != 2; ++Index) {
+ auto ABI = libcallABIValue(PartRegisters[Index], *FragmentType, ABIValueCategory::Integer,
+ OriginalValue, Index, 2, static_cast<std::uint32_t>(Index) * 8U,
+ *OriginalAlignment, ABIExtensionKind::None);
+ if (!ABI)
+ return std::unexpected(ABI.error());
+ Values.push_back(*ABI);
+ }
+ if (Fragments != nullptr)
+ *Fragments = std::move(PartRegisters);
+ return {};
+ };
+
+ std::vector<ABIValue> Arguments;
+ Arguments.reserve((Instruction.operandCount() - 1U) *
+ (SourceType->scalarBitWidth() == 128 ? 2U : 1U));
+ std::uint32_t OriginalArgument = 0;
+ for (std::size_t Index = 1; Index != Instruction.operandCount(); ++Index) {
+ const MachineOperand *Operand = Instruction.operandAt(Index);
+ const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Source == nullptr || !Operand->isUse()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "libcall requires register source operands");
+ }
+ auto ArgumentType = MRI.getType(*Source);
+ if (!ArgumentType)
+ return std::unexpected(ArgumentType.error());
+ if (*ArgumentType != *SourceType) {
+ return makeUnexpected(ErrorCode::VerificationError, "libcall source types must match");
+ }
+ if (!IsConversion && *ArgumentType != *DestinationType) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "binary libcall source and destination types must match");
+ }
+ const ABIExtensionKind Extension =
+ ExtensionFor(SourceCategory, false, ArgumentType->scalarBitWidth());
+ if (auto Appended = AppendValue(*Source, *ArgumentType, SourceCategory, Extension,
+ OriginalArgument++, false, Arguments, nullptr);
+ !Appended) {
+ return std::unexpected(Appended.error());
+ }
+ }
+
+ std::vector<ABIValue> Results;
+ std::vector<Register> ResultFragments;
+ const ABIExtensionKind ResultExtension =
+ ExtensionFor(DestinationCategory, true, DestinationType->scalarBitWidth());
+ if (auto Appended = AppendValue(*Destination, *DestinationType, DestinationCategory,
+ ResultExtension, 0, true, Results, &ResultFragments);
+ !Appended) {
+ return std::unexpected(Appended.error());
+ }
+
+ auto Callee = MachineCallTarget::externalSymbol(std::move(*Name));
+ if (!Callee)
+ return std::unexpected(Callee.error());
+ CallLoweringRequest Request;
+ Request.Convention = Libcalls->Convention;
+ Request.Callee = std::move(*Callee);
+ Request.Arguments = std::move(Arguments);
+ Request.Results = std::move(Results);
+ Request.FixedArgumentCount = OriginalArgument;
+ if (auto Lowered = Libcalls->CallLowering->lowerCall(MIRBuilder, Request); !Lowered)
+ return std::unexpected(Lowered.error());
+
+ if (DestinationType->scalarBitWidth() == 128) {
+ auto Merged = MIRBuilder.buildMergeLikeInstr(*Destination, ResultFragments);
+ if (!Merged)
+ return std::unexpected(Merged.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerSignedAddSubOverflow(MachineInstr &Instruction) {
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (Generic == GenericOpcode::G_SADDE || Generic == GenericOpcode::G_SSUBE) {
+ if (Instruction.operandCount() != 5)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "signed extended add/sub lowering requires five register operands");
+
+ const bool IsAdd = Generic == GenericOpcode::G_SADDE;
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *OverflowOperand = Instruction.operandAt(1);
+ const MachineOperand *LeftOperand = Instruction.operandAt(2);
+ const MachineOperand *RightOperand = Instruction.operandAt(3);
+ const MachineOperand *CarryInOperand = Instruction.operandAt(4);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Overflow =
+ OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ const Register *CarryIn =
+ CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf<Register>();
+ if (Destination == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr ||
+ CarryIn == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "signed extended add/sub lowering requires register operands");
+ }
+
+ auto Type = MRI.getType(*Destination);
+ if (!Type)
+ return std::unexpected(Type.error());
+ auto Result = MRI.cloneVirtualRegister(*Destination);
+ auto ExtendedCarry = MRI.createVirtualRegister(*Type);
+ auto FirstXor = MRI.createVirtualRegister(*Type);
+ auto SecondXor = MRI.createVirtualRegister(*Type);
+ auto OverflowBits = MRI.createVirtualRegister(*Type);
+ if (!Result)
+ return std::unexpected(Result.error());
+ if (!ExtendedCarry)
+ return std::unexpected(ExtendedCarry.error());
+ if (!FirstXor)
+ return std::unexpected(FirstXor.error());
+ if (!SecondXor)
+ return std::unexpected(SecondXor.error());
+ if (!OverflowBits)
+ return std::unexpected(OverflowBits.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ if (IsAdd) {
+ auto Temporary = MRI.createVirtualRegister(*Type);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ if (auto Built = MIRBuilder.buildAdd(*Temporary, *Left, *Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildZExt(*ExtendedCarry, *CarryIn); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildAdd(*Result, *Temporary, *ExtendedCarry); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildCopy(*Destination, *Result); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildXor(*FirstXor, *Result, *Left); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildXor(*SecondXor, *Result, *Right); !Built)
+ return std::unexpected(Built.error());
+ } else {
+ auto RightWithCarry = MRI.createVirtualRegister(*Type);
+ if (!RightWithCarry)
+ return std::unexpected(RightWithCarry.error());
+ if (auto Built = MIRBuilder.buildZExt(*ExtendedCarry, *CarryIn); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildAdd(*RightWithCarry, *Right, *ExtendedCarry); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildSub(*Result, *Left, *RightWithCarry); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildCopy(*Destination, *Result); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildXor(*FirstXor, *Left, *Right); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = MIRBuilder.buildXor(*SecondXor, *Left, *Result); !Built)
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildAnd(*OverflowBits, *FirstXor, *SecondXor); !Built)
+ return std::unexpected(Built.error());
+ auto Zero = buildConstantRegister(MIRBuilder, *Type, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *Overflow, *OverflowBits, *Zero);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+ }
+
+ const bool IsAdd = Instruction.opcode() == opcode(GenericOpcode::G_SADDO);
+ const Register Destination = *Instruction.operandAt(0)->getIf<Register>();
+ const Register Overflow = *Instruction.operandAt(1)->getIf<Register>();
+ const Register Left = *Instruction.operandAt(2)->getIf<Register>();
+ const Register Right = *Instruction.operandAt(3)->getIf<Register>();
+ auto Type = MRI.getType(Destination);
+ auto BooleanType = MRI.getType(Overflow);
+ if (!Type)
+ return std::unexpected(Type.error());
+ if (!BooleanType)
+ return std::unexpected(BooleanType.error());
+
+ auto Result = MRI.cloneVirtualRegister(Destination);
+ auto FirstComparison = MRI.createVirtualRegister(*BooleanType);
+ auto SecondComparison = MRI.createVirtualRegister(*BooleanType);
+ if (!Result)
+ return std::unexpected(Result.error());
+ if (!FirstComparison)
+ return std::unexpected(FirstComparison.error());
+ if (!SecondComparison)
+ return std::unexpected(SecondComparison.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = IsAdd ? MIRBuilder.buildAdd(*Result, Left, Right)
+ : MIRBuilder.buildSub(*Result, Left, Right);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ auto Zero = buildConstantRegister(MIRBuilder, *Type, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (IsAdd) {
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *FirstComparison, *Result, Left);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *SecondComparison, Right, *Zero);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildXor(Overflow, *SecondComparison, *FirstComparison); !Built)
+ return std::unexpected(Built.error());
+ } else {
+ if (auto Built =
+ MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *FirstComparison, Left, Right);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildICmp(IntegerPredicate::SignedLessThan, *SecondComparison,
+ *Result, *Zero);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildXor(Overflow, *FirstComparison, *SecondComparison); !Built)
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = MIRBuilder.buildCopy(Destination, *Result); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerSaturatingAddSubToMinMax(MachineInstr &Instruction) {
+ if (Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "saturating add/sub min/max lowering requires three register operands");
+
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ const bool IsSigned = Generic == GenericOpcode::G_SADDSAT || Generic == GenericOpcode::G_SSUBSAT;
+ const bool IsAdd = Generic == GenericOpcode::G_UADDSAT || Generic == GenericOpcode::G_SADDSAT;
+ const GenericOpcode ArithmeticOpcode = IsAdd ? GenericOpcode::G_ADD : GenericOpcode::G_SUB;
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *LeftOperand = Instruction.operandAt(1);
+ const MachineOperand *RightOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ if (Destination == nullptr || Left == nullptr || Right == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "saturating add/sub min/max lowering requires register operands");
+
+ auto Type = MRI.getType(*Destination);
+ if (!Type)
+ return std::unexpected(Type.error());
+ if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "saturating add/sub min/max lowering requires a scalar integer type");
+ const std::uint32_t BitWidth = Type->scalarBitWidth();
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (IsSigned) {
+ auto MaximumValue = APInt::getLowBitsSet(BitWidth, BitWidth - 1U);
+ auto MinimumValue = oneBitSet(BitWidth, BitWidth - 1U);
+ if (!MaximumValue)
+ return std::unexpected(MaximumValue.error());
+ if (!MinimumValue)
+ return std::unexpected(MinimumValue.error());
+ auto Maximum = buildConstantRegister(MIRBuilder, *Type, std::move(*MaximumValue));
+ auto Minimum = buildConstantRegister(MIRBuilder, *Type, std::move(*MinimumValue));
+ if (!Maximum)
+ return std::unexpected(Maximum.error());
+ if (!Minimum)
+ return std::unexpected(Minimum.error());
+
+ Register LowerBound;
+ Register UpperBound;
+ if (IsAdd) {
+ auto Zero = buildConstantRegister(MIRBuilder, *Type, 0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto LeftMaximum =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SMAX, *Type, {*Left, *Zero});
+ if (!LeftMaximum)
+ return std::unexpected(LeftMaximum.error());
+ auto High =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *Type, {*Maximum, *LeftMaximum});
+ if (!High)
+ return std::unexpected(High.error());
+ auto LeftMinimum =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SMIN, *Type, {*Left, *Zero});
+ if (!LeftMinimum)
+ return std::unexpected(LeftMinimum.error());
+ auto Low =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *Type, {*Minimum, *LeftMinimum});
+ if (!Low)
+ return std::unexpected(Low.error());
+ LowerBound = *Low;
+ UpperBound = *High;
+ } else {
+ auto NegativeOne = buildConstantRegister(MIRBuilder, *Type, -1);
+ if (!NegativeOne)
+ return std::unexpected(NegativeOne.error());
+ auto LeftMaximum =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SMAX, *Type, {*Left, *NegativeOne});
+ if (!LeftMaximum)
+ return std::unexpected(LeftMaximum.error());
+ auto Low =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *Type, {*LeftMaximum, *Maximum});
+ if (!Low)
+ return std::unexpected(Low.error());
+ auto LeftMinimum =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SMIN, *Type, {*Left, *NegativeOne});
+ if (!LeftMinimum)
+ return std::unexpected(LeftMinimum.error());
+ auto High =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SUB, *Type, {*LeftMinimum, *Minimum});
+ if (!High)
+ return std::unexpected(High.error());
+ LowerBound = *Low;
+ UpperBound = *High;
+ }
+
+ auto LowerClamped =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_SMAX, *Type, {LowerBound, *Right});
+ if (!LowerClamped)
+ return std::unexpected(LowerClamped.error());
+ auto ClampedRight = buildResultInstruction(MIRBuilder, GenericOpcode::G_SMIN, *Type,
+ {*LowerClamped, UpperBound});
+ if (!ClampedRight)
+ return std::unexpected(ClampedRight.error());
+ if (auto Built =
+ MIRBuilder.buildInstr(opcode(ArithmeticOpcode), {*Destination}, {*Left, *ClampedRight});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ } else {
+ Register Available;
+ if (IsAdd) {
+ auto AllBits = APInt::getLowBitsSet(BitWidth, BitWidth);
+ if (!AllBits)
+ return std::unexpected(AllBits.error());
+ auto AllOnes = buildConstantRegister(MIRBuilder, *Type, std::move(*AllBits));
+ if (!AllOnes)
+ return std::unexpected(AllOnes.error());
+ auto Inverted =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_XOR, *Type, {*Left, *AllOnes});
+ if (!Inverted)
+ return std::unexpected(Inverted.error());
+ Available = *Inverted;
+ } else {
+ Available = *Left;
+ }
+ auto Minimum =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_UMIN, *Type, {Available, *Right});
+ if (!Minimum)
+ return std::unexpected(Minimum.error());
+ if (auto Built =
+ MIRBuilder.buildInstr(opcode(ArithmeticOpcode), {*Destination}, {*Left, *Minimum});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerSaturatingAddSubToOverflow(MachineInstr &Instruction) {
+ if (Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "saturating add/sub overflow lowering requires three register operands");
+
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ const bool IsSigned = Generic == GenericOpcode::G_SADDSAT || Generic == GenericOpcode::G_SSUBSAT;
+ const bool IsAdd = Generic == GenericOpcode::G_UADDSAT || Generic == GenericOpcode::G_SADDSAT;
+ const GenericOpcode OverflowOpcode =
+ IsAdd ? (IsSigned ? GenericOpcode::G_SADDO : GenericOpcode::G_UADDO)
+ : (IsSigned ? GenericOpcode::G_SSUBO : GenericOpcode::G_USUBO);
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *LeftOperand = Instruction.operandAt(1);
+ const MachineOperand *RightOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ if (Destination == nullptr || Left == nullptr || Right == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "saturating add/sub overflow lowering requires register operands");
+
+ auto Type = MRI.getType(*Destination);
+ if (!Type)
+ return std::unexpected(Type.error());
+ if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "saturating add/sub overflow lowering requires a scalar integer type");
+ auto BooleanType = booleanTypeFor(*Type);
+ if (!BooleanType)
+ return std::unexpected(BooleanType.error());
+ auto Temporary = MRI.createVirtualRegister(*Type);
+ auto Overflow = MRI.createVirtualRegister(*BooleanType);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ if (!Overflow)
+ return std::unexpected(Overflow.error());
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built =
+ MIRBuilder.buildInstr(opcode(OverflowOpcode), {*Temporary, *Overflow}, {*Left, *Right});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+
+ Register Clamp;
+ if (IsSigned) {
+ const std::uint32_t BitWidth = Type->scalarBitWidth();
+ auto ShiftAmount =
+ buildConstantRegister(MIRBuilder, *Type, static_cast<std::int64_t>(BitWidth - 1U));
+ auto MinimumValue = oneBitSet(BitWidth, BitWidth - 1U);
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ if (!MinimumValue)
+ return std::unexpected(MinimumValue.error());
+ auto Sign = buildResultInstruction(MIRBuilder, GenericOpcode::G_ASHR, *Type,
+ {*Temporary, *ShiftAmount});
+ auto Minimum = buildConstantRegister(MIRBuilder, *Type, std::move(*MinimumValue));
+ if (!Sign)
+ return std::unexpected(Sign.error());
+ if (!Minimum)
+ return std::unexpected(Minimum.error());
+ auto Saturated =
+ buildResultInstruction(MIRBuilder, GenericOpcode::G_ADD, *Type, {*Sign, *Minimum});
+ if (!Saturated)
+ return std::unexpected(Saturated.error());
+ Clamp = *Saturated;
+ } else if (IsAdd) {
+ auto AllBits = APInt::getLowBitsSet(Type->scalarBitWidth(), Type->scalarBitWidth());
+ if (!AllBits)
+ return std::unexpected(AllBits.error());
+ auto Saturated = buildConstantRegister(MIRBuilder, *Type, std::move(*AllBits));
+ if (!Saturated)
+ return std::unexpected(Saturated.error());
+ Clamp = *Saturated;
+ } else {
+ auto Saturated = buildConstantRegister(MIRBuilder, *Type, 0);
+ if (!Saturated)
+ return std::unexpected(Saturated.error());
+ Clamp = *Saturated;
+ }
+
+ if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {*Destination},
+ {*Overflow, Clamp, *Temporary});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::lowerSaturatingShiftLeft(MachineInstr &Instruction) {
+ if (Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "saturating shift-left lowering requires three register operands");
+
+ const bool IsSigned = Instruction.opcode() == opcode(GenericOpcode::G_SSHLSAT);
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *LeftOperand = Instruction.operandAt(1);
+ const MachineOperand *RightOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ if (Destination == nullptr || Left == nullptr || Right == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "saturating shift-left lowering requires register operands");
+
+ auto Type = MRI.getType(*Destination);
+ if (!Type)
+ return std::unexpected(Type.error());
+ if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "saturating shift-left lowering requires a scalar integer type");
+ auto BooleanType = booleanTypeFor(*Type);
+ if (!BooleanType)
+ return std::unexpected(BooleanType.error());
+ const std::uint32_t BitWidth = Type->scalarBitWidth();
+
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *Type, {*Left, *Right});
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ auto Restored =
+ buildResultInstruction(MIRBuilder, IsSigned ? GenericOpcode::G_ASHR : GenericOpcode::G_LSHR,
+ *Type, {*Shifted, *Right});
+ if (!Restored)
+ return std::unexpected(Restored.error());
+
+ Register Saturated;
+ if (IsSigned) {
+ auto MinimumValue = oneBitSet(BitWidth, BitWidth - 1U);
+ auto MaximumValue = APInt::getLowBitsSet(BitWidth, BitWidth - 1U);
+ if (!MinimumValue)
+ return std::unexpected(MinimumValue.error());
+ if (!MaximumValue)
+ return std::unexpected(MaximumValue.error());
+ auto Minimum = buildConstantRegister(MIRBuilder, *Type, std::move(*MinimumValue));
+ auto Maximum = buildConstantRegister(MIRBuilder, *Type, std::move(*MaximumValue));
+ auto Zero = buildConstantRegister(MIRBuilder, *Type, 0);
+ if (!Minimum)
+ return std::unexpected(Minimum.error());
+ if (!Maximum)
+ return std::unexpected(Maximum.error());
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Negative =
+ buildICmpRegister(MIRBuilder, IntegerPredicate::SignedLessThan, *BooleanType, *Left, *Zero);
+ if (!Negative)
+ return std::unexpected(Negative.error());
+ auto SignSaturated = buildResultInstruction(MIRBuilder, GenericOpcode::G_SELECT, *Type,
+ {*Negative, *Minimum, *Maximum});
+ if (!SignSaturated)
+ return std::unexpected(SignSaturated.error());
+ Saturated = *SignSaturated;
+ } else {
+ auto AllBits = APInt::getLowBitsSet(BitWidth, BitWidth);
+ if (!AllBits)
+ return std::unexpected(AllBits.error());
+ auto Maximum = buildConstantRegister(MIRBuilder, *Type, std::move(*AllBits));
+ if (!Maximum)
+ return std::unexpected(Maximum.error());
+ Saturated = *Maximum;
+ }
+
+ auto Overflow =
+ buildICmpRegister(MIRBuilder, IntegerPredicate::NotEqual, *BooleanType, *Left, *Restored);
+ if (!Overflow)
+ return std::unexpected(Overflow.error());
+ if (auto Built = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SELECT), {*Destination},
+ {*Overflow, Saturated, *Shifted});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> LegalizerHelper::legalize(MachineInstr &Instruction,
+ const LegalizationDecision &Decision) {
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ switch (Decision.Action) {
+ case LegalizationAction::WidenScalar:
+ return widenScalar(Instruction, Decision.TypeIndex, Decision.NewType);
+ case LegalizationAction::NarrowScalar:
+ return narrowScalar(Instruction, Decision.TypeIndex, Decision.NewType);
+ case LegalizationAction::Custom:
+ return Info.legalizeCustom(*this, Instruction, Decision);
+ case LegalizationAction::Lower:
+ return lower(Instruction, Decision);
+ case LegalizationAction::Libcall:
+ return libcall(Instruction);
+ case LegalizationAction::FewerElements:
+ return fewerElements(Instruction, Decision.TypeIndex, Decision.NewType);
+ case LegalizationAction::Legal:
+ return {};
+ case LegalizationAction::Unsupported:
+ case LegalizationAction::MoreElements:
+ case LegalizationAction::Bitcast:
+ case LegalizationAction::ChangeElementType:
+ case LegalizationAction::ChangeAddressSpace:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "legalizer helper does not implement the requested action");
+ }
+ return makeUnexpected(ErrorCode::InternalError, "unknown legalization action");
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
new file mode 100644
index 0000000..bdc783b
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -0,0 +1,497 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/InstructionDescriptor.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <algorithm>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <ranges>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] LegalityPredicate typeInSet(std::size_t TypeIndex, std::vector<LowLevelType> Types) {
+ return [TypeIndex, Types = std::move(Types)](const LegalizationQuery &Query) {
+ return TypeIndex < Query.Types.size() &&
+ std::ranges::find(Types, Query.Types[TypeIndex]) != Types.end();
+ };
+}
+
+[[nodiscard]] LegalityPredicate
+typePairInSet(std::vector<std::pair<LowLevelType, LowLevelType>> Types) {
+ return [Types = std::move(Types)](const LegalizationQuery &Query) {
+ return Query.Types.size() >= 2 &&
+ std::ranges::find(Types, std::pair{Query.Types[0], Query.Types[1]}) != Types.end();
+ };
+}
+
+[[nodiscard]] LegalityPredicate
+typeTripleInSet(std::vector<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) {
+ return [Types = std::move(Types)](const LegalizationQuery &Query) {
+ return Query.Types.size() >= 3 &&
+ std::ranges::find(Types, std::tuple{Query.Types[0], Query.Types[1], Query.Types[2]}) !=
+ Types.end();
+ };
+}
+
+[[nodiscard]] LowLevelType scalarType(std::uint32_t BitWidth) {
+ auto Type = LowLevelType::makeCategoryNeutralScalar(BitWidth);
+ return Type ? *Type : LowLevelType{};
+}
+
+[[nodiscard]] std::uint32_t nextPowerOfTwo(std::uint32_t Width, std::uint32_t MinimumBitWidth) {
+ return std::max(std::bit_ceil(Width), MinimumBitWidth);
+}
+
+[[nodiscard]] bool memoryRuleMatches(const LegalizationQuery &Query,
+ const LegalizeMemoryRule &Rule) {
+ if (Query.Types.size() < 2 || Query.MemoryOperands.size() != 1 ||
+ Query.Types[0] != Rule.ValueType || Query.Types[1] != Rule.AddressType)
+ return false;
+ const LegalizationMemoryDesc &Memory = Query.MemoryOperands.front();
+ if (Memory.AlignmentInBits < Rule.MinimumAlignmentInBits)
+ return false;
+ if (Memory.Type)
+ return Memory.Type->minimumBitWidth() == Rule.MemoryType.minimumBitWidth();
+ return Memory.Size.isKnown() && !Memory.Size.isScalable() &&
+ Memory.Size.minimumBytes() * 8U == Rule.MemoryType.scalarBitWidth();
+}
+
+} // namespace
+
+LegalizeRule::LegalizeRule(LegalityPredicate Predicate, LegalizationAction Action,
+ LegalizeMutation Mutation)
+ : Predicate(std::move(Predicate)), Action(Action), Mutation(std::move(Mutation)) {}
+
+bool LegalizeRule::matches(const LegalizationQuery &Query) const {
+ return Predicate(Query);
+}
+
+LegalizationDecision LegalizeRule::apply(const LegalizationQuery &Query) const {
+ if (!Mutation)
+ return {.Action = Action, .TypeIndex = 0, .NewType = {}};
+ auto [TypeIndex, NewType] = Mutation(Query);
+ return {.Action = Action, .TypeIndex = TypeIndex, .NewType = NewType};
+}
+
+LegalizeRuleSet &LegalizeRuleSet::actionIf(LegalizationAction Action, LegalityPredicate Predicate,
+ LegalizeMutation Mutation) {
+ Rules.emplace_back(std::move(Predicate), Action, std::move(Mutation));
+ return *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::actionFor(LegalizationAction Action,
+ std::initializer_list<LowLevelType> Types) {
+ return actionIf(Action, typeInSet(0, std::vector<LowLevelType>(Types)));
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::actionFor(LegalizationAction Action,
+ std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+ return actionIf(Action, typePairInSet(std::vector<std::pair<LowLevelType, LowLevelType>>(Types)));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::actionFor(
+ LegalizationAction Action,
+ std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) {
+ return actionIf(
+ Action,
+ typeTripleInSet(std::vector<std::tuple<LowLevelType, LowLevelType, LowLevelType>>(Types)));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::legalIf(LegalityPredicate Predicate) {
+ return actionIf(LegalizationAction::Legal, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::customIf(LegalityPredicate Predicate) {
+ return actionIf(LegalizationAction::Custom, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::lowerIf(LegalityPredicate Predicate) {
+ return actionIf(LegalizationAction::Lower, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::libcallIf(LegalityPredicate Predicate) {
+ return actionIf(LegalizationAction::Libcall, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::unsupportedIf(LegalityPredicate Predicate) {
+ return actionIf(LegalizationAction::Unsupported, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::widenScalarIf(LegalityPredicate Predicate,
+ LegalizeMutation Mutation) {
+ return actionIf(LegalizationAction::WidenScalar, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::narrowScalarIf(LegalityPredicate Predicate,
+ LegalizeMutation Mutation) {
+ return actionIf(LegalizationAction::NarrowScalar, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::moreElementsIf(LegalityPredicate Predicate,
+ LegalizeMutation Mutation) {
+ return actionIf(LegalizationAction::MoreElements, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::fewerElementsIf(LegalityPredicate Predicate,
+ LegalizeMutation Mutation) {
+ return actionIf(LegalizationAction::FewerElements, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::bitcastIf(LegalityPredicate Predicate,
+ LegalizeMutation Mutation) {
+ return actionIf(LegalizationAction::Bitcast, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::legalFor(std::initializer_list<LowLevelType> Types) {
+ return actionFor(LegalizationAction::Legal, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::legalFor(bool Enabled,
+ std::initializer_list<LowLevelType> Types) {
+ return Enabled ? legalFor(Types) : *this;
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::legalFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+ return actionFor(LegalizationAction::Legal, Types);
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::legalFor(bool Enabled,
+ std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+ return Enabled ? legalFor(Types) : *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::legalFor(
+ std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) {
+ return actionFor(LegalizationAction::Legal, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::customFor(std::initializer_list<LowLevelType> Types) {
+ return actionFor(LegalizationAction::Custom, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::customFor(bool Enabled,
+ std::initializer_list<LowLevelType> Types) {
+ return Enabled ? customFor(Types) : *this;
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::customFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+ return actionFor(LegalizationAction::Custom, Types);
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::customFor(bool Enabled,
+ std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+ return Enabled ? customFor(Types) : *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::lowerFor(std::initializer_list<LowLevelType> Types) {
+ return actionFor(LegalizationAction::Lower, Types);
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::lowerFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+ return actionFor(LegalizationAction::Lower, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::libcallFor(std::initializer_list<LowLevelType> Types) {
+ return actionFor(LegalizationAction::Libcall, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::libcallFor(bool Enabled,
+ std::initializer_list<LowLevelType> Types) {
+ return Enabled ? libcallFor(Types) : *this;
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::libcallFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+ return actionFor(LegalizationAction::Libcall, Types);
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::libcallFor(bool Enabled,
+ std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+ return Enabled ? libcallFor(Types) : *this;
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::legalForTypesWithMemDesc(std::initializer_list<LegalizeMemoryRule> MemoryRules) {
+ const std::vector<LegalizeMemoryRule> MemoryRuleList(MemoryRules);
+ return legalIf([MemoryRuleList](const LegalizationQuery &Query) {
+ return std::ranges::any_of(MemoryRuleList, [&Query](const LegalizeMemoryRule &Rule) {
+ return memoryRuleMatches(Query, Rule);
+ });
+ });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::widenScalarToNextPow2(std::size_t TypeIndex,
+ std::uint32_t MinimumBitWidth) {
+ return widenScalarIf(
+ [TypeIndex, MinimumBitWidth](const LegalizationQuery &Query) {
+ if (TypeIndex >= Query.Types.size() || !Query.Types[TypeIndex].isCategoryNeutralScalar())
+ return false;
+ const std::uint32_t Width = Query.Types[TypeIndex].scalarBitWidth();
+ return nextPowerOfTwo(Width, MinimumBitWidth) > Width;
+ },
+ [TypeIndex, MinimumBitWidth](const LegalizationQuery &Query) {
+ return std::pair{TypeIndex, scalarType(nextPowerOfTwo(
+ Query.Types[TypeIndex].scalarBitWidth(), MinimumBitWidth))};
+ });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::scalarize(std::size_t TypeIndex) {
+ return fewerElementsIf(
+ [TypeIndex](const LegalizationQuery &Query) {
+ return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isVector();
+ },
+ [TypeIndex](const LegalizationQuery &Query) {
+ return std::pair{TypeIndex, *Query.Types[TypeIndex].scalarElementType()};
+ });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::minScalar(std::size_t TypeIndex, LowLevelType MinimumType) {
+ return widenScalarIf(
+ [TypeIndex, MinimumType](const LegalizationQuery &Query) {
+ return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isCategoryNeutralScalar() &&
+ Query.Types[TypeIndex].scalarBitWidth() < MinimumType.scalarBitWidth();
+ },
+ [TypeIndex, MinimumType](const LegalizationQuery &) {
+ return std::pair{TypeIndex, MinimumType};
+ });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::minScalar(bool Enabled, std::size_t TypeIndex,
+ LowLevelType MinimumType) {
+ return Enabled ? minScalar(TypeIndex, MinimumType) : *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::maxScalar(std::size_t TypeIndex, LowLevelType MaximumType) {
+ return narrowScalarIf(
+ [TypeIndex, MaximumType](const LegalizationQuery &Query) {
+ return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isCategoryNeutralScalar() &&
+ Query.Types[TypeIndex].scalarBitWidth() > MaximumType.scalarBitWidth();
+ },
+ [TypeIndex, MaximumType](const LegalizationQuery &) {
+ return std::pair{TypeIndex, MaximumType};
+ });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::maxScalar(bool Enabled, std::size_t TypeIndex,
+ LowLevelType MaximumType) {
+ return Enabled ? maxScalar(TypeIndex, MaximumType) : *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::clampScalar(std::size_t TypeIndex, LowLevelType MinimumType,
+ LowLevelType MaximumType) {
+ return minScalar(TypeIndex, MinimumType).maxScalar(TypeIndex, MaximumType);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::scalarSameSizeAs(std::size_t TypeIndex,
+ std::size_t ReferenceTypeIndex) {
+ const auto ChangeToReference = [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) {
+ return std::pair{TypeIndex, scalarType(Query.Types[ReferenceTypeIndex].scalarBitWidth())};
+ };
+ widenScalarIf(
+ [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) {
+ return TypeIndex < Query.Types.size() && ReferenceTypeIndex < Query.Types.size() &&
+ Query.Types[TypeIndex].isCategoryNeutralScalar() &&
+ Query.Types[ReferenceTypeIndex].isCategoryNeutralScalar() &&
+ Query.Types[TypeIndex].scalarBitWidth() <
+ Query.Types[ReferenceTypeIndex].scalarBitWidth();
+ },
+ ChangeToReference);
+ return narrowScalarIf(
+ [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) {
+ return TypeIndex < Query.Types.size() && ReferenceTypeIndex < Query.Types.size() &&
+ Query.Types[TypeIndex].isCategoryNeutralScalar() &&
+ Query.Types[ReferenceTypeIndex].isCategoryNeutralScalar() &&
+ Query.Types[TypeIndex].scalarBitWidth() >
+ Query.Types[ReferenceTypeIndex].scalarBitWidth();
+ },
+ ChangeToReference);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::lowerIfMemSizeNotByteSizePow2(std::size_t MemoryIndex) {
+ return lowerIf([MemoryIndex](const LegalizationQuery &Query) {
+ if (MemoryIndex >= Query.MemoryOperands.size())
+ return false;
+ const MemorySize Size = Query.MemoryOperands[MemoryIndex].Size;
+ return !Size.isKnown() || Size.isScalable() || !std::has_single_bit(Size.minimumBytes());
+ });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::custom() {
+ return customIf([](const LegalizationQuery &) { return true; });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::alwaysLegal() {
+ return legalIf([](const LegalizationQuery &) { return true; });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::lower() {
+ return lowerIf([](const LegalizationQuery &) { return true; });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::libcall() {
+ return libcallIf([](const LegalizationQuery &) { return true; });
+}
+
+std::optional<LegalizationDecision> LegalizeRuleSet::apply(const LegalizationQuery &Query) const {
+ for (const LegalizeRule &Rule : Rules) {
+ if (Rule.matches(Query))
+ return Rule.apply(Query);
+ }
+ return std::nullopt;
+}
+
+LegalizeRuleSet &LegalizerInfo::getActionDefinitionsBuilder(Opcode InstructionOpcode) {
+ return RuleSets[InstructionOpcode];
+}
+
+LegalizeRuleSet &
+LegalizerInfo::getActionDefinitionsBuilder(std::initializer_list<Opcode> InstructionOpcodes) {
+ const Opcode Representative = *InstructionOpcodes.begin();
+ for (const Opcode InstructionOpcode : InstructionOpcodes) {
+ if (InstructionOpcode != Representative)
+ RuleAliases.emplace(InstructionOpcode, Representative);
+ }
+ return getActionDefinitionsBuilder(Representative);
+}
+
+Expected<LegalizationQuery> LegalizerInfo::buildQuery(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &Target) {
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode());
+ if (Descriptor == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "legalization query requires a registered instruction opcode");
+
+ std::size_t ExplicitCount = 0;
+ while (ExplicitCount < Instruction.operandCount() &&
+ !Instruction.operandAt(ExplicitCount)->isImplicit()) {
+ ++ExplicitCount;
+ }
+ auto OperandDescriptors = resolveExplicitOperandDescriptors(*Descriptor, ExplicitCount);
+ if (!OperandDescriptors)
+ return std::unexpected(OperandDescriptors.error());
+
+ std::size_t TypeCount = 0;
+ for (const OperandDescriptor *Operand : *OperandDescriptors) {
+ if (Operand->TypeConstraint && Operand->TypeConstraint->EqualityGroup) {
+ TypeCount = std::max(TypeCount,
+ static_cast<std::size_t>(*Operand->TypeConstraint->EqualityGroup) + 1U);
+ }
+ }
+
+ std::vector<std::optional<LowLevelType>> OptionalTypes(TypeCount);
+ std::vector<std::vector<std::size_t>> TypeOperands(TypeCount);
+ for (std::size_t Index = 0; Index < ExplicitCount; ++Index) {
+ const OperandDescriptor &OperandDescriptor = *(*OperandDescriptors)[Index];
+ if (!OperandDescriptor.TypeConstraint || !OperandDescriptor.TypeConstraint->EqualityGroup) {
+ continue;
+ }
+ const std::size_t TypeIndex = *OperandDescriptor.TypeConstraint->EqualityGroup;
+ const Register *Reg = Instruction.operandAt(Index)->getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "legalization type operand must name a virtual register");
+ auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ const std::optional<LowLevelType> &RegisterType = Info->get().Type;
+ if (!RegisterType.has_value())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "legalization type operand has no LowLevelType");
+ std::optional<LowLevelType> &ExistingType = OptionalTypes[TypeIndex];
+ if (ExistingType.has_value() && ExistingType.value() != RegisterType.value())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "legalization type operands disagree within one type index");
+ ExistingType = RegisterType.value();
+ TypeOperands[TypeIndex].push_back(Index);
+ }
+
+ std::vector<LowLevelType> Types;
+ Types.reserve(TypeCount);
+ for (const std::optional<LowLevelType> &Type : OptionalTypes) {
+ if (!Type)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "legalization query contains a sparse type index");
+ Types.push_back(*Type);
+ }
+
+ std::vector<LegalizationMemoryDesc> MemoryOperands;
+ MemoryOperands.reserve(Instruction.memOperands().size());
+ for (const MachineMemOperand *Memory : Instruction.memOperands()) {
+ const MachineMemOperandDesc &Description = Memory->description();
+ MemoryOperands.push_back({Description.MemoryType, Description.Size,
+ Description.Align.bytes() * 8U, Description.AddressSpace,
+ Description.AtomicKind, Description.SuccessOrdering,
+ Description.FailureOrdering, Description.SyncScope});
+ }
+
+ LegalizationQuery Query(Instruction.opcode(), std::move(Types), std::move(MemoryOperands));
+ Query.TypeOperands = std::move(TypeOperands);
+ return Query;
+}
+
+Expected<LegalizationDecision> LegalizerInfo::getDecision(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &Target) const {
+ auto Query = buildQuery(Instruction, Function, Target);
+ if (!Query)
+ return std::unexpected(Query.error());
+ return getDecision(*Query);
+}
+
+Expected<LegalizationDecision> LegalizerInfo::getDecision(const LegalizationQuery &Query) const {
+ if (!isGenericOpcode(Query.InstructionOpcode))
+ return LegalizationDecision{.Action = LegalizationAction::Legal, .TypeIndex = 0, .NewType = {}};
+
+ Opcode RuleOpcode = Query.InstructionOpcode;
+ if (const auto Alias = RuleAliases.find(RuleOpcode); Alias != RuleAliases.end())
+ RuleOpcode = Alias->second;
+ const auto RuleSet = RuleSets.find(RuleOpcode);
+ if (RuleSet == RuleSets.end())
+ return LegalizationDecision{
+ .Action = LegalizationAction::Unsupported, .TypeIndex = 0, .NewType = {}};
+ const std::optional<LegalizationDecision> Decision = RuleSet->second.apply(Query);
+ return Decision.value_or(LegalizationDecision{
+ .Action = LegalizationAction::Unsupported, .TypeIndex = 0, .NewType = {}});
+}
+
+Expected<bool> LegalizerInfo::isLegal(const LegalizationQuery &Query) const {
+ auto Decision = getDecision(Query);
+ if (!Decision)
+ return std::unexpected(Decision.error());
+ return Decision->Action == LegalizationAction::Legal;
+}
+
+Expected<bool> LegalizerInfo::isUnsupported(const LegalizationQuery &Query) const {
+ auto Decision = getDecision(Query);
+ if (!Decision)
+ return std::unexpected(Decision.error());
+ return Decision->Action == LegalizationAction::Unsupported;
+}
+
+GenericOpcode LegalizerInfo::getExtOpcodeForWideningConstant(LowLevelType SmallType) const {
+ return SmallType.minimumBitWidth() % 8U == 0 ? GenericOpcode::G_SEXT : GenericOpcode::G_ZEXT;
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp
new file mode 100644
index 0000000..32f2f9f
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/RegBankSelect.cpp
@@ -0,0 +1,403 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/RegBankSelect.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] const MachinePassContract &bankSelectContract() {
+ static const MachinePassContract Contract{
+ .Name = "regbankselect",
+ .Requires = {MachineFunctionProperty::Legalized},
+ .Sets = {MachineFunctionProperty::RegBankSelected},
+ .Clears = {},
+ .RequiredAnalyses = {},
+ .VerificationStage = PipelineStage::RegBankSelected,
+ };
+ return Contract;
+}
+
+void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept {
+ State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U);
+}
+
+[[nodiscard]] std::uint64_t
+registerConfigurationFingerprint(const MachineRegisterInfo &RegInfo) noexcept {
+ std::uint64_t Result = 0xcbf29ce484222325ULL;
+ for (const VirtualRegisterInfo &Info : RegInfo.virtualRegisters()) {
+ combineFingerprint(Result, Info.IsTombstone ? 1U : 0U);
+ combineFingerprint(Result, Info.Bank ? Info.Bank->value() : 0U);
+ combineFingerprint(Result, Info.RegClass ? Info.RegClass->value() : 0U);
+ }
+ return Result;
+}
+
+[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) {
+ std::vector<MachineInstr *> Result;
+ for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next())
+ for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next())
+ Result.push_back(Instruction);
+ return Result;
+}
+
+[[nodiscard]] Expected<void> restoreRegisterConfiguration(
+ MachineRegisterInfo &RegInfo,
+ const std::vector<std::pair<Register, std::pair<std::optional<RegisterBankID>,
+ std::optional<RegisterClassID>>>> &Snapshot) {
+ for (const auto &[Reg, Configuration] : Snapshot) {
+ auto ClearBank = RegInfo.clearRegisterBank(Reg);
+ if (!ClearBank)
+ return std::unexpected(ClearBank.error());
+ if (Configuration.first) {
+ auto SetBank = RegInfo.setRegisterBank(Reg, *Configuration.first);
+ if (!SetBank)
+ return std::unexpected(SetBank.error());
+ }
+ auto ClearClass = RegInfo.clearRegisterClass(Reg);
+ if (!ClearClass)
+ return std::unexpected(ClearClass.error());
+ if (Configuration.second) {
+ auto SetClass = RegInfo.setRegisterClass(Reg, *Configuration.second);
+ if (!SetClass)
+ return std::unexpected(SetClass.error());
+ }
+ }
+ return {};
+}
+
+[[nodiscard]] Error passFailure(std::string Message,
+ ErrorCode Code = ErrorCode::VerificationError) {
+ return Error(Code, std::move(Message));
+}
+
+struct PlannedRegisterBank final {
+ Register Reg;
+ RegisterBankID Bank;
+ bool IsConstrained = false;
+};
+
+struct MappingMutation final {
+ MachineOperand *Operand = nullptr;
+ Register Original;
+ Register Fresh;
+ MachineInstr *Copy = nullptr;
+};
+
+[[nodiscard]] Expected<MachineInstr *> buildBankCopy(MachineFunction &Function,
+ const TargetDescription &Target,
+ MachineInstr &Instruction, bool InsertBefore,
+ Register Destination, Register Source) {
+ MachineIRBuilder Builder(Function, Target);
+ auto Point = InsertBefore ? Builder.setInsertPointBefore(Instruction)
+ : Builder.setInsertPointAfter(Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(opcode(GenericOpcode::COPY));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DestinationOperand = Pending->addDef(Destination);
+ if (!DestinationOperand)
+ return std::unexpected(DestinationOperand.error());
+ auto SourceOperand = Pending->addUse(Source);
+ if (!SourceOperand)
+ return std::unexpected(SourceOperand.error());
+ if (Instruction.debugLocation()) {
+ auto Debug = Pending->setDebugLocation(*Instruction.debugLocation());
+ if (!Debug)
+ return std::unexpected(Debug.error());
+ }
+ return Pending->build();
+}
+
+[[nodiscard]] Expected<void> repairMappedOperand(MachineFunction &Function,
+ const TargetDescription &Target,
+ MachineInstr &Instruction, MachineOperand &Operand,
+ RegisterBankID RequiredBank,
+ std::vector<MappingMutation> &Mutations) {
+ const Register *Original = Operand.getIf<Register>();
+ if (Original == nullptr || !Original->isVirtual() || Operand.isDef() == Operand.isUse()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "register-bank repair requires a virtual def or use operand");
+ }
+ const Register OriginalReg = *Original;
+ auto OriginalInfo = Function.registerInfo().virtualRegisterInfo(OriginalReg);
+ if (!OriginalInfo)
+ return std::unexpected(OriginalInfo.error());
+ const std::optional<LowLevelType> &OriginalType = OriginalInfo->get().Type;
+ if (!OriginalType) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "register-bank repair requires a virtual-register type");
+ }
+ if (Operand.subReg().value() != 0) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "register-bank repair does not support subregister operands");
+ }
+
+ auto Fresh = Function.registerInfo().createVirtualRegister(*OriginalType);
+ if (!Fresh)
+ return std::unexpected(Fresh.error());
+ auto SetBank = Function.registerInfo().setRegisterBank(*Fresh, RequiredBank);
+ if (!SetBank) {
+ auto Erased = Function.registerInfo().eraseVirtualRegister(*Fresh);
+ (void)Erased;
+ return std::unexpected(SetBank.error());
+ }
+
+ auto Copy = Operand.isDef()
+ ? buildBankCopy(Function, Target, Instruction, false, OriginalReg, *Fresh)
+ : buildBankCopy(Function, Target, Instruction, true, *Fresh, OriginalReg);
+ if (!Copy) {
+ auto Erased = Function.registerInfo().eraseVirtualRegister(*Fresh);
+ (void)Erased;
+ return std::unexpected(Copy.error());
+ }
+ auto Replaced = Operand.setRegister(*Fresh);
+ if (!Replaced) {
+ if ((*Copy)->parent() != nullptr) {
+ auto Removed = (*Copy)->eraseFromParent();
+ (void)Removed;
+ }
+ auto Erased = Function.registerInfo().eraseVirtualRegister(*Fresh);
+ (void)Erased;
+ return std::unexpected(Replaced.error());
+ }
+ Mutations.push_back(MappingMutation{
+ .Operand = &Operand, .Original = OriginalReg, .Fresh = *Fresh, .Copy = *Copy});
+ return {};
+}
+
+[[nodiscard]] Expected<void> rollbackMappingMutations(MachineRegisterInfo &RegInfo,
+ std::vector<MappingMutation> &Mutations) {
+ for (auto Iterator = Mutations.rbegin(); Iterator != Mutations.rend(); ++Iterator) {
+ auto Restored = Iterator->Operand->setRegister(Iterator->Original);
+ if (!Restored)
+ return std::unexpected(Restored.error());
+ if (Iterator->Copy != nullptr && Iterator->Copy->parent() != nullptr) {
+ auto Removed = Iterator->Copy->eraseFromParent();
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ }
+ auto Erased = RegInfo.eraseVirtualRegister(Iterator->Fresh);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ }
+ Mutations.clear();
+ return {};
+}
+
+[[nodiscard]] Expected<void>
+applyInstructionMappings(MachineFunction &Function, const TargetDescription &Target,
+ const RegisterBankInfo &Info,
+ std::unordered_set<Register> &ConstrainedRegisters,
+ std::vector<MappingMutation> &Mutations, MachinePassResult &Result) {
+ MachineRegisterInfo &RegInfo = Function.registerInfo();
+ const std::vector<MachineInstr *> Instructions = layoutInstructions(Function);
+ for (MachineInstr *Instruction : Instructions) {
+ auto Mapping = Info.getInstructionMapping(*Instruction, Function, Target);
+ if (!Mapping)
+ return std::unexpected(Mapping.error());
+ if (Mapping->empty())
+ continue;
+ if (Mapping->size() != Instruction->operandCount()) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "target register-bank mapping has the wrong operand count");
+ }
+ for (std::size_t Index = 0; Index < Mapping->size(); ++Index) {
+ const std::optional<RegisterBankID> &MappedBank = (*Mapping)[Index];
+ if (!MappedBank)
+ continue;
+ const RegisterBankID RequiredBank = *MappedBank;
+ if (Target.registers().registerBank(RequiredBank) == nullptr) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "target instruction mapping names an unknown register bank");
+ }
+ MachineOperand *Operand = Instruction->operandAt(Index);
+ const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "target register-bank mapping requires a virtual-register operand");
+ }
+ auto RegisterInfo = RegInfo.virtualRegisterInfo(*Reg);
+ if (!RegisterInfo)
+ return std::unexpected(RegisterInfo.error());
+ const std::optional<RegisterBankID> &CurrentBank = RegisterInfo->get().Bank;
+ if (!CurrentBank) {
+ auto SetBank = RegInfo.setRegisterBank(*Reg, RequiredBank);
+ if (!SetBank)
+ return std::unexpected(SetBank.error());
+ ConstrainedRegisters.insert(*Reg);
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Registers);
+ continue;
+ }
+ if (*CurrentBank == RequiredBank) {
+ ConstrainedRegisters.insert(*Reg);
+ continue;
+ }
+ if (!ConstrainedRegisters.contains(*Reg) && !RegisterInfo->get().RegClass) {
+ auto SetBank = RegInfo.setRegisterBank(*Reg, RequiredBank);
+ if (!SetBank)
+ return std::unexpected(SetBank.error());
+ ConstrainedRegisters.insert(*Reg);
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Registers);
+ continue;
+ }
+ auto Repaired =
+ repairMappedOperand(Function, Target, *Instruction, *Operand, RequiredBank, Mutations);
+ if (!Repaired)
+ return std::unexpected(Repaired.error());
+ ConstrainedRegisters.insert(Mutations.back().Fresh);
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Instructions);
+ Result.Changes.add(MachinePassChange::Operands);
+ Result.Changes.add(MachinePassChange::Registers);
+ }
+ }
+ return {};
+}
+
+} // namespace
+
+const MachinePassContract &RegBankSelect::contract() const noexcept {
+ return bankSelectContract();
+}
+
+Expected<MachinePassResult> RegBankSelect::run(MachineFunction &Function,
+ const TargetDescription &Target,
+ MachineFunctionAnalysisManager &) {
+ if (Function.properties().has(MachineFunctionProperty::FailedISel))
+ return MachinePassResult{
+ .Changed = false,
+ .Changes = {},
+ .Preserved = PreservedAnalyses::all(),
+ };
+ if (Info == nullptr)
+ return std::unexpected(passFailure("register-bank select has no target behavior interface",
+ ErrorCode::InvalidArgument));
+ MachinePassResult Result;
+ MachineRegisterInfo &RegInfo = Function.registerInfo();
+ const std::uint64_t RegisterConfigurationBefore = registerConfigurationFingerprint(RegInfo);
+ using RegisterConfiguration =
+ std::pair<std::optional<RegisterBankID>, std::optional<RegisterClassID>>;
+ std::vector<std::pair<Register, RegisterConfiguration>> Snapshot;
+ std::vector<PlannedRegisterBank> PlannedBanks;
+ std::unordered_set<Register> ConstrainedRegisters;
+ for (std::size_t Index = 0; Index < RegInfo.virtualRegisters().size(); ++Index) {
+ auto Reg = Register::makeVirtual(Index);
+ if (!Reg)
+ return std::unexpected(Reg.error());
+ const VirtualRegisterInfo &StoredInfo = RegInfo.virtualRegisters()[Index];
+ if (StoredInfo.IsTombstone)
+ continue;
+ Snapshot.emplace_back(*Reg, RegisterConfiguration{StoredInfo.Bank, StoredInfo.RegClass});
+ auto InfoResult = RegInfo.virtualRegisterInfo(*Reg);
+ if (!InfoResult)
+ return std::unexpected(InfoResult.error());
+ const VirtualRegisterInfo &InfoValue = InfoResult->get();
+ const MachineInstr *Context = nullptr;
+ for (const MachineOperand &Operand : RegInfo.regOperands(*Reg)) {
+ Context = Operand.parent();
+ if (Context != nullptr)
+ break;
+ }
+ auto Bank = Info->selectBank(Function, *Reg, InfoValue, Context, Target);
+ if (!Bank)
+ return std::unexpected(Bank.error());
+ if (Target.registers().registerBank(Bank->Bank) == nullptr)
+ return std::unexpected(passFailure("target register-bank selector returned an unknown bank"));
+ if (InfoValue.Bank && *InfoValue.Bank != Bank->Bank)
+ return std::unexpected(
+ passFailure("target register-bank selector conflicts with an existing bank"));
+ if (InfoValue.RegClass && !Target.registers().bankCoversClass(Bank->Bank, *InfoValue.RegClass))
+ return std::unexpected(
+ passFailure("existing register class is not covered by selected bank"));
+ PlannedBanks.push_back(PlannedRegisterBank{.Reg = *Reg,
+ .Bank = Bank->Bank,
+ .IsConstrained = Bank->IsConstrained ||
+ InfoValue.RegClass.has_value()});
+ }
+
+ for (MachineInstr *Instruction : layoutInstructions(Function)) {
+ auto Validation = Info->validateInstruction(*Instruction, Function, Target);
+ if (!Validation)
+ return std::unexpected(
+ withErrorContext(Validation.error(), "register-bank mapping precondition failed: "));
+ }
+
+ for (const PlannedRegisterBank &Plan : PlannedBanks) {
+ const auto InfoResult = RegInfo.virtualRegisterInfo(Plan.Reg);
+ if (!InfoResult)
+ return std::unexpected(InfoResult.error());
+ if (!InfoResult->get().Bank) {
+ auto SetBank = RegInfo.setRegisterBank(Plan.Reg, Plan.Bank);
+ if (!SetBank)
+ return std::unexpected(SetBank.error());
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Registers);
+ }
+ if (Plan.IsConstrained)
+ ConstrainedRegisters.insert(Plan.Reg);
+ }
+
+ std::vector<MappingMutation> MappingMutations;
+ const auto Restore = [&]() -> Expected<void> {
+ auto MappingRestore = rollbackMappingMutations(RegInfo, MappingMutations);
+ if (!MappingRestore)
+ return std::unexpected(MappingRestore.error());
+ return restoreRegisterConfiguration(RegInfo, Snapshot);
+ };
+ auto Mapped = applyInstructionMappings(Function, Target, *Info, ConstrainedRegisters,
+ MappingMutations, Result);
+ if (!Mapped) {
+ auto Restored = Restore();
+ if (!Restored)
+ return std::unexpected(withErrorContext(Restored.error(), "register-bank rollback failed: "));
+ return std::unexpected(withErrorContext(Mapped.error(), "register-bank mapping failed: "));
+ }
+ for (MachineInstr *Instruction : layoutInstructions(Function)) {
+ auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo());
+ if (!Validation) {
+ auto Restored = Restore();
+ if (!Restored)
+ return std::unexpected(
+ withErrorContext(Restored.error(), "register-bank rollback failed: "));
+ return std::unexpected(withErrorContext(
+ Validation.error(), "register-bank selection produced invalid MachineIR: "));
+ }
+ }
+ auto RegisterValidation = Target.validateRegisterInfo(RegInfo);
+ if (!RegisterValidation) {
+ auto Restored = Restore();
+ if (!Restored)
+ return std::unexpected(withErrorContext(Restored.error(), "register-bank rollback failed: "));
+ return std::unexpected(withErrorContext(
+ RegisterValidation.error(), "register-bank selection produced invalid registers: "));
+ }
+
+ if (registerConfigurationFingerprint(RegInfo) != RegisterConfigurationBefore) {
+ Result.Changed = true;
+ Result.Changes.add(MachinePassChange::Registers);
+ }
+
+ Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
+ return Result;
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/Utils.cpp b/backend/lib/CodeGen/GlobalISel/Utils.cpp
new file mode 100644
index 0000000..f9d6b0a
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/Utils.cpp
@@ -0,0 +1,492 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/Utils.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/InstructionDescriptor.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <cassert>
+#include <cstdint>
+#include <numeric>
+#include <optional>
+#include <utility>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] bool hasProtectedAttachment(const MachineInstr &Instruction) noexcept {
+ return Instruction.flags().has(MachineInstrFlag::FrameSetup) ||
+ Instruction.flags().has(MachineInstrFlag::FrameDestroy) ||
+ Instruction.debugInstrNumber() != 0 || Instruction.preInstrSymbol().has_value() ||
+ Instruction.postInstrSymbol().has_value() || Instruction.heapAllocMarker().has_value() ||
+ Instruction.pcSections().has_value() || Instruction.mmra().has_value() ||
+ Instruction.cfiType().has_value() || Instruction.deactivationSymbol().has_value();
+}
+
+[[nodiscard]] bool wouldBeTriviallyDead(const MachineInstr &Instruction,
+ const InstructionDescriptor &Descriptor) noexcept {
+ switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+ case GenericOpcode::LOCAL_ESCAPE:
+ case GenericOpcode::FAKE_USE:
+ case GenericOpcode::LIFETIME_START:
+ case GenericOpcode::LIFETIME_END:
+ return false;
+ default:
+ break;
+ }
+
+ const InstructionFlags Flags = Descriptor.Flags;
+ if (Flags.has(InstructionFlag::Phi))
+ return true;
+ if (Flags.has(InstructionFlag::Terminator) || Flags.has(InstructionFlag::Branch) ||
+ Flags.has(InstructionFlag::Call) || Flags.has(InstructionFlag::Return) ||
+ Flags.has(InstructionFlag::Barrier) || Flags.has(InstructionFlag::MayLoad) ||
+ Flags.has(InstructionFlag::MayStore) || Flags.has(InstructionFlag::HasSideEffects) ||
+ Flags.has(InstructionFlag::UnmodeledSideEffects) ||
+ Flags.has(InstructionFlag::MayRaiseFPException) || hasProtectedAttachment(Instruction))
+ return false;
+ return true;
+}
+
+[[nodiscard]] Expected<void> saveUsesAndErase(MachineInstr &Instruction, MachineRegisterInfo &MRI,
+ std::vector<MachineInstr *> &DeadChain) {
+ for (const MachineOperand &Operand : Instruction.operands()) {
+ if (!Operand.isRegister() || !Operand.isUse())
+ continue;
+ const std::optional<Register> Reg = Operand.registerValue();
+ if (!Reg || !Reg->isVirtual())
+ continue;
+ MachineInstr *Definition = MRI.getVRegDef(*Reg);
+ if (Definition != nullptr && Definition != &Instruction)
+ DeadChain.push_back(Definition);
+ }
+ return Instruction.eraseFromParent();
+}
+
+[[nodiscard]] Expected<LowLevelType> makeVectorType(std::uint64_t ElementCount, bool IsScalable,
+ LowLevelType ElementType) {
+ if (!IsScalable && ElementCount == 1)
+ return ElementType;
+ return IsScalable ? LowLevelType::makeScalableVector(ElementCount, ElementType)
+ : LowLevelType::makeFixedVector(ElementCount, ElementType);
+}
+
+[[nodiscard]] Expected<void> extractVectorParts(Register Reg, std::size_t ElementCount,
+ std::vector<Register> &Parts,
+ MachineIRBuilder &MIRBuilder,
+ MachineRegisterInfo &MRI) {
+ auto RegisterType = MRI.getType(Reg);
+ if (!RegisterType)
+ return std::unexpected(RegisterType.error());
+ if (!RegisterType->isVector() || RegisterType->isScalableVector() || ElementCount == 0) {
+ return makeUnexpected(
+ ErrorCode::InvalidArgument,
+ "vector part extraction requires a fixed vector and a nonzero element count");
+ }
+
+ const std::optional<LowLevelType> ElementTypeValue = RegisterType->scalarElementType();
+ assert(ElementTypeValue && "validated vector type must have an element type");
+ const LowLevelType ElementType = *ElementTypeValue;
+ auto NarrowType = makeVectorType(ElementCount, false, ElementType);
+ if (!NarrowType)
+ return std::unexpected(NarrowType.error());
+ const std::size_t RegisterElementCount = RegisterType->minimumElementCount();
+ const std::size_t LeftoverElementCount = RegisterElementCount % ElementCount;
+ const std::size_t NarrowPartCount = RegisterElementCount / ElementCount;
+
+ if (LeftoverElementCount == 0)
+ return extractParts(Reg, *NarrowType, static_cast<int>(NarrowPartCount), Parts, MIRBuilder,
+ MRI);
+
+ std::vector<Register> Elements;
+ auto Extracted = extractParts(Reg, ElementType, static_cast<int>(RegisterElementCount), Elements,
+ MIRBuilder, MRI);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+
+ std::size_t Offset = 0;
+ for (std::size_t Index = 0; Index != NarrowPartCount; ++Index, Offset += ElementCount) {
+ std::vector<Register> Piece(Elements.begin() + static_cast<std::ptrdiff_t>(Offset),
+ Elements.begin() +
+ static_cast<std::ptrdiff_t>(Offset + ElementCount));
+ auto Part = MRI.createVirtualRegister(*NarrowType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ auto Built = MIRBuilder.buildMergeLikeInstr(*Part, Piece);
+ if (!Built)
+ return std::unexpected(Built.error());
+ Parts.push_back(*Part);
+ }
+
+ if (LeftoverElementCount == 1) {
+ Parts.push_back(Elements[Offset]);
+ return {};
+ }
+
+ auto LeftoverType = LowLevelType::makeFixedVector(LeftoverElementCount, ElementType);
+ if (!LeftoverType)
+ return std::unexpected(LeftoverType.error());
+ std::vector<Register> LeftoverElements(Elements.begin() + static_cast<std::ptrdiff_t>(Offset),
+ Elements.end());
+ auto Leftover = MRI.createVirtualRegister(*LeftoverType);
+ if (!Leftover)
+ return std::unexpected(Leftover.error());
+ auto Built = MIRBuilder.buildMergeLikeInstr(*Leftover, LeftoverElements);
+ if (!Built)
+ return std::unexpected(Built.error());
+ Parts.push_back(*Leftover);
+ return {};
+}
+
+} // namespace
+
+std::optional<ValueAndVReg> getIConstantVRegValWithLookThrough(Register VReg,
+ const MachineRegisterInfo &MRI,
+ bool LookThroughInstrs) {
+ std::vector<std::pair<GenericOpcode, std::uint32_t>> SeenOpcodes;
+ const MachineInstr *Instruction = nullptr;
+
+ while ((Instruction = MRI.getVRegDef(VReg)) != nullptr &&
+ Instruction->opcode() != opcode(GenericOpcode::G_CONSTANT) && LookThroughInstrs) {
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction->opcode().value());
+ switch (Generic) {
+ case GenericOpcode::G_ANYEXT:
+ return std::nullopt;
+ case GenericOpcode::G_TRUNC:
+ case GenericOpcode::G_SEXT:
+ case GenericOpcode::G_ZEXT: {
+ const MachineOperand *DestinationOperand = Instruction->operandAt(0);
+ const MachineOperand *SourceOperand = Instruction->operandAt(1);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source =
+ SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr)
+ return std::nullopt;
+ auto Type = MRI.getType(*Destination);
+ if (!Type)
+ return std::nullopt;
+ SeenOpcodes.emplace_back(Generic, Type->scalarBitWidth());
+ VReg = *Source;
+ break;
+ }
+ case GenericOpcode::COPY: {
+ const MachineOperand *SourceOperand = Instruction->operandAt(1);
+ const Register *Source =
+ SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Source == nullptr || Source->isPhysical())
+ return std::nullopt;
+ VReg = *Source;
+ break;
+ }
+ case GenericOpcode::G_INTTOPTR: {
+ const MachineOperand *SourceOperand = Instruction->operandAt(1);
+ const Register *Source =
+ SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Source == nullptr)
+ return std::nullopt;
+ VReg = *Source;
+ break;
+ }
+ default:
+ return std::nullopt;
+ }
+ }
+
+ if (Instruction == nullptr || Instruction->opcode() != opcode(GenericOpcode::G_CONSTANT))
+ return std::nullopt;
+ const MachineOperand *ValueOperand = Instruction->operandAt(1);
+ const auto *Immediate =
+ ValueOperand == nullptr ? nullptr : ValueOperand->getIf<IntegerImmediateOperand>();
+ if (Immediate == nullptr)
+ return std::nullopt;
+
+ APInt Value = Immediate->Value;
+ for (auto Iterator = SeenOpcodes.rbegin(); Iterator != SeenOpcodes.rend(); ++Iterator) {
+ Expected<APInt> Transformed = [&]() -> Expected<APInt> {
+ switch (Iterator->first) {
+ case GenericOpcode::G_TRUNC:
+ return Value.trunc(Iterator->second);
+ case GenericOpcode::G_ANYEXT:
+ case GenericOpcode::G_SEXT:
+ return Value.sext(Iterator->second);
+ case GenericOpcode::G_ZEXT:
+ return Value.zext(Iterator->second);
+ default:
+ return makeUnexpected(ErrorCode::InternalError,
+ "unexpected opcode in constant look-through chain");
+ }
+ }();
+ if (!Transformed)
+ return std::nullopt;
+ Value = std::move(*Transformed);
+ }
+ return ValueAndVReg{.Value = std::move(Value), .VReg = VReg};
+}
+
+bool isTriviallyDead(const MachineInstr &Instruction, const MachineRegisterInfo &MRI,
+ const TargetDescription &Target) noexcept {
+ for (const MachineOperand &Operand : Instruction.operands()) {
+ if (!Operand.isRegister() || !Operand.isDef())
+ continue;
+ const std::optional<Register> Reg = Operand.registerValue();
+ if (!Reg || Reg->isPhysical() || !MRI.useEmpty(*Reg, false))
+ return false;
+ }
+
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode());
+ return Descriptor != nullptr && wouldBeTriviallyDead(Instruction, *Descriptor);
+}
+
+Expected<void> eraseInstrs(std::span<MachineInstr *const> Instructions, MachineRegisterInfo &MRI,
+ const TargetDescription &Target) {
+ std::vector<MachineInstr *> DeadChain;
+ for (MachineInstr *Instruction : Instructions) {
+ if (Instruction == nullptr)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "instruction erasure list contains a null instruction");
+ auto Removed = saveUsesAndErase(*Instruction, MRI, DeadChain);
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ }
+
+ while (!DeadChain.empty()) {
+ MachineInstr *Instruction = DeadChain.back();
+ DeadChain.pop_back();
+ if (Instruction->parent() == nullptr || !isTriviallyDead(*Instruction, MRI, Target))
+ continue;
+ auto Removed = saveUsesAndErase(*Instruction, MRI, DeadChain);
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ }
+ return {};
+}
+
+Expected<void> eraseInstr(MachineInstr &Instruction, MachineRegisterInfo &MRI,
+ const TargetDescription &Target) {
+ MachineInstr *Instructions[] = {&Instruction};
+ return eraseInstrs(Instructions, MRI, Target);
+}
+
+Expected<void> extractParts(Register Reg, LowLevelType Type, int PartCount,
+ std::vector<Register> &Parts, MachineIRBuilder &MIRBuilder,
+ MachineRegisterInfo &MRI) {
+ std::vector<Register> NewParts;
+ if (PartCount < 0)
+ return makeUnexpected(ErrorCode::InvalidArgument, "part count cannot be negative");
+ NewParts.reserve(static_cast<std::size_t>(PartCount));
+ for (int Index = 0; Index != PartCount; ++Index) {
+ auto Part = MRI.createVirtualRegister(Type);
+ if (!Part)
+ return std::unexpected(Part.error());
+ NewParts.push_back(*Part);
+ }
+ auto Built = MIRBuilder.buildUnmergeValues(NewParts, Reg);
+ if (!Built)
+ return std::unexpected(Built.error());
+ Parts.insert(Parts.end(), NewParts.begin(), NewParts.end());
+ return {};
+}
+
+Expected<bool> extractParts(Register Reg, LowLevelType RegisterType, LowLevelType MainType,
+ LowLevelType &LeftoverType, std::vector<Register> &Parts,
+ std::vector<Register> &LeftoverParts, MachineIRBuilder &MIRBuilder,
+ MachineRegisterInfo &MRI) {
+ if (LeftoverType.isValid()) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "part extraction leftover type must be an output");
+ }
+
+ const std::uint64_t RegisterSize = RegisterType.minimumBitWidth();
+ const std::uint64_t MainSize = MainType.minimumBitWidth();
+ if (MainSize == 0 || MainSize > RegisterSize) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "part extraction requires a smaller nonzero main type");
+ }
+ const std::size_t PartCount = RegisterSize / MainSize;
+ const std::uint64_t LeftoverSize = RegisterSize - PartCount * MainSize;
+
+ if (LeftoverSize == 0) {
+ auto Extracted =
+ extractParts(Reg, MainType, static_cast<int>(PartCount), Parts, MIRBuilder, MRI);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ return true;
+ }
+
+ if (RegisterType.isVector() && MainType.isVector() && !RegisterType.isScalableVector() &&
+ !MainType.isScalableVector()) {
+ const std::size_t RegisterElementCount = RegisterType.minimumElementCount();
+ const std::size_t MainElementCount = MainType.minimumElementCount();
+ const std::size_t LeftoverElementCount = RegisterElementCount % MainElementCount;
+ if (LeftoverElementCount > 1 && MainElementCount % LeftoverElementCount == 0 &&
+ RegisterElementCount % LeftoverElementCount == 0 &&
+ RegisterType.scalarBitWidth() == MainType.scalarBitWidth()) {
+ const std::optional<LowLevelType> ElementType = RegisterType.scalarElementType();
+ assert(ElementType && "validated vector type must have an element type");
+ auto Candidate = LowLevelType::makeFixedVector(LeftoverElementCount, *ElementType);
+ if (!Candidate)
+ return std::unexpected(Candidate.error());
+ LeftoverType = *Candidate;
+
+ std::vector<Register> Unmerged;
+ auto Extracted = extractParts(Reg, LeftoverType,
+ static_cast<int>(RegisterElementCount / LeftoverElementCount),
+ Unmerged, MIRBuilder, MRI);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+
+ const std::size_t LeftoversPerMain = MainElementCount / LeftoverElementCount;
+ const std::size_t LeftoverValueCount =
+ (RegisterElementCount % MainElementCount) / LeftoverElementCount;
+ std::vector<Register> MergeValues;
+ for (std::size_t Index = 0; Index != Unmerged.size() - LeftoverValueCount; ++Index) {
+ MergeValues.push_back(Unmerged[Index]);
+ if (MergeValues.size() != LeftoversPerMain)
+ continue;
+ auto Part = MRI.createVirtualRegister(MainType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ auto Built = MIRBuilder.buildMergeLikeInstr(*Part, MergeValues);
+ if (!Built)
+ return std::unexpected(Built.error());
+ Parts.push_back(*Part);
+ MergeValues.clear();
+ }
+ LeftoverParts.insert(LeftoverParts.end(),
+ Unmerged.end() - static_cast<std::ptrdiff_t>(LeftoverValueCount),
+ Unmerged.end());
+ return true;
+ }
+ }
+
+ if (MainType.isVector()) {
+ std::vector<Register> RegisterPieces;
+ auto Extracted =
+ extractVectorParts(Reg, MainType.minimumElementCount(), RegisterPieces, MIRBuilder, MRI);
+ if (!Extracted)
+ return std::unexpected(Extracted.error());
+ Parts.insert(Parts.end(), RegisterPieces.begin(), std::prev(RegisterPieces.end()));
+ LeftoverParts.push_back(RegisterPieces.back());
+ auto Type = MRI.getType(LeftoverParts.front());
+ if (!Type)
+ return std::unexpected(Type.error());
+ LeftoverType = *Type;
+ return true;
+ }
+
+ auto ScalarLeftover = LowLevelType::makeCategoryNeutralScalar(LeftoverSize);
+ if (!ScalarLeftover)
+ return std::unexpected(ScalarLeftover.error());
+ LeftoverType = *ScalarLeftover;
+ for (std::size_t Index = 0; Index != PartCount; ++Index) {
+ auto Part = MRI.createVirtualRegister(MainType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ auto Built = MIRBuilder.buildExtract(*Part, Reg, MainSize * Index);
+ if (!Built)
+ return std::unexpected(Built.error());
+ Parts.push_back(*Part);
+ }
+ for (std::uint64_t Offset = MainSize * PartCount; Offset < RegisterSize; Offset += LeftoverSize) {
+ auto Part = MRI.createVirtualRegister(LeftoverType);
+ if (!Part)
+ return std::unexpected(Part.error());
+ auto Built = MIRBuilder.buildExtract(*Part, Reg, Offset);
+ if (!Built)
+ return std::unexpected(Built.error());
+ LeftoverParts.push_back(*Part);
+ }
+ return true;
+}
+
+Expected<LowLevelType> getLCMType(LowLevelType OriginalType, LowLevelType TargetType) {
+ if (OriginalType.minimumBitWidth() == TargetType.minimumBitWidth())
+ return OriginalType;
+ if ((OriginalType.isScalableVector() && TargetType.isVector() &&
+ !TargetType.isScalableVector()) ||
+ (TargetType.isScalableVector() && OriginalType.isVector() &&
+ !OriginalType.isScalableVector())) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "LCM type is undefined between fixed and scalable vectors");
+ }
+
+ if (OriginalType.isVector() && TargetType.isVector()) {
+ const LowLevelType OriginalElement = *OriginalType.scalarElementType();
+ const LowLevelType TargetElement = *TargetType.scalarElementType();
+ if (OriginalElement.minimumBitWidth() == TargetElement.minimumBitWidth()) {
+ const std::uint64_t ElementCount =
+ std::lcm(static_cast<std::uint64_t>(OriginalType.minimumElementCount()),
+ static_cast<std::uint64_t>(TargetType.minimumElementCount()));
+ return makeVectorType(ElementCount, OriginalType.isScalableVector(), OriginalElement);
+ }
+ const std::uint64_t Width =
+ std::lcm(OriginalType.minimumBitWidth(), TargetType.minimumBitWidth());
+ return makeVectorType(Width / OriginalElement.minimumBitWidth(),
+ OriginalType.isScalableVector(), OriginalElement);
+ }
+
+ if (OriginalType.isVector() || TargetType.isVector()) {
+ const LowLevelType VectorType = OriginalType.isVector() ? OriginalType : TargetType;
+ const LowLevelType ScalarType = OriginalType.isVector() ? TargetType : OriginalType;
+ const LowLevelType VectorElement = *VectorType.scalarElementType();
+ const LowLevelType OriginalElement =
+ OriginalType.isVector() ? *OriginalType.scalarElementType() : OriginalType;
+ if (VectorElement.minimumBitWidth() == ScalarType.minimumBitWidth())
+ return makeVectorType(VectorType.minimumElementCount(), VectorType.isScalableVector(),
+ OriginalElement);
+ const std::uint64_t Width =
+ std::lcm(VectorType.minimumBitWidth(), ScalarType.minimumBitWidth());
+ return makeVectorType(Width / OriginalElement.minimumBitWidth(), VectorType.isScalableVector(),
+ OriginalElement);
+ }
+
+ const std::uint64_t Width =
+ std::lcm(OriginalType.minimumBitWidth(), TargetType.minimumBitWidth());
+ if (Width == OriginalType.minimumBitWidth())
+ return OriginalType;
+ if (Width == TargetType.minimumBitWidth())
+ return TargetType;
+ return LowLevelType::makeCategoryNeutralScalar(Width);
+}
+
+Expected<LowLevelType> getGCDType(LowLevelType OriginalType, LowLevelType TargetType) {
+ if (OriginalType.minimumBitWidth() == TargetType.minimumBitWidth())
+ return OriginalType;
+ if ((OriginalType.isScalableVector() && TargetType.isVector() &&
+ !TargetType.isScalableVector()) ||
+ (TargetType.isScalableVector() && OriginalType.isVector() &&
+ !OriginalType.isScalableVector())) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "GCD type is undefined between fixed and scalable vectors");
+ }
+
+ if (OriginalType.isVector() && TargetType.isVector()) {
+ const LowLevelType OriginalElement = *OriginalType.scalarElementType();
+ const std::uint64_t Width =
+ std::gcd(OriginalType.minimumBitWidth(), TargetType.minimumBitWidth());
+ if (Width == OriginalElement.minimumBitWidth())
+ return makeVectorType(1, OriginalType.isScalableVector(), OriginalElement);
+ if (Width < OriginalElement.minimumBitWidth())
+ return LowLevelType::makeCategoryNeutralScalar(Width);
+ return makeVectorType(Width / OriginalElement.minimumBitWidth(),
+ OriginalType.isScalableVector(), OriginalElement);
+ }
+
+ if (OriginalType.isVector() && OriginalType.scalarBitWidth() == TargetType.minimumBitWidth())
+ return *OriginalType.scalarElementType();
+ if (TargetType.isVector() && TargetType.scalarBitWidth() == OriginalType.minimumBitWidth())
+ return OriginalType;
+
+ const std::uint64_t OriginalScalarWidth = OriginalType.scalarBitWidth();
+ const std::uint64_t TargetScalarWidth = TargetType.scalarBitWidth();
+ return LowLevelType::makeCategoryNeutralScalar(std::gcd(OriginalScalarWidth, TargetScalarWidth));
+}
+
+} // namespace rcc
generated by cgit v1.2.3 (git 2.46.0) at 2026-10-30 07:14:18 +0000