diff options
Diffstat (limited to 'backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp')
| -rw-r--r-- | backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 759 |
1 files changed, 759 insertions, 0 deletions
diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index b2cfdf1..b6ba7d1 100644 --- a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -17,6 +17,7 @@ #include <cstddef> #include <cstdint> #include <initializer_list> +#include <limits> #include <numeric> #include <optional> #include <span> @@ -163,6 +164,101 @@ buildFlaggedBinaryInstruction(MachineIRBuilder &Builder, GenericOpcode Generic, return *Destination; } +[[nodiscard]] Expected<std::int64_t> addMemoryOffset(std::int64_t Offset, + std::uint64_t ByteOffset) { + if (ByteOffset > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) { + return makeUnexpected(ErrorCode::Unsupported, + "memory split offset exceeds the representable machine memory range"); + } + const auto SignedByteOffset = static_cast<std::int64_t>(ByteOffset); + if (Offset > std::numeric_limits<std::int64_t>::max() - SignedByteOffset) { + return makeUnexpected(ErrorCode::Unsupported, + "memory split offset exceeds the representable machine memory range"); + } + return Offset + SignedByteOffset; +} + +[[nodiscard]] Expected<const MachineMemOperand *> +deriveMemoryOperand(MachineFunction &Function, const MachineMemOperand &Original, + LowLevelType MemoryType, std::uint64_t ByteOffset) { + if (!MemoryType.isScalarValue() || MemoryType.minimumBitWidth() == 0 || + MemoryType.minimumBitWidth() % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "memory splitting requires a byte-sized scalar memory type"); + } + + const std::uint64_t ByteSize = MemoryType.minimumBitWidth() / 8U; + auto Size = MemorySize::fixed(ByteSize); + if (!Size) + return std::unexpected(Size.error()); + + MachineMemOperandDesc Description = Original.description(); + auto Offset = addMemoryOffset(Description.Offset, ByteOffset); + if (!Offset) + return std::unexpected(Offset.error()); + Description.Offset = *Offset; + if (Description.OffsetRange) { + auto Lower = addMemoryOffset(Description.OffsetRange->Lower, ByteOffset); + auto Upper = addMemoryOffset(Description.OffsetRange->Upper, ByteOffset); + if (!Lower) + return std::unexpected(Lower.error()); + if (!Upper) + return std::unexpected(Upper.error()); + Description.OffsetRange = MemoryOffsetRange{*Lower, *Upper}; + } + + const Alignment BaseAlignment = Description.BaseAlign.value_or(Description.Align); + std::uint64_t EffectiveAlignment = BaseAlignment.bytes(); + if (Description.Offset != 0) { + const auto OffsetBits = static_cast<std::uint64_t>(Description.Offset); + const std::uint64_t OffsetAlignment = std::uint64_t{1} << std::countr_zero(OffsetBits); + EffectiveAlignment = std::min(EffectiveAlignment, OffsetAlignment); + } + auto AlignmentValue = Alignment::create(EffectiveAlignment); + if (!AlignmentValue) + return std::unexpected(AlignmentValue.error()); + + Description.Size = *Size; + Description.MemoryType = MemoryType; + Description.Align = *AlignmentValue; + if (BaseAlignment == Description.Align) + Description.BaseAlign.reset(); + else + Description.BaseAlign = BaseAlignment; + return Function.createMemOperand(std::move(Description)); +} + +[[nodiscard]] Expected<Register> +buildObjectPointerOffset(MachineIRBuilder &Builder, Register Pointer, std::uint64_t ByteOffset) { + if (ByteOffset == 0) + return Pointer; + + auto PointerType = Builder.function().registerInfo().getType(Pointer); + if (!PointerType) + return std::unexpected(PointerType.error()); + if (!PointerType->isPointer()) { + return makeUnexpected(ErrorCode::VerificationError, + "memory splitting requires a pointer address operand"); + } + const std::uint32_t PointerBits = PointerType->scalarBitWidth(); + if (PointerBits < 64U && (ByteOffset >> PointerBits) != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "memory split offset does not fit the pointer index type"); + } + auto OffsetType = LowLevelType::makeCategoryNeutralScalar(PointerBits); + if (!OffsetType) + return std::unexpected(OffsetType.error()); + auto OffsetValue = APInt::fromUnsigned(PointerBits, ByteOffset); + if (!OffsetValue) + return std::unexpected(OffsetValue.error()); + auto Offset = buildConstantRegister(Builder, *OffsetType, std::move(*OffsetValue)); + if (!Offset) + return std::unexpected(Offset.error()); + return buildFlaggedBinaryInstruction( + Builder, GenericOpcode::G_PTR_ADD, *PointerType, Pointer, *Offset, + MachineInstrFlags{MachineInstrFlag::NoUnsignedWrap, MachineInstrFlag::InBounds}); +} + [[nodiscard]] Expected<APInt> repeatedByte(std::uint32_t BitWidth, std::uint8_t Byte) { if (BitWidth == 0 || BitWidth > APInt::MaxBitWidth || BitWidth % 8U != 0) return makeUnexpected(ErrorCode::InvalidArgument, @@ -3332,6 +3428,256 @@ Expected<void> LegalizerHelper::narrowScalarShiftMultiway(MachineInstr &Instruct return Block->removeInstruction(Instruction); } +Expected<void> LegalizerHelper::narrowScalarLoadStore(MachineInstr &Instruction, + std::size_t TypeIndex, + LowLevelType NarrowType) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic != GenericOpcode::G_LOAD && Generic != GenericOpcode::G_STORE) { + return makeUnexpected(ErrorCode::InvalidArgument, + "memory scalar narrowing received an unsupported opcode"); + } + if (TypeIndex != 0 || !NarrowType.isCategoryNeutralScalar() || + NarrowType.minimumBitWidth() == 0 || NarrowType.minimumBitWidth() % 8U != 0) { + return makeUnexpected(ErrorCode::VerificationError, + "memory scalar narrowing requires a byte-sized scalar value type"); + } + if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 || + Instruction.memOperands().front() == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory scalar narrowing requires one address and one memory operand"); + } + + const MachineOperand *ValueOperand = Instruction.operandAt(0); + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>(); + if (Value == nullptr || Address == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory scalar narrowing requires register value and address operands"); + } + auto ValueType = MRI.getType(*Value); + if (!ValueType) + return std::unexpected(ValueType.error()); + if (!ValueType->isCategoryNeutralScalar() || + NarrowType.minimumBitWidth() >= ValueType->minimumBitWidth()) { + return makeUnexpected(ErrorCode::Unsupported, + "memory scalar narrowing requires a smaller integer scalar type"); + } + + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType) { + return makeUnexpected(ErrorCode::Unsupported, + "memory scalar narrowing requires a fixed typed memory operand"); + } + const std::uint64_t MemorySizeInBits = Description.Size.minimumBytes() * 8U; + if (MemorySizeInBits != ValueType->minimumBitWidth()) { + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + if (Generic == GenericOpcode::G_LOAD) { + auto Temporary = MRI.createVirtualRegister(NarrowType); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto Load = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!Load) + return std::unexpected(Load.error()); + if (auto Added = Load->addDef(*Temporary); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addUse(*Address); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addMemOperand(Memory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Load->build(); !Built) + return std::unexpected(Built.error()); + if (auto Extended = MIRBuilder.buildAnyExt(*Value, *Temporary); !Extended) + return std::unexpected(Extended.error()); + } else { + auto Temporary = MRI.createVirtualRegister(NarrowType); + if (!Temporary) + return std::unexpected(Temporary.error()); + if (auto Truncated = MIRBuilder.buildTrunc(*Temporary, *Value); !Truncated) + return std::unexpected(Truncated.error()); + auto Store = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!Store) + return std::unexpected(Store.error()); + if (auto Added = Store->addUse(*Temporary); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addUse(*Address); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addMemOperand(Memory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Store->build(); !Built) + return std::unexpected(Built.error()); + } + + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "memory instruction became detached during scalar narrowing"); + } + return Block->removeInstruction(Instruction); + } + + return reduceLoadStoreWidth(Instruction, TypeIndex, NarrowType); +} + +Expected<void> LegalizerHelper::reduceLoadStoreWidth(MachineInstr &Instruction, + std::size_t TypeIndex, + LowLevelType NarrowType) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic != GenericOpcode::G_LOAD && Generic != GenericOpcode::G_STORE) { + return makeUnexpected(ErrorCode::InvalidArgument, + "memory width reduction received an unsupported opcode"); + } + if (TypeIndex != 0 || !NarrowType.isCategoryNeutralScalar() || + NarrowType.minimumBitWidth() == 0 || NarrowType.minimumBitWidth() % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction requires a byte-sized scalar target type"); + } + if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 || + Instruction.memOperands().front() == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory width reduction requires one address and one memory operand"); + } + + const bool IsLoad = Generic == GenericOpcode::G_LOAD; + const MachineOperand *ValueOperand = Instruction.operandAt(0); + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>(); + if (Value == nullptr || Address == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "memory width reduction requires register value and address operands"); + } + auto ValueType = MRI.getType(*Value); + if (!ValueType) + return std::unexpected(ValueType.error()); + if (!ValueType->isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction does not support pointer or vector values"); + } + + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + if (Memory.isAtomic()) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction does not support atomic operations"); + } + if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType || + Description.MemoryType->isVector()) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction requires a fixed scalar memory operand"); + } + const std::uint64_t ValueBits = ValueType->minimumBitWidth(); + if (ValueBits != Description.Size.minimumBytes() * 8U || + NarrowType.minimumBitWidth() >= ValueBits) { + return makeUnexpected( + ErrorCode::Unsupported, + "memory width reduction cannot split the requested value and memory widths"); + } + + const std::size_t PartCount = static_cast<std::size_t>(ValueBits / NarrowType.minimumBitWidth()); + const std::uint64_t LeftoverBits = ValueBits - PartCount * NarrowType.minimumBitWidth(); + LowLevelType ExpectedLeftoverType; + if (LeftoverBits != 0) { + auto Type = LowLevelType::makeCategoryNeutralScalar(LeftoverBits); + if (!Type) + return std::unexpected(Type.error()); + if (LeftoverBits % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction leaves a non-byte-sized scalar fragment"); + } + ExpectedLeftoverType = *Type; + } + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + std::vector<Register> Parts; + std::vector<Register> Leftovers; + LowLevelType LeftoverType; + if (IsLoad) { + Parts.reserve(PartCount); + for (std::size_t Index = 0; Index != PartCount; ++Index) { + auto Part = MRI.createVirtualRegister(NarrowType); + if (!Part) + return std::unexpected(Part.error()); + Parts.push_back(*Part); + } + LeftoverType = ExpectedLeftoverType; + if (ExpectedLeftoverType.isValid()) { + auto Leftover = MRI.createVirtualRegister(ExpectedLeftoverType); + if (!Leftover) + return std::unexpected(Leftover.error()); + Leftovers.push_back(*Leftover); + } + } else { + auto Extracted = extractParts(*Value, *ValueType, NarrowType, LeftoverType, Parts, Leftovers, + MIRBuilder, MRI); + if (!Extracted) + return std::unexpected(Extracted.error()); + if (!*Extracted || Parts.size() != PartCount || LeftoverType != ExpectedLeftoverType || + (ExpectedLeftoverType.isValid() && Leftovers.size() != 1U)) { + return makeUnexpected(ErrorCode::Unsupported, + "memory width reduction cannot split the store value consistently"); + } + } + + const auto SplitPieces = [&](LowLevelType PartType, const std::vector<Register> &Values, + std::uint64_t &Offset) -> Expected<void> { + const std::uint64_t PartBytes = PartType.minimumBitWidth() / 8U; + for (Register Part : Values) { + auto AddressPart = buildObjectPointerOffset(MIRBuilder, *Address, Offset); + if (!AddressPart) + return std::unexpected(AddressPart.error()); + auto MemoryPart = deriveMemoryOperand(Function, Memory, PartType, Offset); + if (!MemoryPart) + return std::unexpected(MemoryPart.error()); + + auto Access = + MIRBuilder.buildInstr(opcode(IsLoad ? GenericOpcode::G_LOAD : GenericOpcode::G_STORE)); + if (!Access) + return std::unexpected(Access.error()); + if (IsLoad) { + if (auto Added = Access->addDef(Part); !Added) + return std::unexpected(Added.error()); + } else if (auto Added = Access->addUse(Part); !Added) { + return std::unexpected(Added.error()); + } + if (auto Added = Access->addUse(*AddressPart); !Added) + return std::unexpected(Added.error()); + if (auto Added = Access->addMemOperand(**MemoryPart); !Added) + return std::unexpected(Added.error()); + if (auto Built = Access->build(); !Built) + return std::unexpected(Built.error()); + Offset += PartBytes; + } + return {}; + }; + + std::uint64_t Offset = 0; + if (auto Split = SplitPieces(NarrowType, Parts, Offset); !Split) + return std::unexpected(Split.error()); + if (LeftoverType.isValid()) { + if (auto Split = SplitPieces(LeftoverType, Leftovers, Offset); !Split) + return std::unexpected(Split.error()); + } + + if (IsLoad) { + if (auto Inserted = insertParts(*Value, *ValueType, NarrowType, Parts, LeftoverType, Leftovers); + !Inserted) { + return std::unexpected(Inserted.error()); + } + } + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "memory instruction became detached during width reduction"); + } + return Block->removeInstruction(Instruction); +} + Expected<void> LegalizerHelper::narrowScalar(MachineInstr &Instruction, std::size_t TypeIndex, LowLevelType NarrowType) { switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { @@ -3480,6 +3826,9 @@ Expected<void> LegalizerHelper::narrowScalar(MachineInstr &Instruction, std::siz case GenericOpcode::G_MUL: case GenericOpcode::G_UMULH: return narrowScalarMul(Instruction, TypeIndex, NarrowType); + case GenericOpcode::G_LOAD: + case GenericOpcode::G_STORE: + return narrowScalarLoadStore(Instruction, TypeIndex, NarrowType); case GenericOpcode::G_SEXT: case GenericOpcode::G_ZEXT: case GenericOpcode::G_ANYEXT: @@ -4156,6 +4505,410 @@ Expected<void> LegalizerHelper::lowerVAArg(MachineInstr &Instruction) { return Block->removeInstruction(Instruction); } +Expected<void> LegalizerHelper::lowerLoad(MachineInstr &Instruction) { + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + if (Generic != GenericOpcode::G_LOAD && Generic != GenericOpcode::G_SEXTLOAD && + Generic != GenericOpcode::G_ZEXTLOAD) { + return makeUnexpected(ErrorCode::InvalidArgument, + "load lowering received an unsupported opcode"); + } + if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 || + Instruction.memOperands().front() == nullptr) { + return makeUnexpected( + ErrorCode::VerificationError, + "load lowering requires one destination, one address, and one memory operand"); + } + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>(); + if (Destination == nullptr || Address == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "load lowering requires register destination and address operands"); + } + auto DestinationType = MRI.getType(*Destination); + if (!DestinationType) + return std::unexpected(DestinationType.error()); + if (!DestinationType->isScalarValue()) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering does not support vector result types"); + } + + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + if (Memory.isAtomic()) + return makeUnexpected(ErrorCode::Unsupported, + "load lowering does not support atomic operations"); + if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType || + Description.MemoryType->isVector()) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering requires a fixed scalar memory operand"); + } + const LowLevelType MemoryType = *Description.MemoryType; + const std::uint64_t MemoryBits = MemoryType.minimumBitWidth(); + const std::uint64_t MemoryStoreBits = Description.Size.minimumBytes() * 8U; + if (MemoryBits == 0 || MemoryBits > MemoryStoreBits) { + return makeUnexpected(ErrorCode::VerificationError, + "load lowering has an inconsistent memory type and size"); + } + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + const auto BuildLoad = [&](GenericOpcode LoadOpcode, Register Result, Register Pointer, + const MachineMemOperand &LoadMemory) -> Expected<void> { + auto Load = MIRBuilder.buildInstr(opcode(LoadOpcode)); + if (!Load) + return std::unexpected(Load.error()); + if (auto Added = Load->addDef(Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addUse(Pointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addMemOperand(LoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Load->build(); !Built) + return std::unexpected(Built.error()); + return {}; + }; + + if (MemoryBits != MemoryStoreBits) { + if (!MemoryType.isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "non-byte-sized pointer loads are not supported"); + } + auto WideMemoryType = LowLevelType::makeCategoryNeutralScalar(MemoryStoreBits); + if (!WideMemoryType) + return std::unexpected(WideMemoryType.error()); + auto WideMemory = deriveMemoryOperand(Function, Memory, *WideMemoryType, 0); + if (!WideMemory) + return std::unexpected(WideMemory.error()); + + LowLevelType LoadType = *DestinationType; + Register LoadDestination = *Destination; + if (MemoryStoreBits > DestinationType->minimumBitWidth()) { + LoadType = *WideMemoryType; + auto Temporary = MRI.createVirtualRegister(LoadType); + if (!Temporary) + return std::unexpected(Temporary.error()); + LoadDestination = *Temporary; + } + + if (Generic == GenericOpcode::G_SEXTLOAD) { + auto Loaded = MRI.createVirtualRegister(LoadType); + if (!Loaded) + return std::unexpected(Loaded.error()); + if (auto Built = BuildLoad(GenericOpcode::G_LOAD, *Loaded, *Address, **WideMemory); !Built) + return std::unexpected(Built.error()); + if (auto Extended = MIRBuilder.buildSExtInReg(LoadDestination, *Loaded, + static_cast<std::uint32_t>(MemoryBits)); + !Extended) { + return std::unexpected(Extended.error()); + } + } else if (Generic == GenericOpcode::G_ZEXTLOAD || *WideMemoryType == LoadType) { + auto Loaded = MRI.createVirtualRegister(LoadType); + if (!Loaded) + return std::unexpected(Loaded.error()); + if (auto Built = BuildLoad(GenericOpcode::G_LOAD, *Loaded, *Address, **WideMemory); !Built) + return std::unexpected(Built.error()); + auto Assert = MIRBuilder.buildInstr(opcode(GenericOpcode::G_ASSERT_ZEXT)); + if (!Assert) + return std::unexpected(Assert.error()); + if (auto Added = Assert->addDef(LoadDestination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Assert->addUse(*Loaded); !Added) + return std::unexpected(Added.error()); + if (auto Added = Assert->addImmediate(static_cast<std::int64_t>(MemoryBits)); !Added) + return std::unexpected(Added.error()); + if (auto Built = Assert->build(); !Built) + return std::unexpected(Built.error()); + } else if (auto Built = + BuildLoad(GenericOpcode::G_LOAD, LoadDestination, *Address, **WideMemory); + !Built) { + return std::unexpected(Built.error()); + } + + if (*DestinationType != LoadType) { + if (auto Truncated = MIRBuilder.buildTrunc(*Destination, LoadDestination); !Truncated) + return std::unexpected(Truncated.error()); + } + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "load became detached during lowering"); + return Block->removeInstruction(Instruction); + } + + std::uint64_t LargeSplitBits = 0; + std::uint64_t SmallSplitBits = 0; + if (!std::has_single_bit(MemoryBits)) { + LargeSplitBits = std::bit_floor(MemoryBits); + SmallSplitBits = MemoryBits - LargeSplitBits; + } else { + if (MemoryBits < 16U) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering cannot split a byte-sized memory operation"); + } + LargeSplitBits = MemoryBits / 2U; + SmallSplitBits = LargeSplitBits; + } + if (LargeSplitBits == 0 || SmallSplitBits == 0 || LargeSplitBits % 8U != 0 || + SmallSplitBits % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering cannot split the requested memory width into bytes"); + } + + const std::uint64_t AnyExtBits = std::bit_ceil(DestinationType->minimumBitWidth()); + if (AnyExtBits == 0 || AnyExtBits > LowLevelType::MaxBitWidth) { + return makeUnexpected(ErrorCode::Unsupported, + "load lowering result width cannot be represented by the machine IR"); + } + auto AnyExtType = LowLevelType::makeCategoryNeutralScalar(AnyExtBits); + auto LargeMemoryType = LowLevelType::makeCategoryNeutralScalar(LargeSplitBits); + auto SmallMemoryType = LowLevelType::makeCategoryNeutralScalar(SmallSplitBits); + if (!AnyExtType) + return std::unexpected(AnyExtType.error()); + if (!LargeMemoryType) + return std::unexpected(LargeMemoryType.error()); + if (!SmallMemoryType) + return std::unexpected(SmallMemoryType.error()); + auto LargeMemory = deriveMemoryOperand(Function, Memory, *LargeMemoryType, 0); + auto SmallMemory = deriveMemoryOperand(Function, Memory, *SmallMemoryType, LargeSplitBits / 8U); + if (!LargeMemory) + return std::unexpected(LargeMemory.error()); + if (!SmallMemory) + return std::unexpected(SmallMemory.error()); + + auto LargeLoad = MRI.createVirtualRegister(*AnyExtType); + if (!LargeLoad) + return std::unexpected(LargeLoad.error()); + if (auto Built = BuildLoad(GenericOpcode::G_ZEXTLOAD, *LargeLoad, *Address, **LargeMemory); + !Built) + return std::unexpected(Built.error()); + auto SmallAddress = buildObjectPointerOffset(MIRBuilder, *Address, LargeSplitBits / 8U); + if (!SmallAddress) + return std::unexpected(SmallAddress.error()); + auto SmallLoad = MRI.createVirtualRegister(*AnyExtType); + if (!SmallLoad) + return std::unexpected(SmallLoad.error()); + if (auto Built = BuildLoad(Generic, *SmallLoad, *SmallAddress, **SmallMemory); !Built) + return std::unexpected(Built.error()); + auto ShiftAmount = + buildConstantRegister(MIRBuilder, *AnyExtType, static_cast<std::int64_t>(LargeSplitBits)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto Shifted = buildResultInstruction(MIRBuilder, GenericOpcode::G_SHL, *AnyExtType, + {*SmallLoad, *ShiftAmount}); + if (!Shifted) + return std::unexpected(Shifted.error()); + + if (*AnyExtType == *DestinationType) { + if (auto Combined = MIRBuilder.buildOr(*Destination, *Shifted, *LargeLoad); !Combined) + return std::unexpected(Combined.error()); + } else { + auto Combined = buildResultInstruction(MIRBuilder, GenericOpcode::G_OR, *AnyExtType, + {*Shifted, *LargeLoad}); + if (!Combined) + return std::unexpected(Combined.error()); + if (AnyExtBits != DestinationType->minimumBitWidth()) { + if (auto Truncated = MIRBuilder.buildTrunc(*Destination, *Combined); !Truncated) + return std::unexpected(Truncated.error()); + } else if (DestinationType->isPointer()) { + if (auto Converted = MIRBuilder.buildIntToPtr(*Destination, *Combined); !Converted) + return std::unexpected(Converted.error()); + } else { + return makeUnexpected(ErrorCode::InternalError, + "load lowering produced an incompatible scalar result type"); + } + } + + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "load became detached during lowering"); + return Block->removeInstruction(Instruction); +} + +Expected<void> LegalizerHelper::lowerStore(MachineInstr &Instruction) { + if (Instruction.opcode() != opcode(GenericOpcode::G_STORE)) { + return makeUnexpected(ErrorCode::InvalidArgument, + "store lowering received an unsupported opcode"); + } + if (Instruction.operandCount() != 2 || Instruction.memOperands().size() != 1 || + Instruction.memOperands().front() == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "store lowering requires one value, one address, and one memory operand"); + } + + const MachineOperand *ValueOperand = Instruction.operandAt(0); + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>(); + if (Value == nullptr || Address == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "store lowering requires register value and address operands"); + } + auto ValueType = MRI.getType(*Value); + if (!ValueType) + return std::unexpected(ValueType.error()); + if (!ValueType->isScalarValue()) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering does not support vector value types"); + } + + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + if (Memory.isAtomic()) + return makeUnexpected(ErrorCode::Unsupported, + "store lowering does not support atomic operations"); + if (!Description.Size.isKnown() || Description.Size.isScalable() || !Description.MemoryType || + Description.MemoryType->isVector()) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering requires a fixed scalar memory operand"); + } + const LowLevelType MemoryType = *Description.MemoryType; + const std::uint64_t StoreBits = MemoryType.minimumBitWidth(); + const std::uint64_t StoreSizeBits = Description.Size.minimumBytes() * 8U; + if (StoreBits == 0 || StoreBits > StoreSizeBits) { + return makeUnexpected(ErrorCode::VerificationError, + "store lowering has an inconsistent memory type and size"); + } + + if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + const auto BuildStore = [&](Register StoredValue, Register Pointer, + const MachineMemOperand &StoreMemory) -> Expected<void> { + auto Store = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!Store) + return std::unexpected(Store.error()); + if (auto Added = Store->addUse(StoredValue); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addUse(Pointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addMemOperand(StoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Store->build(); !Built) + return std::unexpected(Built.error()); + return {}; + }; + + Register Source = *Value; + LowLevelType SourceType = *ValueType; + if (StoreBits != StoreSizeBits) { + if (!MemoryType.isCategoryNeutralScalar() || !SourceType.isCategoryNeutralScalar()) { + return makeUnexpected(ErrorCode::Unsupported, + "non-byte-sized pointer stores are not supported"); + } + auto WideMemoryType = LowLevelType::makeCategoryNeutralScalar(StoreSizeBits); + if (!WideMemoryType) + return std::unexpected(WideMemoryType.error()); + if (StoreSizeBits > SourceType.minimumBitWidth()) { + auto Extended = + buildResultInstruction(MIRBuilder, GenericOpcode::G_ANYEXT, *WideMemoryType, {Source}); + if (!Extended) + return std::unexpected(Extended.error()); + Source = *Extended; + SourceType = *WideMemoryType; + } + auto MaskValue = + APInt::getLowBitsSet(SourceType.scalarBitWidth(), static_cast<std::uint32_t>(StoreBits)); + if (!MaskValue) + return std::unexpected(MaskValue.error()); + auto Mask = buildConstantRegister(MIRBuilder, SourceType, std::move(*MaskValue)); + if (!Mask) + return std::unexpected(Mask.error()); + auto ZeroExtended = + buildResultInstruction(MIRBuilder, GenericOpcode::G_AND, SourceType, {Source, *Mask}); + if (!ZeroExtended) + return std::unexpected(ZeroExtended.error()); + auto WideMemory = deriveMemoryOperand(Function, Memory, *WideMemoryType, 0); + if (!WideMemory) + return std::unexpected(WideMemory.error()); + if (auto Built = BuildStore(*ZeroExtended, *Address, **WideMemory); !Built) + return std::unexpected(Built.error()); + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "store became detached during lowering"); + return Block->removeInstruction(Instruction); + } + + std::uint64_t LargeSplitBits = 0; + std::uint64_t SmallSplitBits = 0; + if (!std::has_single_bit(StoreBits)) { + LargeSplitBits = std::bit_floor(StoreBits); + SmallSplitBits = StoreBits - LargeSplitBits; + } else { + if (StoreBits < 16U) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering cannot split a byte-sized memory operation"); + } + LargeSplitBits = StoreBits / 2U; + SmallSplitBits = LargeSplitBits; + } + if (LargeSplitBits == 0 || SmallSplitBits == 0 || LargeSplitBits % 8U != 0 || + SmallSplitBits % 8U != 0) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering cannot split the requested memory width into bytes"); + } + + const std::uint64_t AnyExtBits = std::bit_ceil(StoreBits); + if (AnyExtBits == 0 || AnyExtBits > LowLevelType::MaxBitWidth) { + return makeUnexpected(ErrorCode::Unsupported, + "store lowering value width cannot be represented by the machine IR"); + } + auto NewSourceType = LowLevelType::makeCategoryNeutralScalar(AnyExtBits); + auto LargeMemoryType = LowLevelType::makeCategoryNeutralScalar(LargeSplitBits); + auto SmallMemoryType = LowLevelType::makeCategoryNeutralScalar(SmallSplitBits); + if (!NewSourceType) + return std::unexpected(NewSourceType.error()); + if (!LargeMemoryType) + return std::unexpected(LargeMemoryType.error()); + if (!SmallMemoryType) + return std::unexpected(SmallMemoryType.error()); + + if (SourceType.isPointer()) { + auto IntegerPointerType = LowLevelType::makeCategoryNeutralScalar(SourceType.scalarBitWidth()); + if (!IntegerPointerType) + return std::unexpected(IntegerPointerType.error()); + auto Converted = buildResultInstruction(MIRBuilder, GenericOpcode::G_PTRTOINT, + *IntegerPointerType, {Source}); + if (!Converted) + return std::unexpected(Converted.error()); + Source = *Converted; + SourceType = *IntegerPointerType; + } + auto Extended = MIRBuilder.buildAnyExtOrTrunc(*NewSourceType, Source); + if (!Extended) + return std::unexpected(Extended.error()); + auto ExtendedValue = resultRegister(**Extended); + if (!ExtendedValue) + return std::unexpected(ExtendedValue.error()); + auto ShiftAmount = + buildConstantRegister(MIRBuilder, *NewSourceType, static_cast<std::int64_t>(LargeSplitBits)); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto SmallValue = buildResultInstruction(MIRBuilder, GenericOpcode::G_LSHR, *NewSourceType, + {*ExtendedValue, *ShiftAmount}); + if (!SmallValue) + return std::unexpected(SmallValue.error()); + auto SmallAddress = buildObjectPointerOffset(MIRBuilder, *Address, LargeSplitBits / 8U); + if (!SmallAddress) + return std::unexpected(SmallAddress.error()); + auto LargeMemory = deriveMemoryOperand(Function, Memory, *LargeMemoryType, 0); + auto SmallMemory = deriveMemoryOperand(Function, Memory, *SmallMemoryType, LargeSplitBits / 8U); + if (!LargeMemory) + return std::unexpected(LargeMemory.error()); + if (!SmallMemory) + return std::unexpected(SmallMemory.error()); + if (auto Built = BuildStore(*ExtendedValue, *Address, **LargeMemory); !Built) + return std::unexpected(Built.error()); + if (auto Built = BuildStore(*SmallValue, *SmallAddress, **SmallMemory); !Built) + return std::unexpected(Built.error()); + + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::InternalError, "store became detached during lowering"); + return Block->removeInstruction(Instruction); +} + Expected<void> LegalizerHelper::lower(MachineInstr &Instruction, const LegalizationDecision &Decision) { if (Decision.Action != LegalizationAction::Lower) @@ -4164,6 +4917,12 @@ Expected<void> LegalizerHelper::lower(MachineInstr &Instruction, switch (static_cast<GenericOpcode>(Instruction.opcode().value())) { case GenericOpcode::G_ABS: return lowerAbsToAddXor(Instruction); + case GenericOpcode::G_LOAD: + case GenericOpcode::G_SEXTLOAD: + case GenericOpcode::G_ZEXTLOAD: + return lowerLoad(Instruction); + case GenericOpcode::G_STORE: + return lowerStore(Instruction); case GenericOpcode::G_SMIN: case GenericOpcode::G_SMAX: case GenericOpcode::G_UMIN: |