[GlobalISel] Legalize constants and implicit definitions

3 files changed, 195 insertions, 34 deletions

diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index a0b0981..335669e 100644
--- a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -928,6 +928,14 @@ Expected<void> LegalizerHelper::widenScalar(MachineInstr &Instruction, std::size
     break;
   }
 
+  case GenericOpcode::G_IMPLICIT_DEF:
+    if (TypeIndex != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "implicit definition widening has an invalid type index");
+    if (auto Result = Apply(WidenDestination()); !Result)
+      return Result;
+    break;
+
   case GenericOpcode::G_SADDO:
   case GenericOpcode::G_SSUBO:
   case GenericOpcode::G_UADDO:
@@ -1357,6 +1365,137 @@ Expected<void> LegalizerHelper::narrowScalarExt(MachineInstr &Instruction, std::
 Expected<void> LegalizerHelper::narrowScalar(MachineInstr &Instruction, std::size_t TypeIndex,
                                              LowLevelType NarrowType) {
   switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+  case GenericOpcode::G_IMPLICIT_DEF: {
+    const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+    if (Destination == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "G_IMPLICIT_DEF narrowing requires a register destination");
+    auto DestinationType = MRI.getType(*Destination);
+    if (!DestinationType)
+      return std::unexpected(DestinationType.error());
+
+    const std::uint64_t DestinationSize = DestinationType->minimumBitWidth();
+    const std::uint64_t NarrowSize = NarrowType.minimumBitWidth();
+    if (NarrowSize == 0 || NarrowSize >= DestinationSize)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "G_IMPLICIT_DEF narrowing requires a smaller nonzero type");
+
+    if (DestinationSize % NarrowSize != 0) {
+      LowLevelType ImplicitType = NarrowType;
+      if (DestinationType->isVector()) {
+        auto VectorType =
+            DestinationType->isScalableVector()
+                ? LowLevelType::makeScalableVector(DestinationType->minimumElementCount(),
+                                                   NarrowType)
+                : LowLevelType::makeFixedVector(DestinationType->minimumElementCount(), NarrowType);
+        if (!VectorType)
+          return std::unexpected(VectorType.error());
+        ImplicitType = *VectorType;
+      }
+      auto Implicit = MIRBuilder.buildUndef(ImplicitType);
+      if (!Implicit)
+        return std::unexpected(Implicit.error());
+      auto ImplicitReg = resultRegister(**Implicit);
+      if (!ImplicitReg)
+        return std::unexpected(ImplicitReg.error());
+      if (auto Extended = MIRBuilder.buildAnyExt(*Destination, *ImplicitReg); !Extended)
+        return std::unexpected(Extended.error());
+    } else {
+      const std::size_t PartCount = static_cast<std::size_t>(DestinationSize / NarrowSize);
+      std::vector<Register> Parts;
+      Parts.reserve(PartCount);
+      for (std::size_t Index = 0; Index != PartCount; ++Index) {
+        auto Part = MIRBuilder.buildUndef(NarrowType);
+        if (!Part)
+          return std::unexpected(Part.error());
+        auto PartReg = resultRegister(**Part);
+        if (!PartReg)
+          return std::unexpected(PartReg.error());
+        Parts.push_back(*PartReg);
+      }
+      if (auto Merged = MIRBuilder.buildMergeLikeInstr(*Destination, Parts); !Merged)
+        return std::unexpected(Merged.error());
+    }
+
+    MachineBasicBlock *Block = Instruction.parent();
+    if (Block == nullptr)
+      return makeUnexpected(ErrorCode::InternalError,
+                            "G_IMPLICIT_DEF became detached during narrowing");
+    return Block->removeInstruction(Instruction);
+  }
+  case GenericOpcode::G_CONSTANT: {
+    const Register *Destination = Instruction.operandAt(0)->getIf<Register>();
+    const auto *Immediate = Instruction.operandAt(1)->getIf<IntegerImmediateOperand>();
+    if (Destination == nullptr || Immediate == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "G_CONSTANT narrowing requires register and integer operands");
+    auto DestinationType = MRI.getType(*Destination);
+    if (!DestinationType)
+      return std::unexpected(DestinationType.error());
+
+    const std::uint64_t TotalSize = DestinationType->minimumBitWidth();
+    const std::uint64_t NarrowSize = NarrowType.minimumBitWidth();
+    if (NarrowSize == 0 || NarrowSize >= TotalSize || NarrowSize > APInt::MaxBitWidth ||
+        TotalSize > APInt::MaxBitWidth) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "G_CONSTANT narrowing requires a smaller representable type");
+    }
+
+    const std::size_t PartCount = static_cast<std::size_t>(TotalSize / NarrowSize);
+    std::vector<Register> Parts;
+    Parts.reserve(PartCount);
+    for (std::size_t Index = 0; Index != PartCount; ++Index) {
+      auto Shifted = Immediate->Value.logicalShiftRight(Index * NarrowSize);
+      if (!Shifted)
+        return std::unexpected(Shifted.error());
+      APInt PartValue = std::move(*Shifted);
+      if (PartValue.bitWidth() > NarrowSize) {
+        auto Truncated = PartValue.trunc(static_cast<std::uint32_t>(NarrowSize));
+        if (!Truncated)
+          return std::unexpected(Truncated.error());
+        PartValue = std::move(*Truncated);
+      }
+      auto Part = buildConstantRegister(MIRBuilder, NarrowType, std::move(PartValue));
+      if (!Part)
+        return std::unexpected(Part.error());
+      Parts.push_back(*Part);
+    }
+
+    LowLevelType LeftoverType;
+    std::vector<Register> Leftovers;
+    const std::uint64_t LeftoverBits = TotalSize - PartCount * NarrowSize;
+    if (LeftoverBits != 0) {
+      auto Type = LowLevelType::makeCategoryNeutralScalar(LeftoverBits);
+      if (!Type)
+        return std::unexpected(Type.error());
+      LeftoverType = *Type;
+      auto Shifted = Immediate->Value.logicalShiftRight(PartCount * NarrowSize);
+      if (!Shifted)
+        return std::unexpected(Shifted.error());
+      APInt LeftoverValue = std::move(*Shifted);
+      if (LeftoverValue.bitWidth() > LeftoverBits) {
+        auto Truncated = LeftoverValue.trunc(static_cast<std::uint32_t>(LeftoverBits));
+        if (!Truncated)
+          return std::unexpected(Truncated.error());
+        LeftoverValue = std::move(*Truncated);
+      }
+      auto Leftover = buildConstantRegister(MIRBuilder, LeftoverType, std::move(LeftoverValue));
+      if (!Leftover)
+        return std::unexpected(Leftover.error());
+      Leftovers.push_back(*Leftover);
+    }
+
+    if (auto Inserted =
+            insertParts(*Destination, *DestinationType, NarrowType, Parts, LeftoverType, Leftovers);
+        !Inserted) {
+      return std::unexpected(Inserted.error());
+    }
+    MachineBasicBlock *Block = Instruction.parent();
+    if (Block == nullptr)
+      return makeUnexpected(ErrorCode::InternalError,
+                            "G_CONSTANT became detached during narrowing");
+    return Block->removeInstruction(Instruction);
+  }
   case GenericOpcode::G_ADD:
   case GenericOpcode::G_SUB:
   case GenericOpcode::G_SADDO:
diff --git a/backend/lib/Support/APInt.cpp b/backend/lib/Support/APInt.cpp
index 74cbc56..9a6a49c 100644
--- a/backend/lib/Support/APInt.cpp
+++ b/backend/lib/Support/APInt.cpp
@@ -348,6 +348,26 @@ Expected<APInt> APInt::sext(std::uint32_t NewBitWidth) const {
   return create(NewBitWidth, ResultWords);
 }
 
+Expected<APInt> APInt::logicalShiftRight(std::size_t Amount) const {
+  if (BitWidth == 0)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "APInt logical shift requires a nonzero width");
+  std::vector<std::uint64_t> ResultWords(wordCount(BitWidth), 0);
+  if (Amount >= BitWidth)
+    return create(BitWidth, ResultWords);
+
+  const std::size_t WordOffset = Amount / 64U;
+  const unsigned BitOffset = static_cast<unsigned>(Amount % 64U);
+  for (std::size_t Index = 0; Index + WordOffset < Words.size(); ++Index) {
+    std::uint64_t Value = Words[Index + WordOffset] >> BitOffset;
+    if (BitOffset != 0 && Index + WordOffset + 1U < Words.size())
+      Value |= Words[Index + WordOffset + 1U] << (64U - BitOffset);
+    ResultWords[Index] = Value;
+  }
+  ResultWords.back() &= highWordMask(BitWidth);
+  return create(BitWidth, ResultWords);
+}
+
 int APInt::compare(const APInt &Other) const noexcept {
   const std::size_t LeftCount = significantWordCount(Words);
   const std::size_t RightCount = significantWordCount(Other.Words);
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index dec460a..37fad6d 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -693,54 +693,50 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) {
   auto &StoreRules = getActionDefinitionsBuilder(opcode(GenericOpcode::G_STORE));
   auto &ExtLoadRules = getActionDefinitionsBuilder(
       {opcode(GenericOpcode::G_SEXTLOAD), opcode(GenericOpcode::G_ZEXTLOAD)});
-  const std::uint64_t Align8 =
-      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 8;
-  const std::uint64_t Align16 =
-      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 16;
-  const std::uint64_t Align32 =
-      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 32;
-  const std::uint64_t Align64 =
-      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 64;
+  const auto GetScalarMemAlign = [&Subtarget](std::uint64_t Size) {
+    return Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : Size;
+  };
   LoadRules
-      .legalForTypesWithMemDesc({{S16, P0, S8, Align8},
-                                 {S32, P0, S8, Align8},
-                                 {S16, P0, S16, Align16},
-                                 {S32, P0, S16, Align16},
-                                 {S32, P0, S32, Align32},
-                                 {S64, P0, S8, Align8},
-                                 {S64, P0, S16, Align16},
-                                 {S64, P0, S32, Align32},
-                                 {S64, P0, S64, Align64},
-                                 {P0, P0, S64, Align64}})
+      .legalForTypesWithMemDesc({{S16, P0, S8, GetScalarMemAlign(8)},
+                                 {S32, P0, S8, GetScalarMemAlign(8)},
+                                 {S16, P0, S16, GetScalarMemAlign(16)},
+                                 {S32, P0, S16, GetScalarMemAlign(16)},
+                                 {S32, P0, S32, GetScalarMemAlign(32)},
+                                 {S64, P0, S8, GetScalarMemAlign(8)},
+                                 {S64, P0, S16, GetScalarMemAlign(16)},
+                                 {S64, P0, S32, GetScalarMemAlign(32)},
+                                 {S64, P0, S64, GetScalarMemAlign(64)},
+                                 {P0, P0, S64, GetScalarMemAlign(64)}})
       .widenScalarToNextPow2(0, 8)
       .lowerIfMemSizeNotByteSizePow2()
       .clampScalar(0, S16, S64)
       .lower();
   StoreRules
-      .legalForTypesWithMemDesc({{S16, P0, S8, Align8},
-                                 {S32, P0, S8, Align8},
-                                 {S16, P0, S16, Align16},
-                                 {S32, P0, S16, Align16},
-                                 {S32, P0, S32, Align32},
-                                 {S64, P0, S8, Align8},
-                                 {S64, P0, S16, Align16},
-                                 {S64, P0, S32, Align32},
-                                 {S64, P0, S64, Align64},
-                                 {P0, P0, S64, Align64}})
+      .legalForTypesWithMemDesc({{S16, P0, S8, GetScalarMemAlign(8)},
+                                 {S32, P0, S8, GetScalarMemAlign(8)},
+                                 {S16, P0, S16, GetScalarMemAlign(16)},
+                                 {S32, P0, S16, GetScalarMemAlign(16)},
+                                 {S32, P0, S32, GetScalarMemAlign(32)},
+                                 {S64, P0, S8, GetScalarMemAlign(8)},
+                                 {S64, P0, S16, GetScalarMemAlign(16)},
+                                 {S64, P0, S32, GetScalarMemAlign(32)},
+                                 {S64, P0, S64, GetScalarMemAlign(64)},
+                                 {P0, P0, S64, GetScalarMemAlign(64)}})
       .clampScalar(0, S16, S64)
       .lowerIfMemSizeNotByteSizePow2()
       .lower();
   ExtLoadRules
-      .legalForTypesWithMemDesc(
-          {{S64, P0, S8, Align8}, {S64, P0, S16, Align16}, {S64, P0, S32, Align32}})
+      .legalForTypesWithMemDesc({{S64, P0, S8, GetScalarMemAlign(8)},
+                                 {S64, P0, S16, GetScalarMemAlign(16)},
+                                 {S64, P0, S32, GetScalarMemAlign(32)}})
       .widenScalarToNextPow2(0)
       .clampScalar(0, S64, S64)
       .lower();
   getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPEXTLOAD))
-      .legalForTypesWithMemDesc({{S64, P0, S32, Align32}})
+      .legalForTypesWithMemDesc({{S64, P0, S32, GetScalarMemAlign(32)}})
       .lower();
   getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPTRUNCSTORE))
-      .legalForTypesWithMemDesc({{S64, P0, S32, Align32}})
+      .legalForTypesWithMemDesc({{S64, P0, S32, GetScalarMemAlign(32)}})
       .lower();
 
   getActionDefinitionsBuilder({opcode(GenericOpcode::G_PTR_ADD), opcode(GenericOpcode::G_PTRMASK)})
@@ -1096,8 +1092,14 @@ Expected<void> RISCVLegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
     if (Immediate != nullptr && Type && Type->isCategoryNeutralScalar() &&
         Type->scalarBitWidth() == 64) {
       auto Value = signedImmediate(*Immediate);
-      if (Value && shouldUseRV64IntegerConstantPool(*Value))
-        return lowerConstantToPool(Instruction, Function, Target);
+      if (Value && shouldUseRV64IntegerConstantPool(*Value)) {
+        Helper.Observer.changingInstr(Instruction);
+        auto Lowered = lowerConstantToPool(Instruction, Function, Target);
+        if (!Lowered)
+          return std::unexpected(Lowered.error());
+        Helper.Observer.changedInstr(Instruction);
+        return {};
+      }
       return {};
     }
     return makeUnexpected(ErrorCode::Unsupported,