[RISCV][GlobalISel] Generate integer load selection

4 files changed, 310 insertions, 75 deletions

diff --git a/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp b/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp
index 3597509..dd8d920 100644
--- a/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp
+++ b/backend/lib/CodeGen/GlobalISel/GIMatchTableExecutor.cpp
@@ -8,6 +8,7 @@
 #include "rcc/CodeGen/MachineFunction.h"
 #include "rcc/CodeGen/MachineIRBuilder.h"
 #include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
 #include "rcc/CodeGen/MachineOperand.h"
 #include "rcc/CodeGen/MachineRegisterInfo.h"
 #include "rcc/CodeGen/TargetDescription.h"
@@ -61,6 +62,21 @@ namespace {
                         "generated match table has an invalid type predicate");
 }
 
+[[nodiscard]] Expected<bool> matchesMemorySize(const MachineInstr &Instruction,
+                                               const GIMatchTableInstruction &Step) {
+  if (Step.Value == 0) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "generated match table has an invalid memory size");
+  }
+  if (Step.OperandIndex >= Instruction.memOperands().size())
+    return false;
+  const MachineMemOperand *Memory = Instruction.memOperands()[Step.OperandIndex];
+  if (Memory == nullptr)
+    return false;
+  const MemorySize Size = Memory->description().Size;
+  return Size.isKnown() && !Size.isScalable() && Size.minimumBytes() == Step.Value;
+}
+
 [[nodiscard]] Expected<bool> matchesBank(const MachineInstr &Instruction,
                                          const MachineFunction &Function,
                                          const GIMatchTableInstruction &Step) {
@@ -410,6 +426,18 @@ matchRule(const GIMatchTable &Table, const GIMatchTableRule &Rule, const Machine
         return std::optional<MatchedRule>{};
       break;
     }
+    case GIMatchTableOpcode::CheckMemorySize: {
+      if (Applying) {
+        return makeUnexpected(ErrorCode::VerificationError,
+                              "generated match table performs a predicate after an action");
+      }
+      auto Matched = matchesMemorySize(Instruction, Step);
+      if (!Matched)
+        return std::unexpected(Matched.error());
+      if (!*Matched)
+        return std::optional<MatchedRule>{};
+      break;
+    }
     case GIMatchTableOpcode::CheckTargetPredicate: {
       if (Applying || Step.Symbol.empty()) {
         return makeUnexpected(ErrorCode::VerificationError,
diff --git a/backend/lib/TableGen/Parser.cpp b/backend/lib/TableGen/Parser.cpp
index 680104e..50588fb 100644
--- a/backend/lib/TableGen/Parser.cpp
+++ b/backend/lib/TableGen/Parser.cpp
@@ -1204,6 +1204,29 @@ private:
   }
 
   [[nodiscard]] bool
+  parseSelectionMemoryPredicate(const yaml::Node &Node,
+                                std::optional<SourceSelectionMemoryPredicate> &Result) {
+    if (!requireMapping(&Node, "selection memory predicate") ||
+        !checkKeys(Node, {"index", "bytes"}, "selection memory predicate")) {
+      return false;
+    }
+    const yaml::Node *IndexNode = requiredNode(Node, "index", "selection memory predicate");
+    const yaml::Node *BytesNode = requiredNode(Node, "bytes", "selection memory predicate");
+    if (IndexNode == nullptr || BytesNode == nullptr)
+      return false;
+    const auto Index = unsignedInteger32(*IndexNode, "selection memory predicate index");
+    const auto Bytes = unsignedInteger32(*BytesNode, "selection memory predicate bytes");
+    if (!Index || !Bytes)
+      return false;
+    Result = SourceSelectionMemoryPredicate{
+        .MemoryOperandIndex = *Index,
+        .Bytes = *Bytes,
+        .Range = Node.range(),
+    };
+    return true;
+  }
+
+  [[nodiscard]] bool
   parseSelectionTargetPredicates(const yaml::Node &Node,
                                  std::vector<SourceSelectionTargetPredicate> &Result) {
     if (!requireSequence(&Node, "selection target predicates"))
@@ -1574,7 +1597,7 @@ private:
   [[nodiscard]] bool parseSelectionRuleMatch(const yaml::Node &Node, SourceSelectionRule &Result) {
     if (!requireMapping(&Node, "selection rule match") ||
         !checkKeys(Node,
-                   {"opcode", "result_type", "integer_predicate", "floating_predicate",
+                   {"opcode", "result_type", "integer_predicate", "floating_predicate", "memory",
                     "target_predicates", "operand_types", "operand_banks", "operand_classes",
                     "operand_immediates", "operand_def_constants", "def_use"},
                    "selection rule match")) {
@@ -1598,6 +1621,10 @@ private:
       if (!parseSelectionFloatingPredicate(*Predicate, Result.FloatingPredicate))
         return false;
     }
+    if (const yaml::Node *Predicate = optionalNode(Node, "memory")) {
+      if (!parseSelectionMemoryPredicate(*Predicate, Result.MemoryPredicate))
+        return false;
+    }
     if (const yaml::Node *TargetPredicates = optionalNode(Node, "target_predicates")) {
       if (!parseSelectionTargetPredicates(*TargetPredicates, Result.TargetPredicates))
         return false;
diff --git a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 2cceb92..18b986d 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -212,6 +212,55 @@ directIntegerConstant(const MachineRegisterInfo &RegisterInfo, Register ValueReg
   return Value ? std::optional<std::int64_t>(*Value) : std::nullopt;
 }
 
+struct SelectedAddress final {
+  MachineOperand Base;
+  std::int64_t Offset = 0;
+};
+
+[[nodiscard]] std::optional<SelectedAddress> selectAddress(const MachineInstr &Instruction,
+                                                           const MachineFunction &Function,
+                                                           std::uint32_t OperandIndex) {
+  const MachineOperand *Root = Instruction.operandAt(OperandIndex);
+  const Register *RootRegister = Root == nullptr ? nullptr : Root->getIf<Register>();
+  if (RootRegister == nullptr || !RootRegister->isVirtual())
+    return std::nullopt;
+
+  const MachineRegisterInfo &Registers = Function.registerInfo();
+  const MachineInstr *RootDefinition = Registers.getVRegDef(*RootRegister);
+  if (RootDefinition != nullptr &&
+      RootDefinition->opcode() == opcode(GenericOpcode::G_FRAME_INDEX)) {
+    const MachineOperand *FrameIndex = RootDefinition->operandAt(1);
+    if (FrameIndex == nullptr)
+      return std::nullopt;
+    return SelectedAddress{.Base = FrameIndex->cloneDetached(), .Offset = 0};
+  }
+
+  if (RootDefinition != nullptr && RootDefinition->opcode() == opcode(GenericOpcode::G_PTR_ADD)) {
+    const MachineOperand *Base = RootDefinition->operandAt(1);
+    const MachineOperand *Offset = RootDefinition->operandAt(2);
+    const Register *BaseRegister = Base == nullptr ? nullptr : Base->getIf<Register>();
+    const Register *OffsetRegister = Offset == nullptr ? nullptr : Offset->getIf<Register>();
+    if (BaseRegister != nullptr && OffsetRegister != nullptr) {
+      const std::optional<std::int64_t> Constant =
+          directIntegerConstant(Registers, *OffsetRegister);
+      if (Constant && *Constant >= -2048 && *Constant <= 2047) {
+        const MachineInstr *BaseDefinition =
+            BaseRegister->isVirtual() ? Registers.getVRegDef(*BaseRegister) : nullptr;
+        if (BaseDefinition != nullptr &&
+            BaseDefinition->opcode() == opcode(GenericOpcode::G_FRAME_INDEX)) {
+          const MachineOperand *FrameIndex = BaseDefinition->operandAt(1);
+          if (FrameIndex == nullptr)
+            return std::nullopt;
+          return SelectedAddress{.Base = FrameIndex->cloneDetached(), .Offset = *Constant};
+        }
+        return SelectedAddress{.Base = Base->cloneDetached(), .Offset = *Constant};
+      }
+    }
+  }
+
+  return SelectedAddress{.Base = Root->cloneDetached(), .Offset = 0};
+}
+
 struct SelectedIntegerCondition final {
   Register Left;
   Register Right;
@@ -337,6 +386,21 @@ public:
   renderComplexOperand(std::string_view Name, const MachineInstr &Instruction,
                        const MachineFunction &Function, const TargetDescription &Target,
                        std::uint32_t OperandIndex) const override {
+    if (Name == "addr_reg_imm_base" || Name == "addr_reg_imm_offset") {
+      std::optional<SelectedAddress> Address = selectAddress(Instruction, Function, OperandIndex);
+      if (!Address)
+        return std::optional<GIMatchComplexOperand>{};
+      if (Name == "addr_reg_imm_offset") {
+        return GIMatchComplexOperand{
+            .Operand = MachineOperand::immediate(Address->Offset),
+            .Instruction = std::nullopt,
+        };
+      }
+      return GIMatchComplexOperand{
+          .Operand = std::move(Address->Base),
+          .Instruction = std::nullopt,
+      };
+    }
     if (Name == "select_condition_lhs" || Name == "select_condition_rhs" ||
         Name == "select_condition_code") {
       auto Condition = selectIntegerCondition(Instruction, Function, OperandIndex);
@@ -551,9 +615,6 @@ public:
                                 "RISC-V integer constant requires an immediate operand");
       }
       return {};
-    case GenericOpcode::G_LOAD:
-    case GenericOpcode::G_SEXTLOAD:
-    case GenericOpcode::G_ZEXTLOAD:
     case GenericOpcode::G_FPEXTLOAD:
     case GenericOpcode::G_FPTRUNCSTORE:
     case GenericOpcode::G_STORE:
@@ -617,12 +678,6 @@ public:
     const std::optional<LowLevelType> Type = resultType(Function, Instruction);
 
     switch (Generic) {
-    case GenericOpcode::G_LOAD:
-      return selectLoad(Instruction, Function, Type);
-    case GenericOpcode::G_SEXTLOAD:
-      return selectExtLoad(Instruction, Type, true);
-    case GenericOpcode::G_ZEXTLOAD:
-      return selectExtLoad(Instruction, Type, false);
     case GenericOpcode::G_FPEXTLOAD:
       return selectFloatExtLoad(Instruction, Function, Target);
     case GenericOpcode::G_STORE:
@@ -1211,71 +1266,6 @@ private:
     return Instruction.setOpcode(instructionOpcode(InstructionOpcode::FENCE));
   }
 
-  [[nodiscard]] Expected<void> selectLoad(MachineInstr &Instruction, MachineFunction &Function,
-                                          std::optional<LowLevelType> Type) const {
-    if (!Type)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "load result has no virtual-register type");
-    const auto Bytes = memoryAccessBytes(Instruction);
-    if (!Bytes)
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V load size does not match its result type");
-    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-    const Register *Destination =
-        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-    if (Destination == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError, "RISC-V load has no destination");
-    auto Bank = registerBank(Function, *Destination);
-    if (!Bank)
-      return std::unexpected(Bank.error());
-    if (Instruction.memOperands().front()->isAtomic()) {
-      if (*Bank == fprBank() || !isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8))
-        return makeUnexpected(ErrorCode::Unsupported,
-                              "RISC-V atomic load requires a 32-bit or 64-bit GPR value");
-      return Instruction.setOpcode(
-          instructionOpcode(*Bytes == 4 ? InstructionOpcode::LR_W : InstructionOpcode::LR_D));
-    }
-    auto AppendOffset = appendImmediate(Instruction, 0);
-    if (!AppendOffset)
-      return std::unexpected(AppendOffset.error());
-    if (*Bank != fprBank() && isSupportedIntegerType(*Type)) {
-      const InstructionOpcode Selected = *Bytes == 1   ? InstructionOpcode::LB
-                                         : *Bytes == 2 ? InstructionOpcode::LH
-                                         : *Bytes == 4 ? InstructionOpcode::LW
-                                         : *Bytes == 8 ? InstructionOpcode::LD
-                                                       : InstructionOpcode::Invalid;
-      if (Selected == InstructionOpcode::Invalid)
-        return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer load width is unsupported");
-      return Instruction.setOpcode(instructionOpcode(Selected));
-    }
-    if (*Bank == fprBank() && Type->scalarBitWidth() == 32 && *Bytes == 4)
-      return Instruction.setOpcode(instructionOpcode(InstructionOpcode::FLW));
-    if (*Bank == fprBank() && Type->scalarBitWidth() == 64 && *Bytes == 8)
-      return Instruction.setOpcode(instructionOpcode(InstructionOpcode::FLD));
-    return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic load type");
-  }
-
-  [[nodiscard]] Expected<void> selectExtLoad(MachineInstr &Instruction,
-                                             std::optional<LowLevelType> Type, bool Signed) const {
-    if (!Type || !isSupportedIntegerType(*Type))
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V extension load requires an integer result");
-    const auto Bytes = memoryAccessBytes(Instruction);
-    if (!Bytes)
-      return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load has an invalid width");
-    const InstructionOpcode Selected =
-        *Bytes == 1   ? (Signed ? InstructionOpcode::LB : InstructionOpcode::LBU)
-        : *Bytes == 2 ? (Signed ? InstructionOpcode::LH : InstructionOpcode::LHU)
-        : *Bytes == 4 ? (Signed ? InstructionOpcode::LW : InstructionOpcode::LWU)
-                      : InstructionOpcode::Invalid;
-    if (Selected == InstructionOpcode::Invalid)
-      return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load width is unsupported");
-    auto Immediate = appendImmediate(Instruction, 0);
-    if (!Immediate)
-      return std::unexpected(Immediate.error());
-    return Instruction.setOpcode(instructionOpcode(Selected));
-  }
-
   [[nodiscard]] Expected<void> selectFloatExtLoad(MachineInstr &Instruction,
                                                   MachineFunction &Function,
                                                   const TargetDescription &Target) const {
diff --git a/backend/lib/Target/RISCV/RISCVGISelAddressing.rccdesc b/backend/lib/Target/RISCV/RISCVGISelAddressing.rccdesc
index f69ab81..1ff9593 100644
--- a/backend/lib/Target/RISCV/RISCVGISelAddressing.rccdesc
+++ b/backend/lib/Target/RISCV/RISCVGISelAddressing.rccdesc
@@ -2,6 +2,196 @@
 
 selection_rules:
   -
+    id: select_g_load_m8
+    priority: 300
+    match:
+      opcode: G_LOAD
+      memory: { index: 0, bytes: 1 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LBU
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
+    id: select_g_load_m16
+    priority: 300
+    match:
+      opcode: G_LOAD
+      memory: { index: 0, bytes: 2 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LH
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
+    id: select_g_load_m32
+    priority: 300
+    match:
+      opcode: G_LOAD
+      memory: { index: 0, bytes: 4 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LW
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
+    id: select_g_load_m64
+    priority: 300
+    match:
+      opcode: G_LOAD
+      memory: { index: 0, bytes: 8 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LD
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
+    id: select_g_sextload_m8
+    priority: 300
+    match:
+      opcode: G_SEXTLOAD
+      memory: { index: 0, bytes: 1 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LB
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
+    id: select_g_sextload_m16
+    priority: 300
+    match:
+      opcode: G_SEXTLOAD
+      memory: { index: 0, bytes: 2 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LH
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
+    id: select_g_sextload_m32
+    priority: 300
+    match:
+      opcode: G_SEXTLOAD
+      memory: { index: 0, bytes: 4 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LW
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
+    id: select_g_zextload_m8
+    priority: 300
+    match:
+      opcode: G_ZEXTLOAD
+      memory: { index: 0, bytes: 1 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LBU
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
+    id: select_g_zextload_m16
+    priority: 300
+    match:
+      opcode: G_ZEXTLOAD
+      memory: { index: 0, bytes: 2 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LHU
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
+    id: select_g_zextload_m32
+    priority: 300
+    match:
+      opcode: G_ZEXTLOAD
+      memory: { index: 0, bytes: 4 }
+      operand_types:
+        - { index: 1, kind: pointer, bits: 64 }
+      operand_banks:
+        - { index: 0, bank: GPR }
+        - { index: 1, bank: GPR }
+    apply:
+      instructions:
+        - opcode: LWU
+          root: true
+          operands:
+            - { operand: 0 }
+            - { complex: addr_reg_imm_base, operand: 1 }
+            - { complex: addr_reg_imm_offset, operand: 1 }
+  -
     id: select_g_frame_index
     priority: 100
     match: