llvm/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp

//===- AArch64FalkorHWPFFix.cpp - Avoid HW prefetcher pitfalls on Falkor --===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
/// \file For Falkor, we want to avoid HW prefetcher instruction tag collisions
/// that may inhibit the HW prefetching.  This is done in two steps.  Before
/// ISel, we mark strided loads (i.e. those that will likely benefit from
/// prefetching) with metadata.  Then, after opcodes have been finalized, we
/// insert MOVs and re-write loads to prevent unintentional tag collisions.
// ===---------------------------------------------------------------------===//

#include "AArch64.h"
#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
#include "AArch64TargetMachine.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Metadata.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/raw_ostream.h"
#include <iterator>
#include <utility>

usingnamespacellvm;

#define DEBUG_TYPE …

STATISTIC(NumStridedLoadsMarked, "Number of strided loads marked");
STATISTIC(NumCollisionsAvoided,
          "Number of HW prefetch tag collisions avoided");
STATISTIC(NumCollisionsNotAvoided,
          "Number of HW prefetch tag collisions not avoided due to lack of registers");
DEBUG_COUNTER(FixCounter, "falkor-hwpf",
              "Controls which tag collisions are avoided");

namespace {

class FalkorMarkStridedAccesses { … };

class FalkorMarkStridedAccessesLegacy : public FunctionPass { … };

} // end anonymous namespace

char FalkorMarkStridedAccessesLegacy::ID = …;

INITIALIZE_PASS_BEGIN(FalkorMarkStridedAccessesLegacy, DEBUG_TYPE,
                      "Falkor HW Prefetch Fix", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
INITIALIZE_PASS_END(FalkorMarkStridedAccessesLegacy, DEBUG_TYPE,
                    "Falkor HW Prefetch Fix", false, false)

FunctionPass *llvm::createFalkorMarkStridedAccessesPass() { … }

bool FalkorMarkStridedAccessesLegacy::runOnFunction(Function &F) { … }

bool FalkorMarkStridedAccesses::run() { … }

bool FalkorMarkStridedAccesses::runOnLoop(Loop &L) { … }

namespace {

class FalkorHWPFFix : public MachineFunctionPass { … };

/// Bits from load opcodes used to compute HW prefetcher instruction tags.
struct LoadInfo { … };

} // end anonymous namespace

char FalkorHWPFFix::ID = …;

INITIALIZE_PASS_BEGIN(FalkorHWPFFix, "aarch64-falkor-hwpf-fix-late",
                      "Falkor HW Prefetch Fix Late Phase", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
INITIALIZE_PASS_END(FalkorHWPFFix, "aarch64-falkor-hwpf-fix-late",
                    "Falkor HW Prefetch Fix Late Phase", false, false)

static unsigned makeTag(unsigned Dest, unsigned Base, unsigned Offset) { … }

static std::optional<LoadInfo> getLoadInfo(const MachineInstr &MI) { … }

static std::optional<unsigned> getTag(const TargetRegisterInfo *TRI,
                                      const MachineInstr &MI,
                                      const LoadInfo &LI) { … }

void FalkorHWPFFix::runOnLoop(MachineLoop &L, MachineFunction &Fn) { … }

bool FalkorHWPFFix::runOnMachineFunction(MachineFunction &Fn) { … }

FunctionPass *llvm::createFalkorHWPFFixPass() { … }