llvm/llvm/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp

//===- X86AvoidStoreForwardingBlocks.cpp - Avoid HW Store Forward Block ---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// If a load follows a store and reloads data that the store has written to
// memory, Intel microarchitectures can in many cases forward the data directly
// from the store to the load, This "store forwarding" saves cycles by enabling
// the load to directly obtain the data instead of accessing the data from
// cache or memory.
// A "store forward block" occurs in cases that a store cannot be forwarded to
// the load. The most typical case of store forward block on Intel Core
// microarchitecture that a small store cannot be forwarded to a large load.
// The estimated penalty for a store forward block is ~13 cycles.
//
// This pass tries to recognize and handle cases where "store forward block"
// is created by the compiler when lowering memcpy calls to a sequence
// of a load and a store.
//
// The pass currently only handles cases where memcpy is lowered to
// XMM/YMM registers, it tries to break the memcpy into smaller copies.
// breaking the memcpy should be possible since there is no atomicity
// guarantee for loads and stores to XMM/YMM.
//
// It could be better for performance to solve the problem by loading
// to XMM/YMM then inserting the partial store before storing back from XMM/YMM
// to memory, but this will result in a more conservative optimization since it
// requires we prove that all memory accesses between the blocking store and the
// load must alias/don't alias before we can move the store, whereas the
// transformation done here is correct regardless to other memory accesses.
//===----------------------------------------------------------------------===//

#include "X86.h"
#include "X86InstrInfo.h"
#include "X86Subtarget.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Function.h"
#include "llvm/InitializePasses.h"
#include "llvm/MC/MCInstrDesc.h"

usingnamespacellvm;

#define DEBUG_TYPE …

static cl::opt<bool> DisableX86AvoidStoreForwardBlocks(
    "x86-disable-avoid-SFB", cl::Hidden,
    cl::desc("X86: Disable Store Forwarding Blocks fixup."), cl::init(false));

static cl::opt<unsigned> X86AvoidSFBInspectionLimit(
    "x86-sfb-inspection-limit",
    cl::desc("X86: Number of instructions backward to "
             "inspect for store forwarding blocks."),
    cl::init(20), cl::Hidden);

namespace {

DisplacementSizeMap;

class X86AvoidSFBPass : public MachineFunctionPass { … };

} // end anonymous namespace

char X86AvoidSFBPass::ID = …;

INITIALIZE_PASS_BEGIN(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking",
                      false, false)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
INITIALIZE_PASS_END(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking", false,
                    false)

FunctionPass *llvm::createX86AvoidStoreForwardingBlocks() { … }

static bool isXMMLoadOpcode(unsigned Opcode) { … }
static bool isYMMLoadOpcode(unsigned Opcode) { … }

static bool isPotentialBlockedMemCpyLd(unsigned Opcode) { … }

static bool isPotentialBlockedMemCpyPair(unsigned LdOpcode, unsigned StOpcode) { … }

static bool isPotentialBlockingStoreInst(unsigned Opcode, unsigned LoadOpcode) { … }

static const int MOV128SZ = …;
static const int MOV64SZ = …;
static const int MOV32SZ = …;
static const int MOV16SZ = …;
static const int MOV8SZ = …;

static unsigned getYMMtoXMMLoadOpcode(unsigned LoadOpcode) { … }

static unsigned getYMMtoXMMStoreOpcode(unsigned StoreOpcode) { … }

static int getAddrOffset(const MachineInstr *MI) { … }

static MachineOperand &getBaseOperand(MachineInstr *MI) { … }

static MachineOperand &getDispOperand(MachineInstr *MI) { … }

// Relevant addressing modes contain only base register and immediate
// displacement or frameindex and immediate displacement.
// TODO: Consider expanding to other addressing modes in the future
static bool isRelevantAddressingMode(MachineInstr *MI) { … }

// Collect potentially blocking stores.
// Limit the number of instructions backwards we want to inspect
// since the effect of store block won't be visible if the store
// and load instructions have enough instructions in between to
// keep the core busy.
static SmallVector<MachineInstr *, 2>
findPotentialBlockers(MachineInstr *LoadInst) { … }

void X86AvoidSFBPass::buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode,
                                int64_t LoadDisp, MachineInstr *StoreInst,
                                unsigned NStoreOpcode, int64_t StoreDisp,
                                unsigned Size, int64_t LMMOffset,
                                int64_t SMMOffset) { … }

void X86AvoidSFBPass::buildCopies(int Size, MachineInstr *LoadInst,
                                  int64_t LdDispImm, MachineInstr *StoreInst,
                                  int64_t StDispImm, int64_t LMMOffset,
                                  int64_t SMMOffset) { … }

static void updateKillStatus(MachineInstr *LoadInst, MachineInstr *StoreInst) { … }

bool X86AvoidSFBPass::alias(const MachineMemOperand &Op1,
                            const MachineMemOperand &Op2) const { … }

void X86AvoidSFBPass::findPotentiallylBlockedCopies(MachineFunction &MF) { … }

unsigned X86AvoidSFBPass::getRegSizeInBytes(MachineInstr *LoadInst) { … }

void X86AvoidSFBPass::breakBlockedCopies(
    MachineInstr *LoadInst, MachineInstr *StoreInst,
    const DisplacementSizeMap &BlockingStoresDispSizeMap) { … }

static bool hasSameBaseOpValue(MachineInstr *LoadInst,
                               MachineInstr *StoreInst) { … }

static bool isBlockingStore(int64_t LoadDispImm, unsigned LoadSize,
                            int64_t StoreDispImm, unsigned StoreSize) { … }

// Keep track of all stores blocking a load
static void
updateBlockingStoresDispSizeMap(DisplacementSizeMap &BlockingStoresDispSizeMap,
                                int64_t DispImm, unsigned Size) { … }

// Remove blocking stores contained in each other.
static void
removeRedundantBlockingStores(DisplacementSizeMap &BlockingStoresDispSizeMap) { … }

bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) { … }