llvm/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp

//===-- SIOptimizeExecMasking.cpp -----------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/InitializePasses.h"

usingnamespacellvm;

#define DEBUG_TYPE …

namespace {

class SIOptimizeExecMasking : public MachineFunctionPass { … };

} // End anonymous namespace.

INITIALIZE_PASS_BEGIN(SIOptimizeExecMasking, DEBUG_TYPE,
                      "SI optimize exec mask operations", false, false)
INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
INITIALIZE_PASS_END(SIOptimizeExecMasking, DEBUG_TYPE,
                    "SI optimize exec mask operations", false, false)

char SIOptimizeExecMasking::ID = …;

char &llvm::SIOptimizeExecMaskingID = …;

/// If \p MI is a copy from exec, return the register copied to.
Register SIOptimizeExecMasking::isCopyFromExec(const MachineInstr &MI) const { … }

/// If \p MI is a copy to exec, return the register copied from.
Register SIOptimizeExecMasking::isCopyToExec(const MachineInstr &MI) const { … }

/// If \p MI is a logical operation on an exec value,
/// return the register copied to.
static Register isLogicalOpOnExec(const MachineInstr &MI) { … }

static unsigned getSaveExecOp(unsigned Opc) { … }

// These are only terminators to get correct spill code placement during
// register allocation, so turn them back into normal instructions.
bool SIOptimizeExecMasking::removeTerminatorBit(MachineInstr &MI) const { … }

// Turn all pseudoterminators in the block into their equivalent non-terminator
// instructions. Returns the reverse iterator to the first non-terminator
// instruction in the block.
MachineBasicBlock::reverse_iterator
SIOptimizeExecMasking::fixTerminators(MachineBasicBlock &MBB) const { … }

MachineBasicBlock::reverse_iterator SIOptimizeExecMasking::findExecCopy(
    MachineBasicBlock &MBB, MachineBasicBlock::reverse_iterator I) const { … }

// XXX - Seems LiveRegUnits doesn't work correctly since it will incorrectly
// report the register as unavailable because a super-register with a lane mask
// is unavailable.
static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) { … }

// Backwards-iterate from Origin (for n=MaxInstructions iterations) until either
// the beginning of the BB is reached or Pred evaluates to true - which can be
// an arbitrary condition based on the current MachineInstr, for instance an
// target instruction. Breaks prematurely by returning nullptr if one of the
// registers given in NonModifiableRegs is modified by the current instruction.
MachineInstr *SIOptimizeExecMasking::findInstrBackwards(
    MachineInstr &Origin, std::function<bool(MachineInstr *)> Pred,
    ArrayRef<MCRegister> NonModifiableRegs, MachineInstr *Terminator,
    SmallVectorImpl<MachineOperand *> *KillFlagCandidates,
    unsigned MaxInstructions) const { … }

// Determine if a register Reg is not re-defined and still in use
// in the range (Stop..Start].
// It does so by backwards calculating liveness from the end of the BB until
// either Stop or the beginning of the BB is reached.
// After liveness is calculated, we can determine if Reg is still in use and not
// defined inbetween the instructions.
bool SIOptimizeExecMasking::isRegisterInUseBetween(MachineInstr &Stop,
                                                   MachineInstr &Start,
                                                   MCRegister Reg,
                                                   bool UseLiveOuts,
                                                   bool IgnoreStart) const { … }

// Determine if a register Reg is not re-defined and still in use
// in the range (Stop..BB.end].
bool SIOptimizeExecMasking::isRegisterInUseAfter(MachineInstr &Stop,
                                                 MCRegister Reg) const { … }

// Optimize sequences emitted for control flow lowering. They are originally
// emitted as the separate operations because spill code may need to be
// inserted for the saved copy of exec.
//
//     x = copy exec
//     z = s_<op>_b64 x, y
//     exec = copy z
// =>
//     x = s_<op>_saveexec_b64 y
//
bool SIOptimizeExecMasking::optimizeExecSequence() { … }

// Inserts the optimized s_mov_b32 / v_cmpx sequence based on the
// operands extracted from a v_cmp ..., s_and_saveexec pattern.
bool SIOptimizeExecMasking::optimizeVCMPSaveExecSequence(
    MachineInstr &SaveExecInstr, MachineInstr &VCmp, MCRegister Exec) const { … }

// Record (on GFX10.3 and later) occurences of
// v_cmp_* SGPR, IMM, VGPR
// s_and_saveexec_b32 EXEC_SGPR_DEST, SGPR
// to be replaced with
// s_mov_b32 EXEC_SGPR_DEST, exec_lo
// v_cmpx_* IMM, VGPR
// to reduce pipeline stalls.
void SIOptimizeExecMasking::tryRecordVCmpxAndSaveexecSequence(
    MachineInstr &MI) { … }

// Record occurences of
// s_or_saveexec s_o, s_i
// s_xor exec, exec, s_o
// to be replaced with
// s_andn2_saveexec s_o, s_i.
void SIOptimizeExecMasking::tryRecordOrSaveexecXorSequence(MachineInstr &MI) { … }

bool SIOptimizeExecMasking::optimizeOrSaveexecXorSequences() { … }

bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) { … }