Target.cpp | Explore in Territory

//===-- Target.cpp ----------------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "../Target.h"

#include "../Error.h"
#include "../MmapUtils.h"
#include "../ParallelSnippetGenerator.h"
#include "../SerialSnippetGenerator.h"
#include "../SnippetGenerator.h"
#include "../SubprocessMemory.h"
#include "MCTargetDesc/X86BaseInfo.h"
#include "MCTargetDesc/X86MCTargetDesc.h"
#include "X86.h"
#include "X86Counter.h"
#include "X86RegisterInfo.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/TargetParser/Host.h"

#include <memory>
#include <string>
#include <vector>
#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))
#include <immintrin.h>
#include <intrin.h>
#endif
#if defined(_MSC_VER) && defined(_M_X64)
#include <float.h> // For _clearfp in ~X86SavedState().
#endif

#ifdef __linux__
#ifdef __x86_64__
#include <asm/prctl.h>
#endif // __x86_64__
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
#ifdef HAVE_LIBPFM
#include <perfmon/perf_event.h>
#endif // HAVE_LIBPFM
#endif

#define GET_AVAILABLE_OPCODE_CHECKER
#include "X86GenInstrInfo.inc"

namespace llvm {
namespace exegesis {

// If a positive value is specified, we are going to use the LBR in
// latency-mode.
//
// Note:
//  -  A small value is preferred, but too low a value could result in
//     throttling.
//  -  A prime number is preferred to avoid always skipping certain blocks.
//
static cl::opt<unsigned> LbrSamplingPeriod(
    "x86-lbr-sample-period",
    cl::desc("The sample period (nbranches/sample), used for LBR sampling"),
    cl::cat(BenchmarkOptions), cl::init(0));

static cl::opt<bool>
    DisableUpperSSERegisters("x86-disable-upper-sse-registers",
                             cl::desc("Disable XMM8-XMM15 register usage"),
                             cl::cat(BenchmarkOptions), cl::init(false));

// FIXME: Validates that repetition-mode is loop if LBR is requested.

// Returns a non-null reason if we cannot handle the memory references in this
// instruction.
static const char *isInvalidMemoryInstr(const Instruction &Instr) { … }

// If the opcode is invalid, returns a pointer to a character literal indicating
// the reason. nullptr indicates a valid opcode.
static const char *isInvalidOpcode(const Instruction &Instr) { … }

static unsigned getX86FPFlags(const Instruction &Instr) { … }

// Helper to fill a memory operand with a value.
static void setMemOp(InstructionTemplate &IT, int OpIdx,
                     const MCOperand &OpVal) { … }

// Common (latency, uops) code for LEA templates. `GetDestReg` takes the
// addressing base and index registers and returns the LEA destination register.
static Expected<std::vector<CodeTemplate>> generateLEATemplatesCommon(
    const Instruction &Instr, const BitVector &ForbiddenRegisters,
    const LLVMState &State, const SnippetGenerator::Options &Opts,
    std::function<void(unsigned, unsigned, BitVector &CandidateDestRegs)>
        RestrictDestRegs) { … }

namespace {
class X86SerialSnippetGenerator : public SerialSnippetGenerator { … };
} // namespace

Expected<std::vector<CodeTemplate>>
X86SerialSnippetGenerator::generateCodeTemplates(
    InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const { … }

namespace {
class X86ParallelSnippetGenerator : public ParallelSnippetGenerator { … };

} // namespace

Expected<std::vector<CodeTemplate>>
X86ParallelSnippetGenerator::generateCodeTemplates(
    InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const { … }

static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) { … }

// Generates instruction to load an immediate value into a register.
static MCInst loadImmediate(unsigned Reg, unsigned RegBitWidth,
                            const APInt &Value) { … }

// Allocates scratch memory on the stack.
static MCInst allocateStackSpace(unsigned Bytes) { … }

// Fills scratch memory at offset `OffsetBytes` with value `Imm`.
static MCInst fillStackSpace(unsigned MovOpcode, unsigned OffsetBytes,
                             uint64_t Imm) { … }

// Loads scratch memory into register `Reg` using opcode `RMOpcode`.
static MCInst loadToReg(unsigned Reg, unsigned RMOpcode) { … }

// Releases scratch memory.
static MCInst releaseStackSpace(unsigned Bytes) { … }

// Reserves some space on the stack, fills it with the content of the provided
// constant and provide methods to load the stack value into a register.
namespace {
struct ConstantInliner { … };
} // namespace

std::vector<MCInst> ConstantInliner::loadAndFinalize(unsigned Reg,
                                                     unsigned RegBitWidth,
                                                     unsigned Opcode) { … }

std::vector<MCInst> ConstantInliner::loadX87STAndFinalize(unsigned Reg) { … }

std::vector<MCInst> ConstantInliner::loadX87FPAndFinalize(unsigned Reg) { … }

std::vector<MCInst> ConstantInliner::popFlagAndFinalize() { … }

std::vector<MCInst>
ConstantInliner::loadImplicitRegAndFinalize(unsigned Opcode, unsigned Value) { … }

void ConstantInliner::initStack(unsigned Bytes) { … }

#include "X86GenExegesis.inc"

namespace {

class X86SavedState : public ExegesisTarget::SavedState { … };

class ExegesisX86Target : public ExegesisTarget { … };

// We disable a few registers that cannot be encoded on instructions with a REX
// prefix.
const unsigned ExegesisX86Target::kUnavailableRegisters[4] = …;

// Optionally, also disable the upper (x86_64) SSE registers to reduce frontend
// decoder load.
const unsigned ExegesisX86Target::kUnavailableRegistersSSE[12] = …;

// We're using one of R8-R15 because these registers are never hardcoded in
// instructions (e.g. MOVS writes to EDI, ESI, EDX), so they have less
// conflicts.
constexpr const unsigned kDefaultLoopCounterReg = …;

} // namespace

void ExegesisX86Target::addTargetSpecificPasses(PassManagerBase &PM) const { … }

unsigned ExegesisX86Target::getScratchMemoryRegister(const Triple &TT) const { … }

unsigned
ExegesisX86Target::getDefaultLoopCounterRegister(const Triple &TT) const { … }

Error ExegesisX86Target::randomizeTargetMCOperand(
    const Instruction &Instr, const Variable &Var, MCOperand &AssignedValue,
    const BitVector &ForbiddenRegs) const { … }

void ExegesisX86Target::fillMemoryOperands(InstructionTemplate &IT,
                                           unsigned Reg,
                                           unsigned Offset) const { … }

void ExegesisX86Target::decrementLoopCounterAndJump(
    MachineBasicBlock &MBB, MachineBasicBlock &TargetMBB,
    const MCInstrInfo &MII, unsigned LoopRegister) const { … }

void generateRegisterStackPush(unsigned int Register,
                               std::vector<MCInst> &GeneratedCode) { … }

void generateRegisterStackPop(unsigned int Register,
                              std::vector<MCInst> &GeneratedCode) { … }

void generateSyscall(long SyscallNumber, std::vector<MCInst> &GeneratedCode) { … }

// The functions below for saving and restoring system call registers are only
// used when llvm-exegesis is built on Linux.
#ifdef __linux__
constexpr std::array<unsigned, 6> SyscallArgumentRegisters{ … };

static void saveSyscallRegisters(std::vector<MCInst> &GeneratedCode,
                                 unsigned ArgumentCount) { … }

static void restoreSyscallRegisters(std::vector<MCInst> &GeneratedCode,
                                    unsigned ArgumentCount) { … }
#endif // __linux__

static std::vector<MCInst> loadImmediateSegmentRegister(unsigned Reg,
                                                        const APInt &Value) { … }

std::vector<MCInst> ExegesisX86Target::setRegTo(const MCSubtargetInfo &STI,
                                                unsigned Reg,
                                                const APInt &Value) const { … }

#ifdef __linux__

#ifdef __arm__
static constexpr const uintptr_t VAddressSpaceCeiling = 0xC0000000;
#else
static constexpr const uintptr_t VAddressSpaceCeiling = …;
#endif

void generateRoundToNearestPage(unsigned int Register,
                                std::vector<MCInst> &GeneratedCode) { … }

void generateGetInstructionPointer(unsigned int ResultRegister,
                                   std::vector<MCInst> &GeneratedCode) { … }

void ExegesisX86Target::generateLowerMunmap(
    std::vector<MCInst> &GeneratedCode) const { … }

void ExegesisX86Target::generateUpperMunmap(
    std::vector<MCInst> &GeneratedCode) const { … }

std::vector<MCInst>
ExegesisX86Target::generateExitSyscall(unsigned ExitCode) const { … }

std::vector<MCInst>
ExegesisX86Target::generateMmap(uintptr_t Address, size_t Length,
                                uintptr_t FileDescriptorAddress) const { … }

void ExegesisX86Target::generateMmapAuxMem(
    std::vector<MCInst> &GeneratedCode) const { … }

void ExegesisX86Target::moveArgumentRegisters(
    std::vector<MCInst> &GeneratedCode) const { … }

std::vector<MCInst> ExegesisX86Target::generateMemoryInitialSetup() const { … }

std::vector<MCInst> ExegesisX86Target::setStackRegisterToAuxMem() const { … }

uintptr_t ExegesisX86Target::getAuxiliaryMemoryStartAddress() const { … }

std::vector<MCInst>
ExegesisX86Target::configurePerfCounter(long Request, bool SaveRegisters) const { … }

std::vector<unsigned> ExegesisX86Target::getArgumentRegisters() const { … }

std::vector<unsigned> ExegesisX86Target::getRegistersNeedSaving() const { … }

#endif // __linux__

// Instruction can have some variable operands, and we may want to see how
// different operands affect performance. So for each operand position,
// precompute all the possible choices we might care about,
// and greedily generate all the possible combinations of choices.
std::vector<InstructionTemplate> ExegesisX86Target::generateInstructionVariants(
    const Instruction &Instr, unsigned MaxConfigsPerOpcode) const { … }

static ExegesisTarget *getTheExegesisX86Target() { … }

void InitializeX86ExegesisTarget() { … }

} // namespace exegesis
} // namespace llvm
llvm/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
