llvm/llvm/lib/ProfileData/SampleProfReader.cpp

//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the class that reads LLVM sample profiles. It
// supports three file formats: text, binary and gcov.
//
// The textual representation is useful for debugging and testing purposes. The
// binary representation is more compact, resulting in smaller file sizes.
//
// The gcov encoding is the one generated by GCC's AutoFDO profile creation
// tool (https://github.com/google/autofdo)
//
// All three encodings can be used interchangeably as an input sample profile.
//
//===----------------------------------------------------------------------===//

#include "llvm/ProfileData/SampleProfReader.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/ProfileData/SampleProf.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compression.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/JSON.h"
#include "llvm/Support/LEB128.h"
#include "llvm/Support/LineIterator.h"
#include "llvm/Support/MD5.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <limits>
#include <memory>
#include <system_error>
#include <vector>

usingnamespacellvm;
usingnamespacesampleprof;

#define DEBUG_TYPE …

// This internal option specifies if the profile uses FS discriminators.
// It only applies to text, and binary format profiles.
// For ext-binary format profiles, the flag is set in the summary.
static cl::opt<bool> ProfileIsFSDisciminator(
    "profile-isfs", cl::Hidden, cl::init(false),
    cl::desc("Profile uses flow sensitive discriminators"));

/// Dump the function profile for \p FName.
///
/// \param FContext Name + context of the function to print.
/// \param OS Stream to emit the output to.
void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
                                              raw_ostream &OS) { … }

/// Dump all the function profiles found on stream \p OS.
void SampleProfileReader::dump(raw_ostream &OS) { … }

static void dumpFunctionProfileJson(const FunctionSamples &S,
                                    json::OStream &JOS, bool TopLevel = false) { … }

/// Dump all the function profiles found on stream \p OS in the JSON format.
void SampleProfileReader::dumpJson(raw_ostream &OS) { … }

/// Parse \p Input as function head.
///
/// Parse one line of \p Input, and update function name in \p FName,
/// function's total sample count in \p NumSamples, function's entry
/// count in \p NumHeadSamples.
///
/// \returns true if parsing is successful.
static bool ParseHead(const StringRef &Input, StringRef &FName,
                      uint64_t &NumSamples, uint64_t &NumHeadSamples) { … }

/// Returns true if line offset \p L is legal (only has 16 bits).
static bool isOffsetLegal(unsigned L) { … }

/// Parse \p Input that contains metadata.
/// Possible metadata:
/// - CFG Checksum information:
///     !CFGChecksum: 12345
/// - CFG Checksum information:
///     !Attributes: 1
/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
                          uint32_t &Attributes) { … }

enum class LineType { … };

/// Parse \p Input as line sample.
///
/// \param Input input line.
/// \param LineTy Type of this line.
/// \param Depth the depth of the inline stack.
/// \param NumSamples total samples of the line/inlined callsite.
/// \param LineOffset line offset to the start of the function.
/// \param Discriminator discriminator of the line.
/// \param TargetCountMap map from indirect call target to count.
/// \param FunctionHash the function's CFG hash, used by pseudo probe.
///
/// returns true if parsing is successful.
static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
                      uint64_t &NumSamples, uint32_t &LineOffset,
                      uint32_t &Discriminator, StringRef &CalleeName,
                      DenseMap<StringRef, uint64_t> &TargetCountMap,
                      uint64_t &FunctionHash, uint32_t &Attributes,
                      bool &IsFlat) { … }

/// Load samples from a text file.
///
/// See the documentation at the top of the file for an explanation of
/// the expected format.
///
/// \returns true if the file was loaded successfully, false otherwise.
std::error_code SampleProfileReaderText::readImpl() { … }

bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) { … }

template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() { … }

ErrorOr<StringRef> SampleProfileReaderBinary::readString() { … }

template <typename T>
ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() { … }

template <typename T>
inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) { … }

ErrorOr<FunctionId>
SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) { … }

ErrorOr<SampleContextFrames>
SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) { … }

ErrorOr<std::pair<SampleContext, uint64_t>>
SampleProfileReaderBinary::readSampleContextFromTable() { … }

std::error_code
SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { … }

std::error_code
SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start,
                                           SampleProfileMap &Profiles) { … }

std::error_code
SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) { … }

std::error_code SampleProfileReaderBinary::readImpl() { … }

std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
    const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) { … }

bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const { … }

std::error_code
SampleProfileReaderExtBinaryBase::read(const DenseSet<StringRef> &FuncsToUse,
                                       SampleProfileMap &Profiles) { … }

bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() { … }

std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() { … }

std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles(
    const DenseSet<StringRef> &FuncsToUse, SampleProfileMap &Profiles) { … }

std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { … }

std::error_code SampleProfileReaderExtBinaryBase::readProfileSymbolList() { … }

std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
    const uint8_t *SecStart, const uint64_t SecSize,
    const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) { … }

std::error_code SampleProfileReaderExtBinaryBase::readImpl() { … }

std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) { … }

std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) { … }

std::error_code SampleProfileReaderBinary::readNameTable() { … }

std::error_code
SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
                                                   bool FixedLengthMD5) { … }

// Read in the CS name table section, which basically contains a list of context
// vectors. Each element of a context vector, aka a frame, refers to the
// underlying raw function names that are stored in the name table, as well as
// a callsite identifier that only makes sense for non-leaf frames.
std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() { … }

std::error_code
SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
                                                   FunctionSamples *FProfile) { … }

std::error_code
SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
                                                   SampleProfileMap &Profiles) { … }

std::error_code
SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) { … }

std::error_code
SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) { … }

std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() { … }

std::error_code SampleProfileReaderExtBinaryBase::readHeader() { … }

uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) { … }

uint64_t SampleProfileReaderExtBinaryBase::getFileSize() { … }

static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) { … }

bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) { … }

std::error_code SampleProfileReaderBinary::readMagicIdent() { … }

std::error_code SampleProfileReaderBinary::readHeader() { … }

std::error_code SampleProfileReaderBinary::readSummaryEntry(
    std::vector<ProfileSummaryEntry> &Entries) { … }

std::error_code SampleProfileReaderBinary::readSummary() { … }

bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) { … }

bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) { … }

std::error_code SampleProfileReaderGCC::skipNextWord() { … }

template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() { … }

ErrorOr<StringRef> SampleProfileReaderGCC::readString() { … }

std::error_code SampleProfileReaderGCC::readHeader() { … }

std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) { … }

std::error_code SampleProfileReaderGCC::readNameTable() { … }

std::error_code SampleProfileReaderGCC::readFunctionProfiles() { … }

std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
    const InlineCallStack &InlineStack, bool Update, uint32_t Offset) { … }

/// Read a GCC AutoFDO profile.
///
/// This format is generated by the Linux Perf conversion tool at
/// https://github.com/google/autofdo.
std::error_code SampleProfileReaderGCC::readImpl() { … }

bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) { … }

void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) { … }

std::optional<StringRef>
SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) { … }

/// Prepare a memory buffer for the contents of \p Filename.
///
/// \returns an error code indicating the status of the buffer.
static ErrorOr<std::unique_ptr<MemoryBuffer>>
setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) { … }

/// Create a sample profile reader based on the format of the input file.
///
/// \param Filename The file to open.
///
/// \param C The LLVM context to use to emit diagnostics.
///
/// \param P The FSDiscriminatorPass.
///
/// \param RemapFilename The file used for profile remapping.
///
/// \returns an error code indicating the status of the created reader.
ErrorOr<std::unique_ptr<SampleProfileReader>>
SampleProfileReader::create(StringRef Filename, LLVMContext &C,
                            vfs::FileSystem &FS, FSDiscriminatorPass P,
                            StringRef RemapFilename) { … }

/// Create a sample profile remapper from the given input, to remap the
/// function names in the given profile data.
///
/// \param Filename The file to open.
///
/// \param Reader The profile reader the remapper is going to be applied to.
///
/// \param C The LLVM context to use to emit diagnostics.
///
/// \returns an error code indicating the status of the created reader.
ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
SampleProfileReaderItaniumRemapper::create(StringRef Filename,
                                           vfs::FileSystem &FS,
                                           SampleProfileReader &Reader,
                                           LLVMContext &C) { … }

/// Create a sample profile remapper from the given input, to remap the
/// function names in the given profile data.
///
/// \param B The memory buffer to create the reader from (assumes ownership).
///
/// \param C The LLVM context to use to emit diagnostics.
///
/// \param Reader The profile reader the remapper is going to be applied to.
///
/// \returns an error code indicating the status of the created reader.
ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
                                           SampleProfileReader &Reader,
                                           LLVMContext &C) { … }

/// Create a sample profile reader based on the format of the input data.
///
/// \param B The memory buffer to create the reader from (assumes ownership).
///
/// \param C The LLVM context to use to emit diagnostics.
///
/// \param P The FSDiscriminatorPass.
///
/// \param RemapFilename The file used for profile remapping.
///
/// \returns an error code indicating the status of the created reader.
ErrorOr<std::unique_ptr<SampleProfileReader>>
SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
                            vfs::FileSystem &FS, FSDiscriminatorPass P,
                            StringRef RemapFilename) { … }

// For text and GCC file formats, we compute the summary after reading the
// profile. Binary format has the profile summary in its header.
void SampleProfileReader::computeSummary() { … }