OffloadBundler.cpp | Explore in Territory

//===- OffloadBundler.cpp - File Bundling and Unbundling ------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file implements an offload bundling API that bundles different files
/// that relate with the same source code but different targets into a single
/// one. Also the implements the opposite functionality, i.e. unbundle files
/// previous created by this API.
///
//===----------------------------------------------------------------------===//

#include "clang/Driver/OffloadBundler.h"
#include "clang/Basic/Cuda.h"
#include "clang/Basic/TargetID.h"
#include "clang/Basic/Version.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/BinaryFormat/Magic.h"
#include "llvm/Object/Archive.h"
#include "llvm/Object/ArchiveWriter.h"
#include "llvm/Object/Binary.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Compression.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/EndianStream.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MD5.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/Host.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <forward_list>
#include <llvm/Support/Process.h>
#include <memory>
#include <set>
#include <string>
#include <system_error>
#include <utility>

usingnamespacellvm;
usingnamespacellvm::object;
usingnamespaceclang;

static llvm::TimerGroup
    ClangOffloadBundlerTimerGroup("Clang Offload Bundler Timer Group",
                                  "Timer group for clang offload bundler");

/// Magic string that marks the existence of offloading data.
#define OFFLOAD_BUNDLER_MAGIC_STR …

OffloadTargetInfo::OffloadTargetInfo(const StringRef Target,
                                     const OffloadBundlerConfig &BC)
    : … { … }

bool OffloadTargetInfo::hasHostKind() const { … }

bool OffloadTargetInfo::isOffloadKindValid() const { … }

bool OffloadTargetInfo::isOffloadKindCompatible(
    const StringRef TargetOffloadKind) const { … }

bool OffloadTargetInfo::isTripleValid() const { … }

bool OffloadTargetInfo::operator==(const OffloadTargetInfo &Target) const { … }

std::string OffloadTargetInfo::str() const { … }

static StringRef getDeviceFileExtension(StringRef Device,
                                        StringRef BundleFileName) { … }

static std::string getDeviceLibraryFileName(StringRef BundleFileName,
                                            StringRef Device) { … }

namespace {
/// Generic file handler interface.
class FileHandler { … };

/// Handler for binary files. The bundled file will have the following format
/// (all integers are stored in little-endian format):
///
/// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string)
///
/// NumberOfOffloadBundles (8-byte integer)
///
/// OffsetOfBundle1 (8-byte integer)
/// SizeOfBundle1 (8-byte integer)
/// NumberOfBytesInTripleOfBundle1 (8-byte integer)
/// TripleOfBundle1 (byte length defined before)
///
/// ...
///
/// OffsetOfBundleN (8-byte integer)
/// SizeOfBundleN (8-byte integer)
/// NumberOfBytesInTripleOfBundleN (8-byte integer)
/// TripleOfBundleN (byte length defined before)
///
/// Bundle1
/// ...
/// BundleN

/// Read 8-byte integers from a buffer in little-endian format.
static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) { … }

/// Write 8-byte integers to a buffer in little-endian format.
static void Write8byteIntegerToBuffer(raw_ostream &OS, uint64_t Val) { … }

class BinaryFileHandler final : public FileHandler { … };

// This class implements a list of temporary files that are removed upon
// object destruction.
class TempFileHandlerRAII { … };

/// Handler for object files. The bundles are organized by sections with a
/// designated name.
///
/// To unbundle, we just copy the contents of the designated section.
class ObjectFileHandler final : public FileHandler { … };

/// Handler for text files. The bundled file will have the following format.
///
/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
/// Bundle 1
/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
/// ...
/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
/// Bundle N
/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
class TextFileHandler final : public FileHandler { … };
} // namespace

/// Return an appropriate object file handler. We use the specific object
/// handler if we know how to deal with that format, otherwise we use a default
/// binary file handler.
static std::unique_ptr<FileHandler>
CreateObjectFileHandler(MemoryBuffer &FirstInput,
                        const OffloadBundlerConfig &BundlerConfig) { … }

/// Return an appropriate handler given the input files and options.
static Expected<std::unique_ptr<FileHandler>>
CreateFileHandler(MemoryBuffer &FirstInput,
                  const OffloadBundlerConfig &BundlerConfig) { … }

OffloadBundlerConfig::OffloadBundlerConfig() { … }

// Utility function to format numbers with commas
static std::string formatWithCommas(unsigned long long Value) { … }

llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
CompressedOffloadBundle::compress(llvm::compression::Params P,
                                  const llvm::MemoryBuffer &Input,
                                  bool Verbose) { … }

llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
CompressedOffloadBundle::decompress(const llvm::MemoryBuffer &Input,
                                    bool Verbose) { … }

// List bundle IDs. Return true if an error was found.
Error OffloadBundler::ListBundleIDsInFile(
    StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig) { … }

/// @brief Checks if a code object \p CodeObjectInfo is compatible with a given
/// target \p TargetInfo.
/// @link https://clang.llvm.org/docs/ClangOffloadBundler.html#bundle-entry-id
bool isCodeObjectCompatible(const OffloadTargetInfo &CodeObjectInfo,
                            const OffloadTargetInfo &TargetInfo) { … }

/// Bundle the files. Return true if an error was found.
Error OffloadBundler::BundleFiles() { … }

// Unbundle the files. Return true if an error was found.
Error OffloadBundler::UnbundleFiles() { … }

static Archive::Kind getDefaultArchiveKindForHost() { … }

/// @brief Computes a list of targets among all given targets which are
/// compatible with this code object
/// @param [in] CodeObjectInfo Code Object
/// @param [out] CompatibleTargets List of all compatible targets among all
/// given targets
/// @return false, if no compatible target is found.
static bool
getCompatibleOffloadTargets(OffloadTargetInfo &CodeObjectInfo,
                            SmallVectorImpl<StringRef> &CompatibleTargets,
                            const OffloadBundlerConfig &BundlerConfig) { … }

// Check that each code object file in the input archive conforms to following
// rule: for a specific processor, a feature either shows up in all target IDs,
// or does not show up in any target IDs. Otherwise the target ID combination is
// invalid.
static Error
CheckHeterogeneousArchive(StringRef ArchiveName,
                          const OffloadBundlerConfig &BundlerConfig) { … }

/// UnbundleArchive takes an archive file (".a") as input containing bundled
/// code object files, and a list of offload targets (not host), and extracts
/// the code objects into a new archive file for each offload target. Each
/// resulting archive file contains all code object files corresponding to that
/// particular offload target. The created archive file does not
/// contain an index of the symbols and code object files are named as
/// <<Parent Bundle Name>-<CodeObject's TargetID>>, with ':' replaced with '_'.
Error OffloadBundler::UnbundleArchive() { … }
llvm/clang/lib/Driver/OffloadBundler.cpp
