StackColoring.cpp | Explore in Territory

//===- StackColoring.cpp --------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass implements the stack-coloring optimization that looks for
// lifetime markers machine instructions (LIFETIME_START and LIFETIME_END),
// which represent the possible lifetime of stack slots. It attempts to
// merge disjoint stack slots and reduce the used stack space.
// NOTE: This pass is not StackSlotColoring, which optimizes spill slots.
//
// TODO: In the future we plan to improve stack coloring in the following ways:
// 1. Allow merging multiple small slots into a single larger slot at different
//    offsets.
// 2. Merge this pass with StackSlotColoring and allow merging of allocas with
//    spill slots.
//
//===----------------------------------------------------------------------===//

#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/LiveInterval.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/PseudoSourceValueManager.h"
#include "llvm/CodeGen/SlotIndexes.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
#include <limits>
#include <memory>
#include <utility>

usingnamespacellvm;

#define DEBUG_TYPE …

static cl::opt<bool>
DisableColoring("no-stack-coloring",
        cl::init(false), cl::Hidden,
        cl::desc("Disable stack coloring"));

/// The user may write code that uses allocas outside of the declared lifetime
/// zone. This can happen when the user returns a reference to a local
/// data-structure. We can detect these cases and decide not to optimize the
/// code. If this flag is enabled, we try to save the user. This option
/// is treated as overriding LifetimeStartOnFirstUse below.
static cl::opt<bool>
ProtectFromEscapedAllocas("protect-from-escaped-allocas",
                          cl::init(false), cl::Hidden,
                          cl::desc("Do not optimize lifetime zones that "
                                   "are broken"));

/// Enable enhanced dataflow scheme for lifetime analysis (treat first
/// use of stack slot as start of slot lifetime, as opposed to looking
/// for LIFETIME_START marker). See "Implementation notes" below for
/// more info.
static cl::opt<bool>
LifetimeStartOnFirstUse("stackcoloring-lifetime-start-on-first-use",
        cl::init(true), cl::Hidden,
        cl::desc("Treat stack lifetimes as starting on first use, not on START marker."));


STATISTIC(NumMarkerSeen,  "Number of lifetime markers found.");
STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots.");
STATISTIC(StackSlotMerged, "Number of stack slot merged.");
STATISTIC(EscapedAllocas, "Number of allocas that escaped the lifetime region");

//===----------------------------------------------------------------------===//
//                           StackColoring Pass
//===----------------------------------------------------------------------===//
//
// Stack Coloring reduces stack usage by merging stack slots when they
// can't be used together. For example, consider the following C program:
//
//     void bar(char *, int);
//     void foo(bool var) {
//         A: {
//             char z[4096];
//             bar(z, 0);
//         }
//
//         char *p;
//         char x[4096];
//         char y[4096];
//         if (var) {
//             p = x;
//         } else {
//             bar(y, 1);
//             p = y + 1024;
//         }
//     B:
//         bar(p, 2);
//     }
//
// Naively-compiled, this program would use 12k of stack space. However, the
// stack slot corresponding to `z` is always destroyed before either of the
// stack slots for `x` or `y` are used, and then `x` is only used if `var`
// is true, while `y` is only used if `var` is false. So in no time are 2
// of the stack slots used together, and therefore we can merge them,
// compiling the function using only a single 4k alloca:
//
//     void foo(bool var) { // equivalent
//         char x[4096];
//         char *p;
//         bar(x, 0);
//         if (var) {
//             p = x;
//         } else {
//             bar(x, 1);
//             p = x + 1024;
//         }
//         bar(p, 2);
//     }
//
// This is an important optimization if we want stack space to be under
// control in large functions, both open-coded ones and ones created by
// inlining.
//
// Implementation Notes:
// ---------------------
//
// An important part of the above reasoning is that `z` can't be accessed
// while the latter 2 calls to `bar` are running. This is justified because
// `z`'s lifetime is over after we exit from block `A:`, so any further
// accesses to it would be UB. The way we represent this information
// in LLVM is by having frontends delimit blocks with `lifetime.start`
// and `lifetime.end` intrinsics.
//
// The effect of these intrinsics seems to be as follows (maybe I should
// specify this in the reference?):
//
//   L1) at start, each stack-slot is marked as *out-of-scope*, unless no
//   lifetime intrinsic refers to that stack slot, in which case
//   it is marked as *in-scope*.
//   L2) on a `lifetime.start`, a stack slot is marked as *in-scope* and
//   the stack slot is overwritten with `undef`.
//   L3) on a `lifetime.end`, a stack slot is marked as *out-of-scope*.
//   L4) on function exit, all stack slots are marked as *out-of-scope*.
//   L5) `lifetime.end` is a no-op when called on a slot that is already
//   *out-of-scope*.
//   L6) memory accesses to *out-of-scope* stack slots are UB.
//   L7) when a stack-slot is marked as *out-of-scope*, all pointers to it
//   are invalidated, unless the slot is "degenerate". This is used to
//   justify not marking slots as in-use until the pointer to them is
//   used, but feels a bit hacky in the presence of things like LICM. See
//   the "Degenerate Slots" section for more details.
//
// Now, let's ground stack coloring on these rules. We'll define a slot
// as *in-use* at a (dynamic) point in execution if it either can be
// written to at that point, or if it has a live and non-undef content
// at that point.
//
// Obviously, slots that are never *in-use* together can be merged, and
// in our example `foo`, the slots for `x`, `y` and `z` are never
// in-use together (of course, sometimes slots that *are* in-use together
// might still be mergable, but we don't care about that here).
//
// In this implementation, we successively merge pairs of slots that are
// not *in-use* together. We could be smarter - for example, we could merge
// a single large slot with 2 small slots, or we could construct the
// interference graph and run a "smart" graph coloring algorithm, but with
// that aside, how do we find out whether a pair of slots might be *in-use*
// together?
//
// From our rules, we see that *out-of-scope* slots are never *in-use*,
// and from (L7) we see that "non-degenerate" slots remain non-*in-use*
// until their address is taken. Therefore, we can approximate slot activity
// using dataflow.
//
// A subtle point: naively, we might try to figure out which pairs of
// stack-slots interfere by propagating `S in-use` through the CFG for every
// stack-slot `S`, and having `S` and `T` interfere if there is a CFG point in
// which they are both *in-use*.
//
// That is sound, but overly conservative in some cases: in our (artificial)
// example `foo`, either `x` or `y` might be in use at the label `B:`, but
// as `x` is only in use if we came in from the `var` edge and `y` only
// if we came from the `!var` edge, they still can't be in use together.
// See PR32488 for an important real-life case.
//
// If we wanted to find all points of interference precisely, we could
// propagate `S in-use` and `S&T in-use` predicates through the CFG. That
// would be precise, but requires propagating `O(n^2)` dataflow facts.
//
// However, we aren't interested in the *set* of points of interference
// between 2 stack slots, only *whether* there *is* such a point. So we
// can rely on a little trick: for `S` and `T` to be in-use together,
// one of them needs to become in-use while the other is in-use (or
// they might both become in use simultaneously). We can check this
// by also keeping track of the points at which a stack slot might *start*
// being in-use.
//
// Exact first use:
// ----------------
//
// Consider the following motivating example:
//
//     int foo() {
//       char b1[1024], b2[1024];
//       if (...) {
//         char b3[1024];
//         <uses of b1, b3>;
//         return x;
//       } else {
//         char b4[1024], b5[1024];
//         <uses of b2, b4, b5>;
//         return y;
//       }
//     }
//
// In the code above, "b3" and "b4" are declared in distinct lexical
// scopes, meaning that it is easy to prove that they can share the
// same stack slot. Variables "b1" and "b2" are declared in the same
// scope, meaning that from a lexical point of view, their lifetimes
// overlap. From a control flow pointer of view, however, the two
// variables are accessed in disjoint regions of the CFG, thus it
// should be possible for them to share the same stack slot. An ideal
// stack allocation for the function above would look like:
//
//     slot 0: b1, b2
//     slot 1: b3, b4
//     slot 2: b5
//
// Achieving this allocation is tricky, however, due to the way
// lifetime markers are inserted. Here is a simplified view of the
// control flow graph for the code above:
//
//                +------  block 0 -------+
//               0| LIFETIME_START b1, b2 |
//               1| <test 'if' condition> |
//                +-----------------------+
//                   ./              \.
//   +------  block 1 -------+   +------  block 2 -------+
//  2| LIFETIME_START b3     |  5| LIFETIME_START b4, b5 |
//  3| <uses of b1, b3>      |  6| <uses of b2, b4, b5>  |
//  4| LIFETIME_END b3       |  7| LIFETIME_END b4, b5   |
//   +-----------------------+   +-----------------------+
//                   \.              /.
//                +------  block 3 -------+
//               8| <cleanupcode>         |
//               9| LIFETIME_END b1, b2   |
//              10| return                |
//                +-----------------------+
//
// If we create live intervals for the variables above strictly based
// on the lifetime markers, we'll get the set of intervals on the
// left. If we ignore the lifetime start markers and instead treat a
// variable's lifetime as beginning with the first reference to the
// var, then we get the intervals on the right.
//
//            LIFETIME_START      First Use
//     b1:    [0,9]               [3,4] [8,9]
//     b2:    [0,9]               [6,9]
//     b3:    [2,4]               [3,4]
//     b4:    [5,7]               [6,7]
//     b5:    [5,7]               [6,7]
//
// For the intervals on the left, the best we can do is overlap two
// variables (b3 and b4, for example); this gives us a stack size of
// 4*1024 bytes, not ideal. When treating first-use as the start of a
// lifetime, we can additionally overlap b1 and b5, giving us a 3*1024
// byte stack (better).
//
// Degenerate Slots:
// -----------------
//
// Relying entirely on first-use of stack slots is problematic,
// however, due to the fact that optimizations can sometimes migrate
// uses of a variable outside of its lifetime start/end region. Here
// is an example:
//
//     int bar() {
//       char b1[1024], b2[1024];
//       if (...) {
//         <uses of b2>
//         return y;
//       } else {
//         <uses of b1>
//         while (...) {
//           char b3[1024];
//           <uses of b3>
//         }
//       }
//     }
//
// Before optimization, the control flow graph for the code above
// might look like the following:
//
//                +------  block 0 -------+
//               0| LIFETIME_START b1, b2 |
//               1| <test 'if' condition> |
//                +-----------------------+
//                   ./              \.
//   +------  block 1 -------+    +------- block 2 -------+
//  2| <uses of b2>          |   3| <uses of b1>          |
//   +-----------------------+    +-----------------------+
//              |                            |
//              |                 +------- block 3 -------+ <-\.
//              |                4| <while condition>     |    |
//              |                 +-----------------------+    |
//              |               /          |                   |
//              |              /  +------- block 4 -------+
//              \             /  5| LIFETIME_START b3     |    |
//               \           /   6| <uses of b3>          |    |
//                \         /    7| LIFETIME_END b3       |    |
//                 \        |    +------------------------+    |
//                  \       |                 \                /
//                +------  block 5 -----+      \---------------
//               8| <cleanupcode>       |
//               9| LIFETIME_END b1, b2 |
//              10| return              |
//                +---------------------+
//
// During optimization, however, it can happen that an instruction
// computing an address in "b3" (for example, a loop-invariant GEP) is
// hoisted up out of the loop from block 4 to block 2.  [Note that
// this is not an actual load from the stack, only an instruction that
// computes the address to be loaded]. If this happens, there is now a
// path leading from the first use of b3 to the return instruction
// that does not encounter the b3 LIFETIME_END, hence b3's lifetime is
// now larger than if we were computing live intervals strictly based
// on lifetime markers. In the example above, this lengthened lifetime
// would mean that it would appear illegal to overlap b3 with b2.
//
// To deal with this such cases, the code in ::collectMarkers() below
// tries to identify "degenerate" slots -- those slots where on a single
// forward pass through the CFG we encounter a first reference to slot
// K before we hit the slot K lifetime start marker. For such slots,
// we fall back on using the lifetime start marker as the beginning of
// the variable's lifetime.  NB: with this implementation, slots can
// appear degenerate in cases where there is unstructured control flow:
//
//    if (q) goto mid;
//    if (x > 9) {
//         int b[100];
//         memcpy(&b[0], ...);
//    mid: b[k] = ...;
//         abc(&b);
//    }
//
// If in RPO ordering chosen to walk the CFG  we happen to visit the b[k]
// before visiting the memcpy block (which will contain the lifetime start
// for "b" then it will appear that 'b' has a degenerate lifetime.

namespace {

/// StackColoring - A machine pass for merging disjoint stack allocations,
/// marked by the LIFETIME_START and LIFETIME_END pseudo instructions.
class StackColoring : public MachineFunctionPass { … };

} // end anonymous namespace

char StackColoring::ID = …;

char &llvm::StackColoringID = …;

INITIALIZE_PASS_BEGIN(StackColoring, DEBUG_TYPE,
                      "Merge disjoint stack slots", false, false)
INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
INITIALIZE_PASS_END(StackColoring, DEBUG_TYPE,
                    "Merge disjoint stack slots", false, false)

void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const { … }

#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void StackColoring::dumpBV(const char *tag,
                                            const BitVector &BV) const {
  dbgs() << tag << " : { ";
  for (unsigned I = 0, E = BV.size(); I != E; ++I)
    dbgs() << BV.test(I) << " ";
  dbgs() << "}\n";
}

LLVM_DUMP_METHOD void StackColoring::dumpBB(MachineBasicBlock *MBB) const {
  LivenessMap::const_iterator BI = BlockLiveness.find(MBB);
  assert(BI != BlockLiveness.end() && "Block not found");
  const BlockLifetimeInfo &BlockInfo = BI->second;

  dumpBV("BEGIN", BlockInfo.Begin);
  dumpBV("END", BlockInfo.End);
  dumpBV("LIVE_IN", BlockInfo.LiveIn);
  dumpBV("LIVE_OUT", BlockInfo.LiveOut);
}

LLVM_DUMP_METHOD void StackColoring::dump() const {
  for (MachineBasicBlock *MBB : depth_first(MF)) {
    dbgs() << "Inspecting block #" << MBB->getNumber() << " ["
           << MBB->getName() << "]\n";
    dumpBB(MBB);
  }
}

LLVM_DUMP_METHOD void StackColoring::dumpIntervals() const {
  for (unsigned I = 0, E = Intervals.size(); I != E; ++I) {
    dbgs() << "Interval[" << I << "]:\n";
    Intervals[I]->dump();
  }
}
#endif

static inline int getStartOrEndSlot(const MachineInstr &MI)
{ … }

// At the moment the only way to end a variable lifetime is with
// a VARIABLE_LIFETIME op (which can't contain a start). If things
// change and the IR allows for a single inst that both begins
// and ends lifetime(s), this interface will need to be reworked.
bool StackColoring::isLifetimeStartOrEnd(const MachineInstr &MI,
                                         SmallVector<int, 4> &slots,
                                         bool &isStart) { … }

unsigned StackColoring::collectMarkers(unsigned NumSlot) { … }

void StackColoring::calculateLocalLiveness() { … }

void StackColoring::calculateLiveIntervals(unsigned NumSlots) { … }

bool StackColoring::removeAllMarkers() { … }

void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) { … }

void StackColoring::removeInvalidSlotRanges() { … }

void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
                                   unsigned NumSlots) { … }

bool StackColoring::runOnMachineFunction(MachineFunction &Func) { … }
llvm/llvm/lib/CodeGen/StackColoring.cpp
