Tokens.cpp | Explore in Territory

//===- Tokens.cpp - collect tokens from preprocessing ---------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "clang/Tooling/Syntax/Tokens.h"

#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/IdentifierTable.h"
#include "clang/Basic/LLVM.h"
#include "clang/Basic/LangOptions.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TokenKinds.h"
#include "clang/Lex/PPCallbacks.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/Token.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
#include <iterator>
#include <optional>
#include <string>
#include <utility>
#include <vector>

usingnamespaceclang;
usingnamespaceclang::syntax;

namespace {
// Finds the smallest consecutive subsuquence of Toks that covers R.
llvm::ArrayRef<syntax::Token>
getTokensCovering(llvm::ArrayRef<syntax::Token> Toks, SourceRange R,
                  const SourceManager &SM) { … }

// Finds the range within FID corresponding to expanded tokens [First, Last].
// Prev precedes First and Next follows Last, these must *not* be included.
// If no range satisfies the criteria, returns an invalid range.
//
// #define ID(x) x
// ID(ID(ID(a1) a2))
//          ~~       -> a1
//              ~~   -> a2
//       ~~~~~~~~~   -> a1 a2
SourceRange spelledForExpandedSlow(SourceLocation First, SourceLocation Last,
                                   SourceLocation Prev, SourceLocation Next,
                                   FileID TargetFile,
                                   const SourceManager &SM) { … }

} // namespace

syntax::Token::Token(SourceLocation Location, unsigned Length,
                     tok::TokenKind Kind)
    : … { … }

syntax::Token::Token(const clang::Token &T)
    : … { … }

llvm::StringRef syntax::Token::text(const SourceManager &SM) const { … }

FileRange syntax::Token::range(const SourceManager &SM) const { … }

FileRange syntax::Token::range(const SourceManager &SM,
                               const syntax::Token &First,
                               const syntax::Token &Last) { … }

llvm::raw_ostream &syntax::operator<<(llvm::raw_ostream &OS, const Token &T) { … }

FileRange::FileRange(FileID File, unsigned BeginOffset, unsigned EndOffset)
    : … { … }

FileRange::FileRange(const SourceManager &SM, SourceLocation BeginLoc,
                     unsigned Length) { … }
FileRange::FileRange(const SourceManager &SM, SourceLocation BeginLoc,
                     SourceLocation EndLoc) { … }

llvm::raw_ostream &syntax::operator<<(llvm::raw_ostream &OS,
                                      const FileRange &R) { … }

llvm::StringRef FileRange::text(const SourceManager &SM) const { … }

void TokenBuffer::indexExpandedTokens() { … }

llvm::ArrayRef<syntax::Token> TokenBuffer::expandedTokens(SourceRange R) const { … }

CharSourceRange FileRange::toCharRange(const SourceManager &SM) const { … }

std::pair<const syntax::Token *, const TokenBuffer::Mapping *>
TokenBuffer::spelledForExpandedToken(const syntax::Token *Expanded) const { … }

const TokenBuffer::Mapping *
TokenBuffer::mappingStartingBeforeSpelled(const MarkedFile &F,
                                          const syntax::Token *Spelled) { … }

llvm::SmallVector<llvm::ArrayRef<syntax::Token>, 1>
TokenBuffer::expandedForSpelled(llvm::ArrayRef<syntax::Token> Spelled) const { … }

llvm::ArrayRef<syntax::Token> TokenBuffer::spelledTokens(FileID FID) const { … }

const syntax::Token *
TokenBuffer::spelledTokenContaining(SourceLocation Loc) const { … }

std::string TokenBuffer::Mapping::str() const { … }

std::optional<llvm::ArrayRef<syntax::Token>>
TokenBuffer::spelledForExpanded(llvm::ArrayRef<syntax::Token> Expanded) const { … }

TokenBuffer::Expansion TokenBuffer::makeExpansion(const MarkedFile &F,
                                                  const Mapping &M) const { … }

const TokenBuffer::MarkedFile &
TokenBuffer::fileForSpelled(llvm::ArrayRef<syntax::Token> Spelled) const { … }

std::optional<TokenBuffer::Expansion>
TokenBuffer::expansionStartingAt(const syntax::Token *Spelled) const { … }

std::vector<TokenBuffer::Expansion> TokenBuffer::expansionsOverlapping(
    llvm::ArrayRef<syntax::Token> Spelled) const { … }

llvm::ArrayRef<syntax::Token>
syntax::spelledTokensTouching(SourceLocation Loc,
                              llvm::ArrayRef<syntax::Token> Tokens) { … }

llvm::ArrayRef<syntax::Token>
syntax::spelledTokensTouching(SourceLocation Loc,
                              const syntax::TokenBuffer &Tokens) { … }

const syntax::Token *
syntax::spelledIdentifierTouching(SourceLocation Loc,
                                  llvm::ArrayRef<syntax::Token> Tokens) { … }

const syntax::Token *
syntax::spelledIdentifierTouching(SourceLocation Loc,
                                  const syntax::TokenBuffer &Tokens) { … }

std::vector<const syntax::Token *>
TokenBuffer::macroExpansions(FileID FID) const { … }

std::vector<syntax::Token> syntax::tokenize(const FileRange &FR,
                                            const SourceManager &SM,
                                            const LangOptions &LO) { … }

std::vector<syntax::Token> syntax::tokenize(FileID FID, const SourceManager &SM,
                                            const LangOptions &LO) { … }

/// Records information reqired to construct mappings for the token buffer that
/// we are collecting.
class TokenCollector::CollectPPExpansions : public PPCallbacks { … };

/// Fills in the TokenBuffer by tracing the run of a preprocessor. The
/// implementation tracks the tokens, macro expansions and directives coming
/// from the preprocessor and:
/// - for each token, figures out if it is a part of an expanded token stream,
///   spelled token stream or both. Stores the tokens appropriately.
/// - records mappings from the spelled to expanded token ranges, e.g. for macro
///   expansions.
/// FIXME: also properly record:
///          - #include directives,
///          - #pragma, #line and other PP directives,
///          - skipped pp regions,
///          - ...

TokenCollector::TokenCollector(Preprocessor &PP) : … { … }

/// Builds mappings and spelled tokens in the TokenBuffer based on the expanded
/// token stream.
class TokenCollector::Builder { … };

TokenBuffer TokenCollector::consume() && { … }

std::string syntax::Token::str() const { … }

std::string syntax::Token::dumpForTests(const SourceManager &SM) const { … }

std::string TokenBuffer::dumpForTests() const { … }
llvm/clang/lib/Tooling/Syntax/Tokens.cpp
