llvm/clang/lib/Lex/TokenLexer.cpp

//===- TokenLexer.cpp - Lex from a token stream ---------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the TokenLexer interface.
//
//===----------------------------------------------------------------------===//

#include "clang/Lex/TokenLexer.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/IdentifierTable.h"
#include "clang/Basic/LangOptions.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TokenKinds.h"
#include "clang/Lex/LexDiagnostic.h"
#include "clang/Lex/Lexer.h"
#include "clang/Lex/MacroArgs.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/Token.h"
#include "clang/Lex/VariadicMacroSupport.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/iterator_range.h"
#include <cassert>
#include <cstring>
#include <optional>

usingnamespaceclang;

/// Create a TokenLexer for the specified macro with the specified actual
/// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
                      MacroArgs *Actuals) { … }

/// Create a TokenLexer for the specified token stream.  This does not
/// take ownership of the specified token vector.
void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
                      bool disableMacroExpansion, bool ownsTokens,
                      bool isReinject) { … }

void TokenLexer::destroy() { … }

bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
    SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
    unsigned MacroArgNo, Preprocessor &PP) { … }

void TokenLexer::stringifyVAOPTContents(
    SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
    const SourceLocation VAOPTClosingParenLoc) { … }

/// Expand the arguments of a function-like macro so that we can quickly
/// return preexpanded tokens from Tokens.
void TokenLexer::ExpandFunctionArguments() { … }

/// Checks if two tokens form wide string literal.
static bool isWideStringLiteralFromMacro(const Token &FirstTok,
                                         const Token &SecondTok) { … }

/// Lex - Lex and return a token from this macro stream.
bool TokenLexer::Lex(Token &Tok) { … }

bool TokenLexer::pasteTokens(Token &Tok) { … }

/// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ##
/// operator.  Read the ## and RHS, and paste the LHS/RHS together.  If there
/// are more ## after it, chomp them iteratively.  Return the result as LHSTok.
/// If this returns true, the caller should immediately return the token.
bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
                             unsigned int &CurIdx) { … }

/// isNextTokenLParen - If the next token lexed will pop this macro off the
/// expansion stack, return 2.  If the next unexpanded token is a '(', return
/// 1, otherwise return 0.
unsigned TokenLexer::isNextTokenLParen() const { … }

/// isParsingPreprocessorDirective - Return true if we are in the middle of a
/// preprocessor directive.
bool TokenLexer::isParsingPreprocessorDirective() const { … }

/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
/// together to form a comment that comments out everything in the current
/// macro, other active macros, and anything left on the current physical
/// source line of the expanded buffer.  Handle this by returning the
/// first token on the next line.
void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) { … }

/// If \arg loc is a file ID and points inside the current macro
/// definition, returns the appropriate source location pointing at the
/// macro expansion source location entry, otherwise it returns an invalid
/// SourceLocation.
SourceLocation
TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const { … }

/// Finds the tokens that are consecutive (from the same FileID)
/// creates a single SLocEntry, and assigns SourceLocations to each token that
/// point to that SLocEntry. e.g for
///   assert(foo == bar);
/// There will be a single SLocEntry for the "foo == bar" chunk and locations
/// for the 'foo', '==', 'bar' tokens will point inside that chunk.
///
/// \arg begin_tokens will be updated to a position past all the found
/// consecutive tokens.
static void updateConsecutiveMacroArgTokens(SourceManager &SM,
                                            SourceLocation ExpandLoc,
                                            Token *&begin_tokens,
                                            Token * end_tokens) { … }

/// Creates SLocEntries and updates the locations of macro argument
/// tokens to their new expanded locations.
///
/// \param ArgIdSpellLoc the location of the macro argument id inside the macro
/// definition.
void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
                                            Token *begin_tokens,
                                            Token *end_tokens) { … }

void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { … }