llvm/clang/lib/Format/FormatTokenLexer.cpp

//===--- FormatTokenLexer.cpp - Lex FormatTokens -------------*- C++ ----*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file implements FormatTokenLexer, which tokenizes a source file
/// into a FormatToken stream suitable for ClangFormat.
///
//===----------------------------------------------------------------------===//

#include "FormatTokenLexer.h"
#include "FormatToken.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Format/Format.h"
#include "llvm/Support/Regex.h"

namespace clang {
namespace format {

FormatTokenLexer::FormatTokenLexer(
    const SourceManager &SourceMgr, FileID ID, unsigned Column,
    const FormatStyle &Style, encoding::Encoding Encoding,
    llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
    IdentifierTable &IdentTable)
    : … { … }

ArrayRef<FormatToken *> FormatTokenLexer::lex() { … }

void FormatTokenLexer::tryMergePreviousTokens() { … }

bool FormatTokenLexer::tryMergeNSStringLiteral() { … }

bool FormatTokenLexer::tryMergeJSPrivateIdentifier() { … }

// Search for verbatim or interpolated string literals @"ABC" or
// $"aaaaa{abc}aaaaa" i and mark the token as TT_CSharpStringLiteral, and to
// prevent splitting of @, $ and ".
// Merging of multiline verbatim strings with embedded '"' is handled in
// handleCSharpVerbatimAndInterpolatedStrings with lower-level lexing.
bool FormatTokenLexer::tryMergeCSharpStringLiteral() { … }

// Valid C# attribute targets:
// https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/concepts/attributes/#attribute-targets
const llvm::StringSet<> FormatTokenLexer::CSharpAttributeTargets = …;

bool FormatTokenLexer::tryMergeNullishCoalescingEqual() { … }

bool FormatTokenLexer::tryMergeCSharpKeywordVariables() { … }

// In C# transform identifier foreach into kw_foreach
bool FormatTokenLexer::tryTransformCSharpForEach() { … }

bool FormatTokenLexer::tryMergeForEach() { … }

bool FormatTokenLexer::tryTransformTryUsageForC() { … }

bool FormatTokenLexer::tryMergeLessLess() { … }

bool FormatTokenLexer::tryMergeGreaterGreater() { … }

bool FormatTokenLexer::tryMergeTokens(ArrayRef<tok::TokenKind> Kinds,
                                      TokenType NewType) { … }

bool FormatTokenLexer::tryMergeTokens(size_t Count, TokenType NewType) { … }

bool FormatTokenLexer::tryMergeTokensAny(
    ArrayRef<ArrayRef<tok::TokenKind>> Kinds, TokenType NewType) { … }

// Returns \c true if \p Tok can only be followed by an operand in JavaScript.
bool FormatTokenLexer::precedesOperand(FormatToken *Tok) { … }

bool FormatTokenLexer::canPrecedeRegexLiteral(FormatToken *Prev) { … }

// Tries to parse a JavaScript Regex literal starting at the current token,
// if that begins with a slash and is in a location where JavaScript allows
// regex literals. Changes the current token to a regex literal and updates
// its text if successful.
void FormatTokenLexer::tryParseJSRegexLiteral() { … }

static auto lexCSharpString(const char *Begin, const char *End, bool Verbatim,
                            bool Interpolated) { … }

void FormatTokenLexer::handleCSharpVerbatimAndInterpolatedStrings() { … }

void FormatTokenLexer::handleTableGenMultilineString() { … }

void FormatTokenLexer::handleTableGenNumericLikeIdentifier() { … }

void FormatTokenLexer::handleTemplateStrings() { … }

void FormatTokenLexer::tryParsePythonComment() { … }

bool FormatTokenLexer::tryMerge_TMacro() { … }

bool FormatTokenLexer::tryMergeConflictMarkers() { … }

FormatToken *FormatTokenLexer::getStashedToken() { … }

/// Truncate the current token to the new length and make the lexer continue
/// from the end of the truncated token. Used for other languages that have
/// different token boundaries, like JavaScript in which a comment ends at a
/// line break regardless of whether the line break follows a backslash. Also
/// used to set the lexer to the end of whitespace if the lexer regards
/// whitespace and an unrecognized symbol as one token.
void FormatTokenLexer::truncateToken(size_t NewLen) { … }

/// Count the length of leading whitespace in a token.
static size_t countLeadingWhitespace(StringRef Text) { … }

FormatToken *FormatTokenLexer::getNextToken() { … }

bool FormatTokenLexer::readRawTokenVerilogSpecific(Token &Tok) { … }

void FormatTokenLexer::readRawToken(FormatToken &Tok) { … }

void FormatTokenLexer::resetLexer(unsigned Offset) { … }

} // namespace format
} // namespace clang