chromium/third_party/dawn/src/tint/lang/wgsl/reader/parser/lexer.cc

// Copyright 2020 The Dawn & Tint Authors
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its
//    contributors may be used to endorse or promote products derived from
//    this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "src/tint/lang/wgsl/reader/parser/lexer.h"

#include <cctype>
#include <charconv>
#include <cmath>
#include <cstring>
#include <limits>
#include <optional>
#include <string>
#include <tuple>
#include <utility>

#include "src/tint/lang/core/fluent_types.h"
#include "src/tint/lang/core/number.h"
#include "src/tint/utils/ice/ice.h"
#include "src/tint/utils/strconv/parse_num.h"
#include "src/tint/utils/text/unicode.h"

usingnamespacetint::core::fluent_types;  // NOLINT

namespace tint::wgsl::reader {
namespace {

// Unicode parsing code assumes that the size of a single std::string element is
// 1 byte.
static_assert …;

// A token is ~80bytes. The 4k here comes from looking at the number of tokens in the benchmark
// programs and being a bit bigger then those need (atan2-const-eval is the outlier here).
static constexpr size_t kDefaultListSize = …;

bool read_blankspace(std::string_view str,
                     size_t i,
                     bool* is_blankspace,
                     uint32_t* blankspace_size) { … }

uint32_t dec_value(char c) { … }

uint32_t hex_value(char c) { … }

}  // namespace

Lexer::Lexer(const Source::File* file) : … { … }

Lexer::~Lexer() = default;

std::vector<Token> Lexer::Lex() { … }

std::string_view Lexer::line() const { … }

uint32_t Lexer::pos() const { … }

uint32_t Lexer::length() const { … }

const char& Lexer::at(uint32_t pos) const { … }

std::string_view Lexer::substr(uint32_t offset, uint32_t count) { … }

void Lexer::advance(uint32_t offset) { … }

void Lexer::set_pos(uint32_t pos) { … }

void Lexer::advance_line() { … }

bool Lexer::is_eof() const { … }

bool Lexer::is_eol() const { … }

Token Lexer::next() { … }

Source Lexer::begin_source() const { … }

void Lexer::end_source(Source& src) const { … }

bool Lexer::is_null() const { … }

bool Lexer::is_digit(char ch) const { … }
bool Lexer::is_hex(char ch) const { … }

bool Lexer::matches(uint32_t pos, std::string_view sub_string) { … }

bool Lexer::matches(uint32_t pos, char ch) { … }

std::optional<Token> Lexer::skip_blankspace_and_comments() { … }

std::optional<Token> Lexer::skip_comment() { … }

std::optional<Token> Lexer::try_float() { … }

std::optional<Token> Lexer::try_hex_float() { … }

Token Lexer::build_token_from_int_if_possible(Source source,
                                              uint32_t start,
                                              uint32_t prefix_count,
                                              int32_t base) { … }

std::optional<Token> Lexer::try_hex_integer() { … }

std::optional<Token> Lexer::try_integer() { … }

std::optional<Token> Lexer::try_ident() { … }

std::optional<Token> Lexer::try_punctuation() { … }

std::optional<Token::Type> Lexer::parse_keyword(std::string_view str) { … }

}  // namespace tint::wgsl::reader