scanner-character-streams.cc | Explore in Territory

// Copyright 2011 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/parsing/scanner-character-streams.h"

#include <memory>
#include <vector>

#include "include/v8-callbacks.h"
#include "include/v8-primitive.h"
#include "src/base/strings.h"
#include "src/common/globals.h"
#include "src/execution/isolate-utils.h"
#include "src/handles/handles.h"
#include "src/logging/runtime-call-stats-scope.h"
#include "src/objects/objects-inl.h"
#include "src/parsing/scanner.h"
#include "src/strings/unicode-inl.h"

namespace v8 {
namespace internal {

class V8_NODISCARD ScopedExternalStringLock { … };

namespace {
const unibrow::uchar kUtf8Bom = …;
}  // namespace

template <typename Char>
struct Range { … };

// A Char stream backed by an on-heap SeqOneByteString or SeqTwoByteString.
template <typename Char>
class OnHeapStream { … };

// A Char stream backed by an off-heap ExternalOneByteString or
// ExternalTwoByteString.
template <typename Char>
class ExternalStringStream { … };

// A Char stream backed by a C array. Testing only.
template <typename Char>
class TestingStream { … };

// A Char stream backed by multiple source-stream provided off-heap chunks.
template <typename Char>
class ChunkedStream { … };

// Provides a buffered utf-16 view on the bytes from the underlying ByteStream.
// Chars are buffered if either the underlying stream isn't utf-16 or the
// underlying utf-16 stream might move (is on-heap).
template <template <typename T> class ByteStream>
class BufferedCharacterStream : public Utf16CharacterStream { … };

// Provides an unbuffered utf-16 view on the bytes from the underlying
// ByteStream.
template <template <typename T> class ByteStream>
class UnbufferedCharacterStream : public Utf16CharacterStream { … };

// Provides an unbuffered utf-16 view on the bytes from the underlying
// ByteStream.
class RelocatingCharacterStream final
    : public UnbufferedCharacterStream<OnHeapStream> { … };

// ----------------------------------------------------------------------------
// BufferedUtf16CharacterStreams
//
// A buffered character stream based on a random access character
// source (ReadBlock can be called with pos() pointing to any position,
// even positions before the current).
//
// TODO(verwaest): Remove together with Utf8 external streaming streams.
class BufferedUtf16CharacterStream : public Utf16CharacterStream { … };

BufferedUtf16CharacterStream::BufferedUtf16CharacterStream()
    : … { … }

bool BufferedUtf16CharacterStream::ReadBlock(size_t position) { … }

// ----------------------------------------------------------------------------
// Windows1252CharacterStream - chunked streaming of windows-1252 data.
//
// Similar to BufferedCharacterStream, but does the translation of
// windows-1252 that are incompatible with their latin-1 equivalents.

namespace {

static const base::uc16 kWindows1252ToUC16[256] = …;

}  // namespace

class Windows1252CharacterStream final : public Utf16CharacterStream { … };

// ----------------------------------------------------------------------------
// Utf8ExternalStreamingStream - chunked streaming of Utf-8 data.
//
// This implementation is fairly complex, since data arrives in chunks which
// may 'cut' arbitrarily into utf-8 characters. Also, seeking to a given
// character position is tricky because the byte position cannot be derived
// from the character position.
//
// TODO(verwaest): Decode utf8 chunks into utf16 chunks on the blink side
// instead so we don't need to buffer.

class Utf8ExternalStreamingStream final : public BufferedUtf16CharacterStream { … };

bool Utf8ExternalStreamingStream::SkipToPosition(size_t position) { … }

void Utf8ExternalStreamingStream::FillBufferFromCurrentChunk() { … }

bool Utf8ExternalStreamingStream::FetchChunk() { … }

void Utf8ExternalStreamingStream::SearchPosition(size_t position) { … }

size_t Utf8ExternalStreamingStream::FillBuffer(size_t position) { … }

// ----------------------------------------------------------------------------
// ScannerStream: Create stream instances.

Utf16CharacterStream* ScannerStream::For(Isolate* isolate,
                                         Handle<String> data) { … }

Utf16CharacterStream* ScannerStream::For(Isolate* isolate, Handle<String> data,
                                         int start_pos, int end_pos) { … }

std::unique_ptr<Utf16CharacterStream> ScannerStream::ForTesting(
    const char* data) { … }

std::unique_ptr<Utf16CharacterStream> ScannerStream::ForTesting(
    const char* data, size_t length) { … }

std::unique_ptr<Utf16CharacterStream> ScannerStream::ForTesting(
    const uint16_t* data, size_t length) { … }

Utf16CharacterStream* ScannerStream::For(
    ScriptCompiler::ExternalSourceStream* source_stream,
    v8::ScriptCompiler::StreamedSource::Encoding encoding) { … }

}  // namespace internal
}  // namespace v8
chromium/v8/src/parsing/scanner-character-streams.cc
