streaming_line_reader.h | Explore in Territory

/*
 * Copyright (C) 2022 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef SRC_TRACE_PROCESSOR_UTIL_STREAMING_LINE_READER_H_
#define SRC_TRACE_PROCESSOR_UTIL_STREAMING_LINE_READER_H_

#include <functional>
#include <vector>

#include "perfetto/ext/base/string_view.h"

namespace perfetto {
namespace trace_processor {
namespace util {

// A streaming line tokenizer for efficiently processing large text files on a
// line-by-line basis. It's designed to be used in conjunction with ZipReader to
// stream lines out of a compressed file (think of a bugreport) without having
// to decompress the whole file in memory upfront.
// Internally it deals with the necessary buffering and line-merging across
// different chunks.
// Usage:
// - The caller should pass a callback into the ctor. The callback is invoked
//   whenever a batch of lines has been tokenized. This happens after calls to
//   either BeginWrite()+EndWrite() or Tokenize(). In order to avoid too much
//   virtual dispatch overhead, the callback argument is a vector of lines, not
//   a single line.
// - The caller can call either:
//   - Tokenize(whole input): this exist to avoid a copy in the case of
//     non-compressed (STORE) files in zip archive.
//   - A sequence of BeginWrite() + EndWrite() as follows:
//     - BeginWrite(n) guarantees that the caller can write at least `n` char.
//       `n` is typically the decompression buffer passed to zlib.
//     - The caller writes at most `n` bytes into the pointer returned above.
//     - The caller calls EndWrite(m) passing the number of bytes actually
//       written (`m` <= `n`);
// NOTE:
// This implementation slightly diverges from base::StringSplitter as follows:
// 1. It does NOT skip empty lines. SS coalesces empty tokens, this doesn't.
// 2. it won't output the last line unless it terminates with a \n. SS doesn't
//    tell the difference between "foo\nbar" and "foo\nbar\n". This is
//    fundamental for streaming, where we cannot tell upfront if we got the end.
class StreamingLineReader { … };

}  // namespace util
}  // namespace trace_processor
}  // namespace perfetto

#endif  // SRC_TRACE_PROCESSOR_UTIL_STREAMING_LINE_READER_H_
chromium/third_party/perfetto/src/trace_processor/util/streaming_line_reader.h
