#include "mediapipe/tasks/cc/text/utils/vocab_utils.h"
#include <fstream>
#include "absl/strings/str_split.h"
#include "mediapipe/util/resource_util.h"
namespace mediapipe {
namespace tasks {
namespace text {
namespace {
struct membuf : std::streambuf { … };
void ReadIStreamLineByLine(
std::istream* istream,
const std::function<void(std::string)>& line_processor) { … }
absl::node_hash_map<std::string, int> ReadIStreamLineSplits(
std::istream* istream) { … }
std::vector<std::string> ReadIStreamByLine(std::istream* istream) { … }
}
std::vector<std::string> LoadVocabFromFile(const std::string& path_to_vocab) { … }
std::vector<std::string> LoadVocabFromBuffer(const char* vocab_buffer_data,
const size_t vocab_buffer_size) { … }
absl::node_hash_map<std::string, int> LoadVocabAndIndexFromFile(
const std::string& path_to_vocab) { … }
absl::node_hash_map<std::string, int> LoadVocabAndIndexFromBuffer(
const char* vocab_buffer_data, const size_t vocab_buffer_size) { … }
}
}
}