IOBuf.cpp | Explore in Territory

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
#endif

#include <folly/io/IOBuf.h>

#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <limits>
#include <stdexcept>
#include <type_traits>

#include <folly/Conv.h>
#include <folly/Likely.h>
#include <folly/Memory.h>
#include <folly/ScopeGuard.h>
#include <folly/hash/SpookyHashV2.h>
#include <folly/io/Cursor.h>
#include <folly/lang/Align.h>
#include <folly/lang/CheckedMath.h>
#include <folly/lang/Exception.h>
#include <folly/lang/Hint.h>
#include <folly/memory/Malloc.h>
#include <folly/memory/SanitizeAddress.h>

/*
 * Callbacks that will be invoked when IOBuf allocates or frees memory.
 * Note that io_buf_alloc_cb() will also be invoked when IOBuf takes ownership
 * of a malloc-allocated buffer, even if it was allocated earlier by another
 * part of the code.
 *
 * By default these are unimplemented, but programs can define these functions
 * to perform their own custom logic on memory allocation.  This is intended
 * primarily to help programs track memory usage and possibly take action
 * when thresholds are hit.  Callers should generally avoid performing any
 * expensive work in these callbacks, since they may be called from arbitrary
 * locations in the code that use IOBuf, possibly while holding locks.
 */

#if FOLLY_HAVE_WEAK_SYMBOLS
FOLLY_ATTR_WEAK void io_buf_alloc_cb(void* /*ptr*/, size_t /*size*/) noexcept;
FOLLY_ATTR_WEAK void io_buf_free_cb(void* /*ptr*/, size_t /*size*/) noexcept;
#else
static void (*io_buf_alloc_cb)(void* /*ptr*/, size_t /*size*/) noexcept = …;
static void (*io_buf_free_cb)(void* /*ptr*/, size_t /*size*/) noexcept = …;
#endif

unique_ptr;

namespace {

constexpr uint16_t kHeapMagic = …;

// When create() is called for buffers less than kDefaultCombinedBufSize,
// we allocate a single combined memory segment for the IOBuf and the data
// together.  See the comments for createCombined()/createSeparate() for more
// details.
//
// (The size of 1k is largely just a guess here.  We could could probably do
// benchmarks of real applications to see if adjusting this number makes a
// difference.  Callers that know their exact use case can also explicitly
// call createCombined() or createSeparate().)
constexpr size_t kDefaultCombinedBufSize = …;
constexpr size_t kMaxIOBufSize = …;

// Helper function for IOBuf::takeOwnership()
// The user's free function is not allowed to throw.
// (We are already in the middle of throwing an exception, so
// we cannot let this exception go unhandled.)
void takeOwnershipError(
    bool freeOnError,
    void* buf,
    folly::IOBuf::FreeFunction freeFn,
    void* userData) noexcept { … }

} // namespace

namespace folly {

// use free for size >= 4GB
// since we can store only 32 bits in the size var
struct IOBuf::HeapPrefix { … };

struct IOBuf::HeapStorage { … };

struct alignas(folly::max_align_v) IOBuf::HeapFullStorage { … };

// Only IOBuf is in use.
constexpr uint8_t kHeapStorageRefcount = …;
// Both IOBuf and SharedInfo (possibly with attached data) are in use.
constexpr uint8_t kHeapFullStorageRefcount = …;

IOBuf::SharedInfo::SharedInfo(FreeFunction fn, void* arg, StorageType st)
    : … { … }

void IOBuf::SharedInfo::invokeAndDeleteEachObserver(
    SharedInfoObserverEntryBase* observerListHead, ObserverCb cb) noexcept { … }

// storageType is stored in info, but info may be already deleted in the
// kExtBuffer case, so we need to pass it separately.
void IOBuf::SharedInfo::releaseStorage(
    IOBuf* parent, StorageType storageType, SharedInfo* info) noexcept { … }

void* IOBuf::operator new(size_t size) { … }

void* IOBuf::operator new(size_t /* size */, void* ptr) { … }

void IOBuf::operator delete(void* ptr) { … }

void IOBuf::operator delete(void* /* ptr */, void* /* placement */) { … }

void IOBuf::decrementStorageRefcount(HeapStorage* storage) noexcept { … }

IOBuf::IOBuf(CreateOp, std::size_t capacity)
    : … { … }

IOBuf::IOBuf(
    CopyBufferOp /* op */,
    const void* buf,
    std::size_t size,
    std::size_t headroom,
    std::size_t minTailroom)
    : … { … }

IOBuf::IOBuf(
    CopyBufferOp op,
    ByteRange br,
    std::size_t headroom,
    std::size_t minTailroom)
    : … { … }

unique_ptr<IOBuf> IOBuf::create(std::size_t capacity) { … }

unique_ptr<IOBuf> IOBuf::createCombined(std::size_t capacity) { … }

unique_ptr<IOBuf> IOBuf::createSeparate(std::size_t capacity) { … }

unique_ptr<IOBuf> IOBuf::createChain(
    size_t totalCapacity, std::size_t maxBufCapacity) { … }

size_t IOBuf::goodSize(size_t minCapacity, CombinedOption combined) { … }

IOBuf::IOBuf(
    TakeOwnershipOp,
    void* buf,
    std::size_t capacity,
    std::size_t offset,
    std::size_t length,
    FreeFunction freeFn,
    void* userData,
    bool freeOnError)
    : … { … }

IOBuf::IOBuf(
    TakeOwnershipOp,
    SizedFree,
    void* buf,
    std::size_t capacity,
    std::size_t offset,
    std::size_t length,
    bool freeOnError)
    : … { … }

unique_ptr<IOBuf> IOBuf::takeOwnership(
    void* buf,
    std::size_t capacity,
    std::size_t offset,
    std::size_t length,
    FreeFunction freeFn,
    void* userData,
    bool freeOnError,
    TakeOwnershipOption option) { … }

IOBuf::IOBuf(WrapBufferOp, const void* buf, std::size_t capacity) noexcept
    : … { … }

IOBuf::IOBuf(WrapBufferOp op, ByteRange br) noexcept
    : … { … }

unique_ptr<IOBuf> IOBuf::wrapBuffer(const void* buf, std::size_t capacity) { … }

IOBuf IOBuf::wrapBufferAsValue(const void* buf, std::size_t capacity) noexcept { … }

IOBuf::IOBuf() noexcept = default;

IOBuf::IOBuf(IOBuf&& other) noexcept
    : … { … }

IOBuf::IOBuf(const IOBuf& other) { … }

IOBuf::IOBuf(
    InternalConstructor,
    SharedInfo* sharedInfo,
    uint8_t* buf,
    std::size_t capacity,
    uint8_t* data,
    std::size_t length) noexcept
    : … { … }

IOBuf::~IOBuf() { … }

IOBuf& IOBuf::operator=(IOBuf&& other) noexcept { … }

IOBuf& IOBuf::operator=(const IOBuf& other) { … }

bool IOBuf::empty() const { … }

size_t IOBuf::countChainElements() const { … }

std::size_t IOBuf::computeChainDataLength() const { … }

std::size_t IOBuf::computeChainCapacity() const { … }

void IOBuf::appendToChain(unique_ptr<IOBuf>&& iobuf) { … }

unique_ptr<IOBuf> IOBuf::clone() const { … }

unique_ptr<IOBuf> IOBuf::cloneOne() const { … }

unique_ptr<IOBuf> IOBuf::cloneCoalesced() const { … }

unique_ptr<IOBuf> IOBuf::cloneCoalescedWithHeadroomTailroom(
    std::size_t newHeadroom, std::size_t newTailroom) const { … }

IOBuf IOBuf::cloneAsValue() const { … }

IOBuf IOBuf::cloneOneAsValue() const { … }

IOBuf IOBuf::cloneCoalescedAsValue() const { … }

IOBuf IOBuf::cloneCoalescedAsValueWithHeadroomTailroom(
    std::size_t newHeadroom, std::size_t newTailroom) const { … }

std::unique_ptr<IOBuf> IOBuf::maybeSplitTail() { … }

void IOBuf::unshareOneSlow() { … }

void IOBuf::unshareChained() { … }

void IOBuf::markExternallyShared() { … }

void IOBuf::makeManagedChained() { … }

void IOBuf::coalesceSlow() { … }

void IOBuf::coalesceSlow(size_t maxLength) { … }

void IOBuf::coalesceAndReallocate(
    size_t newHeadroom, size_t newLength, IOBuf* end, size_t newTailroom) { … }

void IOBuf::decrementRefcount() noexcept { … }

void IOBuf::reserveSlow(std::size_t minHeadroom, std::size_t minTailroom) { … }

// The user's free function should never throw. Otherwise we might throw from
// the IOBuf destructor. Other code paths like coalesce() also assume that
// decrementRefcount() cannot throw.
void IOBuf::freeExtBuffer() noexcept { … }

void IOBuf::allocExtBuffer(
    std::size_t minCapacity,
    uint8_t** bufReturn,
    SharedInfo** infoReturn,
    std::size_t* capacityReturn) { … }

size_t IOBuf::goodExtBufferSize(std::size_t minCapacity) { … }

void IOBuf::initExtBuffer(
    uint8_t* buf,
    size_t mallocSize,
    SharedInfo** infoReturn,
    std::size_t* capacityReturn) { … }

fbstring IOBuf::moveToFbString() { … }

IOBuf::Iterator IOBuf::cbegin() const { … }

IOBuf::Iterator IOBuf::cend() const { … }

folly::fbvector<struct iovec> IOBuf::getIov() const { … }

void IOBuf::appendToIov(folly::fbvector<struct iovec>* iov) const { … }

unique_ptr<IOBuf> IOBuf::wrapIov(const iovec* vec, size_t count) { … }

std::unique_ptr<IOBuf> IOBuf::takeOwnershipIov(
    const iovec* vec,
    size_t count,
    FreeFunction freeFn,
    void* userData,
    bool freeOnError) { … }

IOBuf::FillIovResult IOBuf::fillIov(struct iovec* iov, size_t len) const { … }

uint32_t IOBuf::approximateShareCountOne() const { … }

size_t IOBufHash::operator()(const IOBuf& buf) const noexcept { … }

ordering IOBufCompare::impl(const IOBuf& a, const IOBuf& b) const noexcept { … }

} // namespace folly
folly/folly/io/IOBuf.cpp
