use core::iter::FromIterator;
use core::mem::{self, ManuallyDrop, MaybeUninit};
use core::ops::{Deref, DerefMut};
use core::ptr::{self, NonNull};
use core::{cmp, fmt, hash, isize, slice, usize};
use alloc::{
borrow::{Borrow, BorrowMut},
boxed::Box,
string::String,
vec,
vec::Vec,
};
use crate::buf::{IntoIter, UninitSlice};
use crate::bytes::Vtable;
#[allow(unused)]
use crate::loom::sync::atomic::AtomicMut;
use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};
use crate::{offset_from, Buf, BufMut, Bytes};
/// A unique reference to a contiguous slice of memory.
///
/// `BytesMut` represents a unique view into a potentially shared memory region.
/// Given the uniqueness guarantee, owners of `BytesMut` handles are able to
/// mutate the memory.
///
/// `BytesMut` can be thought of as containing a `buf: Arc<Vec<u8>>`, an offset
/// into `buf`, a slice length, and a guarantee that no other `BytesMut` for the
/// same `buf` overlaps with its slice. That guarantee means that a write lock
/// is not required.
///
/// # Growth
///
/// `BytesMut`'s `BufMut` implementation will implicitly grow its buffer as
/// necessary. However, explicitly reserving the required space up-front before
/// a series of inserts will be more efficient.
///
/// # Examples
///
/// ```
/// use bytes::{BytesMut, BufMut};
///
/// let mut buf = BytesMut::with_capacity(64);
///
/// buf.put_u8(b'h');
/// buf.put_u8(b'e');
/// buf.put(&b"llo"[..]);
///
/// assert_eq!(&buf[..], b"hello");
///
/// // Freeze the buffer so that it can be shared
/// let a = buf.freeze();
///
/// // This does not allocate, instead `b` points to the same memory.
/// let b = a.clone();
///
/// assert_eq!(&a[..], b"hello");
/// assert_eq!(&b[..], b"hello");
/// ```
pub struct BytesMut {
ptr: NonNull<u8>,
len: usize,
cap: usize,
data: *mut Shared,
}
// Thread-safe reference-counted container for the shared storage. This mostly
// the same as `core::sync::Arc` but without the weak counter. The ref counting
// fns are based on the ones found in `std`.
//
// The main reason to use `Shared` instead of `core::sync::Arc` is that it ends
// up making the overall code simpler and easier to reason about. This is due to
// some of the logic around setting `Inner::arc` and other ways the `arc` field
// is used. Using `Arc` ended up requiring a number of funky transmutes and
// other shenanigans to make it work.
struct Shared {
vec: Vec<u8>,
original_capacity_repr: usize,
ref_count: AtomicUsize,
}
// Assert that the alignment of `Shared` is divisible by 2.
// This is a necessary invariant since we depend on allocating `Shared` a
// shared object to implicitly carry the `KIND_ARC` flag in its pointer.
// This flag is set when the LSB is 0.
const _: [(); 0 - mem::align_of::<Shared>() % 2] = []; // Assert that the alignment of `Shared` is divisible by 2.
// Buffer storage strategy flags.
const KIND_ARC: usize = 0b0;
const KIND_VEC: usize = 0b1;
const KIND_MASK: usize = 0b1;
// The max original capacity value. Any `Bytes` allocated with a greater initial
// capacity will default to this.
const MAX_ORIGINAL_CAPACITY_WIDTH: usize = 17;
// The original capacity algorithm will not take effect unless the originally
// allocated capacity was at least 1kb in size.
const MIN_ORIGINAL_CAPACITY_WIDTH: usize = 10;
// The original capacity is stored in powers of 2 starting at 1kb to a max of
// 64kb. Representing it as such requires only 3 bits of storage.
const ORIGINAL_CAPACITY_MASK: usize = 0b11100;
const ORIGINAL_CAPACITY_OFFSET: usize = 2;
const VEC_POS_OFFSET: usize = 5;
// When the storage is in the `Vec` representation, the pointer can be advanced
// at most this value. This is due to the amount of storage available to track
// the offset is usize - number of KIND bits and number of ORIGINAL_CAPACITY
// bits.
const MAX_VEC_POS: usize = usize::MAX >> VEC_POS_OFFSET;
const NOT_VEC_POS_MASK: usize = 0b11111;
#[cfg(target_pointer_width = "64")]
const PTR_WIDTH: usize = 64;
#[cfg(target_pointer_width = "32")]
const PTR_WIDTH: usize = 32;
/*
*
* ===== BytesMut =====
*
*/
impl BytesMut {
/// Creates a new `BytesMut` with the specified capacity.
///
/// The returned `BytesMut` will be able to hold at least `capacity` bytes
/// without reallocating.
///
/// It is important to note that this function does not specify the length
/// of the returned `BytesMut`, but only the capacity.
///
/// # Examples
///
/// ```
/// use bytes::{BytesMut, BufMut};
///
/// let mut bytes = BytesMut::with_capacity(64);
///
/// // `bytes` contains no data, even though there is capacity
/// assert_eq!(bytes.len(), 0);
///
/// bytes.put(&b"hello world"[..]);
///
/// assert_eq!(&bytes[..], b"hello world");
/// ```
#[inline]
pub fn with_capacity(capacity: usize) -> BytesMut {
BytesMut::from_vec(Vec::with_capacity(capacity))
}
/// Creates a new `BytesMut` with default capacity.
///
/// Resulting object has length 0 and unspecified capacity.
/// This function does not allocate.
///
/// # Examples
///
/// ```
/// use bytes::{BytesMut, BufMut};
///
/// let mut bytes = BytesMut::new();
///
/// assert_eq!(0, bytes.len());
///
/// bytes.reserve(2);
/// bytes.put_slice(b"xy");
///
/// assert_eq!(&b"xy"[..], &bytes[..]);
/// ```
#[inline]
pub fn new() -> BytesMut {
BytesMut::with_capacity(0)
}
/// Returns the number of bytes contained in this `BytesMut`.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let b = BytesMut::from(&b"hello"[..]);
/// assert_eq!(b.len(), 5);
/// ```
#[inline]
pub fn len(&self) -> usize {
self.len
}
/// Returns true if the `BytesMut` has a length of 0.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let b = BytesMut::with_capacity(64);
/// assert!(b.is_empty());
/// ```
#[inline]
pub fn is_empty(&self) -> bool {
self.len == 0
}
/// Returns the number of bytes the `BytesMut` can hold without reallocating.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let b = BytesMut::with_capacity(64);
/// assert_eq!(b.capacity(), 64);
/// ```
#[inline]
pub fn capacity(&self) -> usize {
self.cap
}
/// Converts `self` into an immutable `Bytes`.
///
/// The conversion is zero cost and is used to indicate that the slice
/// referenced by the handle will no longer be mutated. Once the conversion
/// is done, the handle can be cloned and shared across threads.
///
/// # Examples
///
/// ```
/// use bytes::{BytesMut, BufMut};
/// use std::thread;
///
/// let mut b = BytesMut::with_capacity(64);
/// b.put(&b"hello world"[..]);
/// let b1 = b.freeze();
/// let b2 = b1.clone();
///
/// let th = thread::spawn(move || {
/// assert_eq!(&b1[..], b"hello world");
/// });
///
/// assert_eq!(&b2[..], b"hello world");
/// th.join().unwrap();
/// ```
#[inline]
pub fn freeze(self) -> Bytes {
let bytes = ManuallyDrop::new(self);
if bytes.kind() == KIND_VEC {
// Just re-use `Bytes` internal Vec vtable
unsafe {
let off = bytes.get_vec_pos();
let vec = rebuild_vec(bytes.ptr.as_ptr(), bytes.len, bytes.cap, off);
let mut b: Bytes = vec.into();
b.advance(off);
b
}
} else {
debug_assert_eq!(bytes.kind(), KIND_ARC);
let ptr = bytes.ptr.as_ptr();
let len = bytes.len;
let data = AtomicPtr::new(bytes.data.cast());
unsafe { Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE) }
}
}
/// Creates a new `BytesMut` containing `len` zeros.
///
/// The resulting object has a length of `len` and a capacity greater
/// than or equal to `len`. The entire length of the object will be filled
/// with zeros.
///
/// On some platforms or allocators this function may be faster than
/// a manual implementation.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let zeros = BytesMut::zeroed(42);
///
/// assert!(zeros.capacity() >= 42);
/// assert_eq!(zeros.len(), 42);
/// zeros.into_iter().for_each(|x| assert_eq!(x, 0));
/// ```
pub fn zeroed(len: usize) -> BytesMut {
BytesMut::from_vec(vec![0; len])
}
/// Splits the bytes into two at the given index.
///
/// Afterwards `self` contains elements `[0, at)`, and the returned
/// `BytesMut` contains elements `[at, capacity)`.
///
/// This is an `O(1)` operation that just increases the reference count
/// and sets a few indices.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let mut a = BytesMut::from(&b"hello world"[..]);
/// let mut b = a.split_off(5);
///
/// a[0] = b'j';
/// b[0] = b'!';
///
/// assert_eq!(&a[..], b"jello");
/// assert_eq!(&b[..], b"!world");
/// ```
///
/// # Panics
///
/// Panics if `at > capacity`.
#[must_use = "consider BytesMut::truncate if you don't need the other half"]
pub fn split_off(&mut self, at: usize) -> BytesMut {
assert!(
at <= self.capacity(),
"split_off out of bounds: {:?} <= {:?}",
at,
self.capacity(),
);
unsafe {
let mut other = self.shallow_clone();
// SAFETY: We've checked that `at` <= `self.capacity()` above.
other.advance_unchecked(at);
self.cap = at;
self.len = cmp::min(self.len, at);
other
}
}
/// Removes the bytes from the current view, returning them in a new
/// `BytesMut` handle.
///
/// Afterwards, `self` will be empty, but will retain any additional
/// capacity that it had before the operation. This is identical to
/// `self.split_to(self.len())`.
///
/// This is an `O(1)` operation that just increases the reference count and
/// sets a few indices.
///
/// # Examples
///
/// ```
/// use bytes::{BytesMut, BufMut};
///
/// let mut buf = BytesMut::with_capacity(1024);
/// buf.put(&b"hello world"[..]);
///
/// let other = buf.split();
///
/// assert!(buf.is_empty());
/// assert_eq!(1013, buf.capacity());
///
/// assert_eq!(other, b"hello world"[..]);
/// ```
#[must_use = "consider BytesMut::clear if you don't need the other half"]
pub fn split(&mut self) -> BytesMut {
let len = self.len();
self.split_to(len)
}
/// Splits the buffer into two at the given index.
///
/// Afterwards `self` contains elements `[at, len)`, and the returned `BytesMut`
/// contains elements `[0, at)`.
///
/// This is an `O(1)` operation that just increases the reference count and
/// sets a few indices.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let mut a = BytesMut::from(&b"hello world"[..]);
/// let mut b = a.split_to(5);
///
/// a[0] = b'!';
/// b[0] = b'j';
///
/// assert_eq!(&a[..], b"!world");
/// assert_eq!(&b[..], b"jello");
/// ```
///
/// # Panics
///
/// Panics if `at > len`.
#[must_use = "consider BytesMut::advance if you don't need the other half"]
pub fn split_to(&mut self, at: usize) -> BytesMut {
assert!(
at <= self.len(),
"split_to out of bounds: {:?} <= {:?}",
at,
self.len(),
);
unsafe {
let mut other = self.shallow_clone();
// SAFETY: We've checked that `at` <= `self.len()` and we know that `self.len()` <=
// `self.capacity()`.
self.advance_unchecked(at);
other.cap = at;
other.len = at;
other
}
}
/// Shortens the buffer, keeping the first `len` bytes and dropping the
/// rest.
///
/// If `len` is greater than the buffer's current length, this has no
/// effect.
///
/// Existing underlying capacity is preserved.
///
/// The [split_off](`Self::split_off()`) method can emulate `truncate`, but this causes the
/// excess bytes to be returned instead of dropped.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let mut buf = BytesMut::from(&b"hello world"[..]);
/// buf.truncate(5);
/// assert_eq!(buf, b"hello"[..]);
/// ```
pub fn truncate(&mut self, len: usize) {
if len <= self.len() {
// SAFETY: Shrinking the buffer cannot expose uninitialized bytes.
unsafe { self.set_len(len) };
}
}
/// Clears the buffer, removing all data. Existing capacity is preserved.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let mut buf = BytesMut::from(&b"hello world"[..]);
/// buf.clear();
/// assert!(buf.is_empty());
/// ```
pub fn clear(&mut self) {
// SAFETY: Setting the length to zero cannot expose uninitialized bytes.
unsafe { self.set_len(0) };
}
/// Resizes the buffer so that `len` is equal to `new_len`.
///
/// If `new_len` is greater than `len`, the buffer is extended by the
/// difference with each additional byte set to `value`. If `new_len` is
/// less than `len`, the buffer is simply truncated.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let mut buf = BytesMut::new();
///
/// buf.resize(3, 0x1);
/// assert_eq!(&buf[..], &[0x1, 0x1, 0x1]);
///
/// buf.resize(2, 0x2);
/// assert_eq!(&buf[..], &[0x1, 0x1]);
///
/// buf.resize(4, 0x3);
/// assert_eq!(&buf[..], &[0x1, 0x1, 0x3, 0x3]);
/// ```
pub fn resize(&mut self, new_len: usize, value: u8) {
let additional = if let Some(additional) = new_len.checked_sub(self.len()) {
additional
} else {
self.truncate(new_len);
return;
};
if additional == 0 {
return;
}
self.reserve(additional);
let dst = self.spare_capacity_mut().as_mut_ptr();
// SAFETY: `spare_capacity_mut` returns a valid, properly aligned pointer and we've
// reserved enough space to write `additional` bytes.
unsafe { ptr::write_bytes(dst, value, additional) };
// SAFETY: There are at least `new_len` initialized bytes in the buffer so no
// uninitialized bytes are being exposed.
unsafe { self.set_len(new_len) };
}
/// Sets the length of the buffer.
///
/// This will explicitly set the size of the buffer without actually
/// modifying the data, so it is up to the caller to ensure that the data
/// has been initialized.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let mut b = BytesMut::from(&b"hello world"[..]);
///
/// unsafe {
/// b.set_len(5);
/// }
///
/// assert_eq!(&b[..], b"hello");
///
/// unsafe {
/// b.set_len(11);
/// }
///
/// assert_eq!(&b[..], b"hello world");
/// ```
#[inline]
pub unsafe fn set_len(&mut self, len: usize) {
debug_assert!(len <= self.cap, "set_len out of bounds");
self.len = len;
}
/// Reserves capacity for at least `additional` more bytes to be inserted
/// into the given `BytesMut`.
///
/// More than `additional` bytes may be reserved in order to avoid frequent
/// reallocations. A call to `reserve` may result in an allocation.
///
/// Before allocating new buffer space, the function will attempt to reclaim
/// space in the existing buffer. If the current handle references a view
/// into a larger original buffer, and all other handles referencing part
/// of the same original buffer have been dropped, then the current view
/// can be copied/shifted to the front of the buffer and the handle can take
/// ownership of the full buffer, provided that the full buffer is large
/// enough to fit the requested additional capacity.
///
/// This optimization will only happen if shifting the data from the current
/// view to the front of the buffer is not too expensive in terms of the
/// (amortized) time required. The precise condition is subject to change;
/// as of now, the length of the data being shifted needs to be at least as
/// large as the distance that it's shifted by. If the current view is empty
/// and the original buffer is large enough to fit the requested additional
/// capacity, then reallocations will never happen.
///
/// # Examples
///
/// In the following example, a new buffer is allocated.
///
/// ```
/// use bytes::BytesMut;
///
/// let mut buf = BytesMut::from(&b"hello"[..]);
/// buf.reserve(64);
/// assert!(buf.capacity() >= 69);
/// ```
///
/// In the following example, the existing buffer is reclaimed.
///
/// ```
/// use bytes::{BytesMut, BufMut};
///
/// let mut buf = BytesMut::with_capacity(128);
/// buf.put(&[0; 64][..]);
///
/// let ptr = buf.as_ptr();
/// let other = buf.split();
///
/// assert!(buf.is_empty());
/// assert_eq!(buf.capacity(), 64);
///
/// drop(other);
/// buf.reserve(128);
///
/// assert_eq!(buf.capacity(), 128);
/// assert_eq!(buf.as_ptr(), ptr);
/// ```
///
/// # Panics
///
/// Panics if the new capacity overflows `usize`.
#[inline]
pub fn reserve(&mut self, additional: usize) {
let len = self.len();
let rem = self.capacity() - len;
if additional <= rem {
// The handle can already store at least `additional` more bytes, so
// there is no further work needed to be done.
return;
}
// will always succeed
let _ = self.reserve_inner(additional, true);
}
// In separate function to allow the short-circuits in `reserve` and `try_reclaim` to
// be inline-able. Significantly helps performance. Returns false if it did not succeed.
fn reserve_inner(&mut self, additional: usize, allocate: bool) -> bool {
let len = self.len();
let kind = self.kind();
if kind == KIND_VEC {
// If there's enough free space before the start of the buffer, then
// just copy the data backwards and reuse the already-allocated
// space.
//
// Otherwise, since backed by a vector, use `Vec::reserve`
//
// We need to make sure that this optimization does not kill the
// amortized runtimes of BytesMut's operations.
unsafe {
let off = self.get_vec_pos();
// Only reuse space if we can satisfy the requested additional space.
//
// Also check if the value of `off` suggests that enough bytes
// have been read to account for the overhead of shifting all
// the data (in an amortized analysis).
// Hence the condition `off >= self.len()`.
//
// This condition also already implies that the buffer is going
// to be (at least) half-empty in the end; so we do not break
// the (amortized) runtime with future resizes of the underlying
// `Vec`.
//
// [For more details check issue #524, and PR #525.]
if self.capacity() - self.len() + off >= additional && off >= self.len() {
// There's enough space, and it's not too much overhead:
// reuse the space!
//
// Just move the pointer back to the start after copying
// data back.
let base_ptr = self.ptr.as_ptr().sub(off);
// Since `off >= self.len()`, the two regions don't overlap.
ptr::copy_nonoverlapping(self.ptr.as_ptr(), base_ptr, self.len);
self.ptr = vptr(base_ptr);
self.set_vec_pos(0);
// Length stays constant, but since we moved backwards we
// can gain capacity back.
self.cap += off;
} else {
if !allocate {
return false;
}
// Not enough space, or reusing might be too much overhead:
// allocate more space!
let mut v =
ManuallyDrop::new(rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off));
v.reserve(additional);
// Update the info
self.ptr = vptr(v.as_mut_ptr().add(off));
self.cap = v.capacity() - off;
debug_assert_eq!(self.len, v.len() - off);
}
return true;
}
}
debug_assert_eq!(kind, KIND_ARC);
let shared: *mut Shared = self.data;
// Reserving involves abandoning the currently shared buffer and
// allocating a new vector with the requested capacity.
//
// Compute the new capacity
let mut new_cap = match len.checked_add(additional) {
Some(new_cap) => new_cap,
None if !allocate => return false,
None => panic!("overflow"),
};
unsafe {
// First, try to reclaim the buffer. This is possible if the current
// handle is the only outstanding handle pointing to the buffer.
if (*shared).is_unique() {
// This is the only handle to the buffer. It can be reclaimed.
// However, before doing the work of copying data, check to make
// sure that the vector has enough capacity.
let v = &mut (*shared).vec;
let v_capacity = v.capacity();
let ptr = v.as_mut_ptr();
let offset = offset_from(self.ptr.as_ptr(), ptr);
// Compare the condition in the `kind == KIND_VEC` case above
// for more details.
if v_capacity >= new_cap + offset {
self.cap = new_cap;
// no copy is necessary
} else if v_capacity >= new_cap && offset >= len {
// The capacity is sufficient, and copying is not too much
// overhead: reclaim the buffer!
// `offset >= len` means: no overlap
ptr::copy_nonoverlapping(self.ptr.as_ptr(), ptr, len);
self.ptr = vptr(ptr);
self.cap = v.capacity();
} else {
if !allocate {
return false;
}
// calculate offset
let off = (self.ptr.as_ptr() as usize) - (v.as_ptr() as usize);
// new_cap is calculated in terms of `BytesMut`, not the underlying
// `Vec`, so it does not take the offset into account.
//
// Thus we have to manually add it here.
new_cap = new_cap.checked_add(off).expect("overflow");
// The vector capacity is not sufficient. The reserve request is
// asking for more than the initial buffer capacity. Allocate more
// than requested if `new_cap` is not much bigger than the current
// capacity.
//
// There are some situations, using `reserve_exact` that the
// buffer capacity could be below `original_capacity`, so do a
// check.
let double = v.capacity().checked_shl(1).unwrap_or(new_cap);
new_cap = cmp::max(double, new_cap);
// No space - allocate more
//
// The length field of `Shared::vec` is not used by the `BytesMut`;
// instead we use the `len` field in the `BytesMut` itself. However,
// when calling `reserve`, it doesn't guarantee that data stored in
// the unused capacity of the vector is copied over to the new
// allocation, so we need to ensure that we don't have any data we
// care about in the unused capacity before calling `reserve`.
debug_assert!(off + len <= v.capacity());
v.set_len(off + len);
v.reserve(new_cap - v.len());
// Update the info
self.ptr = vptr(v.as_mut_ptr().add(off));
self.cap = v.capacity() - off;
}
return true;
}
}
if !allocate {
return false;
}
let original_capacity_repr = unsafe { (*shared).original_capacity_repr };
let original_capacity = original_capacity_from_repr(original_capacity_repr);
new_cap = cmp::max(new_cap, original_capacity);
// Create a new vector to store the data
let mut v = ManuallyDrop::new(Vec::with_capacity(new_cap));
// Copy the bytes
v.extend_from_slice(self.as_ref());
// Release the shared handle. This must be done *after* the bytes are
// copied.
unsafe { release_shared(shared) };
// Update self
let data = (original_capacity_repr << ORIGINAL_CAPACITY_OFFSET) | KIND_VEC;
self.data = invalid_ptr(data);
self.ptr = vptr(v.as_mut_ptr());
self.cap = v.capacity();
debug_assert_eq!(self.len, v.len());
return true;
}
/// Attempts to cheaply reclaim already allocated capacity for at least `additional` more
/// bytes to be inserted into the given `BytesMut` and returns `true` if it succeeded.
///
/// `try_reclaim` behaves exactly like `reserve`, except that it never allocates new storage
/// and returns a `bool` indicating whether it was successful in doing so:
///
/// `try_reclaim` returns false under these conditions:
/// - The spare capacity left is less than `additional` bytes AND
/// - The existing allocation cannot be reclaimed cheaply or it was less than
/// `additional` bytes in size
///
/// Reclaiming the allocation cheaply is possible if the `BytesMut` has no outstanding
/// references through other `BytesMut`s or `Bytes` which point to the same underlying
/// storage.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let mut buf = BytesMut::with_capacity(64);
/// assert_eq!(true, buf.try_reclaim(64));
/// assert_eq!(64, buf.capacity());
///
/// buf.extend_from_slice(b"abcd");
/// let mut split = buf.split();
/// assert_eq!(60, buf.capacity());
/// assert_eq!(4, split.capacity());
/// assert_eq!(false, split.try_reclaim(64));
/// assert_eq!(false, buf.try_reclaim(64));
/// // The split buffer is filled with "abcd"
/// assert_eq!(false, split.try_reclaim(4));
/// // buf is empty and has capacity for 60 bytes
/// assert_eq!(true, buf.try_reclaim(60));
///
/// drop(buf);
/// assert_eq!(false, split.try_reclaim(64));
///
/// split.clear();
/// assert_eq!(4, split.capacity());
/// assert_eq!(true, split.try_reclaim(64));
/// assert_eq!(64, split.capacity());
/// ```
// I tried splitting out try_reclaim_inner after the short circuits, but it was inlined
// regardless with Rust 1.78.0 so probably not worth it
#[inline]
#[must_use = "consider BytesMut::reserve if you need an infallible reservation"]
pub fn try_reclaim(&mut self, additional: usize) -> bool {
let len = self.len();
let rem = self.capacity() - len;
if additional <= rem {
// The handle can already store at least `additional` more bytes, so
// there is no further work needed to be done.
return true;
}
self.reserve_inner(additional, false)
}
/// Appends given bytes to this `BytesMut`.
///
/// If this `BytesMut` object does not have enough capacity, it is resized
/// first.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let mut buf = BytesMut::with_capacity(0);
/// buf.extend_from_slice(b"aaabbb");
/// buf.extend_from_slice(b"cccddd");
///
/// assert_eq!(b"aaabbbcccddd", &buf[..]);
/// ```
#[inline]
pub fn extend_from_slice(&mut self, extend: &[u8]) {
let cnt = extend.len();
self.reserve(cnt);
unsafe {
let dst = self.spare_capacity_mut();
// Reserved above
debug_assert!(dst.len() >= cnt);
ptr::copy_nonoverlapping(extend.as_ptr(), dst.as_mut_ptr().cast(), cnt);
}
unsafe {
self.advance_mut(cnt);
}
}
/// Absorbs a `BytesMut` that was previously split off.
///
/// If the two `BytesMut` objects were previously contiguous and not mutated
/// in a way that causes re-allocation i.e., if `other` was created by
/// calling `split_off` on this `BytesMut`, then this is an `O(1)` operation
/// that just decreases a reference count and sets a few indices.
/// Otherwise this method degenerates to
/// `self.extend_from_slice(other.as_ref())`.
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// let mut buf = BytesMut::with_capacity(64);
/// buf.extend_from_slice(b"aaabbbcccddd");
///
/// let split = buf.split_off(6);
/// assert_eq!(b"aaabbb", &buf[..]);
/// assert_eq!(b"cccddd", &split[..]);
///
/// buf.unsplit(split);
/// assert_eq!(b"aaabbbcccddd", &buf[..]);
/// ```
pub fn unsplit(&mut self, other: BytesMut) {
if self.is_empty() {
*self = other;
return;
}
if let Err(other) = self.try_unsplit(other) {
self.extend_from_slice(other.as_ref());
}
}
// private
// For now, use a `Vec` to manage the memory for us, but we may want to
// change that in the future to some alternate allocator strategy.
//
// Thus, we don't expose an easy way to construct from a `Vec` since an
// internal change could make a simple pattern (`BytesMut::from(vec)`)
// suddenly a lot more expensive.
#[inline]
pub(crate) fn from_vec(vec: Vec<u8>) -> BytesMut {
let mut vec = ManuallyDrop::new(vec);
let ptr = vptr(vec.as_mut_ptr());
let len = vec.len();
let cap = vec.capacity();
let original_capacity_repr = original_capacity_to_repr(cap);
let data = (original_capacity_repr << ORIGINAL_CAPACITY_OFFSET) | KIND_VEC;
BytesMut {
ptr,
len,
cap,
data: invalid_ptr(data),
}
}
#[inline]
fn as_slice(&self) -> &[u8] {
unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len) }
}
#[inline]
fn as_slice_mut(&mut self) -> &mut [u8] {
unsafe { slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) }
}
/// Advance the buffer without bounds checking.
///
/// # SAFETY
///
/// The caller must ensure that `count` <= `self.cap`.
pub(crate) unsafe fn advance_unchecked(&mut self, count: usize) {
// Setting the start to 0 is a no-op, so return early if this is the
// case.
if count == 0 {
return;
}
debug_assert!(count <= self.cap, "internal: set_start out of bounds");
let kind = self.kind();
if kind == KIND_VEC {
// Setting the start when in vec representation is a little more
// complicated. First, we have to track how far ahead the
// "start" of the byte buffer from the beginning of the vec. We
// also have to ensure that we don't exceed the maximum shift.
let pos = self.get_vec_pos() + count;
if pos <= MAX_VEC_POS {
self.set_vec_pos(pos);
} else {
// The repr must be upgraded to ARC. This will never happen
// on 64 bit systems and will only happen on 32 bit systems
// when shifting past 134,217,727 bytes. As such, we don't
// worry too much about performance here.
self.promote_to_shared(/*ref_count = */ 1);
}
}
// Updating the start of the view is setting `ptr` to point to the
// new start and updating the `len` field to reflect the new length
// of the view.
self.ptr = vptr(self.ptr.as_ptr().add(count));
self.len = self.len.checked_sub(count).unwrap_or(0);
self.cap -= count;
}
fn try_unsplit(&mut self, other: BytesMut) -> Result<(), BytesMut> {
if other.capacity() == 0 {
return Ok(());
}
let ptr = unsafe { self.ptr.as_ptr().add(self.len) };
if ptr == other.ptr.as_ptr()
&& self.kind() == KIND_ARC
&& other.kind() == KIND_ARC
&& self.data == other.data
{
// Contiguous blocks, just combine directly
self.len += other.len;
self.cap += other.cap;
Ok(())
} else {
Err(other)
}
}
#[inline]
fn kind(&self) -> usize {
self.data as usize & KIND_MASK
}
unsafe fn promote_to_shared(&mut self, ref_cnt: usize) {
debug_assert_eq!(self.kind(), KIND_VEC);
debug_assert!(ref_cnt == 1 || ref_cnt == 2);
let original_capacity_repr =
(self.data as usize & ORIGINAL_CAPACITY_MASK) >> ORIGINAL_CAPACITY_OFFSET;
// The vec offset cannot be concurrently mutated, so there
// should be no danger reading it.
let off = (self.data as usize) >> VEC_POS_OFFSET;
// First, allocate a new `Shared` instance containing the
// `Vec` fields. It's important to note that `ptr`, `len`,
// and `cap` cannot be mutated without having `&mut self`.
// This means that these fields will not be concurrently
// updated and since the buffer hasn't been promoted to an
// `Arc`, those three fields still are the components of the
// vector.
let shared = Box::new(Shared {
vec: rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off),
original_capacity_repr,
ref_count: AtomicUsize::new(ref_cnt),
});
let shared = Box::into_raw(shared);
// The pointer should be aligned, so this assert should
// always succeed.
debug_assert_eq!(shared as usize & KIND_MASK, KIND_ARC);
self.data = shared;
}
/// Makes an exact shallow clone of `self`.
///
/// The kind of `self` doesn't matter, but this is unsafe
/// because the clone will have the same offsets. You must
/// be sure the returned value to the user doesn't allow
/// two views into the same range.
#[inline]
unsafe fn shallow_clone(&mut self) -> BytesMut {
if self.kind() == KIND_ARC {
increment_shared(self.data);
ptr::read(self)
} else {
self.promote_to_shared(/*ref_count = */ 2);
ptr::read(self)
}
}
#[inline]
unsafe fn get_vec_pos(&self) -> usize {
debug_assert_eq!(self.kind(), KIND_VEC);
self.data as usize >> VEC_POS_OFFSET
}
#[inline]
unsafe fn set_vec_pos(&mut self, pos: usize) {
debug_assert_eq!(self.kind(), KIND_VEC);
debug_assert!(pos <= MAX_VEC_POS);
self.data = invalid_ptr((pos << VEC_POS_OFFSET) | (self.data as usize & NOT_VEC_POS_MASK));
}
/// Returns the remaining spare capacity of the buffer as a slice of `MaybeUninit<u8>`.
///
/// The returned slice can be used to fill the buffer with data (e.g. by
/// reading from a file) before marking the data as initialized using the
/// [`set_len`] method.
///
/// [`set_len`]: BytesMut::set_len
///
/// # Examples
///
/// ```
/// use bytes::BytesMut;
///
/// // Allocate buffer big enough for 10 bytes.
/// let mut buf = BytesMut::with_capacity(10);
///
/// // Fill in the first 3 elements.
/// let uninit = buf.spare_capacity_mut();
/// uninit[0].write(0);
/// uninit[1].write(1);
/// uninit[2].write(2);
///
/// // Mark the first 3 bytes of the buffer as being initialized.
/// unsafe {
/// buf.set_len(3);
/// }
///
/// assert_eq!(&buf[..], &[0, 1, 2]);
/// ```
#[inline]
pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit<u8>] {
unsafe {
let ptr = self.ptr.as_ptr().add(self.len);
let len = self.cap - self.len;
slice::from_raw_parts_mut(ptr.cast(), len)
}
}
}
impl Drop for BytesMut {
fn drop(&mut self) {
let kind = self.kind();
if kind == KIND_VEC {
unsafe {
let off = self.get_vec_pos();
// Vector storage, free the vector
let _ = rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off);
}
} else if kind == KIND_ARC {
unsafe { release_shared(self.data) };
}
}
}
impl Buf for BytesMut {
#[inline]
fn remaining(&self) -> usize {
self.len()
}
#[inline]
fn chunk(&self) -> &[u8] {
self.as_slice()
}
#[inline]
fn advance(&mut self, cnt: usize) {
assert!(
cnt <= self.remaining(),
"cannot advance past `remaining`: {:?} <= {:?}",
cnt,
self.remaining(),
);
unsafe {
// SAFETY: We've checked that `cnt` <= `self.remaining()` and we know that
// `self.remaining()` <= `self.cap`.
self.advance_unchecked(cnt);
}
}
fn copy_to_bytes(&mut self, len: usize) -> Bytes {
self.split_to(len).freeze()
}
}
unsafe impl BufMut for BytesMut {
#[inline]
fn remaining_mut(&self) -> usize {
usize::MAX - self.len()
}
#[inline]
unsafe fn advance_mut(&mut self, cnt: usize) {
let remaining = self.cap - self.len();
if cnt > remaining {
super::panic_advance(cnt, remaining);
}
// Addition won't overflow since it is at most `self.cap`.
self.len = self.len() + cnt;
}
#[inline]
fn chunk_mut(&mut self) -> &mut UninitSlice {
if self.capacity() == self.len() {
self.reserve(64);
}
self.spare_capacity_mut().into()
}
// Specialize these methods so they can skip checking `remaining_mut`
// and `advance_mut`.
fn put<T: Buf>(&mut self, mut src: T)
where
Self: Sized,
{
while src.has_remaining() {
let s = src.chunk();
let l = s.len();
self.extend_from_slice(s);
src.advance(l);
}
}
fn put_slice(&mut self, src: &[u8]) {
self.extend_from_slice(src);
}
fn put_bytes(&mut self, val: u8, cnt: usize) {
self.reserve(cnt);
unsafe {
let dst = self.spare_capacity_mut();
// Reserved above
debug_assert!(dst.len() >= cnt);
ptr::write_bytes(dst.as_mut_ptr(), val, cnt);
self.advance_mut(cnt);
}
}
}
impl AsRef<[u8]> for BytesMut {
#[inline]
fn as_ref(&self) -> &[u8] {
self.as_slice()
}
}
impl Deref for BytesMut {
type Target = [u8];
#[inline]
fn deref(&self) -> &[u8] {
self.as_ref()
}
}
impl AsMut<[u8]> for BytesMut {
#[inline]
fn as_mut(&mut self) -> &mut [u8] {
self.as_slice_mut()
}
}
impl DerefMut for BytesMut {
#[inline]
fn deref_mut(&mut self) -> &mut [u8] {
self.as_mut()
}
}
impl<'a> From<&'a [u8]> for BytesMut {
fn from(src: &'a [u8]) -> BytesMut {
BytesMut::from_vec(src.to_vec())
}
}
impl<'a> From<&'a str> for BytesMut {
fn from(src: &'a str) -> BytesMut {
BytesMut::from(src.as_bytes())
}
}
impl From<BytesMut> for Bytes {
fn from(src: BytesMut) -> Bytes {
src.freeze()
}
}
impl PartialEq for BytesMut {
fn eq(&self, other: &BytesMut) -> bool {
self.as_slice() == other.as_slice()
}
}
impl PartialOrd for BytesMut {
fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> {
self.as_slice().partial_cmp(other.as_slice())
}
}
impl Ord for BytesMut {
fn cmp(&self, other: &BytesMut) -> cmp::Ordering {
self.as_slice().cmp(other.as_slice())
}
}
impl Eq for BytesMut {}
impl Default for BytesMut {
#[inline]
fn default() -> BytesMut {
BytesMut::new()
}
}
impl hash::Hash for BytesMut {
fn hash<H>(&self, state: &mut H)
where
H: hash::Hasher,
{
let s: &[u8] = self.as_ref();
s.hash(state);
}
}
impl Borrow<[u8]> for BytesMut {
fn borrow(&self) -> &[u8] {
self.as_ref()
}
}
impl BorrowMut<[u8]> for BytesMut {
fn borrow_mut(&mut self) -> &mut [u8] {
self.as_mut()
}
}
impl fmt::Write for BytesMut {
#[inline]
fn write_str(&mut self, s: &str) -> fmt::Result {
if self.remaining_mut() >= s.len() {
self.put_slice(s.as_bytes());
Ok(())
} else {
Err(fmt::Error)
}
}
#[inline]
fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> fmt::Result {
fmt::write(self, args)
}
}
impl Clone for BytesMut {
fn clone(&self) -> BytesMut {
BytesMut::from(&self[..])
}
}
impl IntoIterator for BytesMut {
type Item = u8;
type IntoIter = IntoIter<BytesMut>;
fn into_iter(self) -> Self::IntoIter {
IntoIter::new(self)
}
}
impl<'a> IntoIterator for &'a BytesMut {
type Item = &'a u8;
type IntoIter = core::slice::Iter<'a, u8>;
fn into_iter(self) -> Self::IntoIter {
self.as_ref().iter()
}
}
impl Extend<u8> for BytesMut {
fn extend<T>(&mut self, iter: T)
where
T: IntoIterator<Item = u8>,
{
let iter = iter.into_iter();
let (lower, _) = iter.size_hint();
self.reserve(lower);
// TODO: optimize
// 1. If self.kind() == KIND_VEC, use Vec::extend
for b in iter {
self.put_u8(b);
}
}
}
impl<'a> Extend<&'a u8> for BytesMut {
fn extend<T>(&mut self, iter: T)
where
T: IntoIterator<Item = &'a u8>,
{
self.extend(iter.into_iter().copied())
}
}
impl Extend<Bytes> for BytesMut {
fn extend<T>(&mut self, iter: T)
where
T: IntoIterator<Item = Bytes>,
{
for bytes in iter {
self.extend_from_slice(&bytes)
}
}
}
impl FromIterator<u8> for BytesMut {
fn from_iter<T: IntoIterator<Item = u8>>(into_iter: T) -> Self {
BytesMut::from_vec(Vec::from_iter(into_iter))
}
}
impl<'a> FromIterator<&'a u8> for BytesMut {
fn from_iter<T: IntoIterator<Item = &'a u8>>(into_iter: T) -> Self {
BytesMut::from_iter(into_iter.into_iter().copied())
}
}
/*
*
* ===== Inner =====
*
*/
unsafe fn increment_shared(ptr: *mut Shared) {
let old_size = (*ptr).ref_count.fetch_add(1, Ordering::Relaxed);
if old_size > isize::MAX as usize {
crate::abort();
}
}
unsafe fn release_shared(ptr: *mut Shared) {
// `Shared` storage... follow the drop steps from Arc.
if (*ptr).ref_count.fetch_sub(1, Ordering::Release) != 1 {
return;
}
// This fence is needed to prevent reordering of use of the data and
// deletion of the data. Because it is marked `Release`, the decreasing
// of the reference count synchronizes with this `Acquire` fence. This
// means that use of the data happens before decreasing the reference
// count, which happens before this fence, which happens before the
// deletion of the data.
//
// As explained in the [Boost documentation][1],
//
// > It is important to enforce any possible access to the object in one
// > thread (through an existing reference) to *happen before* deleting
// > the object in a different thread. This is achieved by a "release"
// > operation after dropping a reference (any access to the object
// > through this reference must obviously happened before), and an
// > "acquire" operation before deleting the object.
//
// [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
//
// Thread sanitizer does not support atomic fences. Use an atomic load
// instead.
(*ptr).ref_count.load(Ordering::Acquire);
// Drop the data
drop(Box::from_raw(ptr));
}
impl Shared {
fn is_unique(&self) -> bool {
// The goal is to check if the current handle is the only handle
// that currently has access to the buffer. This is done by
// checking if the `ref_count` is currently 1.
//
// The `Acquire` ordering synchronizes with the `Release` as
// part of the `fetch_sub` in `release_shared`. The `fetch_sub`
// operation guarantees that any mutations done in other threads
// are ordered before the `ref_count` is decremented. As such,
// this `Acquire` will guarantee that those mutations are
// visible to the current thread.
self.ref_count.load(Ordering::Acquire) == 1
}
}
#[inline]
fn original_capacity_to_repr(cap: usize) -> usize {
let width = PTR_WIDTH - ((cap >> MIN_ORIGINAL_CAPACITY_WIDTH).leading_zeros() as usize);
cmp::min(
width,
MAX_ORIGINAL_CAPACITY_WIDTH - MIN_ORIGINAL_CAPACITY_WIDTH,
)
}
fn original_capacity_from_repr(repr: usize) -> usize {
if repr == 0 {
return 0;
}
1 << (repr + (MIN_ORIGINAL_CAPACITY_WIDTH - 1))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_original_capacity_to_repr() {
assert_eq!(original_capacity_to_repr(0), 0);
let max_width = 32;
for width in 1..(max_width + 1) {
let cap = 1 << width - 1;
let expected = if width < MIN_ORIGINAL_CAPACITY_WIDTH {
0
} else if width < MAX_ORIGINAL_CAPACITY_WIDTH {
width - MIN_ORIGINAL_CAPACITY_WIDTH
} else {
MAX_ORIGINAL_CAPACITY_WIDTH - MIN_ORIGINAL_CAPACITY_WIDTH
};
assert_eq!(original_capacity_to_repr(cap), expected);
if width > 1 {
assert_eq!(original_capacity_to_repr(cap + 1), expected);
}
// MIN_ORIGINAL_CAPACITY_WIDTH must be bigger than 7 to pass tests below
if width == MIN_ORIGINAL_CAPACITY_WIDTH + 1 {
assert_eq!(original_capacity_to_repr(cap - 24), expected - 1);
assert_eq!(original_capacity_to_repr(cap + 76), expected);
} else if width == MIN_ORIGINAL_CAPACITY_WIDTH + 2 {
assert_eq!(original_capacity_to_repr(cap - 1), expected - 1);
assert_eq!(original_capacity_to_repr(cap - 48), expected - 1);
}
}
}
#[test]
fn test_original_capacity_from_repr() {
assert_eq!(0, original_capacity_from_repr(0));
let min_cap = 1 << MIN_ORIGINAL_CAPACITY_WIDTH;
assert_eq!(min_cap, original_capacity_from_repr(1));
assert_eq!(min_cap * 2, original_capacity_from_repr(2));
assert_eq!(min_cap * 4, original_capacity_from_repr(3));
assert_eq!(min_cap * 8, original_capacity_from_repr(4));
assert_eq!(min_cap * 16, original_capacity_from_repr(5));
assert_eq!(min_cap * 32, original_capacity_from_repr(6));
assert_eq!(min_cap * 64, original_capacity_from_repr(7));
}
}
unsafe impl Send for BytesMut {}
unsafe impl Sync for BytesMut {}
/*
*
* ===== PartialEq / PartialOrd =====
*
*/
impl PartialEq<[u8]> for BytesMut {
fn eq(&self, other: &[u8]) -> bool {
&**self == other
}
}
impl PartialOrd<[u8]> for BytesMut {
fn partial_cmp(&self, other: &[u8]) -> Option<cmp::Ordering> {
(**self).partial_cmp(other)
}
}
impl PartialEq<BytesMut> for [u8] {
fn eq(&self, other: &BytesMut) -> bool {
*other == *self
}
}
impl PartialOrd<BytesMut> for [u8] {
fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> {
<[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
}
}
impl PartialEq<str> for BytesMut {
fn eq(&self, other: &str) -> bool {
&**self == other.as_bytes()
}
}
impl PartialOrd<str> for BytesMut {
fn partial_cmp(&self, other: &str) -> Option<cmp::Ordering> {
(**self).partial_cmp(other.as_bytes())
}
}
impl PartialEq<BytesMut> for str {
fn eq(&self, other: &BytesMut) -> bool {
*other == *self
}
}
impl PartialOrd<BytesMut> for str {
fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> {
<[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
}
}
impl PartialEq<Vec<u8>> for BytesMut {
fn eq(&self, other: &Vec<u8>) -> bool {
*self == other[..]
}
}
impl PartialOrd<Vec<u8>> for BytesMut {
fn partial_cmp(&self, other: &Vec<u8>) -> Option<cmp::Ordering> {
(**self).partial_cmp(&other[..])
}
}
impl PartialEq<BytesMut> for Vec<u8> {
fn eq(&self, other: &BytesMut) -> bool {
*other == *self
}
}
impl PartialOrd<BytesMut> for Vec<u8> {
fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> {
other.partial_cmp(self)
}
}
impl PartialEq<String> for BytesMut {
fn eq(&self, other: &String) -> bool {
*self == other[..]
}
}
impl PartialOrd<String> for BytesMut {
fn partial_cmp(&self, other: &String) -> Option<cmp::Ordering> {
(**self).partial_cmp(other.as_bytes())
}
}
impl PartialEq<BytesMut> for String {
fn eq(&self, other: &BytesMut) -> bool {
*other == *self
}
}
impl PartialOrd<BytesMut> for String {
fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> {
<[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
}
}
impl<'a, T: ?Sized> PartialEq<&'a T> for BytesMut
where
BytesMut: PartialEq<T>,
{
fn eq(&self, other: &&'a T) -> bool {
*self == **other
}
}
impl<'a, T: ?Sized> PartialOrd<&'a T> for BytesMut
where
BytesMut: PartialOrd<T>,
{
fn partial_cmp(&self, other: &&'a T) -> Option<cmp::Ordering> {
self.partial_cmp(*other)
}
}
impl PartialEq<BytesMut> for &[u8] {
fn eq(&self, other: &BytesMut) -> bool {
*other == *self
}
}
impl PartialOrd<BytesMut> for &[u8] {
fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> {
<[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
}
}
impl PartialEq<BytesMut> for &str {
fn eq(&self, other: &BytesMut) -> bool {
*other == *self
}
}
impl PartialOrd<BytesMut> for &str {
fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> {
other.partial_cmp(self)
}
}
impl PartialEq<BytesMut> for Bytes {
fn eq(&self, other: &BytesMut) -> bool {
other[..] == self[..]
}
}
impl PartialEq<Bytes> for BytesMut {
fn eq(&self, other: &Bytes) -> bool {
other[..] == self[..]
}
}
impl From<BytesMut> for Vec<u8> {
fn from(bytes: BytesMut) -> Self {
let kind = bytes.kind();
let bytes = ManuallyDrop::new(bytes);
let mut vec = if kind == KIND_VEC {
unsafe {
let off = bytes.get_vec_pos();
rebuild_vec(bytes.ptr.as_ptr(), bytes.len, bytes.cap, off)
}
} else {
let shared = bytes.data as *mut Shared;
if unsafe { (*shared).is_unique() } {
let vec = mem::replace(unsafe { &mut (*shared).vec }, Vec::new());
unsafe { release_shared(shared) };
vec
} else {
return ManuallyDrop::into_inner(bytes).deref().to_vec();
}
};
let len = bytes.len;
unsafe {
ptr::copy(bytes.ptr.as_ptr(), vec.as_mut_ptr(), len);
vec.set_len(len);
}
vec
}
}
#[inline]
fn vptr(ptr: *mut u8) -> NonNull<u8> {
if cfg!(debug_assertions) {
NonNull::new(ptr).expect("Vec pointer should be non-null")
} else {
unsafe { NonNull::new_unchecked(ptr) }
}
}
/// Returns a dangling pointer with the given address. This is used to store
/// integer data in pointer fields.
///
/// It is equivalent to `addr as *mut T`, but this fails on miri when strict
/// provenance checking is enabled.
#[inline]
fn invalid_ptr<T>(addr: usize) -> *mut T {
let ptr = core::ptr::null_mut::<u8>().wrapping_add(addr);
debug_assert_eq!(ptr as usize, addr);
ptr.cast::<T>()
}
unsafe fn rebuild_vec(ptr: *mut u8, mut len: usize, mut cap: usize, off: usize) -> Vec<u8> {
let ptr = ptr.sub(off);
len += off;
cap += off;
Vec::from_raw_parts(ptr, len, cap)
}
// ===== impl SharedVtable =====
static SHARED_VTABLE: Vtable = Vtable {
clone: shared_v_clone,
to_vec: shared_v_to_vec,
to_mut: shared_v_to_mut,
is_unique: shared_v_is_unique,
drop: shared_v_drop,
};
unsafe fn shared_v_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
let shared = data.load(Ordering::Relaxed) as *mut Shared;
increment_shared(shared);
let data = AtomicPtr::new(shared as *mut ());
Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE)
}
unsafe fn shared_v_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
let shared: *mut Shared = data.load(Ordering::Relaxed).cast();
if (*shared).is_unique() {
let shared = &mut *shared;
// Drop shared
let mut vec = mem::replace(&mut shared.vec, Vec::new());
release_shared(shared);
// Copy back buffer
ptr::copy(ptr, vec.as_mut_ptr(), len);
vec.set_len(len);
vec
} else {
let v = slice::from_raw_parts(ptr, len).to_vec();
release_shared(shared);
v
}
}
unsafe fn shared_v_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut {
let shared: *mut Shared = data.load(Ordering::Relaxed).cast();
if (*shared).is_unique() {
let shared = &mut *shared;
// The capacity is always the original capacity of the buffer
// minus the offset from the start of the buffer
let v = &mut shared.vec;
let v_capacity = v.capacity();
let v_ptr = v.as_mut_ptr();
let offset = offset_from(ptr as *mut u8, v_ptr);
let cap = v_capacity - offset;
let ptr = vptr(ptr as *mut u8);
BytesMut {
ptr,
len,
cap,
data: shared,
}
} else {
let v = slice::from_raw_parts(ptr, len).to_vec();
release_shared(shared);
BytesMut::from_vec(v)
}
}
unsafe fn shared_v_is_unique(data: &AtomicPtr<()>) -> bool {
let shared = data.load(Ordering::Acquire);
let ref_count = (*shared.cast::<Shared>()).ref_count.load(Ordering::Relaxed);
ref_count == 1
}
unsafe fn shared_v_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) {
data.with_mut(|shared| {
release_shared(*shared as *mut Shared);
});
}
// compile-fails
/// ```compile_fail
/// use bytes::BytesMut;
/// #[deny(unused_must_use)]
/// {
/// let mut b1 = BytesMut::from("hello world");
/// b1.split_to(6);
/// }
/// ```
fn _split_to_must_use() {}
/// ```compile_fail
/// use bytes::BytesMut;
/// #[deny(unused_must_use)]
/// {
/// let mut b1 = BytesMut::from("hello world");
/// b1.split_off(6);
/// }
/// ```
fn _split_off_must_use() {}
/// ```compile_fail
/// use bytes::BytesMut;
/// #[deny(unused_must_use)]
/// {
/// let mut b1 = BytesMut::from("hello world");
/// b1.split();
/// }
/// ```
fn _split_must_use() {}
// fuzz tests
#[cfg(all(test, loom))]
mod fuzz {
use loom::sync::Arc;
use loom::thread;
use super::BytesMut;
use crate::Bytes;
#[test]
fn bytes_mut_cloning_frozen() {
loom::model(|| {
let a = BytesMut::from(&b"abcdefgh"[..]).split().freeze();
let addr = a.as_ptr() as usize;
// test the Bytes::clone is Sync by putting it in an Arc
let a1 = Arc::new(a);
let a2 = a1.clone();
let t1 = thread::spawn(move || {
let b: Bytes = (*a1).clone();
assert_eq!(b.as_ptr() as usize, addr);
});
let t2 = thread::spawn(move || {
let b: Bytes = (*a2).clone();
assert_eq!(b.as_ptr() as usize, addr);
});
t1.join().unwrap();
t2.join().unwrap();
});
}
}
Codebase Browser
chromium