/* SPDX-License-Identifier: GPL-2.0 */ /* XDP user-space ring structure * Copyright(c) 2018 Intel Corporation. */ #ifndef _LINUX_XSK_QUEUE_H #define _LINUX_XSK_QUEUE_H #include <linux/types.h> #include <linux/if_xdp.h> #include <net/xdp_sock.h> #include <net/xsk_buff_pool.h> #include "xsk.h" struct xdp_ring { … }; /* Used for the RX and TX queues for packets */ struct xdp_rxtx_ring { … }; /* Used for the fill and completion queues for buffers */ struct xdp_umem_ring { … }; struct xsk_queue { … }; struct parsed_desc { … }; /* The structure of the shared state of the rings are a simple * circular buffer, as outlined in * Documentation/core-api/circular-buffers.rst. For the Rx and * completion ring, the kernel is the producer and user space is the * consumer. For the Tx and fill rings, the kernel is the consumer and * user space is the producer. * * producer consumer * * if (LOAD ->consumer) { (A) LOAD.acq ->producer (C) * STORE $data LOAD $data * STORE.rel ->producer (B) STORE.rel ->consumer (D) * } * * (A) pairs with (D), and (B) pairs with (C). * * Starting with (B), it protects the data from being written after * the producer pointer. If this barrier was missing, the consumer * could observe the producer pointer being set and thus load the data * before the producer has written the new data. The consumer would in * this case load the old data. * * (C) protects the consumer from speculatively loading the data before * the producer pointer actually has been read. If we do not have this * barrier, some architectures could load old data as speculative loads * are not discarded as the CPU does not know there is a dependency * between ->producer and data. * * (A) is a control dependency that separates the load of ->consumer * from the stores of $data. In case ->consumer indicates there is no * room in the buffer to store $data we do not. The dependency will * order both of the stores after the loads. So no barrier is needed. * * (D) protects the load of the data to be observed to happen after the * store of the consumer pointer. If we did not have this memory * barrier, the producer could observe the consumer pointer being set * and overwrite the data with a new value before the consumer got the * chance to read the old value. The consumer would thus miss reading * the old entry and very likely read the new entry twice, once right * now and again after circling through the ring. */ /* The operations on the rings are the following: * * producer consumer * * RESERVE entries PEEK in the ring for entries * WRITE data into the ring READ data from the ring * SUBMIT entries RELEASE entries * * The producer reserves one or more entries in the ring. It can then * fill in these entries and finally submit them so that they can be * seen and read by the consumer. * * The consumer peeks into the ring to see if the producer has written * any new entries. If so, the consumer can then read these entries * and when it is done reading them release them back to the producer * so that the producer can use these slots to fill in new entries. * * The function names below reflect these operations. */ /* Functions that read and validate content from consumer rings. */ static inline void __xskq_cons_read_addr_unchecked(struct xsk_queue *q, u32 cached_cons, u64 *addr) { … } static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr) { … } static inline bool xp_unused_options_set(u32 options) { … } static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc) { … } static inline bool xp_unaligned_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc) { … } static inline bool xp_validate_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc) { … } static inline bool xskq_has_descs(struct xsk_queue *q) { … } static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q, struct xdp_desc *d, struct xsk_buff_pool *pool) { … } static inline bool xskq_cons_read_desc(struct xsk_queue *q, struct xdp_desc *desc, struct xsk_buff_pool *pool) { … } static inline void xskq_cons_release_n(struct xsk_queue *q, u32 cnt) { … } static inline void parse_desc(struct xsk_queue *q, struct xsk_buff_pool *pool, struct xdp_desc *desc, struct parsed_desc *parsed) { … } static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool, u32 max) { … } /* Functions for consumers */ static inline void __xskq_cons_release(struct xsk_queue *q) { … } static inline void __xskq_cons_peek(struct xsk_queue *q) { … } static inline void xskq_cons_get_entries(struct xsk_queue *q) { … } static inline u32 xskq_cons_nb_entries(struct xsk_queue *q, u32 max) { … } static inline bool xskq_cons_has_entries(struct xsk_queue *q, u32 cnt) { … } static inline bool xskq_cons_peek_addr_unchecked(struct xsk_queue *q, u64 *addr) { … } static inline bool xskq_cons_peek_desc(struct xsk_queue *q, struct xdp_desc *desc, struct xsk_buff_pool *pool) { … } /* To improve performance in the xskq_cons_release functions, only update local state here. * Reflect this to global state when we get new entries from the ring in * xskq_cons_get_entries() and whenever Rx or Tx processing are completed in the NAPI loop. */ static inline void xskq_cons_release(struct xsk_queue *q) { … } static inline void xskq_cons_cancel_n(struct xsk_queue *q, u32 cnt) { … } static inline u32 xskq_cons_present_entries(struct xsk_queue *q) { … } /* Functions for producers */ static inline u32 xskq_prod_nb_free(struct xsk_queue *q, u32 max) { … } static inline bool xskq_prod_is_full(struct xsk_queue *q) { … } static inline void xskq_prod_cancel_n(struct xsk_queue *q, u32 cnt) { … } static inline int xskq_prod_reserve(struct xsk_queue *q) { … } static inline int xskq_prod_reserve_addr(struct xsk_queue *q, u64 addr) { … } static inline void xskq_prod_write_addr_batch(struct xsk_queue *q, struct xdp_desc *descs, u32 nb_entries) { … } static inline int xskq_prod_reserve_desc(struct xsk_queue *q, u64 addr, u32 len, u32 flags) { … } static inline void __xskq_prod_submit(struct xsk_queue *q, u32 idx) { … } static inline void xskq_prod_submit(struct xsk_queue *q) { … } static inline void xskq_prod_submit_n(struct xsk_queue *q, u32 nb_entries) { … } static inline bool xskq_prod_is_empty(struct xsk_queue *q) { … } /* For both producers and consumers */ static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q) { … } static inline u64 xskq_nb_queue_empty_descs(struct xsk_queue *q) { … } struct xsk_queue *xskq_create(u32 nentries, bool umem_queue); void xskq_destroy(struct xsk_queue *q_ops); #endif /* _LINUX_XSK_QUEUE_H */
Codebase Browser
linux