linux/kernel/events/internal.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _KERNEL_EVENTS_INTERNAL_H
#define _KERNEL_EVENTS_INTERNAL_H

#include <linux/hardirq.h>
#include <linux/uaccess.h>
#include <linux/refcount.h>

/* Buffer handling */

#define RING_BUFFER_WRITABLE …

struct perf_buffer { … };

extern void rb_free(struct perf_buffer *rb);

static inline void rb_free_rcu(struct rcu_head *rcu_head)
{ … }

static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause)
{ … }

extern struct perf_buffer *
rb_alloc(int nr_pages, long watermark, int cpu, int flags);
extern void perf_event_wakeup(struct perf_event *event);
extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event,
			pgoff_t pgoff, int nr_pages, long watermark, int flags);
extern void rb_free_aux(struct perf_buffer *rb);
extern struct perf_buffer *ring_buffer_get(struct perf_event *event);
extern void ring_buffer_put(struct perf_buffer *rb);

static inline bool rb_has_aux(struct perf_buffer *rb)
{ … }

void perf_event_aux_event(struct perf_event *event, unsigned long head,
			  unsigned long size, u64 flags);

extern struct page *
perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff);

#ifdef CONFIG_PERF_USE_VMALLOC
/*
 * Back perf_mmap() with vmalloc memory.
 *
 * Required for architectures that have d-cache aliasing issues.
 */

static inline int page_order(struct perf_buffer *rb)
{ … }

#else

static inline int page_order(struct perf_buffer *rb)
{
	return 0;
}
#endif

static inline int data_page_nr(struct perf_buffer *rb)
{ … }

static inline unsigned long perf_data_size(struct perf_buffer *rb)
{ … }

static inline unsigned long perf_aux_size(struct perf_buffer *rb)
{ … }

#define __DEFINE_OUTPUT_COPY_BODY(advance_buf, memcpy_func, ...) …

#define DEFINE_OUTPUT_COPY(func_name, memcpy_func) …

static inline unsigned long
__output_custom(struct perf_output_handle *handle, perf_copy_f copy_func,
		const void *buf, unsigned long len)
{ … }

static inline unsigned long
memcpy_common(void *dst, const void *src, unsigned long n)
{ … }

DEFINE_OUTPUT_COPY(__output_copy, memcpy_common)

static inline unsigned long
memcpy_skip(void *dst, const void *src, unsigned long n)
{ … }

DEFINE_OUTPUT_COPY(__output_skip, memcpy_skip)

#ifndef arch_perf_out_copy_user
#define arch_perf_out_copy_user …

static inline unsigned long
arch_perf_out_copy_user(void *dst, const void *src, unsigned long n)
{
	unsigned long ret;

	pagefault_disable();
	ret = __copy_from_user_inatomic(dst, src, n);
	pagefault_enable();

	return ret;
}
#endif

DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)

static inline int get_recursion_context(u8 *recursion)
{ … }

static inline void put_recursion_context(u8 *recursion, unsigned char rctx)
{ … }

#ifdef CONFIG_HAVE_PERF_USER_STACK_DUMP
static inline bool arch_perf_have_user_stack_dump(void)
{ … }

#define perf_user_stack_pointer(regs) …
#else
static inline bool arch_perf_have_user_stack_dump(void)
{
	return false;
}

#define perf_user_stack_pointer …
#endif /* CONFIG_HAVE_PERF_USER_STACK_DUMP */

#endif /* _KERNEL_EVENTS_INTERNAL_H */