linux/mm/page_owner.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/debugfs.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/memblock.h>
#include <linux/stacktrace.h>
#include <linux/page_owner.h>
#include <linux/jump_label.h>
#include <linux/migrate.h>
#include <linux/stackdepot.h>
#include <linux/seq_file.h>
#include <linux/memcontrol.h>
#include <linux/sched/clock.h>

#include "internal.h"

/*
 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
 * to use off stack temporal storage
 */
#define PAGE_OWNER_STACK_DEPTH …

struct page_owner { … };

struct stack { … };
static struct stack dummy_stack;
static struct stack failure_stack;
static struct stack *stack_list;
static DEFINE_SPINLOCK(stack_list_lock);

static bool page_owner_enabled __initdata;
DEFINE_STATIC_KEY_FALSE(page_owner_inited);

static depot_stack_handle_t dummy_handle;
static depot_stack_handle_t failure_handle;
static depot_stack_handle_t early_handle;

static void init_early_allocated_pages(void);

static inline void set_current_in_page_owner(void)
{ … }

static inline void unset_current_in_page_owner(void)
{ … }

static int __init early_page_owner_param(char *buf)
{ … }
early_param(…);

static __init bool need_page_owner(void)
{ … }

static __always_inline depot_stack_handle_t create_dummy_stack(void)
{ … }

static noinline void register_dummy_stack(void)
{ … }

static noinline void register_failure_stack(void)
{ … }

static noinline void register_early_stack(void)
{ … }

static __init void init_page_owner(void)
{ … }

struct page_ext_operations page_owner_ops = …;

static inline struct page_owner *get_page_owner(struct page_ext *page_ext)
{ … }

static noinline depot_stack_handle_t save_stack(gfp_t flags)
{ … }

static void add_stack_record_to_list(struct stack_record *stack_record,
				     gfp_t gfp_mask)
{ … }

static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask,
				   int nr_base_pages)
{ … }

static void dec_stack_record_count(depot_stack_handle_t handle,
				   int nr_base_pages)
{ … }

static inline void __update_page_owner_handle(struct page_ext *page_ext,
					      depot_stack_handle_t handle,
					      unsigned short order,
					      gfp_t gfp_mask,
					      short last_migrate_reason, u64 ts_nsec,
					      pid_t pid, pid_t tgid, char *comm)
{ … }

static inline void __update_page_owner_free_handle(struct page_ext *page_ext,
						   depot_stack_handle_t handle,
						   unsigned short order,
						   pid_t pid, pid_t tgid,
						   u64 free_ts_nsec)
{ … }

void __reset_page_owner(struct page *page, unsigned short order)
{ … }

noinline void __set_page_owner(struct page *page, unsigned short order,
					gfp_t gfp_mask)
{ … }

void __set_page_owner_migrate_reason(struct page *page, int reason)
{ … }

void __split_page_owner(struct page *page, int old_order, int new_order)
{ … }

void __folio_copy_owner(struct folio *newfolio, struct folio *old)
{ … }

void pagetypeinfo_showmixedcount_print(struct seq_file *m,
				       pg_data_t *pgdat, struct zone *zone)
{ … }

/*
 * Looking for memcg information and print it out
 */
static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
					 struct page *page)
{ … }

static ssize_t
print_page_owner(char __user *buf, size_t count, unsigned long pfn,
		struct page *page, struct page_owner *page_owner,
		depot_stack_handle_t handle)
{ … }

void __dump_page_owner(const struct page *page)
{ … }

static ssize_t
read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{ … }

static loff_t lseek_page_owner(struct file *file, loff_t offset, int orig)
{ … }

static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
{ … }

static void init_zones_in_node(pg_data_t *pgdat)
{ … }

static void init_early_allocated_pages(void)
{ … }

static const struct file_operations proc_page_owner_operations = …;

static void *stack_start(struct seq_file *m, loff_t *ppos)
{ … }

static void *stack_next(struct seq_file *m, void *v, loff_t *ppos)
{ … }

static unsigned long page_owner_pages_threshold;

static int stack_print(struct seq_file *m, void *v)
{ … }

static void stack_stop(struct seq_file *m, void *v)
{ … }

static const struct seq_operations page_owner_stack_op = …;

static int page_owner_stack_open(struct inode *inode, struct file *file)
{ … }

static const struct file_operations page_owner_stack_operations = …;

static int page_owner_threshold_get(void *data, u64 *val)
{ … }

static int page_owner_threshold_set(void *data, u64 val)
{ … }

DEFINE_SIMPLE_ATTRIBUTE(…);


static int __init pageowner_init(void)
{ … }
late_initcall(…) …