linux/mm/list_lru.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2013 Red Hat, Inc. and Parallels Inc. All rights reserved.
 * Authors: David Chinner and Glauber Costa
 *
 * Generic LRU infrastructure
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/list_lru.h>
#include <linux/slab.h>
#include <linux/mutex.h>
#include <linux/memcontrol.h>
#include "slab.h"
#include "internal.h"

#ifdef CONFIG_MEMCG
static LIST_HEAD(memcg_list_lrus);
static DEFINE_MUTEX(list_lrus_mutex);

static inline bool list_lru_memcg_aware(struct list_lru *lru)
{ … }

static void list_lru_register(struct list_lru *lru)
{ … }

static void list_lru_unregister(struct list_lru *lru)
{ … }

static int lru_shrinker_id(struct list_lru *lru)
{ … }

static inline struct list_lru_one *
list_lru_from_memcg_idx(struct list_lru *lru, int nid, int idx)
{ … }
#else
static void list_lru_register(struct list_lru *lru)
{
}

static void list_lru_unregister(struct list_lru *lru)
{
}

static int lru_shrinker_id(struct list_lru *lru)
{
	return -1;
}

static inline bool list_lru_memcg_aware(struct list_lru *lru)
{
	return false;
}

static inline struct list_lru_one *
list_lru_from_memcg_idx(struct list_lru *lru, int nid, int idx)
{
	return &lru->node[nid].lru;
}
#endif /* CONFIG_MEMCG */

/* The caller must ensure the memcg lifetime. */
bool list_lru_add(struct list_lru *lru, struct list_head *item, int nid,
		    struct mem_cgroup *memcg)
{ … }
EXPORT_SYMBOL_GPL(…);

bool list_lru_add_obj(struct list_lru *lru, struct list_head *item)
{ … }
EXPORT_SYMBOL_GPL(…);

/* The caller must ensure the memcg lifetime. */
bool list_lru_del(struct list_lru *lru, struct list_head *item, int nid,
		    struct mem_cgroup *memcg)
{ … }
EXPORT_SYMBOL_GPL(…);

bool list_lru_del_obj(struct list_lru *lru, struct list_head *item)
{ … }
EXPORT_SYMBOL_GPL(…);

void list_lru_isolate(struct list_lru_one *list, struct list_head *item)
{ … }
EXPORT_SYMBOL_GPL(…);

void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item,
			   struct list_head *head)
{ … }
EXPORT_SYMBOL_GPL(…);

unsigned long list_lru_count_one(struct list_lru *lru,
				 int nid, struct mem_cgroup *memcg)
{ … }
EXPORT_SYMBOL_GPL(…);

unsigned long list_lru_count_node(struct list_lru *lru, int nid)
{ … }
EXPORT_SYMBOL_GPL(…);

static unsigned long
__list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,
		    list_lru_walk_cb isolate, void *cb_arg,
		    unsigned long *nr_to_walk)
{ … }

unsigned long
list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
		  list_lru_walk_cb isolate, void *cb_arg,
		  unsigned long *nr_to_walk)
{ … }
EXPORT_SYMBOL_GPL(…);

unsigned long
list_lru_walk_one_irq(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
		      list_lru_walk_cb isolate, void *cb_arg,
		      unsigned long *nr_to_walk)
{ … }

unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
				 list_lru_walk_cb isolate, void *cb_arg,
				 unsigned long *nr_to_walk)
{ … }
EXPORT_SYMBOL_GPL(…);

static void init_one_lru(struct list_lru_one *l)
{ … }

#ifdef CONFIG_MEMCG
static struct list_lru_memcg *memcg_init_list_lru_one(gfp_t gfp)
{ … }

static void memcg_list_lru_free(struct list_lru *lru, int src_idx)
{ … }

static inline void memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
{ … }

static void memcg_destroy_list_lru(struct list_lru *lru)
{ … }

static void memcg_reparent_list_lru_node(struct list_lru *lru, int nid,
					 int src_idx, struct mem_cgroup *dst_memcg)
{ … }

static void memcg_reparent_list_lru(struct list_lru *lru,
				    int src_idx, struct mem_cgroup *dst_memcg)
{ … }

void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *parent)
{ … }

static inline bool memcg_list_lru_allocated(struct mem_cgroup *memcg,
					    struct list_lru *lru)
{ … }

int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru,
			 gfp_t gfp)
{ … }
#else
static inline void memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
{
}

static void memcg_destroy_list_lru(struct list_lru *lru)
{
}
#endif /* CONFIG_MEMCG */

int __list_lru_init(struct list_lru *lru, bool memcg_aware,
		    struct lock_class_key *key, struct shrinker *shrinker)
{ … }
EXPORT_SYMBOL_GPL(…);

void list_lru_destroy(struct list_lru *lru)
{ … }
EXPORT_SYMBOL_GPL(…);