// SPDX-License-Identifier: GPL-2.0 /* * DAMON Primitives for Virtual Address Spaces * * Author: SeongJae Park <[email protected]> */ #define pr_fmt(fmt) … #include <linux/highmem.h> #include <linux/hugetlb.h> #include <linux/mman.h> #include <linux/mmu_notifier.h> #include <linux/page_idle.h> #include <linux/pagewalk.h> #include <linux/sched/mm.h> #include "ops-common.h" #ifdef CONFIG_DAMON_VADDR_KUNIT_TEST #undef DAMON_MIN_REGION #define DAMON_MIN_REGION … #endif /* * 't->pid' should be the pointer to the relevant 'struct pid' having reference * count. Caller must put the returned task, unless it is NULL. */ static inline struct task_struct *damon_get_task_struct(struct damon_target *t) { … } /* * Get the mm_struct of the given target * * Caller _must_ put the mm_struct after use, unless it is NULL. * * Returns the mm_struct of the target on success, NULL on failure */ static struct mm_struct *damon_get_mm(struct damon_target *t) { … } /* * Functions for the initial monitoring target regions construction */ /* * Size-evenly split a region into 'nr_pieces' small regions * * Returns 0 on success, or negative error code otherwise. */ static int damon_va_evenly_split_region(struct damon_target *t, struct damon_region *r, unsigned int nr_pieces) { … } static unsigned long sz_range(struct damon_addr_range *r) { … } /* * Find three regions separated by two biggest unmapped regions * * vma the head vma of the target address space * regions an array of three address ranges that results will be saved * * This function receives an address space and finds three regions in it which * separated by the two biggest unmapped regions in the space. Please refer to * below comments of '__damon_va_init_regions()' function to know why this is * necessary. * * Returns 0 if success, or negative error code otherwise. */ static int __damon_va_three_regions(struct mm_struct *mm, struct damon_addr_range regions[3]) { … } /* * Get the three regions in the given target (task) * * Returns 0 on success, negative error code otherwise. */ static int damon_va_three_regions(struct damon_target *t, struct damon_addr_range regions[3]) { … } /* * Initialize the monitoring target regions for the given target (task) * * t the given target * * Because only a number of small portions of the entire address space * is actually mapped to the memory and accessed, monitoring the unmapped * regions is wasteful. That said, because we can deal with small noises, * tracking every mapping is not strictly required but could even incur a high * overhead if the mapping frequently changes or the number of mappings is * high. The adaptive regions adjustment mechanism will further help to deal * with the noise by simply identifying the unmapped areas as a region that * has no access. Moreover, applying the real mappings that would have many * unmapped areas inside will make the adaptive mechanism quite complex. That * said, too huge unmapped areas inside the monitoring target should be removed * to not take the time for the adaptive mechanism. * * For the reason, we convert the complex mappings to three distinct regions * that cover every mapped area of the address space. Also the two gaps * between the three regions are the two biggest unmapped areas in the given * address space. In detail, this function first identifies the start and the * end of the mappings and the two biggest unmapped areas of the address space. * Then, it constructs the three regions as below: * * [mappings[0]->start, big_two_unmapped_areas[0]->start) * [big_two_unmapped_areas[0]->end, big_two_unmapped_areas[1]->start) * [big_two_unmapped_areas[1]->end, mappings[nr_mappings - 1]->end) * * As usual memory map of processes is as below, the gap between the heap and * the uppermost mmap()-ed region, and the gap between the lowermost mmap()-ed * region and the stack will be two biggest unmapped regions. Because these * gaps are exceptionally huge areas in usual address space, excluding these * two biggest unmapped regions will be sufficient to make a trade-off. * * <heap> * <BIG UNMAPPED REGION 1> * <uppermost mmap()-ed region> * (other mmap()-ed regions and small unmapped regions) * <lowermost mmap()-ed region> * <BIG UNMAPPED REGION 2> * <stack> */ static void __damon_va_init_regions(struct damon_ctx *ctx, struct damon_target *t) { … } /* Initialize '->regions_list' of every target (task) */ static void damon_va_init(struct damon_ctx *ctx) { … } /* * Update regions for current memory mappings */ static void damon_va_update(struct damon_ctx *ctx) { … } static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long next, struct mm_walk *walk) { … } #ifdef CONFIG_HUGETLB_PAGE static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { … } static int damon_mkold_hugetlb_entry(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long end, struct mm_walk *walk) { … } #else #define damon_mkold_hugetlb_entry … #endif /* CONFIG_HUGETLB_PAGE */ static const struct mm_walk_ops damon_mkold_ops = …; static void damon_va_mkold(struct mm_struct *mm, unsigned long addr) { … } /* * Functions for the access checking of the regions */ static void __damon_va_prepare_access_check(struct mm_struct *mm, struct damon_region *r) { … } static void damon_va_prepare_access_checks(struct damon_ctx *ctx) { … } struct damon_young_walk_private { … }; static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long next, struct mm_walk *walk) { … } #ifdef CONFIG_HUGETLB_PAGE static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long end, struct mm_walk *walk) { … } #else #define damon_young_hugetlb_entry … #endif /* CONFIG_HUGETLB_PAGE */ static const struct mm_walk_ops damon_young_ops = …; static bool damon_va_young(struct mm_struct *mm, unsigned long addr, unsigned long *folio_sz) { … } /* * Check whether the region was accessed after the last preparation * * mm 'mm_struct' for the given virtual address space * r the region to be checked */ static void __damon_va_check_access(struct mm_struct *mm, struct damon_region *r, bool same_target, struct damon_attrs *attrs) { … } static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) { … } /* * Functions for the target validity check and cleanup */ static bool damon_va_target_valid(struct damon_target *t) { … } #ifndef CONFIG_ADVISE_SYSCALLS static unsigned long damos_madvise(struct damon_target *target, struct damon_region *r, int behavior) { return 0; } #else static unsigned long damos_madvise(struct damon_target *target, struct damon_region *r, int behavior) { … } #endif /* CONFIG_ADVISE_SYSCALLS */ static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme) { … } static int damon_va_scheme_score(struct damon_ctx *context, struct damon_target *t, struct damon_region *r, struct damos *scheme) { … } static int __init damon_va_initcall(void) { struct damon_operations ops = { .id = DAMON_OPS_VADDR, .init = damon_va_init, .update = damon_va_update, .prepare_access_checks = damon_va_prepare_access_checks, .check_accesses = damon_va_check_accesses, .reset_aggregated = NULL, .target_valid = damon_va_target_valid, .cleanup = NULL, .apply_scheme = damon_va_apply_scheme, .get_scheme_score = damon_va_scheme_score, }; /* ops for fixed virtual address ranges */ struct damon_operations ops_fvaddr = ops; int err; /* Don't set the monitoring target regions for the entire mapping */ ops_fvaddr.id = DAMON_OPS_FVADDR; ops_fvaddr.init = NULL; ops_fvaddr.update = NULL; err = damon_register_ops(&ops); if (err) return err; return damon_register_ops(&ops_fvaddr); }; subsys_initcall(damon_va_initcall); #include "vaddr-test.h"
Codebase Browser
linux