linux/mm/mremap.c

// SPDX-License-Identifier: GPL-2.0
/*
 *	mm/mremap.c
 *
 *	(C) Copyright 1996 Linus Torvalds
 *
 *	Address space accounting code	<[email protected]>
 *	(C) Copyright 2002 Red Hat Inc, All Rights Reserved
 */

#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
#include <linux/ksm.h>
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/capability.h>
#include <linux/fs.h>
#include <linux/swapops.h>
#include <linux/highmem.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/mmu_notifier.h>
#include <linux/uaccess.h>
#include <linux/userfaultfd_k.h>
#include <linux/mempolicy.h>

#include <asm/cacheflush.h>
#include <asm/tlb.h>
#include <asm/pgalloc.h>

#include "internal.h"

static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr)
{ … }

static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
{ … }

static pud_t *alloc_new_pud(struct mm_struct *mm, struct vm_area_struct *vma,
			    unsigned long addr)
{ … }

static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
			    unsigned long addr)
{ … }

static void take_rmap_locks(struct vm_area_struct *vma)
{ … }

static void drop_rmap_locks(struct vm_area_struct *vma)
{ … }

static pte_t move_soft_dirty_pte(pte_t pte)
{ … }

static int move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
		unsigned long old_addr, unsigned long old_end,
		struct vm_area_struct *new_vma, pmd_t *new_pmd,
		unsigned long new_addr, bool need_rmap_locks)
{ … }

#ifndef arch_supports_page_table_move
#define arch_supports_page_table_move …
static inline bool arch_supports_page_table_move(void)
{ … }
#endif

#ifdef CONFIG_HAVE_MOVE_PMD
static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
		  unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
{ … }
#else
static inline bool move_normal_pmd(struct vm_area_struct *vma,
		unsigned long old_addr, unsigned long new_addr, pmd_t *old_pmd,
		pmd_t *new_pmd)
{
	return false;
}
#endif

#if CONFIG_PGTABLE_LEVELS > 2 && defined(CONFIG_HAVE_MOVE_PUD)
static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
		  unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
{ … }
#else
static inline bool move_normal_pud(struct vm_area_struct *vma,
		unsigned long old_addr, unsigned long new_addr, pud_t *old_pud,
		pud_t *new_pud)
{
	return false;
}
#endif

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr,
			  unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
{ … }
#else
static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr,
			  unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
{
	WARN_ON_ONCE(1);
	return false;

}
#endif

enum pgt_entry { … };

/*
 * Returns an extent of the corresponding size for the pgt_entry specified if
 * valid. Else returns a smaller extent bounded by the end of the source and
 * destination pgt_entry.
 */
static __always_inline unsigned long get_extent(enum pgt_entry entry,
			unsigned long old_addr, unsigned long old_end,
			unsigned long new_addr)
{ … }

/*
 * Attempts to speedup the move by moving entry at the level corresponding to
 * pgt_entry. Returns true if the move was successful, else false.
 */
static bool move_pgt_entry(enum pgt_entry entry, struct vm_area_struct *vma,
			unsigned long old_addr, unsigned long new_addr,
			void *old_entry, void *new_entry, bool need_rmap_locks)
{ … }

/*
 * A helper to check if aligning down is OK. The aligned address should fall
 * on *no mapping*. For the stack moving down, that's a special move within
 * the VMA that is created to span the source and destination of the move,
 * so we make an exception for it.
 */
static bool can_align_down(struct vm_area_struct *vma, unsigned long addr_to_align,
			    unsigned long mask, bool for_stack)
{ … }

/* Opportunistically realign to specified boundary for faster copy. */
static void try_realign_addr(unsigned long *old_addr, struct vm_area_struct *old_vma,
			     unsigned long *new_addr, struct vm_area_struct *new_vma,
			     unsigned long mask, bool for_stack)
{ … }

unsigned long move_page_tables(struct vm_area_struct *vma,
		unsigned long old_addr, struct vm_area_struct *new_vma,
		unsigned long new_addr, unsigned long len,
		bool need_rmap_locks, bool for_stack)
{ … }

static unsigned long move_vma(struct vm_area_struct *vma,
		unsigned long old_addr, unsigned long old_len,
		unsigned long new_len, unsigned long new_addr,
		bool *locked, unsigned long flags,
		struct vm_userfaultfd_ctx *uf, struct list_head *uf_unmap)
{ … }

static struct vm_area_struct *vma_to_resize(unsigned long addr,
	unsigned long old_len, unsigned long new_len, unsigned long flags)
{ … }

static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
		unsigned long new_addr, unsigned long new_len, bool *locked,
		unsigned long flags, struct vm_userfaultfd_ctx *uf,
		struct list_head *uf_unmap_early,
		struct list_head *uf_unmap)
{ … }

static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
{ … }

/*
 * Expand (or shrink) an existing mapping, potentially moving it at the
 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
 *
 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
 * This option implies MREMAP_MAYMOVE.
 */
SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
		unsigned long, new_len, unsigned long, flags,
		unsigned long, new_addr)
{ … }