linux/kernel/fork.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/kernel/fork.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/*
 *  'fork.c' contains the help-routines for the 'fork' system call
 * (see also entry.S and others).
 * Fork is rather simple, once you get the hang of it, but the memory
 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
 */

#include <linux/anon_inodes.h>
#include <linux/slab.h>
#include <linux/sched/autogroup.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/sched/user.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/stat.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/sched/cputime.h>
#include <linux/seq_file.h>
#include <linux/rtmutex.h>
#include <linux/init.h>
#include <linux/unistd.h>
#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/completion.h>
#include <linux/personality.h>
#include <linux/mempolicy.h>
#include <linux/sem.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/iocontext.h>
#include <linux/key.h>
#include <linux/kmsan.h>
#include <linux/binfmts.h>
#include <linux/mman.h>
#include <linux/mmu_notifier.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/memblock.h>
#include <linux/nsproxy.h>
#include <linux/capability.h>
#include <linux/cpu.h>
#include <linux/cgroup.h>
#include <linux/security.h>
#include <linux/hugetlb.h>
#include <linux/seccomp.h>
#include <linux/swap.h>
#include <linux/syscalls.h>
#include <linux/syscall_user_dispatch.h>
#include <linux/jiffies.h>
#include <linux/futex.h>
#include <linux/compat.h>
#include <linux/kthread.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/rcupdate.h>
#include <linux/ptrace.h>
#include <linux/mount.h>
#include <linux/audit.h>
#include <linux/memcontrol.h>
#include <linux/ftrace.h>
#include <linux/proc_fs.h>
#include <linux/profile.h>
#include <linux/rmap.h>
#include <linux/ksm.h>
#include <linux/acct.h>
#include <linux/userfaultfd_k.h>
#include <linux/tsacct_kern.h>
#include <linux/cn_proc.h>
#include <linux/freezer.h>
#include <linux/delayacct.h>
#include <linux/taskstats_kern.h>
#include <linux/tty.h>
#include <linux/fs_struct.h>
#include <linux/magic.h>
#include <linux/perf_event.h>
#include <linux/posix-timers.h>
#include <linux/user-return-notifier.h>
#include <linux/oom.h>
#include <linux/khugepaged.h>
#include <linux/signalfd.h>
#include <linux/uprobes.h>
#include <linux/aio.h>
#include <linux/compiler.h>
#include <linux/sysctl.h>
#include <linux/kcov.h>
#include <linux/livepatch.h>
#include <linux/thread_info.h>
#include <linux/stackleak.h>
#include <linux/kasan.h>
#include <linux/scs.h>
#include <linux/io_uring.h>
#include <linux/bpf.h>
#include <linux/stackprotector.h>
#include <linux/user_events.h>
#include <linux/iommu.h>
#include <linux/rseq.h>
#include <uapi/linux/pidfd.h>
#include <linux/pidfs.h>

#include <asm/pgalloc.h>
#include <linux/uaccess.h>
#include <asm/mmu_context.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>

#include <trace/events/sched.h>

#define CREATE_TRACE_POINTS
#include <trace/events/task.h>

#include <kunit/visibility.h>

/*
 * Minimum number of threads to boot the kernel
 */
#define MIN_THREADS …

/*
 * Maximum number of threads
 */
#define MAX_THREADS …

/*
 * Protected counters by write_lock_irq(&tasklist_lock)
 */
unsigned long total_forks;	/* Handle normal Linux uptimes. */
int nr_threads;			/* The idle threads do not count.. */

static int max_threads;		/* tunable limit on nr_threads */

#define NAMED_ARRAY_INDEX(x) …

static const char * const resident_page_types[] = …;

DEFINE_PER_CPU(unsigned long, process_counts) = …;

__cacheline_aligned DEFINE_RWLOCK(…);  /* outer */

#ifdef CONFIG_PROVE_RCU
int lockdep_tasklist_lock_is_held(void)
{ … }
EXPORT_SYMBOL_GPL(…);
#endif /* #ifdef CONFIG_PROVE_RCU */

int nr_processes(void)
{ … }

void __weak arch_release_task_struct(struct task_struct *tsk)
{ … }

static struct kmem_cache *task_struct_cachep;

static inline struct task_struct *alloc_task_struct_node(int node)
{ … }

static inline void free_task_struct(struct task_struct *tsk)
{ … }

/*
 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
 * kmemcache based allocator.
 */
# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)

#  ifdef CONFIG_VMAP_STACK
/*
 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
 * flush.  Try to minimize the number of calls by caching stacks.
 */
#define NR_CACHED_STACKS …
static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);

struct vm_stack { … };

static bool try_release_thread_stack_to_cache(struct vm_struct *vm)
{ … }

static void thread_stack_free_rcu(struct rcu_head *rh)
{ … }

static void thread_stack_delayed_free(struct task_struct *tsk)
{ … }

static int free_vm_stack_cache(unsigned int cpu)
{ … }

static int memcg_charge_kernel_stack(struct vm_struct *vm)
{ … }

static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{ … }

static void free_thread_stack(struct task_struct *tsk)
{ … }

#  else /* !CONFIG_VMAP_STACK */

static void thread_stack_free_rcu(struct rcu_head *rh)
{
	__free_pages(virt_to_page(rh), THREAD_SIZE_ORDER);
}

static void thread_stack_delayed_free(struct task_struct *tsk)
{
	struct rcu_head *rh = tsk->stack;

	call_rcu(rh, thread_stack_free_rcu);
}

static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
	struct page *page = alloc_pages_node(node, THREADINFO_GFP,
					     THREAD_SIZE_ORDER);

	if (likely(page)) {
		tsk->stack = kasan_reset_tag(page_address(page));
		return 0;
	}
	return -ENOMEM;
}

static void free_thread_stack(struct task_struct *tsk)
{
	thread_stack_delayed_free(tsk);
	tsk->stack = NULL;
}

#  endif /* CONFIG_VMAP_STACK */
# else /* !(THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)) */

static struct kmem_cache *thread_stack_cache;

static void thread_stack_free_rcu(struct rcu_head *rh)
{
	kmem_cache_free(thread_stack_cache, rh);
}

static void thread_stack_delayed_free(struct task_struct *tsk)
{
	struct rcu_head *rh = tsk->stack;

	call_rcu(rh, thread_stack_free_rcu);
}

static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
	unsigned long *stack;
	stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
	stack = kasan_reset_tag(stack);
	tsk->stack = stack;
	return stack ? 0 : -ENOMEM;
}

static void free_thread_stack(struct task_struct *tsk)
{
	thread_stack_delayed_free(tsk);
	tsk->stack = NULL;
}

void thread_stack_cache_init(void)
{
	thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
					THREAD_SIZE, THREAD_SIZE, 0, 0,
					THREAD_SIZE, NULL);
	BUG_ON(thread_stack_cache == NULL);
}

# endif /* THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) */

/* SLAB cache for signal_struct structures (tsk->signal) */
static struct kmem_cache *signal_cachep;

/* SLAB cache for sighand_struct structures (tsk->sighand) */
struct kmem_cache *sighand_cachep;

/* SLAB cache for files_struct structures (tsk->files) */
struct kmem_cache *files_cachep;

/* SLAB cache for fs_struct structures (tsk->fs) */
struct kmem_cache *fs_cachep;

/* SLAB cache for vm_area_struct structures */
static struct kmem_cache *vm_area_cachep;

/* SLAB cache for mm_struct structures (tsk->mm) */
static struct kmem_cache *mm_cachep;

#ifdef CONFIG_PER_VMA_LOCK

/* SLAB cache for vm_area_struct.lock */
static struct kmem_cache *vma_lock_cachep;

static bool vma_lock_alloc(struct vm_area_struct *vma)
{ … }

static inline void vma_lock_free(struct vm_area_struct *vma)
{ … }

#else /* CONFIG_PER_VMA_LOCK */

static inline bool vma_lock_alloc(struct vm_area_struct *vma) { return true; }
static inline void vma_lock_free(struct vm_area_struct *vma) {}

#endif /* CONFIG_PER_VMA_LOCK */

struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
{ … }

struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
{ … }

void __vm_area_free(struct vm_area_struct *vma)
{ … }

#ifdef CONFIG_PER_VMA_LOCK
static void vm_area_free_rcu_cb(struct rcu_head *head)
{ … }
#endif

void vm_area_free(struct vm_area_struct *vma)
{ … }

static void account_kernel_stack(struct task_struct *tsk, int account)
{ … }

void exit_task_stack_account(struct task_struct *tsk)
{ … }

static void release_task_stack(struct task_struct *tsk)
{ … }

#ifdef CONFIG_THREAD_INFO_IN_TASK
void put_task_stack(struct task_struct *tsk)
{ … }
#endif

void free_task(struct task_struct *tsk)
{ … }
EXPORT_SYMBOL(…);

static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm)
{ … }

#ifdef CONFIG_MMU
static __latent_entropy int dup_mmap(struct mm_struct *mm,
					struct mm_struct *oldmm)
{ … }

static inline int mm_alloc_pgd(struct mm_struct *mm)
{ … }

static inline void mm_free_pgd(struct mm_struct *mm)
{ … }
#else
static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
{
	mmap_write_lock(oldmm);
	dup_mm_exe_file(mm, oldmm);
	mmap_write_unlock(oldmm);
	return 0;
}
#define mm_alloc_pgd …
#define mm_free_pgd …
#endif /* CONFIG_MMU */

static void check_mm(struct mm_struct *mm)
{ … }

#define allocate_mm() …
#define free_mm(mm) …

static void do_check_lazy_tlb(void *arg)
{ … }

static void do_shoot_lazy_tlb(void *arg)
{ … }

static void cleanup_lazy_tlbs(struct mm_struct *mm)
{ … }

/*
 * Called when the last reference to the mm
 * is dropped: either by a lazy thread or by
 * mmput. Free the page directory and the mm.
 */
void __mmdrop(struct mm_struct *mm)
{ … }
EXPORT_SYMBOL_GPL(…);

static void mmdrop_async_fn(struct work_struct *work)
{ … }

static void mmdrop_async(struct mm_struct *mm)
{ … }

static inline void free_signal_struct(struct signal_struct *sig)
{ … }

static inline void put_signal_struct(struct signal_struct *sig)
{ … }

void __put_task_struct(struct task_struct *tsk)
{ … }
EXPORT_SYMBOL_GPL(…);

void __put_task_struct_rcu_cb(struct rcu_head *rhp)
{ … }
EXPORT_SYMBOL_GPL(…);

void __init __weak arch_task_cache_init(void) { … }

/*
 * set_max_threads
 */
static void __init set_max_threads(unsigned int max_threads_suggested)
{ … }

#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
/* Initialized by the architecture: */
int arch_task_struct_size __read_mostly;
#endif

static void __init task_struct_whitelist(unsigned long *offset, unsigned long *size)
{ … }

void __init fork_init(void)
{ … }

int __weak arch_dup_task_struct(struct task_struct *dst,
					       struct task_struct *src)
{ … }

void set_task_stack_end_magic(struct task_struct *tsk)
{ … }

static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
{ … }

__cacheline_aligned_in_smp DEFINE_SPINLOCK(…);

static unsigned long default_dump_filter = …;

static int __init coredump_filter_setup(char *s)
{ … }

__setup(…);

#include <linux/init_task.h>

static void mm_init_aio(struct mm_struct *mm)
{ … }

static __always_inline void mm_clear_owner(struct mm_struct *mm,
					   struct task_struct *p)
{ … }

static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
{ … }

static void mm_init_uprobes_state(struct mm_struct *mm)
{ … }

static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
	struct user_namespace *user_ns)
{ … }

/*
 * Allocate and initialize an mm_struct.
 */
struct mm_struct *mm_alloc(void)
{ … }
EXPORT_SYMBOL_IF_KUNIT(…);

static inline void __mmput(struct mm_struct *mm)
{ … }

/*
 * Decrement the use count and release all resources for an mm.
 */
void mmput(struct mm_struct *mm)
{ … }
EXPORT_SYMBOL_GPL(…);

#ifdef CONFIG_MMU
static void mmput_async_fn(struct work_struct *work)
{ … }

void mmput_async(struct mm_struct *mm)
{ … }
EXPORT_SYMBOL_GPL(…);
#endif

/**
 * set_mm_exe_file - change a reference to the mm's executable file
 * @mm: The mm to change.
 * @new_exe_file: The new file to use.
 *
 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
 *
 * Main users are mmput() and sys_execve(). Callers prevent concurrent
 * invocations: in mmput() nobody alive left, in execve it happens before
 * the new mm is made visible to anyone.
 *
 * Can only fail if new_exe_file != NULL.
 */
int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
{ … }

/**
 * replace_mm_exe_file - replace a reference to the mm's executable file
 * @mm: The mm to change.
 * @new_exe_file: The new file to use.
 *
 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
 *
 * Main user is sys_prctl(PR_SET_MM_MAP/EXE_FILE).
 */
int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
{ … }

/**
 * get_mm_exe_file - acquire a reference to the mm's executable file
 * @mm: The mm of interest.
 *
 * Returns %NULL if mm has no associated executable file.
 * User must release file via fput().
 */
struct file *get_mm_exe_file(struct mm_struct *mm)
{ … }

/**
 * get_task_exe_file - acquire a reference to the task's executable file
 * @task: The task.
 *
 * Returns %NULL if task's mm (if any) has no associated executable file or
 * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
 * User must release file via fput().
 */
struct file *get_task_exe_file(struct task_struct *task)
{ … }

/**
 * get_task_mm - acquire a reference to the task's mm
 * @task: The task.
 *
 * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning
 * this kernel workthread has transiently adopted a user mm with use_mm,
 * to do its AIO) is not set and if so returns a reference to it, after
 * bumping up the use count.  User must release the mm via mmput()
 * after use.  Typically used by /proc and ptrace.
 */
struct mm_struct *get_task_mm(struct task_struct *task)
{ … }
EXPORT_SYMBOL_GPL(…);

struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
{ … }

static void complete_vfork_done(struct task_struct *tsk)
{ … }

static int wait_for_vfork_done(struct task_struct *child,
				struct completion *vfork)
{ … }

/* Please note the differences between mmput and mm_release.
 * mmput is called whenever we stop holding onto a mm_struct,
 * error success whatever.
 *
 * mm_release is called after a mm_struct has been removed
 * from the current process.
 *
 * This difference is important for error handling, when we
 * only half set up a mm_struct for a new process and need to restore
 * the old one.  Because we mmput the new mm_struct before
 * restoring the old one. . .
 * Eric Biederman 10 January 1998
 */
static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
{ … }

void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)
{ … }

void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)
{ … }

/**
 * dup_mm() - duplicates an existing mm structure
 * @tsk: the task_struct with which the new mm will be associated.
 * @oldmm: the mm to duplicate.
 *
 * Allocates a new mm structure and duplicates the provided @oldmm structure
 * content into it.
 *
 * Return: the duplicated mm or NULL on failure.
 */
static struct mm_struct *dup_mm(struct task_struct *tsk,
				struct mm_struct *oldmm)
{ … }

static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
{ … }

static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
{ … }

static int copy_files(unsigned long clone_flags, struct task_struct *tsk,
		      int no_files)
{ … }

static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
{ … }

void __cleanup_sighand(struct sighand_struct *sighand)
{ … }

/*
 * Initialize POSIX timer handling for a thread group.
 */
static void posix_cpu_timers_init_group(struct signal_struct *sig)
{ … }

static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
{ … }

static void copy_seccomp(struct task_struct *p)
{ … }

SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
{ … }

static void rt_mutex_init_task(struct task_struct *p)
{ … }

static inline void init_task_pid_links(struct task_struct *task)
{ … }

static inline void
init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
{ … }

static inline void rcu_copy_process(struct task_struct *p)
{ … }

/**
 * __pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
 * @pid:   the struct pid for which to create a pidfd
 * @flags: flags of the new @pidfd
 * @ret: Where to return the file for the pidfd.
 *
 * Allocate a new file that stashes @pid and reserve a new pidfd number in the
 * caller's file descriptor table. The pidfd is reserved but not installed yet.
 *
 * The helper doesn't perform checks on @pid which makes it useful for pidfds
 * created via CLONE_PIDFD where @pid has no task attached when the pidfd and
 * pidfd file are prepared.
 *
 * If this function returns successfully the caller is responsible to either
 * call fd_install() passing the returned pidfd and pidfd file as arguments in
 * order to install the pidfd into its file descriptor table or they must use
 * put_unused_fd() and fput() on the returned pidfd and pidfd file
 * respectively.
 *
 * This function is useful when a pidfd must already be reserved but there
 * might still be points of failure afterwards and the caller wants to ensure
 * that no pidfd is leaked into its file descriptor table.
 *
 * Return: On success, a reserved pidfd is returned from the function and a new
 *         pidfd file is returned in the last argument to the function. On
 *         error, a negative error code is returned from the function and the
 *         last argument remains unchanged.
 */
static int __pidfd_prepare(struct pid *pid, unsigned int flags, struct file **ret)
{ … }

/**
 * pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
 * @pid:   the struct pid for which to create a pidfd
 * @flags: flags of the new @pidfd
 * @ret: Where to return the pidfd.
 *
 * Allocate a new file that stashes @pid and reserve a new pidfd number in the
 * caller's file descriptor table. The pidfd is reserved but not installed yet.
 *
 * The helper verifies that @pid is still in use, without PIDFD_THREAD the
 * task identified by @pid must be a thread-group leader.
 *
 * If this function returns successfully the caller is responsible to either
 * call fd_install() passing the returned pidfd and pidfd file as arguments in
 * order to install the pidfd into its file descriptor table or they must use
 * put_unused_fd() and fput() on the returned pidfd and pidfd file
 * respectively.
 *
 * This function is useful when a pidfd must already be reserved but there
 * might still be points of failure afterwards and the caller wants to ensure
 * that no pidfd is leaked into its file descriptor table.
 *
 * Return: On success, a reserved pidfd is returned from the function and a new
 *         pidfd file is returned in the last argument to the function. On
 *         error, a negative error code is returned from the function and the
 *         last argument remains unchanged.
 */
int pidfd_prepare(struct pid *pid, unsigned int flags, struct file **ret)
{ … }

static void __delayed_free_task(struct rcu_head *rhp)
{ … }

static __always_inline void delayed_free_task(struct task_struct *tsk)
{ … }

static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)
{ … }

#ifdef CONFIG_RV
static void rv_task_fork(struct task_struct *p)
{ … }
#else
#define rv_task_fork …
#endif

/*
 * This creates a new process as a copy of the old one,
 * but does not actually start it yet.
 *
 * It copies the registers, and all the appropriate
 * parts of the process environment (as per the clone
 * flags). The actual kick-off is left to the caller.
 */
__latent_entropy struct task_struct *copy_process(
					struct pid *pid,
					int trace,
					int node,
					struct kernel_clone_args *args)
{ … }

static inline void init_idle_pids(struct task_struct *idle)
{ … }

static int idle_dummy(void *dummy)
{ … }

struct task_struct * __init fork_idle(int cpu)
{ … }

/*
 * This is like kernel_clone(), but shaved down and tailored to just
 * creating io_uring workers. It returns a created task, or an error pointer.
 * The returned task is inactive, and the caller must fire it up through
 * wake_up_new_task(p). All signals are blocked in the created task.
 */
struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node)
{ … }

/*
 *  Ok, this is the main fork-routine.
 *
 * It copies the process, and if successful kick-starts
 * it and waits for it to finish using the VM if required.
 *
 * args->exit_signal is expected to be checked for sanity by the caller.
 */
pid_t kernel_clone(struct kernel_clone_args *args)
{ … }

/*
 * Create a kernel thread.
 */
pid_t kernel_thread(int (*fn)(void *), void *arg, const char *name,
		    unsigned long flags)
{ … }

/*
 * Create a user mode thread.
 */
pid_t user_mode_thread(int (*fn)(void *), void *arg, unsigned long flags)
{ … }

#ifdef __ARCH_WANT_SYS_FORK
SYSCALL_DEFINE0(fork)
{ … }
#endif

#ifdef __ARCH_WANT_SYS_VFORK
SYSCALL_DEFINE0(vfork)
{ … }
#endif

#ifdef __ARCH_WANT_SYS_CLONE
#ifdef CONFIG_CLONE_BACKWARDS
SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
		 int __user *, parent_tidptr,
		 unsigned long, tls,
		 int __user *, child_tidptr)
#elif defined(CONFIG_CLONE_BACKWARDS2)
SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
		 int __user *, parent_tidptr,
		 int __user *, child_tidptr,
		 unsigned long, tls)
#elif defined(CONFIG_CLONE_BACKWARDS3)
SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
		int, stack_size,
		int __user *, parent_tidptr,
		int __user *, child_tidptr,
		unsigned long, tls)
#else
SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
		 int __user *, parent_tidptr,
		 int __user *, child_tidptr,
		 unsigned long, tls)
#endif
{ … }
#endif

noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs,
					      struct clone_args __user *uargs,
					      size_t usize)
{ … }

/**
 * clone3_stack_valid - check and prepare stack
 * @kargs: kernel clone args
 *
 * Verify that the stack arguments userspace gave us are sane.
 * In addition, set the stack direction for userspace since it's easy for us to
 * determine.
 */
static inline bool clone3_stack_valid(struct kernel_clone_args *kargs)
{ … }

static bool clone3_args_valid(struct kernel_clone_args *kargs)
{ … }

/**
 * sys_clone3 - create a new process with specific properties
 * @uargs: argument structure
 * @size:  size of @uargs
 *
 * clone3() is the extensible successor to clone()/clone2().
 * It takes a struct as argument that is versioned by its size.
 *
 * Return: On success, a positive PID for the child process.
 *         On error, a negative errno number.
 */
SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size)
{ … }

void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data)
{ … }

#ifndef ARCH_MIN_MMSTRUCT_ALIGN
#define ARCH_MIN_MMSTRUCT_ALIGN …
#endif

static void sighand_ctor(void *data)
{ … }

void __init mm_cache_init(void)
{ … }

void __init proc_caches_init(void)
{ … }

/*
 * Check constraints on flags passed to the unshare system call.
 */
static int check_unshare_flags(unsigned long unshare_flags)
{ … }

/*
 * Unshare the filesystem structure if it is being shared
 */
static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
{ … }

/*
 * Unshare file descriptor table if it is being shared
 */
int unshare_fd(unsigned long unshare_flags, unsigned int max_fds,
	       struct files_struct **new_fdp)
{ … }

/*
 * unshare allows a process to 'unshare' part of the process
 * context which was originally shared using clone.  copy_*
 * functions used by kernel_clone() cannot be used here directly
 * because they modify an inactive task_struct that is being
 * constructed. Here we are modifying the current, active,
 * task_struct.
 */
int ksys_unshare(unsigned long unshare_flags)
{ … }

SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
{ … }

/*
 *	Helper to unshare the files of the current task.
 *	We don't want to expose copy_files internals to
 *	the exec layer of the kernel.
 */

int unshare_files(void)
{ … }

int sysctl_max_threads(const struct ctl_table *table, int write,
		       void *buffer, size_t *lenp, loff_t *ppos)
{ … }