linux/ipc/shm.c

// SPDX-License-Identifier: GPL-2.0
/*
 * linux/ipc/shm.c
 * Copyright (C) 1992, 1993 Krishna Balasubramanian
 *	 Many improvements/fixes by Bruno Haible.
 * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
 * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
 *
 * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <[email protected]>
 * BIGMEM support, Andrea Arcangeli <[email protected]>
 * SMP thread shm, Jean-Luc Boyard <[email protected]>
 * HIGHMEM support, Ingo Molnar <[email protected]>
 * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <[email protected]>
 * Shared /dev/zero support, Kanoj Sarcar <[email protected]>
 * Move the mm functionality over to mm/shmem.c, Christoph Rohland <[email protected]>
 *
 * support for audit of ipc object properties and permission changes
 * Dustin Kirkland <[email protected]>
 *
 * namespaces support
 * OpenVZ, SWsoft Inc.
 * Pavel Emelianov <[email protected]>
 *
 * Better ipc lock (kern_ipc_perm.lock) handling
 * Davidlohr Bueso <[email protected]>, June 2013.
 */

#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
#include <uapi/linux/shm.h>
#include <linux/init.h>
#include <linux/file.h>
#include <linux/mman.h>
#include <linux/shmem_fs.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
#include <linux/capability.h>
#include <linux/ptrace.h>
#include <linux/seq_file.h>
#include <linux/rwsem.h>
#include <linux/nsproxy.h>
#include <linux/mount.h>
#include <linux/ipc_namespace.h>
#include <linux/rhashtable.h>

#include <linux/uaccess.h>

#include "util.h"

struct shmid_kernel /* private to the kernel */
{ … } __randomize_layout;

/* shm_mode upper byte flags */
#define SHM_DEST …
#define SHM_LOCKED …

struct shm_file_data { … };

#define shm_file_data(file) …

static const struct file_operations shm_file_operations;
static const struct vm_operations_struct shm_vm_ops;

#define shm_ids(ns) …

#define shm_unlock(shp) …

static int newseg(struct ipc_namespace *, struct ipc_params *);
static void shm_open(struct vm_area_struct *vma);
static void shm_close(struct vm_area_struct *vma);
static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
#ifdef CONFIG_PROC_FS
static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
#endif

void shm_init_ns(struct ipc_namespace *ns)
{ … }

/*
 * Called with shm_ids.rwsem (writer) and the shp structure locked.
 * Only shm_ids.rwsem remains locked on exit.
 */
static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
{ … }

#ifdef CONFIG_IPC_NS
void shm_exit_ns(struct ipc_namespace *ns)
{ … }
#endif

static int __init ipc_ns_init(void)
{ … }

pure_initcall(ipc_ns_init);

void __init shm_init(void)
{ … }

static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
{ … }

static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
{ … }

/*
 * shm_lock_(check_) routines are called in the paths where the rwsem
 * is not necessarily held.
 */
static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
{ … }

static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
{ … }

static void shm_rcu_free(struct rcu_head *head)
{ … }

/*
 * It has to be called with shp locked.
 * It must be called before ipc_rmid()
 */
static inline void shm_clist_rm(struct shmid_kernel *shp)
{ … }

static inline void shm_rmid(struct shmid_kernel *s)
{ … }


static int __shm_open(struct shm_file_data *sfd)
{ … }

/* This is called by fork, once for every shm attach. */
static void shm_open(struct vm_area_struct *vma)
{ … }

/*
 * shm_destroy - free the struct shmid_kernel
 *
 * @ns: namespace
 * @shp: struct to free
 *
 * It has to be called with shp and shm_ids.rwsem (writer) locked,
 * but returns with shp unlocked and freed.
 */
static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
{ … }

/*
 * shm_may_destroy - identifies whether shm segment should be destroyed now
 *
 * Returns true if and only if there are no active users of the segment and
 * one of the following is true:
 *
 * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
 *
 * 2) sysctl kernel.shm_rmid_forced is set to 1.
 */
static bool shm_may_destroy(struct shmid_kernel *shp)
{ … }

/*
 * remove the attach descriptor vma.
 * free memory for segment if it is marked destroyed.
 * The descriptor has already been removed from the current->mm->mmap list
 * and will later be kfree()d.
 */
static void __shm_close(struct shm_file_data *sfd)
{ … }

static void shm_close(struct vm_area_struct *vma)
{ … }

/* Called with ns->shm_ids(ns).rwsem locked */
static int shm_try_destroy_orphaned(int id, void *p, void *data)
{ … }

void shm_destroy_orphaned(struct ipc_namespace *ns)
{ … }

/* Locking assumes this will only be called with task == current */
void exit_shm(struct task_struct *task)
{ … }

static vm_fault_t shm_fault(struct vm_fault *vmf)
{ … }

static int shm_may_split(struct vm_area_struct *vma, unsigned long addr)
{ … }

static unsigned long shm_pagesize(struct vm_area_struct *vma)
{ … }

#ifdef CONFIG_NUMA
static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
{ … }

static struct mempolicy *shm_get_policy(struct vm_area_struct *vma,
					unsigned long addr, pgoff_t *ilx)
{ … }
#endif

static int shm_mmap(struct file *file, struct vm_area_struct *vma)
{ … }

static int shm_release(struct inode *ino, struct file *file)
{ … }

static int shm_fsync(struct file *file, loff_t start, loff_t end, int datasync)
{ … }

static long shm_fallocate(struct file *file, int mode, loff_t offset,
			  loff_t len)
{ … }

static unsigned long shm_get_unmapped_area(struct file *file,
	unsigned long addr, unsigned long len, unsigned long pgoff,
	unsigned long flags)
{ … }

static const struct file_operations shm_file_operations = …;

/*
 * shm_file_operations_huge is now identical to shm_file_operations
 * except for fop_flags
 */
static const struct file_operations shm_file_operations_huge = …;

static const struct vm_operations_struct shm_vm_ops = …;

/**
 * newseg - Create a new shared memory segment
 * @ns: namespace
 * @params: ptr to the structure that contains key, size and shmflg
 *
 * Called with shm_ids.rwsem held as a writer.
 */
static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
{ … }

/*
 * Called with shm_ids.rwsem and ipcp locked.
 */
static int shm_more_checks(struct kern_ipc_perm *ipcp, struct ipc_params *params)
{ … }

long ksys_shmget(key_t key, size_t size, int shmflg)
{ … }

SYSCALL_DEFINE3(shmget, key_t, key, size_t, size, int, shmflg)
{ … }

static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
{ … }

static inline unsigned long
copy_shmid_from_user(struct shmid64_ds *out, void __user *buf, int version)
{ … }

static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
{ … }

/*
 * Calculate and add used RSS and swap pages of a shm.
 * Called with shm_ids.rwsem held as a reader
 */
static void shm_add_rss_swap(struct shmid_kernel *shp,
	unsigned long *rss_add, unsigned long *swp_add)
{ … }

/*
 * Called with shm_ids.rwsem held as a reader
 */
static void shm_get_stat(struct ipc_namespace *ns, unsigned long *rss,
		unsigned long *swp)
{ … }

/*
 * This function handles some shmctl commands which require the rwsem
 * to be held in write mode.
 * NOTE: no locks must be held, the rwsem is taken inside this function.
 */
static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
		       struct shmid64_ds *shmid64)
{ … }

static int shmctl_ipc_info(struct ipc_namespace *ns,
			   struct shminfo64 *shminfo)
{ … }

static int shmctl_shm_info(struct ipc_namespace *ns,
			   struct shm_info *shm_info)
{ … }

static int shmctl_stat(struct ipc_namespace *ns, int shmid,
			int cmd, struct shmid64_ds *tbuf)
{ … }

static int shmctl_do_lock(struct ipc_namespace *ns, int shmid, int cmd)
{ … }

static long ksys_shmctl(int shmid, int cmd, struct shmid_ds __user *buf, int version)
{ … }

SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
{ … }

#ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
long ksys_old_shmctl(int shmid, int cmd, struct shmid_ds __user *buf)
{
	int version = ipc_parse_version(&cmd);

	return ksys_shmctl(shmid, cmd, buf, version);
}

SYSCALL_DEFINE3(old_shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
{
	return ksys_old_shmctl(shmid, cmd, buf);
}
#endif

#ifdef CONFIG_COMPAT

struct compat_shmid_ds { … };

struct compat_shminfo64 { … };

struct compat_shm_info { … };

static int copy_compat_shminfo_to_user(void __user *buf, struct shminfo64 *in,
					int version)
{ … }

static int put_compat_shm_info(struct shm_info *ip,
				struct compat_shm_info __user *uip)
{ … }

static int copy_compat_shmid_to_user(void __user *buf, struct shmid64_ds *in,
					int version)
{ … }

static int copy_compat_shmid_from_user(struct shmid64_ds *out, void __user *buf,
					int version)
{ … }

static long compat_ksys_shmctl(int shmid, int cmd, void __user *uptr, int version)
{ … }

COMPAT_SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, void __user *, uptr)
{ … }

#ifdef CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION
long compat_ksys_old_shmctl(int shmid, int cmd, void __user *uptr)
{ … }

COMPAT_SYSCALL_DEFINE3(old_shmctl, int, shmid, int, cmd, void __user *, uptr)
{ … }
#endif
#endif

/*
 * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
 *
 * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
 * "raddr" thing points to kernel space, and there has to be a wrapper around
 * this.
 */
long do_shmat(int shmid, char __user *shmaddr, int shmflg,
	      ulong *raddr, unsigned long shmlba)
{ … }

SYSCALL_DEFINE3(shmat, int, shmid, char __user *, shmaddr, int, shmflg)
{ … }

#ifdef CONFIG_COMPAT

#ifndef COMPAT_SHMLBA
#define COMPAT_SHMLBA …
#endif

COMPAT_SYSCALL_DEFINE3(shmat, int, shmid, compat_uptr_t, shmaddr, int, shmflg)
{ … }
#endif

/*
 * detach and kill segment if marked destroyed.
 * The work is done in shm_close.
 */
long ksys_shmdt(char __user *shmaddr)
{ … }

SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
{ … }

#ifdef CONFIG_PROC_FS
static int sysvipc_shm_proc_show(struct seq_file *s, void *it)
{ … }
#endif