linux/drivers/nvdimm/pmem.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Persistent Memory Driver
 *
 * Copyright (c) 2014-2015, Intel Corporation.
 * Copyright (c) 2015, Christoph Hellwig <[email protected]>.
 * Copyright (c) 2015, Boaz Harrosh <[email protected]>.
 */

#include <linux/blkdev.h>
#include <linux/pagemap.h>
#include <linux/hdreg.h>
#include <linux/init.h>
#include <linux/platform_device.h>
#include <linux/set_memory.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/badblocks.h>
#include <linux/memremap.h>
#include <linux/kstrtox.h>
#include <linux/vmalloc.h>
#include <linux/blk-mq.h>
#include <linux/pfn_t.h>
#include <linux/slab.h>
#include <linux/uio.h>
#include <linux/dax.h>
#include <linux/nd.h>
#include <linux/mm.h>
#include <asm/cacheflush.h>
#include "pmem.h"
#include "btt.h"
#include "pfn.h"
#include "nd.h"

static struct device *to_dev(struct pmem_device *pmem)
{ … }

static struct nd_region *to_region(struct pmem_device *pmem)
{ … }

static phys_addr_t pmem_to_phys(struct pmem_device *pmem, phys_addr_t offset)
{ … }

static sector_t to_sect(struct pmem_device *pmem, phys_addr_t offset)
{ … }

static phys_addr_t to_offset(struct pmem_device *pmem, sector_t sector)
{ … }

static void pmem_mkpage_present(struct pmem_device *pmem, phys_addr_t offset,
		unsigned int len)
{ … }

static void pmem_clear_bb(struct pmem_device *pmem, sector_t sector, long blks)
{ … }

static long __pmem_clear_poison(struct pmem_device *pmem,
		phys_addr_t offset, unsigned int len)
{ … }

static blk_status_t pmem_clear_poison(struct pmem_device *pmem,
		phys_addr_t offset, unsigned int len)
{ … }

static void write_pmem(void *pmem_addr, struct page *page,
		unsigned int off, unsigned int len)
{ … }

static blk_status_t read_pmem(struct page *page, unsigned int off,
		void *pmem_addr, unsigned int len)
{ … }

static blk_status_t pmem_do_read(struct pmem_device *pmem,
			struct page *page, unsigned int page_off,
			sector_t sector, unsigned int len)
{ … }

static blk_status_t pmem_do_write(struct pmem_device *pmem,
			struct page *page, unsigned int page_off,
			sector_t sector, unsigned int len)
{ … }

static void pmem_submit_bio(struct bio *bio)
{ … }

/* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */
__weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
		long nr_pages, enum dax_access_mode mode, void **kaddr,
		pfn_t *pfn)
{ … }

static const struct block_device_operations pmem_fops = …;

static int pmem_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
				    size_t nr_pages)
{ … }

static long pmem_dax_direct_access(struct dax_device *dax_dev,
		pgoff_t pgoff, long nr_pages, enum dax_access_mode mode,
		void **kaddr, pfn_t *pfn)
{ … }

/*
 * The recovery write thread started out as a normal pwrite thread and
 * when the filesystem was told about potential media error in the
 * range, filesystem turns the normal pwrite to a dax_recovery_write.
 *
 * The recovery write consists of clearing media poison, clearing page
 * HWPoison bit, reenable page-wide read-write permission, flush the
 * caches and finally write.  A competing pread thread will be held
 * off during the recovery process since data read back might not be
 * valid, and this is achieved by clearing the badblock records after
 * the recovery write is complete. Competing recovery write threads
 * are already serialized by writer lock held by dax_iomap_rw().
 */
static size_t pmem_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
		void *addr, size_t bytes, struct iov_iter *i)
{ … }

static const struct dax_operations pmem_dax_ops = …;

static ssize_t write_cache_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{ … }

static ssize_t write_cache_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{ … }
static DEVICE_ATTR_RW(write_cache);

static umode_t dax_visible(struct kobject *kobj, struct attribute *a, int n)
{ … }

static struct attribute *dax_attributes[] = …;

static const struct attribute_group dax_attribute_group = …;

static const struct attribute_group *pmem_attribute_groups[] = …;

static void pmem_release_disk(void *__pmem)
{ … }

static int pmem_pagemap_memory_failure(struct dev_pagemap *pgmap,
		unsigned long pfn, unsigned long nr_pages, int mf_flags)
{ … }

static const struct dev_pagemap_ops fsdax_pagemap_ops = …;

static int pmem_attach_disk(struct device *dev,
		struct nd_namespace_common *ndns)
{ … }

static int nd_pmem_probe(struct device *dev)
{ … }

static void nd_pmem_remove(struct device *dev)
{ … }

static void nd_pmem_shutdown(struct device *dev)
{ … }

static void pmem_revalidate_poison(struct device *dev)
{ … }

static void pmem_revalidate_region(struct device *dev)
{ … }

static void nd_pmem_notify(struct device *dev, enum nvdimm_event event)
{ … }

MODULE_ALIAS(…) …;
MODULE_ALIAS_ND_DEVICE(…);
MODULE_ALIAS_ND_DEVICE(…);
static struct nd_device_driver nd_pmem_driver = …;

module_nd_driver(…) …;

MODULE_AUTHOR(…) …;
MODULE_DESCRIPTION(…) …;
MODULE_LICENSE(…) …;