// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
*/
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>
#include <linux/slab.h>
#include <linux/bitmap.h>
#include "dm-core.h"
#define DM_MSG_PREFIX "zone"
/*
* For internal zone reports bypassing the top BIO submission path.
*/
static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t,
sector_t sector, unsigned int nr_zones,
report_zones_cb cb, void *data)
{
struct gendisk *disk = md->disk;
int ret;
struct dm_report_zones_args args = {
.next_sector = sector,
.orig_data = data,
.orig_cb = cb,
};
do {
struct dm_target *tgt;
tgt = dm_table_find_target(t, args.next_sector);
if (WARN_ON_ONCE(!tgt->type->report_zones))
return -EIO;
args.tgt = tgt;
ret = tgt->type->report_zones(tgt, &args,
nr_zones - args.zone_idx);
if (ret < 0)
return ret;
} while (args.zone_idx < nr_zones &&
args.next_sector < get_capacity(disk));
return args.zone_idx;
}
/*
* User facing dm device block device report zone operation. This calls the
* report_zones operation for each target of a device table. This operation is
* generally implemented by targets using dm_report_zones().
*/
int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
unsigned int nr_zones, report_zones_cb cb, void *data)
{
struct mapped_device *md = disk->private_data;
struct dm_table *map;
int srcu_idx, ret;
if (!md->zone_revalidate_map) {
/* Regular user context */
if (dm_suspended_md(md))
return -EAGAIN;
map = dm_get_live_table(md, &srcu_idx);
if (!map)
return -EIO;
} else {
/* Zone revalidation during __bind() */
map = md->zone_revalidate_map;
}
ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data);
if (!md->zone_revalidate_map)
dm_put_live_table(md, srcu_idx);
return ret;
}
static int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx,
void *data)
{
struct dm_report_zones_args *args = data;
sector_t sector_diff = args->tgt->begin - args->start;
/*
* Ignore zones beyond the target range.
*/
if (zone->start >= args->start + args->tgt->len)
return 0;
/*
* Remap the start sector and write pointer position of the zone
* to match its position in the target range.
*/
zone->start += sector_diff;
if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
if (zone->cond == BLK_ZONE_COND_FULL)
zone->wp = zone->start + zone->len;
else if (zone->cond == BLK_ZONE_COND_EMPTY)
zone->wp = zone->start;
else
zone->wp += sector_diff;
}
args->next_sector = zone->start + zone->len;
return args->orig_cb(zone, args->zone_idx++, args->orig_data);
}
/*
* Helper for drivers of zoned targets to implement struct target_type
* report_zones operation.
*/
int dm_report_zones(struct block_device *bdev, sector_t start, sector_t sector,
struct dm_report_zones_args *args, unsigned int nr_zones)
{
/*
* Set the target mapping start sector first so that
* dm_report_zones_cb() can correctly remap zone information.
*/
args->start = start;
return blkdev_report_zones(bdev, sector, nr_zones,
dm_report_zones_cb, args);
}
EXPORT_SYMBOL_GPL(dm_report_zones);
bool dm_is_zone_write(struct mapped_device *md, struct bio *bio)
{
struct request_queue *q = md->queue;
if (!blk_queue_is_zoned(q))
return false;
switch (bio_op(bio)) {
case REQ_OP_WRITE_ZEROES:
case REQ_OP_WRITE:
return !op_is_flush(bio->bi_opf) && bio_sectors(bio);
default:
return false;
}
}
/*
* Revalidate the zones of a mapped device to initialize resource necessary
* for zone append emulation. Note that we cannot simply use the block layer
* blk_revalidate_disk_zones() function here as the mapped device is suspended
* (this is called from __bind() context).
*/
int dm_revalidate_zones(struct dm_table *t, struct request_queue *q)
{
struct mapped_device *md = t->md;
struct gendisk *disk = md->disk;
int ret;
if (!get_capacity(disk))
return 0;
/* Revalidate only if something changed. */
if (!disk->nr_zones || disk->nr_zones != md->nr_zones) {
DMINFO("%s using %s zone append",
disk->disk_name,
queue_emulates_zone_append(q) ? "emulated" : "native");
md->nr_zones = 0;
}
if (md->nr_zones)
return 0;
/*
* Our table is not live yet. So the call to dm_get_live_table()
* in dm_blk_report_zones() will fail. Set a temporary pointer to
* our table for dm_blk_report_zones() to use directly.
*/
md->zone_revalidate_map = t;
ret = blk_revalidate_disk_zones(disk);
md->zone_revalidate_map = NULL;
if (ret) {
DMERR("Revalidate zones failed %d", ret);
return ret;
}
md->nr_zones = disk->nr_zones;
return 0;
}
static int device_not_zone_append_capable(struct dm_target *ti,
struct dm_dev *dev, sector_t start,
sector_t len, void *data)
{
return !bdev_is_zoned(dev->bdev);
}
static bool dm_table_supports_zone_append(struct dm_table *t)
{
for (unsigned int i = 0; i < t->num_targets; i++) {
struct dm_target *ti = dm_table_get_target(t, i);
if (ti->emulate_zone_append)
return false;
if (!ti->type->iterate_devices ||
ti->type->iterate_devices(ti, device_not_zone_append_capable, NULL))
return false;
}
return true;
}
struct dm_device_zone_count {
sector_t start;
sector_t len;
unsigned int total_nr_seq_zones;
unsigned int target_nr_seq_zones;
};
/*
* Count the total number of and the number of mapped sequential zones of a
* target zoned device.
*/
static int dm_device_count_zones_cb(struct blk_zone *zone,
unsigned int idx, void *data)
{
struct dm_device_zone_count *zc = data;
if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
zc->total_nr_seq_zones++;
if (zone->start >= zc->start &&
zone->start < zc->start + zc->len)
zc->target_nr_seq_zones++;
}
return 0;
}
static int dm_device_count_zones(struct dm_dev *dev,
struct dm_device_zone_count *zc)
{
int ret;
ret = blkdev_report_zones(dev->bdev, 0, BLK_ALL_ZONES,
dm_device_count_zones_cb, zc);
if (ret < 0)
return ret;
if (!ret)
return -EIO;
return 0;
}
struct dm_zone_resource_limits {
unsigned int mapped_nr_seq_zones;
struct queue_limits *lim;
bool reliable_limits;
};
static int device_get_zone_resource_limits(struct dm_target *ti,
struct dm_dev *dev, sector_t start,
sector_t len, void *data)
{
struct dm_zone_resource_limits *zlim = data;
struct gendisk *disk = dev->bdev->bd_disk;
unsigned int max_open_zones, max_active_zones;
int ret;
struct dm_device_zone_count zc = {
.start = start,
.len = len,
};
/*
* If the target is not the whole device, the device zone resources may
* be shared between different targets. Check this by counting the
* number of mapped sequential zones: if this number is smaller than the
* total number of sequential zones of the target device, then resource
* sharing may happen and the zone limits will not be reliable.
*/
ret = dm_device_count_zones(dev, &zc);
if (ret) {
DMERR("Count %s zones failed %d", disk->disk_name, ret);
return ret;
}
/*
* If the target does not map any sequential zones, then we do not need
* any zone resource limits.
*/
if (!zc.target_nr_seq_zones)
return 0;
/*
* If the target does not map all sequential zones, the limits
* will not be reliable and we cannot use REQ_OP_ZONE_RESET_ALL.
*/
if (zc.target_nr_seq_zones < zc.total_nr_seq_zones) {
zlim->reliable_limits = false;
ti->zone_reset_all_supported = false;
}
/*
* If the target maps less sequential zones than the limit values, then
* we do not have limits for this target.
*/
max_active_zones = disk->queue->limits.max_active_zones;
if (max_active_zones >= zc.target_nr_seq_zones)
max_active_zones = 0;
zlim->lim->max_active_zones =
min_not_zero(max_active_zones, zlim->lim->max_active_zones);
max_open_zones = disk->queue->limits.max_open_zones;
if (max_open_zones >= zc.target_nr_seq_zones)
max_open_zones = 0;
zlim->lim->max_open_zones =
min_not_zero(max_open_zones, zlim->lim->max_open_zones);
/*
* Also count the total number of sequential zones for the mapped
* device so that when we are done inspecting all its targets, we are
* able to check if the mapped device actually has any sequential zones.
*/
zlim->mapped_nr_seq_zones += zc.target_nr_seq_zones;
return 0;
}
int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q,
struct queue_limits *lim)
{
struct mapped_device *md = t->md;
struct gendisk *disk = md->disk;
struct dm_zone_resource_limits zlim = {
.reliable_limits = true,
.lim = lim,
};
/*
* Check if zone append is natively supported, and if not, set the
* mapped device queue as needing zone append emulation.
*/
WARN_ON_ONCE(queue_is_mq(q));
if (dm_table_supports_zone_append(t)) {
clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
} else {
set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
lim->max_zone_append_sectors = 0;
}
/*
* Determine the max open and max active zone limits for the mapped
* device by inspecting the zone resource limits and the zones mapped
* by each target.
*/
for (unsigned int i = 0; i < t->num_targets; i++) {
struct dm_target *ti = dm_table_get_target(t, i);
/*
* Assume that the target can accept REQ_OP_ZONE_RESET_ALL.
* device_get_zone_resource_limits() may adjust this if one of
* the device used by the target does not have all its
* sequential write required zones mapped.
*/
ti->zone_reset_all_supported = true;
if (!ti->type->iterate_devices ||
ti->type->iterate_devices(ti,
device_get_zone_resource_limits, &zlim)) {
DMERR("Could not determine %s zone resource limits",
disk->disk_name);
return -ENODEV;
}
}
/*
* If we only have conventional zones mapped, expose the mapped device
+ as a regular device.
*/
if (!zlim.mapped_nr_seq_zones) {
lim->max_open_zones = 0;
lim->max_active_zones = 0;
lim->max_zone_append_sectors = 0;
lim->zone_write_granularity = 0;
lim->chunk_sectors = 0;
lim->features &= ~BLK_FEAT_ZONED;
clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
md->nr_zones = 0;
disk->nr_zones = 0;
return 0;
}
/*
* Warn once (when the capacity is not yet set) if the mapped device is
* partially using zone resources of the target devices as that leads to
* unreliable limits, i.e. if another mapped device uses the same
* underlying devices, we cannot enforce zone limits to guarantee that
* writing will not lead to errors. Note that we really should return
* an error for such case but there is no easy way to find out if
* another mapped device uses the same underlying zoned devices.
*/
if (!get_capacity(disk) && !zlim.reliable_limits)
DMWARN("%s zone resource limits may be unreliable",
disk->disk_name);
if (lim->features & BLK_FEAT_ZONED &&
!static_key_enabled(&zoned_enabled.key))
static_branch_enable(&zoned_enabled);
return 0;
}
/*
* IO completion callback called from clone_endio().
*/
void dm_zone_endio(struct dm_io *io, struct bio *clone)
{
struct mapped_device *md = io->md;
struct gendisk *disk = md->disk;
struct bio *orig_bio = io->orig_bio;
/*
* Get the offset within the zone of the written sector
* and add that to the original bio sector position.
*/
if (clone->bi_status == BLK_STS_OK &&
bio_op(clone) == REQ_OP_ZONE_APPEND) {
sector_t mask = bdev_zone_sectors(disk->part0) - 1;
orig_bio->bi_iter.bi_sector += clone->bi_iter.bi_sector & mask;
}
return;
}
static int dm_zone_need_reset_cb(struct blk_zone *zone, unsigned int idx,
void *data)
{
/*
* For an all-zones reset, ignore conventional, empty, read-only
* and offline zones.
*/
switch (zone->cond) {
case BLK_ZONE_COND_NOT_WP:
case BLK_ZONE_COND_EMPTY:
case BLK_ZONE_COND_READONLY:
case BLK_ZONE_COND_OFFLINE:
return 0;
default:
set_bit(idx, (unsigned long *)data);
return 0;
}
}
int dm_zone_get_reset_bitmap(struct mapped_device *md, struct dm_table *t,
sector_t sector, unsigned int nr_zones,
unsigned long *need_reset)
{
int ret;
ret = dm_blk_do_report_zones(md, t, sector, nr_zones,
dm_zone_need_reset_cb, need_reset);
if (ret != nr_zones) {
DMERR("Get %s zone reset bitmap failed\n",
md->disk->disk_name);
return -EIO;
}
return 0;
}
Codebase Browser
linux