// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2022, Oracle and/or its affiliates. * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved */ #include <linux/iova_bitmap.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/highmem.h> #define BITS_PER_PAGE … /* * struct iova_bitmap_map - A bitmap representing an IOVA range * * Main data structure for tracking mapped user pages of bitmap data. * * For example, for something recording dirty IOVAs, it will be provided a * struct iova_bitmap structure, as a general structure for iterating the * total IOVA range. The struct iova_bitmap_map, though, represents the * subset of said IOVA space that is pinned by its parent structure (struct * iova_bitmap). * * The user does not need to exact location of the bits in the bitmap. * From user perspective the only API available is iova_bitmap_set() which * records the IOVA *range* in the bitmap by setting the corresponding * bits. * * The bitmap is an array of u64 whereas each bit represents an IOVA of * range of (1 << pgshift). Thus formula for the bitmap data to be set is: * * data[(iova / page_size) / 64] & (1ULL << (iova % 64)) */ struct iova_bitmap_map { … }; /* * struct iova_bitmap - The IOVA bitmap object * * Main data structure for iterating over the bitmap data. * * Abstracts the pinning work and iterates in IOVA ranges. * It uses a windowing scheme and pins the bitmap in relatively * big ranges e.g. * * The bitmap object uses one base page to store all the pinned pages * pointers related to the bitmap. For sizeof(struct page*) == 8 it stores * 512 struct page pointers which, if the base page size is 4K, it means * 2M of bitmap data is pinned at a time. If the iova_bitmap page size is * also 4K then the range window to iterate is 64G. * * For example iterating on a total IOVA range of 4G..128G, it will walk * through this set of ranges: * * 4G - 68G-1 (64G) * 68G - 128G-1 (64G) * * An example of the APIs on how to use/iterate over the IOVA bitmap: * * bitmap = iova_bitmap_alloc(iova, length, page_size, data); * if (IS_ERR(bitmap)) * return PTR_ERR(bitmap); * * ret = iova_bitmap_for_each(bitmap, arg, dirty_reporter_fn); * * iova_bitmap_free(bitmap); * * Each iteration of the @dirty_reporter_fn is called with a unique @iova * and @length argument, indicating the current range available through the * iova_bitmap. The @dirty_reporter_fn uses iova_bitmap_set() to mark dirty * areas (@iova_length) within that provided range, as following: * * iova_bitmap_set(bitmap, iova, iova_length); * * The internals of the object uses an index @mapped_base_index that indexes * which u64 word of the bitmap is mapped, up to @mapped_total_index. * Those keep being incremented until @mapped_total_index is reached while * mapping up to PAGE_SIZE / sizeof(struct page*) maximum of pages. * * The IOVA bitmap is usually located on what tracks DMA mapped ranges or * some form of IOVA range tracking that co-relates to the user passed * bitmap. */ struct iova_bitmap { … }; /* * Converts a relative IOVA to a bitmap index. * This function provides the index into the u64 array (bitmap::bitmap) * for a given IOVA offset. * Relative IOVA means relative to the bitmap::mapped base IOVA * (stored in mapped::iova). All computations in this file are done using * relative IOVAs and thus avoid an extra subtraction against mapped::iova. * The user API iova_bitmap_set() always uses a regular absolute IOVAs. */ static unsigned long iova_bitmap_offset_to_index(struct iova_bitmap *bitmap, unsigned long iova) { … } /* * Converts a bitmap index to a *relative* IOVA. */ static unsigned long iova_bitmap_index_to_offset(struct iova_bitmap *bitmap, unsigned long index) { … } /* * Returns the base IOVA of the mapped range. */ static unsigned long iova_bitmap_mapped_iova(struct iova_bitmap *bitmap) { … } static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap); /* * Pins the bitmap user pages for the current range window. * This is internal to IOVA bitmap and called when advancing the * index (@mapped_base_index) or allocating the bitmap. */ static int iova_bitmap_get(struct iova_bitmap *bitmap) { … } /* * Unpins the bitmap user pages and clears @npages * (un)pinning is abstracted from API user and it's done when advancing * the index or freeing the bitmap. */ static void iova_bitmap_put(struct iova_bitmap *bitmap) { … } /** * iova_bitmap_alloc() - Allocates an IOVA bitmap object * @iova: Start address of the IOVA range * @length: Length of the IOVA range * @page_size: Page size of the IOVA bitmap. It defines what each bit * granularity represents * @data: Userspace address of the bitmap * * Allocates an IOVA object and initializes all its fields including the * first user pages of @data. * * Return: A pointer to a newly allocated struct iova_bitmap * or ERR_PTR() on error. */ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, unsigned long page_size, u64 __user *data) { … } EXPORT_SYMBOL_NS_GPL(…); /** * iova_bitmap_free() - Frees an IOVA bitmap object * @bitmap: IOVA bitmap to free * * It unpins and releases pages array memory and clears any leftover * state. */ void iova_bitmap_free(struct iova_bitmap *bitmap) { … } EXPORT_SYMBOL_NS_GPL(…); /* * Returns the remaining bitmap indexes from mapped_total_index to process for * the currently pinned bitmap pages. */ static unsigned long iova_bitmap_mapped_remaining(struct iova_bitmap *bitmap) { … } /* * Returns the length of the mapped IOVA range. */ static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap) { … } /* * Returns true if [@iova..@iova+@length-1] is part of the mapped IOVA range. */ static bool iova_bitmap_mapped_range(struct iova_bitmap_map *mapped, unsigned long iova, size_t length) { … } /* * Advances to a selected range, releases the current pinned * pages and pins the next set of bitmap pages. * Returns 0 on success or otherwise errno. */ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, unsigned long iova) { … } /** * iova_bitmap_for_each() - Iterates over the bitmap * @bitmap: IOVA bitmap to iterate * @opaque: Additional argument to pass to the callback * @fn: Function that gets called for each IOVA range * * Helper function to iterate over bitmap data representing a portion of IOVA * space. It hides the complexity of iterating bitmaps and translating the * mapped bitmap user pages into IOVA ranges to process. * * Return: 0 on success, and an error on failure either upon * iteration or when the callback returns an error. */ int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, iova_bitmap_fn_t fn) { … } EXPORT_SYMBOL_NS_GPL(…); /** * iova_bitmap_set() - Records an IOVA range in bitmap * @bitmap: IOVA bitmap * @iova: IOVA to start * @length: IOVA range length * * Set the bits corresponding to the range [iova .. iova+length-1] in * the user bitmap. * */ void iova_bitmap_set(struct iova_bitmap *bitmap, unsigned long iova, size_t length) { … } EXPORT_SYMBOL_NS_GPL(…);
Codebase Browser
linux