linux/drivers/accel/habanalabs/common/memory_mgr.c

// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright 2022 HabanaLabs, Ltd.
 * All Rights Reserved.
 */

#include "habanalabs.h"

/**
 * hl_mmap_mem_buf_get - increase the buffer refcount and return a pointer to
 *                        the buffer descriptor.
 *
 * @mmg: parent unified memory manager
 * @handle: requested buffer handle
 *
 * Find the buffer in the store and return a pointer to its descriptor.
 * Increase buffer refcount. If not found - return NULL.
 */
struct hl_mmap_mem_buf *hl_mmap_mem_buf_get(struct hl_mem_mgr *mmg, u64 handle)
{ … }

/**
 * hl_mmap_mem_buf_destroy - destroy the unused buffer
 *
 * @buf: memory manager buffer descriptor
 *
 * Internal function, used as a final step of buffer release. Shall be invoked
 * only when the buffer is no longer in use (removed from idr). Will call the
 * release callback (if applicable), and free the memory.
 */
static void hl_mmap_mem_buf_destroy(struct hl_mmap_mem_buf *buf)
{ … }

/**
 * hl_mmap_mem_buf_release - release buffer
 *
 * @kref: kref that reached 0.
 *
 * Internal function, used as a kref release callback, when the last user of
 * the buffer is released. Shall be called from an interrupt context.
 */
static void hl_mmap_mem_buf_release(struct kref *kref)
{ … }

/**
 * hl_mmap_mem_buf_remove_idr_locked - remove handle from idr
 *
 * @kref: kref that reached 0.
 *
 * Internal function, used for kref put by handle. Assumes mmg lock is taken.
 * Will remove the buffer from idr, without destroying it.
 */
static void hl_mmap_mem_buf_remove_idr_locked(struct kref *kref)
{ … }

/**
 * hl_mmap_mem_buf_put - decrease the reference to the buffer
 *
 * @buf: memory manager buffer descriptor
 *
 * Decrease the reference to the buffer, and release it if it was the last one.
 * Shall be called from an interrupt context.
 */
int hl_mmap_mem_buf_put(struct hl_mmap_mem_buf *buf)
{ … }

/**
 * hl_mmap_mem_buf_put_handle - decrease the reference to the buffer with the
 *                              given handle.
 *
 * @mmg: parent unified memory manager
 * @handle: requested buffer handle
 *
 * Decrease the reference to the buffer, and release it if it was the last one.
 * Shall not be called from an interrupt context. Return -EINVAL if handle was
 * not found, else return the put outcome (0 or 1).
 */
int hl_mmap_mem_buf_put_handle(struct hl_mem_mgr *mmg, u64 handle)
{ … }

/**
 * hl_mmap_mem_buf_alloc - allocate a new mappable buffer
 *
 * @mmg: parent unified memory manager
 * @behavior: behavior object describing this buffer polymorphic behavior
 * @gfp: gfp flags to use for the memory allocations
 * @args: additional args passed to behavior->alloc
 *
 * Allocate and register a new memory buffer inside the give memory manager.
 * Return the pointer to the new buffer on success or NULL on failure.
 */
struct hl_mmap_mem_buf *
hl_mmap_mem_buf_alloc(struct hl_mem_mgr *mmg,
		      struct hl_mmap_mem_buf_behavior *behavior, gfp_t gfp,
		      void *args)
{ … }

/**
 * hl_mmap_mem_buf_vm_close - handle mmap close
 *
 * @vma: the vma object for which mmap was closed.
 *
 * Put the memory buffer if it is no longer mapped.
 */
static void hl_mmap_mem_buf_vm_close(struct vm_area_struct *vma)
{ … }

static const struct vm_operations_struct hl_mmap_mem_buf_vm_ops = …;

/**
 * hl_mem_mgr_mmap - map the given buffer to the user
 *
 * @mmg: unified memory manager
 * @vma: the vma object for which mmap was closed.
 * @args: additional args passed to behavior->mmap
 *
 * Map the buffer specified by the vma->vm_pgoff to the given vma.
 */
int hl_mem_mgr_mmap(struct hl_mem_mgr *mmg, struct vm_area_struct *vma,
		    void *args)
{ … }

/**
 * hl_mem_mgr_init - initialize unified memory manager
 *
 * @dev: owner device pointer
 * @mmg: structure to initialize
 *
 * Initialize an instance of unified memory manager
 */
void hl_mem_mgr_init(struct device *dev, struct hl_mem_mgr *mmg)
{ … }

static void hl_mem_mgr_fini_stats_reset(struct hl_mem_mgr_fini_stats *stats)
{ … }

static void hl_mem_mgr_fini_stats_inc(u64 mem_id, struct hl_mem_mgr_fini_stats *stats)
{ … }

/**
 * hl_mem_mgr_fini - release unified memory manager
 *
 * @mmg: parent unified memory manager
 * @stats: if non-NULL, will return some counters for handles that could not be removed.
 *
 * Release the unified memory manager. Shall be called from an interrupt context.
 */
void hl_mem_mgr_fini(struct hl_mem_mgr *mmg, struct hl_mem_mgr_fini_stats *stats)
{ … }

/**
 * hl_mem_mgr_idr_destroy() - destroy memory manager IDR.
 * @mmg: parent unified memory manager
 *
 * Destroy the memory manager IDR.
 * Shall be called when IDR is empty and no memory buffers are in use.
 */
void hl_mem_mgr_idr_destroy(struct hl_mem_mgr *mmg)
{ … }