// SPDX-License-Identifier: GPL-2.0
/*
* Copyright 2018-2020 Broadcom.
*/
#include <linux/dma-mapping.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/pgtable.h>
#include <linux/vmalloc.h>
#include <asm/page.h>
#include <linux/unaligned.h>
#include <uapi/linux/misc/bcm_vk.h>
#include "bcm_vk.h"
#include "bcm_vk_msg.h"
#include "bcm_vk_sg.h"
/*
* Valkyrie has a hardware limitation of 16M transfer size.
* So limit the SGL chunks to 16M.
*/
#define BCM_VK_MAX_SGL_CHUNK SZ_16M
static int bcm_vk_dma_alloc(struct device *dev,
struct bcm_vk_dma *dma,
int dir,
struct _vk_data *vkdata);
static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma);
/* Uncomment to dump SGLIST */
/* #define BCM_VK_DUMP_SGLIST */
static int bcm_vk_dma_alloc(struct device *dev,
struct bcm_vk_dma *dma,
int direction,
struct _vk_data *vkdata)
{
dma_addr_t addr, sg_addr;
int err;
int i;
int offset;
u32 size;
u32 remaining_size;
u32 transfer_size;
u64 data;
unsigned long first, last;
struct _vk_data *sgdata;
/* Get 64-bit user address */
data = get_unaligned(&vkdata->address);
/* offset into first page */
offset = offset_in_page(data);
/* Calculate number of pages */
first = (data & PAGE_MASK) >> PAGE_SHIFT;
last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT;
dma->nr_pages = last - first + 1;
/* Allocate DMA pages */
dma->pages = kmalloc_array(dma->nr_pages,
sizeof(struct page *),
GFP_KERNEL);
if (!dma->pages)
return -ENOMEM;
dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n",
data, vkdata->size, dma->nr_pages);
dma->direction = direction;
/* Get user pages into memory */
err = get_user_pages_fast(data & PAGE_MASK,
dma->nr_pages,
direction == DMA_FROM_DEVICE,
dma->pages);
if (err != dma->nr_pages) {
dma->nr_pages = (err >= 0) ? err : 0;
dev_err(dev, "get_user_pages_fast, err=%d [%d]\n",
err, dma->nr_pages);
return err < 0 ? err : -EINVAL;
}
/* Max size of sg list is 1 per mapped page + fields at start */
dma->sglen = (dma->nr_pages * sizeof(*sgdata)) +
(sizeof(u32) * SGLIST_VKDATA_START);
/* Allocate sglist */
dma->sglist = dma_alloc_coherent(dev,
dma->sglen,
&dma->handle,
GFP_KERNEL);
if (!dma->sglist)
return -ENOMEM;
dma->sglist[SGLIST_NUM_SG] = 0;
dma->sglist[SGLIST_TOTALSIZE] = vkdata->size;
remaining_size = vkdata->size;
sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
/* Map all pages into DMA */
size = min_t(size_t, PAGE_SIZE - offset, remaining_size);
remaining_size -= size;
sg_addr = dma_map_page(dev,
dma->pages[0],
offset,
size,
dma->direction);
transfer_size = size;
if (unlikely(dma_mapping_error(dev, sg_addr))) {
__free_page(dma->pages[0]);
return -EIO;
}
for (i = 1; i < dma->nr_pages; i++) {
size = min_t(size_t, PAGE_SIZE, remaining_size);
remaining_size -= size;
addr = dma_map_page(dev,
dma->pages[i],
0,
size,
dma->direction);
if (unlikely(dma_mapping_error(dev, addr))) {
__free_page(dma->pages[i]);
return -EIO;
}
/*
* Compress SG list entry when pages are contiguous
* and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK
*/
if ((addr == (sg_addr + transfer_size)) &&
((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) {
/* pages are contiguous, add to same sg entry */
transfer_size += size;
} else {
/* pages are not contiguous, write sg entry */
sgdata->size = transfer_size;
put_unaligned(sg_addr, (u64 *)&sgdata->address);
dma->sglist[SGLIST_NUM_SG]++;
/* start new sg entry */
sgdata++;
sg_addr = addr;
transfer_size = size;
}
}
/* Write last sg list entry */
sgdata->size = transfer_size;
put_unaligned(sg_addr, (u64 *)&sgdata->address);
dma->sglist[SGLIST_NUM_SG]++;
/* Update pointers and size field to point to sglist */
put_unaligned((u64)dma->handle, &vkdata->address);
vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) +
(sizeof(u32) * SGLIST_VKDATA_START);
#ifdef BCM_VK_DUMP_SGLIST
dev_dbg(dev,
"sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n",
(u64)dma->sglist,
dma->handle,
dma->sglen,
vkdata->size);
for (i = 0; i < vkdata->size / sizeof(u32); i++)
dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]);
#endif
return 0;
}
int bcm_vk_sg_alloc(struct device *dev,
struct bcm_vk_dma *dma,
int dir,
struct _vk_data *vkdata,
int num)
{
int i;
int rc = -EINVAL;
/* Convert user addresses to DMA SG List */
for (i = 0; i < num; i++) {
if (vkdata[i].size && vkdata[i].address) {
/*
* If both size and address are non-zero
* then DMA alloc.
*/
rc = bcm_vk_dma_alloc(dev,
&dma[i],
dir,
&vkdata[i]);
} else if (vkdata[i].size ||
vkdata[i].address) {
/*
* If one of size and address are zero
* there is a problem.
*/
dev_err(dev,
"Invalid vkdata %x 0x%x 0x%llx\n",
i, vkdata[i].size, vkdata[i].address);
rc = -EINVAL;
} else {
/*
* If size and address are both zero
* don't convert, but return success.
*/
rc = 0;
}
if (rc)
goto fail_alloc;
}
return rc;
fail_alloc:
while (i > 0) {
i--;
if (dma[i].sglist)
bcm_vk_dma_free(dev, &dma[i]);
}
return rc;
}
static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma)
{
dma_addr_t addr;
int i;
int num_sg;
u32 size;
struct _vk_data *vkdata;
dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen);
/* Unmap all pages in the sglist */
num_sg = dma->sglist[SGLIST_NUM_SG];
vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
for (i = 0; i < num_sg; i++) {
size = vkdata[i].size;
addr = get_unaligned(&vkdata[i].address);
dma_unmap_page(dev, addr, size, dma->direction);
}
/* Free allocated sglist */
dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle);
/* Release lock on all pages */
for (i = 0; i < dma->nr_pages; i++)
put_page(dma->pages[i]);
/* Free allocated dma pages */
kfree(dma->pages);
dma->sglist = NULL;
return 0;
}
int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num,
int *proc_cnt)
{
int i;
*proc_cnt = 0;
/* Unmap and free all pages and sglists */
for (i = 0; i < num; i++) {
if (dma[i].sglist) {
bcm_vk_dma_free(dev, &dma[i]);
*proc_cnt += 1;
}
}
return 0;
}
Codebase Browser
linux