// SPDX-License-Identifier: GPL-2.0-only
/*
* VFIO PCI I/O Port & MMIO access
*
* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
* Author: Alex Williamson <[email protected]>
*
* Derived from original vfio:
* Copyright 2010 Cisco Systems, Inc. All rights reserved.
* Author: Tom Lyon, [email protected]
*/
#include <linux/fs.h>
#include <linux/pci.h>
#include <linux/uaccess.h>
#include <linux/io.h>
#include <linux/vfio.h>
#include <linux/vgaarb.h>
#include "vfio_pci_priv.h"
#ifdef __LITTLE_ENDIAN
#define vfio_ioread64 ioread64
#define vfio_iowrite64 iowrite64
#define vfio_ioread32 ioread32
#define vfio_iowrite32 iowrite32
#define vfio_ioread16 ioread16
#define vfio_iowrite16 iowrite16
#else
#define vfio_ioread64 ioread64be
#define vfio_iowrite64 iowrite64be
#define vfio_ioread32 ioread32be
#define vfio_iowrite32 iowrite32be
#define vfio_ioread16 ioread16be
#define vfio_iowrite16 iowrite16be
#endif
#define vfio_ioread8 ioread8
#define vfio_iowrite8 iowrite8
#define VFIO_IOWRITE(size) \
int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev, \
bool test_mem, u##size val, void __iomem *io) \
{ \
if (test_mem) { \
down_read(&vdev->memory_lock); \
if (!__vfio_pci_memory_enabled(vdev)) { \
up_read(&vdev->memory_lock); \
return -EIO; \
} \
} \
\
vfio_iowrite##size(val, io); \
\
if (test_mem) \
up_read(&vdev->memory_lock); \
\
return 0; \
} \
EXPORT_SYMBOL_GPL(vfio_pci_core_iowrite##size);
VFIO_IOWRITE(8)
VFIO_IOWRITE(16)
VFIO_IOWRITE(32)
#ifdef iowrite64
VFIO_IOWRITE(64)
#endif
#define VFIO_IOREAD(size) \
int vfio_pci_core_ioread##size(struct vfio_pci_core_device *vdev, \
bool test_mem, u##size *val, void __iomem *io) \
{ \
if (test_mem) { \
down_read(&vdev->memory_lock); \
if (!__vfio_pci_memory_enabled(vdev)) { \
up_read(&vdev->memory_lock); \
return -EIO; \
} \
} \
\
*val = vfio_ioread##size(io); \
\
if (test_mem) \
up_read(&vdev->memory_lock); \
\
return 0; \
} \
EXPORT_SYMBOL_GPL(vfio_pci_core_ioread##size);
VFIO_IOREAD(8)
VFIO_IOREAD(16)
VFIO_IOREAD(32)
#ifdef ioread64
VFIO_IOREAD(64)
#endif
#define VFIO_IORDWR(size) \
static int vfio_pci_iordwr##size(struct vfio_pci_core_device *vdev,\
bool iswrite, bool test_mem, \
void __iomem *io, char __user *buf, \
loff_t off, size_t *filled) \
{ \
u##size val; \
int ret; \
\
if (iswrite) { \
if (copy_from_user(&val, buf, sizeof(val))) \
return -EFAULT; \
\
ret = vfio_pci_core_iowrite##size(vdev, test_mem, \
val, io + off); \
if (ret) \
return ret; \
} else { \
ret = vfio_pci_core_ioread##size(vdev, test_mem, \
&val, io + off); \
if (ret) \
return ret; \
\
if (copy_to_user(buf, &val, sizeof(val))) \
return -EFAULT; \
} \
\
*filled = sizeof(val); \
return 0; \
} \
VFIO_IORDWR(8)
VFIO_IORDWR(16)
VFIO_IORDWR(32)
#if defined(ioread64) && defined(iowrite64)
VFIO_IORDWR(64)
#endif
/*
* Read or write from an __iomem region (MMIO or I/O port) with an excluded
* range which is inaccessible. The excluded range drops writes and fills
* reads with -1. This is intended for handling MSI-X vector tables and
* leftover space for ROM BARs.
*/
ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
void __iomem *io, char __user *buf,
loff_t off, size_t count, size_t x_start,
size_t x_end, bool iswrite)
{
ssize_t done = 0;
int ret;
while (count) {
size_t fillable, filled;
if (off < x_start)
fillable = min(count, (size_t)(x_start - off));
else if (off >= x_end)
fillable = count;
else
fillable = 0;
#if defined(ioread64) && defined(iowrite64)
if (fillable >= 8 && !(off % 8)) {
ret = vfio_pci_iordwr64(vdev, iswrite, test_mem,
io, buf, off, &filled);
if (ret)
return ret;
} else
#endif
if (fillable >= 4 && !(off % 4)) {
ret = vfio_pci_iordwr32(vdev, iswrite, test_mem,
io, buf, off, &filled);
if (ret)
return ret;
} else if (fillable >= 2 && !(off % 2)) {
ret = vfio_pci_iordwr16(vdev, iswrite, test_mem,
io, buf, off, &filled);
if (ret)
return ret;
} else if (fillable) {
ret = vfio_pci_iordwr8(vdev, iswrite, test_mem,
io, buf, off, &filled);
if (ret)
return ret;
} else {
/* Fill reads with -1, drop writes */
filled = min(count, (size_t)(x_end - off));
if (!iswrite) {
u8 val = 0xFF;
size_t i;
for (i = 0; i < filled; i++)
if (copy_to_user(buf + i, &val, 1))
return -EFAULT;
}
}
count -= filled;
done += filled;
off += filled;
buf += filled;
}
return done;
}
EXPORT_SYMBOL_GPL(vfio_pci_core_do_io_rw);
int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
{
struct pci_dev *pdev = vdev->pdev;
int ret;
void __iomem *io;
if (vdev->barmap[bar])
return 0;
ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
if (ret)
return ret;
io = pci_iomap(pdev, bar, 0);
if (!io) {
pci_release_selected_regions(pdev, 1 << bar);
return -ENOMEM;
}
vdev->barmap[bar] = io;
return 0;
}
EXPORT_SYMBOL_GPL(vfio_pci_core_setup_barmap);
ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
size_t count, loff_t *ppos, bool iswrite)
{
struct pci_dev *pdev = vdev->pdev;
loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
int bar = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
size_t x_start = 0, x_end = 0;
resource_size_t end;
void __iomem *io;
struct resource *res = &vdev->pdev->resource[bar];
ssize_t done;
if (pci_resource_start(pdev, bar))
end = pci_resource_len(pdev, bar);
else if (bar == PCI_ROM_RESOURCE &&
pdev->resource[bar].flags & IORESOURCE_ROM_SHADOW)
end = 0x20000;
else
return -EINVAL;
if (pos >= end)
return -EINVAL;
count = min(count, (size_t)(end - pos));
if (bar == PCI_ROM_RESOURCE) {
/*
* The ROM can fill less space than the BAR, so we start the
* excluded range at the end of the actual ROM. This makes
* filling large ROM BARs much faster.
*/
io = pci_map_rom(pdev, &x_start);
if (!io) {
done = -ENOMEM;
goto out;
}
x_end = end;
} else {
int ret = vfio_pci_core_setup_barmap(vdev, bar);
if (ret) {
done = ret;
goto out;
}
io = vdev->barmap[bar];
}
if (bar == vdev->msix_bar) {
x_start = vdev->msix_offset;
x_end = vdev->msix_offset + vdev->msix_size;
}
done = vfio_pci_core_do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos,
count, x_start, x_end, iswrite);
if (done >= 0)
*ppos += done;
if (bar == PCI_ROM_RESOURCE)
pci_unmap_rom(pdev, io);
out:
return done;
}
#ifdef CONFIG_VFIO_PCI_VGA
ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
size_t count, loff_t *ppos, bool iswrite)
{
int ret;
loff_t off, pos = *ppos & VFIO_PCI_OFFSET_MASK;
void __iomem *iomem = NULL;
unsigned int rsrc;
bool is_ioport;
ssize_t done;
if (!vdev->has_vga)
return -EINVAL;
if (pos > 0xbfffful)
return -EINVAL;
switch ((u32)pos) {
case 0xa0000 ... 0xbffff:
count = min(count, (size_t)(0xc0000 - pos));
iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1);
off = pos - 0xa0000;
rsrc = VGA_RSRC_LEGACY_MEM;
is_ioport = false;
break;
case 0x3b0 ... 0x3bb:
count = min(count, (size_t)(0x3bc - pos));
iomem = ioport_map(0x3b0, 0x3bb - 0x3b0 + 1);
off = pos - 0x3b0;
rsrc = VGA_RSRC_LEGACY_IO;
is_ioport = true;
break;
case 0x3c0 ... 0x3df:
count = min(count, (size_t)(0x3e0 - pos));
iomem = ioport_map(0x3c0, 0x3df - 0x3c0 + 1);
off = pos - 0x3c0;
rsrc = VGA_RSRC_LEGACY_IO;
is_ioport = true;
break;
default:
return -EINVAL;
}
if (!iomem)
return -ENOMEM;
ret = vga_get_interruptible(vdev->pdev, rsrc);
if (ret) {
is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
return ret;
}
/*
* VGA MMIO is a legacy, non-BAR resource that hopefully allows
* probing, so we don't currently worry about access in relation
* to the memory enable bit in the command register.
*/
done = vfio_pci_core_do_io_rw(vdev, false, iomem, buf, off, count,
0, 0, iswrite);
vga_put(vdev->pdev, rsrc);
is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
if (done >= 0)
*ppos += done;
return done;
}
#endif
static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
bool test_mem)
{
switch (ioeventfd->count) {
case 1:
vfio_pci_core_iowrite8(ioeventfd->vdev, test_mem,
ioeventfd->data, ioeventfd->addr);
break;
case 2:
vfio_pci_core_iowrite16(ioeventfd->vdev, test_mem,
ioeventfd->data, ioeventfd->addr);
break;
case 4:
vfio_pci_core_iowrite32(ioeventfd->vdev, test_mem,
ioeventfd->data, ioeventfd->addr);
break;
#ifdef iowrite64
case 8:
vfio_pci_core_iowrite64(ioeventfd->vdev, test_mem,
ioeventfd->data, ioeventfd->addr);
break;
#endif
}
}
static int vfio_pci_ioeventfd_handler(void *opaque, void *unused)
{
struct vfio_pci_ioeventfd *ioeventfd = opaque;
struct vfio_pci_core_device *vdev = ioeventfd->vdev;
if (ioeventfd->test_mem) {
if (!down_read_trylock(&vdev->memory_lock))
return 1; /* Lock contended, use thread */
if (!__vfio_pci_memory_enabled(vdev)) {
up_read(&vdev->memory_lock);
return 0;
}
}
vfio_pci_ioeventfd_do_write(ioeventfd, false);
if (ioeventfd->test_mem)
up_read(&vdev->memory_lock);
return 0;
}
static void vfio_pci_ioeventfd_thread(void *opaque, void *unused)
{
struct vfio_pci_ioeventfd *ioeventfd = opaque;
vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
}
int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
uint64_t data, int count, int fd)
{
struct pci_dev *pdev = vdev->pdev;
loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
struct vfio_pci_ioeventfd *ioeventfd;
/* Only support ioeventfds into BARs */
if (bar > VFIO_PCI_BAR5_REGION_INDEX)
return -EINVAL;
if (pos + count > pci_resource_len(pdev, bar))
return -EINVAL;
/* Disallow ioeventfds working around MSI-X table writes */
if (bar == vdev->msix_bar &&
!(pos + count <= vdev->msix_offset ||
pos >= vdev->msix_offset + vdev->msix_size))
return -EINVAL;
#ifndef iowrite64
if (count == 8)
return -EINVAL;
#endif
ret = vfio_pci_core_setup_barmap(vdev, bar);
if (ret)
return ret;
mutex_lock(&vdev->ioeventfds_lock);
list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
ioeventfd->data == data && ioeventfd->count == count) {
if (fd == -1) {
vfio_virqfd_disable(&ioeventfd->virqfd);
list_del(&ioeventfd->next);
vdev->ioeventfds_nr--;
kfree(ioeventfd);
ret = 0;
} else
ret = -EEXIST;
goto out_unlock;
}
}
if (fd < 0) {
ret = -ENODEV;
goto out_unlock;
}
if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
ret = -ENOSPC;
goto out_unlock;
}
ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL_ACCOUNT);
if (!ioeventfd) {
ret = -ENOMEM;
goto out_unlock;
}
ioeventfd->vdev = vdev;
ioeventfd->addr = vdev->barmap[bar] + pos;
ioeventfd->data = data;
ioeventfd->pos = pos;
ioeventfd->bar = bar;
ioeventfd->count = count;
ioeventfd->test_mem = vdev->pdev->resource[bar].flags & IORESOURCE_MEM;
ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler,
vfio_pci_ioeventfd_thread, NULL,
&ioeventfd->virqfd, fd);
if (ret) {
kfree(ioeventfd);
goto out_unlock;
}
list_add(&ioeventfd->next, &vdev->ioeventfds_list);
vdev->ioeventfds_nr++;
out_unlock:
mutex_unlock(&vdev->ioeventfds_lock);
return ret;
}
Codebase Browser
linux