linux/drivers/iommu/intel/pasid.c

// SPDX-License-Identifier: GPL-2.0
/*
 * intel-pasid.c - PASID idr, table and entry manipulation
 *
 * Copyright (C) 2018 Intel Corporation
 *
 * Author: Lu Baolu <[email protected]>
 */

#define pr_fmt(fmt) …

#include <linux/bitops.h>
#include <linux/cpufeature.h>
#include <linux/dmar.h>
#include <linux/iommu.h>
#include <linux/memory.h>
#include <linux/pci.h>
#include <linux/pci-ats.h>
#include <linux/spinlock.h>

#include "iommu.h"
#include "pasid.h"
#include "../iommu-pages.h"

/*
 * Intel IOMMU system wide PASID name space:
 */
u32 intel_pasid_max_id = …;

/*
 * Per device pasid table management:
 */

/*
 * Allocate a pasid table for @dev. It should be called in a
 * single-thread context.
 */
int intel_pasid_alloc_table(struct device *dev)
{ … }

void intel_pasid_free_table(struct device *dev)
{ … }

struct pasid_table *intel_pasid_get_table(struct device *dev)
{ … }

static int intel_pasid_get_dev_max_id(struct device *dev)
{ … }

static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid)
{ … }

/*
 * Interfaces for PASID table entry manipulation:
 */
static void
intel_pasid_clear_entry(struct device *dev, u32 pasid, bool fault_ignore)
{ … }

static void
pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,
				    u16 did, u32 pasid)
{ … }

static void
devtlb_invalidation_with_pasid(struct intel_iommu *iommu,
			       struct device *dev, u32 pasid)
{ … }

void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
				 u32 pasid, bool fault_ignore)
{ … }

/*
 * This function flushes cache for a newly setup pasid table entry.
 * Caller of it should not modify the in-use pasid table entries.
 */
static void pasid_flush_caches(struct intel_iommu *iommu,
				struct pasid_entry *pte,
			       u32 pasid, u16 did)
{ … }

/*
 * Set up the scalable mode pasid table entry for first only
 * translation type.
 */
int intel_pasid_setup_first_level(struct intel_iommu *iommu,
				  struct device *dev, pgd_t *pgd,
				  u32 pasid, u16 did, int flags)
{ … }

/*
 * Skip top levels of page tables for iommu which has less agaw
 * than default. Unnecessary for PT mode.
 */
static int iommu_skip_agaw(struct dmar_domain *domain,
			   struct intel_iommu *iommu,
			   struct dma_pte **pgd)
{ … }

/*
 * Set up the scalable mode pasid entry for second only translation type.
 */
int intel_pasid_setup_second_level(struct intel_iommu *iommu,
				   struct dmar_domain *domain,
				   struct device *dev, u32 pasid)
{ … }

/*
 * Set up dirty tracking on a second only or nested translation type.
 */
int intel_pasid_setup_dirty_tracking(struct intel_iommu *iommu,
				     struct device *dev, u32 pasid,
				     bool enabled)
{ … }

/*
 * Set up the scalable mode pasid entry for passthrough translation type.
 */
int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
				   struct device *dev, u32 pasid)
{ … }

/*
 * Set the page snoop control for a pasid entry which has been set up.
 */
void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,
					  struct device *dev, u32 pasid)
{ … }

/**
 * intel_pasid_setup_nested() - Set up PASID entry for nested translation.
 * @iommu:      IOMMU which the device belong to
 * @dev:        Device to be set up for translation
 * @pasid:      PASID to be programmed in the device PASID table
 * @domain:     User stage-1 domain nested on a stage-2 domain
 *
 * This is used for nested translation. The input domain should be
 * nested type and nested on a parent with 'is_nested_parent' flag
 * set.
 */
int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
			     u32 pasid, struct dmar_domain *domain)
{ … }

/*
 * Interfaces to setup or teardown a pasid table to the scalable-mode
 * context table entry:
 */

static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn)
{ … }

static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias, void *data)
{ … }

void intel_pasid_teardown_sm_context(struct device *dev)
{ … }

/*
 * Get the PASID directory size for scalable mode context entry.
 * Value of X in the PDTS field of a scalable mode context entry
 * indicates PASID directory with 2^(X + 7) entries.
 */
static unsigned long context_get_sm_pds(struct pasid_table *table)
{ … }

static int context_entry_set_pasid_table(struct context_entry *context,
					 struct device *dev)
{ … }

static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn)
{ … }

static int pci_pasid_table_setup(struct pci_dev *pdev, u16 alias, void *data)
{ … }

/*
 * Set the device's PASID table to its context table entry.
 *
 * The PASID table is set to the context entries of both device itself
 * and its alias requester ID for DMA.
 */
int intel_pasid_setup_sm_context(struct device *dev)
{ … }

/*
 * Global Device-TLB invalidation following changes in a context entry which
 * was present.
 */
static void __context_flush_dev_iotlb(struct device_domain_info *info)
{ … }

/*
 * Cache invalidations after change in a context table entry that was present
 * according to the Spec 6.5.3.3 (Guidance to Software for Invalidations). If
 * IOMMU is in scalable mode and all PASID table entries of the device were
 * non-present, set flush_domains to false. Otherwise, true.
 */
void intel_context_flush_present(struct device_domain_info *info,
				 struct context_entry *context,
				 u16 did, bool flush_domains)
{ … }