// SPDX-License-Identifier: GPL-2.0 /* * Resource Director Technology (RDT) * * Pseudo-locking support built on top of Cache Allocation Technology (CAT) * * Copyright (C) 2018 Intel Corporation * * Author: Reinette Chatre <[email protected]> */ #define pr_fmt(fmt) … #include <linux/cpu.h> #include <linux/cpumask.h> #include <linux/debugfs.h> #include <linux/kthread.h> #include <linux/mman.h> #include <linux/perf_event.h> #include <linux/pm_qos.h> #include <linux/slab.h> #include <linux/uaccess.h> #include <asm/cacheflush.h> #include <asm/cpu_device_id.h> #include <asm/resctrl.h> #include <asm/perf_event.h> #include "../../events/perf_event.h" /* For X86_CONFIG() */ #include "internal.h" #define CREATE_TRACE_POINTS #include "trace.h" /* * The bits needed to disable hardware prefetching varies based on the * platform. During initialization we will discover which bits to use. */ static u64 prefetch_disable_bits; /* * Major number assigned to and shared by all devices exposing * pseudo-locked regions. */ static unsigned int pseudo_lock_major; static unsigned long pseudo_lock_minor_avail = …; static char *pseudo_lock_devnode(const struct device *dev, umode_t *mode) { … } static const struct class pseudo_lock_class = …; /** * get_prefetch_disable_bits - prefetch disable bits of supported platforms * @void: It takes no parameters. * * Capture the list of platforms that have been validated to support * pseudo-locking. This includes testing to ensure pseudo-locked regions * with low cache miss rates can be created under variety of load conditions * as well as that these pseudo-locked regions can maintain their low cache * miss rates under variety of load conditions for significant lengths of time. * * After a platform has been validated to support pseudo-locking its * hardware prefetch disable bits are included here as they are documented * in the SDM. * * When adding a platform here also add support for its cache events to * measure_cycles_perf_fn() * * Return: * If platform is supported, the bits to disable hardware prefetchers, 0 * if platform is not supported. */ static u64 get_prefetch_disable_bits(void) { … } /** * pseudo_lock_minor_get - Obtain available minor number * @minor: Pointer to where new minor number will be stored * * A bitmask is used to track available minor numbers. Here the next free * minor number is marked as unavailable and returned. * * Return: 0 on success, <0 on failure. */ static int pseudo_lock_minor_get(unsigned int *minor) { … } /** * pseudo_lock_minor_release - Return minor number to available * @minor: The minor number made available */ static void pseudo_lock_minor_release(unsigned int minor) { … } /** * region_find_by_minor - Locate a pseudo-lock region by inode minor number * @minor: The minor number of the device representing pseudo-locked region * * When the character device is accessed we need to determine which * pseudo-locked region it belongs to. This is done by matching the minor * number of the device to the pseudo-locked region it belongs. * * Minor numbers are assigned at the time a pseudo-locked region is associated * with a cache instance. * * Return: On success return pointer to resource group owning the pseudo-locked * region, NULL on failure. */ static struct rdtgroup *region_find_by_minor(unsigned int minor) { … } /** * struct pseudo_lock_pm_req - A power management QoS request list entry * @list: Entry within the @pm_reqs list for a pseudo-locked region * @req: PM QoS request */ struct pseudo_lock_pm_req { … }; static void pseudo_lock_cstates_relax(struct pseudo_lock_region *plr) { … } /** * pseudo_lock_cstates_constrain - Restrict cores from entering C6 * @plr: Pseudo-locked region * * To prevent the cache from being affected by power management entering * C6 has to be avoided. This is accomplished by requesting a latency * requirement lower than lowest C6 exit latency of all supported * platforms as found in the cpuidle state tables in the intel_idle driver. * At this time it is possible to do so with a single latency requirement * for all supported platforms. * * Since Goldmont is supported, which is affected by X86_BUG_MONITOR, * the ACPI latencies need to be considered while keeping in mind that C2 * may be set to map to deeper sleep states. In this case the latency * requirement needs to prevent entering C2 also. * * Return: 0 on success, <0 on failure */ static int pseudo_lock_cstates_constrain(struct pseudo_lock_region *plr) { … } /** * pseudo_lock_region_clear - Reset pseudo-lock region data * @plr: pseudo-lock region * * All content of the pseudo-locked region is reset - any memory allocated * freed. * * Return: void */ static void pseudo_lock_region_clear(struct pseudo_lock_region *plr) { … } /** * pseudo_lock_region_init - Initialize pseudo-lock region information * @plr: pseudo-lock region * * Called after user provided a schemata to be pseudo-locked. From the * schemata the &struct pseudo_lock_region is on entry already initialized * with the resource, domain, and capacity bitmask. Here the information * required for pseudo-locking is deduced from this data and &struct * pseudo_lock_region initialized further. This information includes: * - size in bytes of the region to be pseudo-locked * - cache line size to know the stride with which data needs to be accessed * to be pseudo-locked * - a cpu associated with the cache instance on which the pseudo-locking * flow can be executed * * Return: 0 on success, <0 on failure. Descriptive error will be written * to last_cmd_status buffer. */ static int pseudo_lock_region_init(struct pseudo_lock_region *plr) { … } /** * pseudo_lock_init - Initialize a pseudo-lock region * @rdtgrp: resource group to which new pseudo-locked region will belong * * A pseudo-locked region is associated with a resource group. When this * association is created the pseudo-locked region is initialized. The * details of the pseudo-locked region are not known at this time so only * allocation is done and association established. * * Return: 0 on success, <0 on failure */ static int pseudo_lock_init(struct rdtgroup *rdtgrp) { … } /** * pseudo_lock_region_alloc - Allocate kernel memory that will be pseudo-locked * @plr: pseudo-lock region * * Initialize the details required to set up the pseudo-locked region and * allocate the contiguous memory that will be pseudo-locked to the cache. * * Return: 0 on success, <0 on failure. Descriptive error will be written * to last_cmd_status buffer. */ static int pseudo_lock_region_alloc(struct pseudo_lock_region *plr) { … } /** * pseudo_lock_free - Free a pseudo-locked region * @rdtgrp: resource group to which pseudo-locked region belonged * * The pseudo-locked region's resources have already been released, or not * yet created at this point. Now it can be freed and disassociated from the * resource group. * * Return: void */ static void pseudo_lock_free(struct rdtgroup *rdtgrp) { … } /** * pseudo_lock_fn - Load kernel memory into cache * @_rdtgrp: resource group to which pseudo-lock region belongs * * This is the core pseudo-locking flow. * * First we ensure that the kernel memory cannot be found in the cache. * Then, while taking care that there will be as little interference as * possible, the memory to be loaded is accessed while core is running * with class of service set to the bitmask of the pseudo-locked region. * After this is complete no future CAT allocations will be allowed to * overlap with this bitmask. * * Local register variables are utilized to ensure that the memory region * to be locked is the only memory access made during the critical locking * loop. * * Return: 0. Waiter on waitqueue will be woken on completion. */ static int pseudo_lock_fn(void *_rdtgrp) { … } /** * rdtgroup_monitor_in_progress - Test if monitoring in progress * @rdtgrp: resource group being queried * * Return: 1 if monitor groups have been created for this resource * group, 0 otherwise. */ static int rdtgroup_monitor_in_progress(struct rdtgroup *rdtgrp) { … } /** * rdtgroup_locksetup_user_restrict - Restrict user access to group * @rdtgrp: resource group needing access restricted * * A resource group used for cache pseudo-locking cannot have cpus or tasks * assigned to it. This is communicated to the user by restricting access * to all the files that can be used to make such changes. * * Permissions restored with rdtgroup_locksetup_user_restore() * * Return: 0 on success, <0 on failure. If a failure occurs during the * restriction of access an attempt will be made to restore permissions but * the state of the mode of these files will be uncertain when a failure * occurs. */ static int rdtgroup_locksetup_user_restrict(struct rdtgroup *rdtgrp) { … } /** * rdtgroup_locksetup_user_restore - Restore user access to group * @rdtgrp: resource group needing access restored * * Restore all file access previously removed using * rdtgroup_locksetup_user_restrict() * * Return: 0 on success, <0 on failure. If a failure occurs during the * restoration of access an attempt will be made to restrict permissions * again but the state of the mode of these files will be uncertain when * a failure occurs. */ static int rdtgroup_locksetup_user_restore(struct rdtgroup *rdtgrp) { … } /** * rdtgroup_locksetup_enter - Resource group enters locksetup mode * @rdtgrp: resource group requested to enter locksetup mode * * A resource group enters locksetup mode to reflect that it would be used * to represent a pseudo-locked region and is in the process of being set * up to do so. A resource group used for a pseudo-locked region would * lose the closid associated with it so we cannot allow it to have any * tasks or cpus assigned nor permit tasks or cpus to be assigned in the * future. Monitoring of a pseudo-locked region is not allowed either. * * The above and more restrictions on a pseudo-locked region are checked * for and enforced before the resource group enters the locksetup mode. * * Returns: 0 if the resource group successfully entered locksetup mode, <0 * on failure. On failure the last_cmd_status buffer is updated with text to * communicate details of failure to the user. */ int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp) { … } /** * rdtgroup_locksetup_exit - resource group exist locksetup mode * @rdtgrp: resource group * * When a resource group exits locksetup mode the earlier restrictions are * lifted. * * Return: 0 on success, <0 on failure */ int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp) { … } /** * rdtgroup_cbm_overlaps_pseudo_locked - Test if CBM or portion is pseudo-locked * @d: RDT domain * @cbm: CBM to test * * @d represents a cache instance and @cbm a capacity bitmask that is * considered for it. Determine if @cbm overlaps with any existing * pseudo-locked region on @d. * * @cbm is unsigned long, even if only 32 bits are used, to make the * bitmap functions work correctly. * * Return: true if @cbm overlaps with pseudo-locked region on @d, false * otherwise. */ bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_ctrl_domain *d, unsigned long cbm) { … } /** * rdtgroup_pseudo_locked_in_hierarchy - Pseudo-locked region in cache hierarchy * @d: RDT domain under test * * The setup of a pseudo-locked region affects all cache instances within * the hierarchy of the region. It is thus essential to know if any * pseudo-locked regions exist within a cache hierarchy to prevent any * attempts to create new pseudo-locked regions in the same hierarchy. * * Return: true if a pseudo-locked region exists in the hierarchy of @d or * if it is not possible to test due to memory allocation issue, * false otherwise. */ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d) { … } /** * measure_cycles_lat_fn - Measure cycle latency to read pseudo-locked memory * @_plr: pseudo-lock region to measure * * There is no deterministic way to test if a memory region is cached. One * way is to measure how long it takes to read the memory, the speed of * access is a good way to learn how close to the cpu the data was. Even * more, if the prefetcher is disabled and the memory is read at a stride * of half the cache line, then a cache miss will be easy to spot since the * read of the first half would be significantly slower than the read of * the second half. * * Return: 0. Waiter on waitqueue will be woken on completion. */ static int measure_cycles_lat_fn(void *_plr) { … } /* * Create a perf_event_attr for the hit and miss perf events that will * be used during the performance measurement. A perf_event maintains * a pointer to its perf_event_attr so a unique attribute structure is * created for each perf_event. * * The actual configuration of the event is set right before use in order * to use the X86_CONFIG macro. */ static struct perf_event_attr perf_miss_attr = …; static struct perf_event_attr perf_hit_attr = …; struct residency_counts { … }; static int measure_residency_fn(struct perf_event_attr *miss_attr, struct perf_event_attr *hit_attr, struct pseudo_lock_region *plr, struct residency_counts *counts) { … } static int measure_l2_residency(void *_plr) { … } static int measure_l3_residency(void *_plr) { … } /** * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region * @rdtgrp: Resource group to which the pseudo-locked region belongs. * @sel: Selector of which measurement to perform on a pseudo-locked region. * * The measurement of latency to access a pseudo-locked region should be * done from a cpu that is associated with that pseudo-locked region. * Determine which cpu is associated with this region and start a thread on * that cpu to perform the measurement, wait for that thread to complete. * * Return: 0 on success, <0 on failure */ static int pseudo_lock_measure_cycles(struct rdtgroup *rdtgrp, int sel) { … } static ssize_t pseudo_lock_measure_trigger(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) { … } static const struct file_operations pseudo_measure_fops = …; /** * rdtgroup_pseudo_lock_create - Create a pseudo-locked region * @rdtgrp: resource group to which pseudo-lock region belongs * * Called when a resource group in the pseudo-locksetup mode receives a * valid schemata that should be pseudo-locked. Since the resource group is * in pseudo-locksetup mode the &struct pseudo_lock_region has already been * allocated and initialized with the essential information. If a failure * occurs the resource group remains in the pseudo-locksetup mode with the * &struct pseudo_lock_region associated with it, but cleared from all * information and ready for the user to re-attempt pseudo-locking by * writing the schemata again. * * Return: 0 if the pseudo-locked region was successfully pseudo-locked, <0 * on failure. Descriptive error will be written to last_cmd_status buffer. */ int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) { … } /** * rdtgroup_pseudo_lock_remove - Remove a pseudo-locked region * @rdtgrp: resource group to which the pseudo-locked region belongs * * The removal of a pseudo-locked region can be initiated when the resource * group is removed from user space via a "rmdir" from userspace or the * unmount of the resctrl filesystem. On removal the resource group does * not go back to pseudo-locksetup mode before it is removed, instead it is * removed directly. There is thus asymmetry with the creation where the * &struct pseudo_lock_region is removed here while it was not created in * rdtgroup_pseudo_lock_create(). * * Return: void */ void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp) { … } static int pseudo_lock_dev_open(struct inode *inode, struct file *filp) { … } static int pseudo_lock_dev_release(struct inode *inode, struct file *filp) { … } static int pseudo_lock_dev_mremap(struct vm_area_struct *area) { … } static const struct vm_operations_struct pseudo_mmap_ops = …; static int pseudo_lock_dev_mmap(struct file *filp, struct vm_area_struct *vma) { … } static const struct file_operations pseudo_lock_dev_fops = …; int rdt_pseudo_lock_init(void) { … } void rdt_pseudo_lock_release(void) { … }
Codebase Browser
linux